Enhance Atmos Configuration with atmos.d Support and Import Functionality

.golangci.yml

@@ -104,7 +104,7 @@ linters-settings:
       - name: add-constant
         arguments:
           - maxLitCount: "3"
-            allowStrs: '""'
+            allowStrs: '"","error","path","import"'
             allowInts: "0,1,2,3,4"
             allowFloats: "0.0,0.,1.0,1.,2.0,2."
       - name: argument-limit
@@ -114,7 +114,7 @@ linters-settings:
       - name: comment-spacings
         severity: warning
         disabled: false
-        exclude: [""]
+        exclude: []
         arguments:
           - mypragma
           - otherpragma

pkg/config/README.md

@@ -0,0 +1,44 @@
+### Import Config
+
+```mermaid
+flowchart TD
+    A["Load Configuration File"] --> B{"Import Section Exists?"}
+
+    B -- Yes --> C["Process Imports in Order"]
+    C --> D{"Import Type?"}
+    D --> E["Remote URL"]
+    D --> F["Specific Path"]
+    D --> G["Wildcard Globs"]
+
+    E --> H["Fetch Config from Remote URL"]
+    F --> I["Read Config from Filesystem"]
+    G --> I["Read Config from Filesystem"]
+
+    H --> J["Call Load Configuration File (Recursively)"]
+    I --> J["Call Load Configuration File (Recursively)"]
+
+    J --> L["Deep Merge with Current Config in Memory"]
+    L --> K{"More Imports to Process?"}
+    K -- Yes --> C
+    K -- No --> M["Configuration Processing Complete"]
+
+    %% Loopback for recursion
+    J -.-> A
+
+    %% Styling for clarity
+    style A fill:#A8DADC,stroke:#1D3557,stroke-width:2px,color:#000000
+    style B fill:#F4A261,stroke:#1D3557,stroke-width:2px,color:#FFFFFF
+    style C fill:#457B9D,stroke:#1D3557,stroke-width:2px,color:#FFFFFF
+    style D fill:#A8DADC,stroke:#1D3557,stroke-width:2px,color:#000000
+    style E fill:#E63946,stroke:#1D3557,stroke-width:2px,color:#FFFFFF
+    style F fill:#E63946,stroke:#1D3557,stroke-width:2px,color:#FFFFFF
+    style G fill:#E63946,stroke:#1D3557,stroke-width:2px,color:#FFFFFF
+    style H fill:#A8DADC,stroke:#1D3557,stroke-width:2px,color:#000000
+    style I fill:#A8DADC,stroke:#1D3557,stroke-width:2px,color:#000000
+    style J fill:#F4A261,stroke:#1D3557,stroke-width:2px,color:#FFFFFF
+    style L fill:#457B9D,stroke:#1D3557,stroke-width:2px,color:#FFFFFF
+    style K fill:#F4A261,stroke:#1D3557,stroke-width:2px,color:#FFFFFF
+    style M fill:#1D3557,stroke:#1D3557,stroke-width:2px,color:#FFFFFF
+
+    classDef recursion stroke-dasharray: 5 5;
+```

pkg/config/config.go

@@ -1,279 +1,24 @@
 package config
 
 import (
-	"bytes"
-	"encoding/json"
 	"fmt"
-	"os"
 	"path/filepath"
-	"runtime"
 
-	"github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
-	"github.com/spf13/viper"
 
-	"github.com/cloudposse/atmos/internal/tui/templates"
 	"github.com/cloudposse/atmos/pkg/schema"
-	"github.com/cloudposse/atmos/pkg/ui/theme"
 	u "github.com/cloudposse/atmos/pkg/utils"
 	"github.com/cloudposse/atmos/pkg/version"
 )
 
-var (
-	NotFound = errors.New("\n'atmos.yaml' CLI config was not found in any of the searched paths: system dir, home dir, current dir, ENV vars." +
-		"\nYou can download a sample config and adapt it to your requirements from " +
-		"https://raw.githubusercontent.com/cloudposse/atmos/main/examples/quick-start-advanced/atmos.yaml")
-
-	defaultCliConfig = schema.AtmosConfiguration{
-		Default:  true,
-		BasePath: ".",
-		Stacks: schema.Stacks{
-			BasePath:    "stacks",
-			NamePattern: "{tenant}-{environment}-{stage}",
-			IncludedPaths: []string{
-				"orgs/**/*",
-			},
-			ExcludedPaths: []string{
-				"**/_defaults.yaml",
-			},
-		},
-		Components: schema.Components{
-			Terraform: schema.Terraform{
-				BasePath:                "components/terraform",
-				ApplyAutoApprove:        false,
-				DeployRunInit:           true,
-				InitRunReconfigure:      true,
-				AutoGenerateBackendFile: true,
-				AppendUserAgent:         fmt.Sprintf("Atmos/%s (Cloud Posse; +https://atmos.tools)", version.Version),
-			},
-			Helmfile: schema.Helmfile{
-				BasePath:              "components/helmfile",
-				KubeconfigPath:        "",
-				HelmAwsProfilePattern: "{namespace}-{tenant}-gbl-{stage}-helm",
-				ClusterNamePattern:    "{namespace}-{tenant}-{environment}-{stage}-eks-cluster",
-				UseEKS:                true,
-			},
-		},
-		Settings: schema.AtmosSettings{
-			ListMergeStrategy: "replace",
-			Terminal: schema.Terminal{
-				MaxWidth: templates.GetTerminalWidth(),
-				Pager:    true,
-				Colors:   true,
-				Unicode:  true,
-				SyntaxHighlighting: schema.SyntaxHighlighting{
-					Enabled:                true,
-					Formatter:              "terminal",
-					Theme:                  "dracula",
-					HighlightedOutputPager: true,
-					LineNumbers:            true,
-					Wrap:                   false,
-				},
-			},
-		},
-		Workflows: schema.Workflows{
-			BasePath: "stacks/workflows",
-		},
-		Logs: schema.Logs{
-			File:  "/dev/stderr",
-			Level: "Info",
-		},
-		Schemas: schema.Schemas{
-			JsonSchema: schema.JsonSchema{
-				BasePath: "stacks/schemas/jsonschema",
-			},
-			Opa: schema.Opa{
-				BasePath: "stacks/schemas/opa",
-			},
-		},
-		Templates: schema.Templates{
-			Settings: schema.TemplatesSettings{
-				Enabled: true,
-				Sprig: schema.TemplatesSettingsSprig{
-					Enabled: true,
-				},
-				Gomplate: schema.TemplatesSettingsGomplate{
-					Enabled:     true,
-					Datasources: make(map[string]schema.TemplatesSettingsGomplateDatasource),
-				},
-			},
-		},
-		Initialized: true,
-		Version: schema.Version{
-			Check: schema.VersionCheck{
-				Enabled:   true,
-				Timeout:   1000,
-				Frequency: "daily",
-			},
-		},
-	}
-)
-
 // InitCliConfig finds and merges CLI configurations in the following order: system dir, home dir, current dir, ENV vars, command-line arguments
 // https://dev.to/techschoolguru/load-config-from-file-environment-variables-in-golang-with-viper-2j2d
 // https://medium.com/@bnprashanth256/reading-configuration-files-and-environment-variables-in-go-golang-c2607f912b63
 func InitCliConfig(configAndStacksInfo schema.ConfigAndStacksInfo, processStacks bool) (schema.AtmosConfiguration, error) {
-	// atmosConfig is loaded from the following locations (from lower to higher priority):
-	// system dir (`/usr/local/etc/atmos` on Linux, `%LOCALAPPDATA%/atmos` on Windows)
-	// home dir (~/.atmos)
-	// current directory
-	// ENV vars
-	// Command-line arguments
-
-	var atmosConfig schema.AtmosConfiguration
-	var err error
-	configFound := false
-	var found bool
-
-	v := viper.New()
-	v.SetConfigType("yaml")
-	v.SetTypeByDefaultValue(true)
-
-	// Default configuration values
-	v.SetDefault("components.helmfile.use_eks", true)
-	v.SetDefault("components.terraform.append_user_agent", fmt.Sprintf("Atmos/%s (Cloud Posse; +https://atmos.tools)", version.Version))
-	v.SetDefault("settings.inject_github_token", true)
-
-	v.SetDefault("logs.file", "/dev/stderr")
-	v.SetDefault("logs.level", "Info")
-
-	// Process config in system folder
-	configFilePath1 := ""
-	atmosConfigFilePath := ""
-	// https://pureinfotech.com/list-environment-variables-windows-10/
-	// https://docs.microsoft.com/en-us/windows/deployment/usmt/usmt-recognized-environment-variables
-	// https://softwareengineering.stackexchange.com/questions/299869/where-is-the-appropriate-place-to-put-application-configuration-files-for-each-p
-	// https://stackoverflow.com/questions/37946282/why-does-appdata-in-windows-7-seemingly-points-to-wrong-folder
-	if runtime.GOOS == "windows" {
-		appDataDir := os.Getenv(WindowsAppDataEnvVar)
-		if len(appDataDir) > 0 {
-			configFilePath1 = appDataDir
-		}
-	} else {
-		configFilePath1 = SystemDirConfigFilePath
-	}
-
-	if len(configFilePath1) > 0 {
-		configFile1 := filepath.Join(configFilePath1, CliConfigFileName)
-		found, err = processConfigFile(atmosConfig, configFile1, v)
-		if err != nil {
-			return atmosConfig, err
-		}
-		if found {
-			configFound = true
-			atmosConfigFilePath = configFile1
-		}
-	}
-
-	// Process config in user's HOME dir
-	configFilePath2, err := homedir.Dir()
-	if err != nil {
-		return atmosConfig, err
-	}
-	configFile2 := filepath.Join(configFilePath2, ".atmos", CliConfigFileName)
-	found, err = processConfigFile(atmosConfig, configFile2, v)
-	if err != nil {
-		return atmosConfig, err
-	}
-	if found {
-		configFound = true
-		atmosConfigFilePath = configFile2
-	}
-
-	// Process config in the current dir
-	configFilePath3, err := os.Getwd()
-	if err != nil {
-		return atmosConfig, err
-	}
-	configFile3 := filepath.Join(configFilePath3, CliConfigFileName)
-	found, err = processConfigFile(atmosConfig, configFile3, v)
-	if err != nil {
-		return atmosConfig, err
-	}
-	if found {
-		configFound = true
-		atmosConfigFilePath = configFile3
-	}
-
-	// Process config from the path in ENV var `ATMOS_CLI_CONFIG_PATH`
-	configFilePath4 := os.Getenv("ATMOS_CLI_CONFIG_PATH")
-	if len(configFilePath4) > 0 {
-		u.LogTrace(fmt.Sprintf("Found ENV var ATMOS_CLI_CONFIG_PATH=%s", configFilePath4))
-		configFile4 := filepath.Join(configFilePath4, CliConfigFileName)
-		found, err = processConfigFile(atmosConfig, configFile4, v)
-		if err != nil {
-			return atmosConfig, err
-		}
-		if found {
-			configFound = true
-			atmosConfigFilePath = configFile4
-		}
-	}
-
-	// Process config from the path specified in the Terraform provider (which calls into the atmos code)
-	if configAndStacksInfo.AtmosCliConfigPath != "" {
-		configFilePath5 := configAndStacksInfo.AtmosCliConfigPath
-		if len(configFilePath5) > 0 {
-			configFile5 := filepath.Join(configFilePath5, CliConfigFileName)
-			found, err = processConfigFile(atmosConfig, configFile5, v)
-			if err != nil {
-				return atmosConfig, err
-			}
-			if found {
-				configFound = true
-				atmosConfigFilePath = configFile5
-			}
-		}
-	}
-
-	if !configFound {
-		// If `atmos.yaml` not found, use the default config
-		// Set `ATMOS_LOGS_LEVEL` ENV var to "Debug" to see the message about Atmos using the default CLI config
-		logsLevelEnvVar := os.Getenv("ATMOS_LOGS_LEVEL")
-		if logsLevelEnvVar == u.LogLevelDebug || logsLevelEnvVar == u.LogLevelTrace {
-			u.PrintMessageInColor("'atmos.yaml' CLI config was not found in any of the searched paths: system dir, home dir, current dir, ENV vars.\n"+
-				"Refer to https://atmos.tools/cli/configuration for details on how to configure 'atmos.yaml'.\n"+
-				"Using the default CLI config:\n\n", theme.Colors.Info)
-
-			err = u.PrintAsYAMLToFileDescriptor(atmosConfig, defaultCliConfig)
-			if err != nil {
-				return atmosConfig, err
-			}
-			fmt.Println()
-		}
-
-		j, err := json.Marshal(defaultCliConfig)
-		if err != nil {
-			return atmosConfig, err
-		}
-
-		reader := bytes.NewReader(j)
-		err = v.MergeConfig(reader)
-		if err != nil {
-			return atmosConfig, err
-		}
-	}
-	// We want the editorconfig color by default to be true
-	atmosConfig.Validate.EditorConfig.Color = true
-	// https://gist.github.com/chazcheadle/45bf85b793dea2b71bd05ebaa3c28644
-	// https://sagikazarmark.hu/blog/decoding-custom-formats-with-viper/
-	err = v.Unmarshal(&atmosConfig)
+	atmosConfig, err := LoadConfig(&configAndStacksInfo)
 	if err != nil {
 		return atmosConfig, err
 	}
-
-	// Set the CLI config path in the atmosConfig struct
-	// get dir of atmosConfigFilePath
-	atmosConfigDir := filepath.Dir(atmosConfigFilePath)
-	if filepath.IsAbs(atmosConfigDir) {
-		atmosConfig.CliConfigPath = atmosConfigDir
-	} else {
-		absPath, err := filepath.Abs(atmosConfigDir)
-		if err != nil {
-			return atmosConfig, err
-		}
-		atmosConfig.CliConfigPath = absPath
-	}
 	// Process ENV vars
 	err = processEnvVars(&atmosConfig)
 	if err != nil {
@@ -388,37 +133,3 @@
 	atmosConfig.Initialized = true
 	return atmosConfig, nil
 }
-
-// https://github.com/NCAR/go-figure
-// https://github.com/spf13/viper/issues/181
-// https://medium.com/@bnprashanth256/reading-configuration-files-and-environment-variables-in-go-golang-c2607f912b63
-func processConfigFile(
-	atmosConfig schema.AtmosConfiguration,
-	path string,
-	v *viper.Viper,
-) (bool, error) {
-	// Check if the config file exists
-	configPath, fileExists := u.SearchConfigFile(path)
-	if !fileExists {
-		return false, nil
-	}
-
-	reader, err := os.Open(configPath)
-	if err != nil {
-		return false, err
-	}
-
-	defer func(reader *os.File) {
-		err := reader.Close()
-		if err != nil {
-			u.LogWarning(fmt.Sprintf("error closing file '%s'. %v", configPath, err))
-		}
-	}(reader)
-
-	err = v.MergeConfig(reader)
-	if err != nil {
-		return false, err
-	}
-
-	return true, nil
-}