leviathonbeast/ionscale
1
0
Fork 0
mirror of https://github.com/jsiebens/ionscale.git synced 2026-03-31 15:07:49 +01:00
Code Issues Packages Projects Releases Wiki Activity

feat: add environment variable substition in configuration, remove implicit use of env variables

Browse Source
This commit is contained in:
Johan Siebens
2025-03-07 09:21:47 +01:00
parent c1c708269c
commit 978b0ecf4f
4 changed files with 220 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/config/config.go
+91 -50
View File
@@ -3,9 +3,7 @@ package config
import (
"encoding/base64"
"fmt"
"github.com/caarlos0/env/v6"
"github.com/caddyserver/certmagic"
"github.com/imdario/mergo"
"github.com/jsiebens/ionscale/internal/domain"
"github.com/jsiebens/ionscale/internal/key"
"github.com/jsiebens/ionscale/internal/util"
@@ -14,6 +12,8 @@ import (
"net/url"
"os"
"path/filepath"
"regexp"
"strings"
"tailscale.com/tailcfg"
tkey "tailscale.com/types/key"
"time"
@@ -61,6 +61,11 @@ func LoadConfig(path string) (*Config, error) {
return nil, err
}
b, err = expandEnvVars(b)
if err != nil {
return nil, err
}
if err := yaml.Unmarshal(b, cfg); err != nil {
return nil, err
}
@@ -73,22 +78,16 @@ func LoadConfig(path string) (*Config, error) {
return nil, err
}
// merge env configuration on top of the default/file configuration
b, err = expandEnvVars(b)
if err != nil {
return nil, err
}
if err := yaml.Unmarshal(b, cfg); err != nil {
return nil, err
}
}
envCfg := &Config{}
if err := env.Parse(envCfg, env.Options{Prefix: "IONSCALE_"}); err != nil {
return nil, err
}
// merge env configuration on top of the default/file configuration
if err := mergo.Merge(cfg, envCfg, mergo.WithOverride); err != nil {
return nil, err
}
keepAliveInterval = cfg.PollNet.KeepAliveInterval
magicDNSSuffix = cfg.DNS.MagicDNSSuffix
@@ -143,19 +142,19 @@ type ServerKeys struct {
}
type Config struct {
ListenAddr string `yaml:"listen_addr,omitempty" env:"LISTEN_ADDR"`
StunListenAddr string `yaml:"stun_listen_addr,omitempty" env:"STUN_LISTEN_ADDR"`
MetricsListenAddr string `yaml:"metrics_listen_addr,omitempty" env:"METRICS_LISTEN_ADDR"`
PublicAddr string `yaml:"public_addr,omitempty" env:"PUBLIC_ADDR"`
StunPublicAddr string `yaml:"stun_public_addr,omitempty" env:"STUN_PUBLIC_ADDR"`
Tls Tls `yaml:"tls,omitempty" envPrefix:"TLS_"`
PollNet PollNet `yaml:"poll_net,omitempty" envPrefix:"POLL_NET_"`
Keys Keys `yaml:"keys,omitempty" envPrefix:"KEYS_"`
Database Database `yaml:"database,omitempty" envPrefix:"DB_"`
Auth Auth `yaml:"auth,omitempty" envPrefix:"AUTH_"`
ListenAddr string `yaml:"listen_addr,omitempty"`
StunListenAddr string `yaml:"stun_listen_addr,omitempty"`
MetricsListenAddr string `yaml:"metrics_listen_addr,omitempty"`
PublicAddr string `yaml:"public_addr,omitempty"`
StunPublicAddr string `yaml:"stun_public_addr,omitempty"`
Tls Tls `yaml:"tls,omitempty"`
PollNet PollNet `yaml:"poll_net,omitempty"`
Keys Keys `yaml:"keys,omitempty"`
Database Database `yaml:"database,omitempty"`
Auth Auth `yaml:"auth,omitempty"`
DNS DNS `yaml:"dns,omitempty"`
DERP DERP `yaml:"derp,omitempty" envPrefix:"DERP_"`
Logging Logging `yaml:"logging,omitempty" envPrefix:"LOGGING_"`
DERP DERP `yaml:"derp,omitempty"`
Logging Logging `yaml:"logging,omitempty"`
PublicUrl *url.URL `yaml:"-"`
@@ -166,50 +165,50 @@ type Config struct {
}
type Tls struct {
Disable bool `yaml:"disable" env:"DISABLE"`
ForceHttps bool `yaml:"force_https" env:"FORCE_HTTPS"`
CertFile string `yaml:"cert_file,omitempty" env:"CERT_FILE"`
KeyFile string `yaml:"key_file,omitempty" env:"KEY_FILE"`
AcmeEnabled bool `yaml:"acme,omitempty" env:"ACME_ENABLED"`
AcmeEmail string `yaml:"acme_email,omitempty" env:"ACME_EMAIL"`
AcmeCA string `yaml:"acme_ca,omitempty" env:"ACME_CA"`
Disable bool `yaml:"disable"`
ForceHttps bool `yaml:"force_https"`
CertFile string `yaml:"cert_file,omitempty"`
KeyFile string `yaml:"key_file,omitempty"`
AcmeEnabled bool `yaml:"acme,omitempty"`
AcmeEmail string `yaml:"acme_email,omitempty"`
AcmeCA string `yaml:"acme_ca,omitempty"`
}
type PollNet struct {
KeepAliveInterval time.Duration `yaml:"keep_alive_interval" env:"KEEP_ALIVE_INTERVAL"`
KeepAliveInterval time.Duration `yaml:"keep_alive_interval"`
}
type Logging struct {
Level string `yaml:"level,omitempty" env:"LEVEL"`
Format string `yaml:"format,omitempty" env:"FORMAT"`
File string `yaml:"file,omitempty" env:"FILE"`
Level string `yaml:"level,omitempty"`
Format string `yaml:"format,omitempty"`
File string `yaml:"file,omitempty"`
}
type Database struct {
Type string `yaml:"type,omitempty" env:"TYPE"`
Url string `yaml:"url,omitempty" env:"URL"`
MaxOpenConns int `yaml:"max_open_conns,omitempty" env:"MAX_OPEN_CONNS"`
MaxIdleConns int `yaml:"max_idle_conns,omitempty" env:"MAX_IDLE_CONNS"`
ConnMaxLifetime time.Duration `yaml:"conn_max_life_time,omitempty" env:"CONN_MAX_LIFE_TIME"`
ConnMaxIdleTime time.Duration `yaml:"conn_max_idle_time,omitempty" env:"CONN_MAX_IDLE_TIME"`
Type string `yaml:"type,omitempty"`
Url string `yaml:"url,omitempty"`
MaxOpenConns int `yaml:"max_open_conns,omitempty"`
MaxIdleConns int `yaml:"max_idle_conns,omitempty"`
ConnMaxLifetime time.Duration `yaml:"conn_max_life_time,omitempty"`
ConnMaxIdleTime time.Duration `yaml:"conn_max_idle_time,omitempty"`
}
type Keys struct {
ControlKey string `yaml:"control_key,omitempty" env:"CONTROL_KEY"`
LegacyControlKey string `yaml:"legacy_control_key,omitempty" env:"LEGACY_CONTROL_KEY"`
SystemAdminKey string `yaml:"system_admin_key,omitempty" env:"SYSTEM_ADMIN_KEY"`
ControlKey string `yaml:"control_key,omitempty"`
LegacyControlKey string `yaml:"legacy_control_key,omitempty"`
SystemAdminKey string `yaml:"system_admin_key,omitempty"`
}
type Auth struct {
Provider AuthProvider `yaml:"provider,omitempty" envPrefix:"PROVIDER_"`
Provider AuthProvider `yaml:"provider,omitempty"`
SystemAdminPolicy SystemAdminPolicy `yaml:"system_admins"`
}
type AuthProvider struct {
Issuer string `yaml:"issuer" env:"ISSUER"`
ClientID string `yaml:"client_id" env:"CLIENT_ID"`
ClientSecret string `yaml:"client_secret" env:"CLIENT_SECRET"`
Scopes []string `yaml:"additional_scopes" env:"SCOPES"`
Issuer string `yaml:"issuer"`
ClientID string `yaml:"client_id"`
ClientSecret string `yaml:"client_secret"`
Scopes []string `yaml:"additional_scopes" `
}
type DNS struct {
@@ -356,3 +355,45 @@ func (c *Config) DefaultDERPMap() *tailcfg.DERPMap {
},
}
}
// Match ${VAR:default} syntax for variables with default values
var optionalEnvRegex = regexp.MustCompile(`\${([a-zA-Z0-9_]+):([^}]*)}`)
// Match ${VAR} syntax (without default) - these are required
var requiredEnvRegex = regexp.MustCompile(`\${([a-zA-Z0-9_]+)}`)
func expandEnvVars(config []byte) ([]byte, error) {
var result = config
var missingVars []string
result = optionalEnvRegex.ReplaceAllFunc(result, func(match []byte) []byte {
parts := optionalEnvRegex.FindSubmatch(match)
envVar := string(parts[1])
defaultValue := parts[2]
envValue := os.Getenv(envVar)
if envValue != "" {
return []byte(envValue)
}
return defaultValue
})
result = requiredEnvRegex.ReplaceAllFunc(result, func(match []byte) []byte {
parts := requiredEnvRegex.FindSubmatch(match)
envVar := string(parts[1])
envValue := os.Getenv(envVar)
if envValue == "" {
missingVars = append(missingVars, envVar)
return match
}
return []byte(envValue)
})
if len(missingVars) > 0 {
return nil, fmt.Errorf("missing required environment variables: %s", strings.Join(missingVars, ", "))
}
return result, nil
}
internal/config/config_test.go
+129
View File
@@ -0,0 +1,129 @@
package config
import (
"bytes"
"github.com/stretchr/testify/require"
"os"
"testing"
)
func TestLoadConfig(t *testing.T) {
// Create a temporary config file
tempFile, err := os.CreateTemp("", "config-*.yaml")
if err != nil {
t.Fatalf("Failed to create temp file: %v", err)
}
defer os.Remove(tempFile.Name())
// Write test configuration
yamlContent := `
public_addr: "ionscale.localtest.me:443"
stun_public_addr: "ionscale.localtest.me:3478"
database:
type: ${DB_TYPE:sqlite}
url: ${DB_URL}
max_open_conns: ${DB_MAX_OPEN_CONNS:5}
`
if _, err := tempFile.Write([]byte(yamlContent)); err != nil {
t.Fatalf("Failed to write to temp file: %v", err)
}
tempFile.Close()
t.Run("With DB_URL set", func(t *testing.T) {
require.NoError(t, os.Setenv("DB_URL", "./ionscale.db"))
config, err := LoadConfig(tempFile.Name())
require.NoError(t, err)
require.Equal(t, "sqlite", config.Database.Type)
require.Equal(t, "./ionscale.db", config.Database.Url)
require.Equal(t, 5, config.Database.MaxOpenConns)
})
t.Run("Without required DB_URL", func(t *testing.T) {
require.NoError(t, os.Unsetenv("DB_URL"))
_, err := LoadConfig(tempFile.Name())
require.Error(t, err)
})
}
func TestExpandEnvVars(t *testing.T) {
// Setup test environment variables
require.NoError(t, os.Setenv("TEST_VAR", "test_value"))
require.NoError(t, os.Setenv("PORT", "9090"))
// Ensure TEST_DEFAULT is not set
require.NoError(t, os.Unsetenv("TEST_DEFAULT"))
tests := []struct {
name string
input []byte
expected []byte
expectError bool
}{
{
name: "Braced variable",
input: []byte("Port: ${PORT}"),
expected: []byte("Port: 9090"),
expectError: false,
},
{
name: "Default value used",
input: []byte("Default: ${TEST_DEFAULT:fallback}"),
expected: []byte("Default: fallback"),
expectError: false,
},
{
name: "Default value not used when env var exists",
input: []byte("Not default: ${PORT:8080}"),
expected: []byte("Not default: 9090"),
expectError: false,
},
{
name: "Multiple replacements",
input: []byte("Config: ${TEST_VAR} ${PORT} ${TEST_DEFAULT:default}"),
expected: []byte("Config: test_value 9090 default"),
expectError: false,
},
{
name: "Missing required variable",
input: []byte("Required: ${MISSING_VAR}"),
expected: nil,
expectError: true,
},
{
name: "Mixed variables with one missing",
input: []byte("Mixed: ${TEST_VAR} ${MISSING_VAR} ${TEST_DEFAULT:default}"),
expected: nil,
expectError: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result, err := expandEnvVars(tt.input)
// Check error expectation
if tt.expectError && err == nil {
t.Errorf("expandEnvVars() expected error but got none")
return
}
if !tt.expectError && err != nil {
t.Errorf("expandEnvVars() got unexpected error: %v", err)
return
}
// If we expected an error, don't check the result further
if tt.expectError {
return
}
// Check result
if !bytes.Equal(result, tt.expected) {
t.Errorf("expandEnvVars() got = %s, want %s", result, tt.expected)
}
})
}
}
internal/config/funcs.go
-12
View File
@@ -25,18 +25,6 @@ func GetString(key, defaultValue string) string {
return defaultValue
}
func GetUint64(key string, defaultValue uint64) uint64 {
v := os.Getenv(key)
if v != "" {
vi, err := strconv.ParseUint(v, 10, 64)
if err != nil {
return defaultValue
}
return vi
}
return defaultValue
}
func validatePublicAddr(addr string) (*url.URL, string, int, error) {
scheme := "https"
internal/dns/provider.go
-63
View File
@@ -3,7 +3,6 @@ package dns
import (
"context"
"fmt"
"github.com/imdario/mergo"
"github.com/jsiebens/ionscale/internal/config"
"github.com/jsiebens/ionscale/internal/mapping"
"github.com/libdns/azure"
@@ -51,20 +50,6 @@ func configureAzureProvider(zone string, values map[string]string) (Provider, er
if err := mapping.CopyViaJson(values, p); err != nil {
return nil, err
}
e := &azure.Provider{
TenantId: config.GetString("IONSCALE_DNS_AZURE_TENANT_ID", ""),
ClientId: config.GetString("IONSCALE_DNS_AZURE_CLIENT_ID", ""),
ClientSecret: config.GetString("IONSCALE_DNS_AZURE_CLIENT_SECRET", ""),
SubscriptionId: config.GetString("IONSCALE_DNS_AZURE_SUBSCRIPTION_ID", ""),
ResourceGroupName: config.GetString("IONSCALE_DNS_AZURE_RESOURCE_GROUP_NAME", ""),
}
// merge env configuration on top of the default/file configuration
if err := mergo.Merge(p, e, mergo.WithOverride); err != nil {
return nil, err
}
return &externalProvider{zone: fqdn(zone), setter: p}, nil
}
@@ -73,16 +58,6 @@ func configureCloudflareProvider(zone string, values map[string]string) (Provide
if err := mapping.CopyViaJson(values, p); err != nil {
return nil, err
}
e := &cloudflare.Provider{
APIToken: config.GetString("IONSCALE_DNS_CLOUDFLARE_API_TOKEN", ""),
}
// merge env configuration on top of the default/file configuration
if err := mergo.Merge(p, e, mergo.WithOverride); err != nil {
return nil, err
}
return &externalProvider{zone: fqdn(zone), setter: p}, nil
}
@@ -91,16 +66,6 @@ func configureDigitalOceanProvider(zone string, values map[string]string) (Provi
if err := mapping.CopyViaJson(values, p); err != nil {
return nil, err
}
e := &digitalocean.Provider{
APIToken: config.GetString("IONSCALE_DNS_DIGITALOCEAN_API_TOKEN", ""),
}
// merge env configuration on top of the default/file configuration
if err := mergo.Merge(p, e, mergo.WithOverride); err != nil {
return nil, err
}
return &externalProvider{zone: fqdn(zone), setter: p}, nil
}
@@ -109,17 +74,6 @@ func configureGoogleCloudDNSProvider(zone string, values map[string]string) (Pro
if err := mapping.CopyViaJson(values, p); err != nil {
return nil, err
}
e := &googleclouddns.Provider{
Project: config.GetString("IONSCALE_DNS_GOOGLECLOUDDNS_PROJECT", ""),
ServiceAccountJSON: config.GetString("IONSCALE_DNS_GOOGLECLOUDDNS_SERVICE_ACCOUNT_JSON", ""),
}
// merge env configuration on top of the default/file configuration
if err := mergo.Merge(p, e, mergo.WithOverride); err != nil {
return nil, err
}
return &externalProvider{zone: fqdn(zone), setter: p}, nil
}
@@ -128,23 +82,6 @@ func configureRoute53Provider(zone string, values map[string]string) (Provider,
if err := mapping.CopyViaJson(values, p); err != nil {
return nil, err
}
e := &route53.Provider{
MaxRetries: 0,
MaxWaitDur: 0,
WaitForPropagation: false,
Region: config.GetString("IONSCALE_DNS_ROUTE53_REGION", ""),
AWSProfile: config.GetString("IONSCALE_DNS_ROUTE53_AWS_PROFILE", ""),
AccessKeyId: config.GetString("IONSCALE_DNS_ROUTE53_ACCESS_KEY_ID", ""),
SecretAccessKey: config.GetString("IONSCALE_DNS_ROUTE53_SECRET_ACCESS_KEY", ""),
Token: config.GetString("IONSCALE_DNS_ROUTE53_TOKEN", ""),
}
// merge env configuration on top of the default/file configuration
if err := mergo.Merge(p, e, mergo.WithOverride); err != nil {
return nil, err
}
return &externalProvider{zone: fqdn(zone), setter: p}, nil
}