cmd/atlas/internal/cmdapi: support multi file schemas in project files (#943)

Author: rotemtam

cmd/atlas/internal/cmdapi/migrate.go

@@ -917,14 +917,17 @@ func migrateFlagsFromEnv(cmd *cobra.Command, _ []string) error {
 		return err
 	}
 	// Transform "src" to a URL.
-	toURL := activeEnv.Source
-	if toURL != "" {
-		if toURL, err = filepath.Abs(activeEnv.Source); err != nil {
-			return fmt.Errorf("finding abs path to source: %q: %w", activeEnv.Source, err)
+	srcs, err := activeEnv.Sources()
+	if err != nil {
+		return err
+	}
+	for i, s := range srcs {
+		if s, err = filepath.Abs(s); err != nil {
+			return fmt.Errorf("finding abs path to source: %q: %w", s, err)
 		}
-		toURL = "file://" + toURL
+		srcs[i] = "file://" + s
 	}
-	if err := maySetFlag(cmd, migrateFlagTo, toURL); err != nil {
+	if err := maySetFlag(cmd, migrateFlagTo, strings.Join(srcs, ",")); err != nil {
 		return err
 	}
 	if s := "[" + strings.Join(activeEnv.Schemas, "") + "]"; len(activeEnv.Schemas) > 0 {

cmd/atlas/internal/cmdapi/project.go

@@ -42,6 +42,9 @@ type MigrationDir struct {
 func (e *Env) asMap() (map[string]string, error) {
 	m := make(map[string]string, len(e.Extra.Attrs))
 	for _, attr := range e.Extra.Attrs {
+		if attr.K == "src" {
+			continue
+		}
 		if v, err := attr.String(); err == nil {
 			m[attr.K] = v
 			continue
@@ -66,9 +69,6 @@ type Env struct {
 	// See: https://atlasgo.io/dev-database
 	DevURL string `spec:"dev"`
 
-	// Path to the file containing the desired schema of the environment.
-	Source string `spec:"src"`
-
 	// List of schemas in this database that are managed by Atlas.
 	Schemas []string `spec:"schemas"`
 
@@ -78,6 +78,21 @@ type Env struct {
 	schemahcl.DefaultExtension
 }
 
+// Sources returns the paths containing the Atlas schema.
+func (e *Env) Sources() ([]string, error) {
+	attr, exists := e.Attr("src")
+	if !exists {
+		return nil, nil
+	}
+	if s, err := attr.String(); err == nil {
+		return []string{s}, nil
+	}
+	if s, err := attr.Strings(); err == nil {
+		return s, nil
+	}
+	return nil, fmt.Errorf("expected src to be either a string or a string array")
+}
+
 var hclState = schemahcl.New(
 	schemahcl.WithScopedEnums("env.migration_dir.format", formatAtlas, formatFlyway, formatLiquibase, formatGoose, formatGolangMigrate),
 )
@@ -108,8 +123,8 @@ func LoadEnv(path string, name string, opts ...LoadOption) (*Env, error) {
 		if e.URL == "" {
 			return nil, fmt.Errorf("no url set for env %q", e.Name)
 		}
-		if e.Source == "" {
-			return nil, fmt.Errorf("no src set for env %q", e.Name)
+		if _, err := e.Sources(); err != nil {
+			return nil, err
 		}
 		projEnvs[e.Name] = e
 	}

cmd/atlas/internal/cmdapi/project_test.go

@@ -37,6 +37,14 @@ env "local" {
 	integer = 42
 	str = var.name
 }
+
+env "multi" {
+	url = "mysql://root:pass@localhost:3306/"
+	src = [
+		"./a.hcl",
+		"./b.hcl",
+	]
+}
 `
 	err := os.WriteFile(filepath.Join(d, projectFileName), []byte(h), 0600)
 	require.NoError(t, err)
@@ -52,7 +60,6 @@ env "local" {
 			Name:    "local",
 			URL:     "mysql://root:pass@localhost:3306/",
 			DevURL:  "docker://mysql/8",
-			Source:  "./app.hcl",
 			Schemas: []string{"hello", "world"},
 			MigrationDir: &MigrationDir{
 				URL:    "file://migrations",
@@ -63,11 +70,22 @@ env "local" {
 					Attrs: []*schemahcl.Attr{
 						{K: "bool", V: &schemahcl.LiteralValue{V: "true"}},
 						{K: "integer", V: &schemahcl.LiteralValue{V: "42"}},
+						{K: "src", V: &schemahcl.LiteralValue{V: `"./app.hcl"`}},
 						{K: "str", V: &schemahcl.LiteralValue{V: `"hello"`}},
 					},
 				},
 			},
 		}, env)
+		sources, err := env.Sources()
+		require.NoError(t, err)
+		require.EqualValues(t, []string{"./app.hcl"}, sources)
+	})
+	t.Run("multi", func(t *testing.T) {
+		env, err := LoadEnv(path, "multi")
+		require.NoError(t, err)
+		srcs, err := env.Sources()
+		require.NoError(t, err)
+		require.EqualValues(t, []string{"./a.hcl", "./b.hcl"}, srcs)
 	})
 	t.Run("with input", func(t *testing.T) {
 		env, err := LoadEnv(path, "local", WithInput(map[string]string{

cmd/atlas/internal/cmdapi/schema.go

@@ -203,7 +203,11 @@ func schemaFlagsFromEnv(cmd *cobra.Command, _ []string) error {
 	if err := maySetFlag(cmd, devURLFlag, activeEnv.DevURL); err != nil {
 		return err
 	}
-	if err := maySetFlag(cmd, fileFlag, activeEnv.Source); err != nil {
+	srcs, err := activeEnv.Sources()
+	if err != nil {
+		return err
+	}
+	if err := maySetFlag(cmd, fileFlag, strings.Join(srcs, "")); err != nil {
 		return err
 	}
 	if s := strings.Join(activeEnv.Schemas, ""); len(activeEnv.Schemas) > 0 {

doc/md/cli/projects.md

@@ -12,7 +12,10 @@ environments when working with Atlas. A project file is a file named
 ```hcl
 // Define an environment named "local"
 env "local" {
-  // Declare where the schema definition file resides.
+  // Declare where the schema definition resides.
+  // Also supported:
+  //   src = "./dir/with/schema" 
+  //   src = ["multi.hcl", "file.hcl"]
   src = "./project/schema.hcl"
   
   // Define the URL of the database which is managed in

internal/integration/testdata/sqlite/cli-migrate-project-multifile.txt

@@ -1,19 +1,17 @@
-exec mkdir migrations
 atlas migrate diff --env local
 cmpmig 0 diff.sql
 
-atlas migrate validate --env local
+# reset
+exec rm -rf migrations
 
-atlas migrate new 1 --env local
-cmpmig 1 empty.sql
-
-exec touch migrations/2.sql
-! atlas migrate validate --env local
-stderr 'Error: checksum mismatch'
+atlas migrate diff --env src_list
+cmpmig 0 diff.sql
 
-atlas migrate hash --force --env local
-atlas migrate validate --env local
+# reset
+exec rm -rf migrations
 
+atlas migrate diff --env single_elem
+cmpmig 0 diff.sql
 -- atlas.hcl --
 env "local" {
     url = "URL"
@@ -24,6 +22,29 @@ env "local" {
         format = atlas
     }
 }
+env "src_list" {
+    url = "URL"
+    dev = "sqlite://test?mode=memory&_fk=1"
+    src = [
+        "./schema/1.hcl",
+        "./schema/2.hcl",
+    ]
+    migration_dir {
+        url = "file://migrations"
+        format = atlas
+    }
+}
+env "single_elem" {
+    url = "URL"
+    dev = "sqlite://test?mode=memory&_fk=1"
+    src = [
+        "./schema/",
+    ]
+    migration_dir {
+        url = "file://migrations"
+        format = atlas
+    }
+}
 -- schema/1.hcl --
 table "users" {
   schema = schema.main