feat: Support multiple providers for Remote Taskfiles and support directories

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/fatih/color v1.17.0
 	github.com/go-task/slim-sprig/v3 v3.0.0
 	github.com/go-task/template v0.1.0
+	github.com/hashicorp/go-getter v1.7.5
 	github.com/joho/godotenv v1.5.1
 	github.com/mattn/go-zglob v0.0.6
 	github.com/mitchellh/hashstructure/v2 v2.0.2
@@ -27,13 +28,42 @@ require (
 )
 
 require (
+	cloud.google.com/go v0.104.0 // indirect
+	cloud.google.com/go/compute v1.10.0 // indirect
+	cloud.google.com/go/iam v0.5.0 // indirect
+	cloud.google.com/go/storage v1.27.0 // indirect
+	github.com/aws/aws-sdk-go v1.44.122 // indirect
+	github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
 	github.com/dlclark/regexp2 v1.11.0 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect
+	github.com/googleapis/gax-go/v2 v2.6.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-safetemp v1.0.0 // indirect
+	github.com/hashicorp/go-version v1.6.0 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/klauspost/compress v1.15.11 // indirect
 	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mitchellh/go-testing-interface v1.14.1 // indirect
 	github.com/muesli/cancelreader v0.2.2 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
+	github.com/ulikunitz/xz v0.5.10 // indirect
+	go.opencensus.io v0.23.0 // indirect
+	golang.org/x/net v0.1.0 // indirect
+	golang.org/x/oauth2 v0.1.0 // indirect
 	golang.org/x/sys v0.25.0 // indirect
-	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
+	golang.org/x/text v0.4.0 // indirect
+	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
+	google.golang.org/api v0.100.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20221025140454-527a21cfbd71 // indirect
+	google.golang.org/grpc v1.50.1 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
 )

internal/compiler/compiler.go

@@ -187,7 +187,7 @@ func (c *Compiler) getSpecialVars(t *ast.Task, call *ast.Call) (map[string]strin
 		"ROOT_TASKFILE":    filepathext.SmartJoin(c.Dir, c.Entrypoint),
 		"ROOT_DIR":         c.Dir,
 		"TASKFILE":         t.Location.Taskfile,
-		"TASKFILE_DIR":     filepath.Dir(t.Location.Taskfile),
+		"TASKFILE_DIR":     t.Location.TaskfileDir,
 		"USER_WORKING_DIR": c.UserWorkingDir,
 		"TASK_VERSION":     version.GetVersion(),
 	}, nil

task_test.go

@@ -1,17 +1,23 @@
 package task_test
 
 import (
+	"archive/zip"
 	"bytes"
 	"context"
 	"fmt"
 	"io"
+	"math/rand"
+	"net/http"
+	"net/http/httptest"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"regexp"
 	"runtime"
 	"strings"
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/Masterminds/semver/v3"
 	"github.com/stretchr/testify/assert"
@@ -21,9 +27,12 @@ import (
 	"github.com/go-task/task/v3/errors"
 	"github.com/go-task/task/v3/internal/experiments"
 	"github.com/go-task/task/v3/internal/filepathext"
+	"github.com/go-task/task/v3/internal/logger"
 	"github.com/go-task/task/v3/taskfile/ast"
 )
 
+var random = rand.New(rand.NewSource(time.Now().UnixNano()))
+
 func init() {
 	_ = os.Setenv("NO_COLOR", "1")
 }
@@ -1042,6 +1051,225 @@ func TestIncludesMultiLevel(t *testing.T) {
 	tt.Run(t)
 }
 
+func TestIncludesRemote(t *testing.T) {
+	dir := "testdata/includes_remote"
+
+	os.RemoveAll(filepath.Join(dir, ".task"))
+
+	srv := httptest.NewServer(http.FileServer(http.Dir(dir)))
+	defer srv.Close()
+
+	createZipFileOfDir(t, filepath.Join(dir, "tasks-root.zip"), dir)
+	createZipFileOfDir(t, filepath.Join(dir, "tasks-first.zip"), filepath.Join(dir, "first"))
+
+	tcs := []struct {
+		rootTaskfile string
+		firstRemote  string
+		secondRemote string
+		extraTasks   []string
+	}{
+		//
+		// NOTE: When adding content for tests that use `getGitRemoteURL`,
+		// you must commit the test data for the tests to be able to find it.
+		//
+		// These tests will not see data in the working tree because they clone
+		// this repo.
+		//
+		{
+			// Ensure non-remote includes still work
+			firstRemote:  "./first/Taskfile.yml",
+			secondRemote: "./second/Taskfile.yml",
+		},
+		{
+			firstRemote:  srv.URL + "/first/Taskfile.yml",
+			secondRemote: srv.URL + "/first/second/Taskfile.yml",
+		},
+		{
+			firstRemote:  srv.URL + "/first/Taskfile.yml",
+			secondRemote: "./second/Taskfile.yml",
+		},
+		{
+			firstRemote:  getGitRemoteURL(t, dir+"/first"),
+			secondRemote: getGitRemoteURL(t, dir+"/first/second"),
+		},
+		{
+			firstRemote:  srv.URL + "/first/Taskfile.yml",
+			secondRemote: getGitRemoteURL(t, dir+"/first/second"),
+		},
+		{
+			firstRemote:  getGitRemoteURL(t, dir+"/first"),
+			secondRemote: srv.URL + "/first/second/Taskfile.yml",
+		},
+		{
+			firstRemote:  getGitRemoteURL(t, dir+"/first"),
+			secondRemote: "./second/Taskfile.yml",
+			extraTasks: []string{
+				"first:check-if-neighbor-file-exists",
+				"first:second:check-if-neighbor-file-exists",
+			},
+		},
+		{
+			firstRemote: getGitRemoteURL(t, dir+"/first") + "&taskfile=Taskfile2.yml",
+			extraTasks: []string{
+				"first:first-taskfile2-task",
+			},
+		},
+		{
+			firstRemote: getGitRemoteURL(t, dir+"/first") + "&taskfile=second/Taskfile2.yml",
+			extraTasks: []string{
+				"first:second-taskfile2-task",
+			},
+		},
+		{
+			firstRemote:  srv.URL + "/tasks-first.zip",
+			secondRemote: "./second/Taskfile.yml",
+			extraTasks: []string{
+				"first:check-if-neighbor-file-exists",
+				"first:second:check-if-neighbor-file-exists",
+			},
+		},
+		{
+			rootTaskfile: srv.URL + "/Taskfile.yml",
+			firstRemote:  "./first/Taskfile.yml",
+			secondRemote: "./second/Taskfile.yml",
+		},
+		{
+			rootTaskfile: getGitRemoteURL(t, dir),
+			firstRemote:  "./first/Taskfile.yml",
+			secondRemote: "./second/Taskfile.yml",
+			extraTasks: []string{
+				"first:check-if-neighbor-file-exists",
+				"first:second:check-if-neighbor-file-exists",
+			},
+		},
+		{
+			rootTaskfile: srv.URL + "/tasks-root.zip",
+			firstRemote:  "./first/Taskfile.yml",
+			secondRemote: "./second/Taskfile.yml",
+			extraTasks: []string{
+				"first:check-if-neighbor-file-exists",
+				"first:second:check-if-neighbor-file-exists",
+			},
+		},
+	}
+
+	tasks := []string{
+		"first:write-file",
+		"first:second:write-file",
+	}
+
+	for i, tc := range tcs {
+		t.Run(fmt.Sprint(i), func(t *testing.T) {
+			t.Setenv("FIRST_REMOTE_URL", tc.firstRemote)
+			t.Setenv("SECOND_REMOTE_URL", tc.secondRemote)
+
+			executors := []struct {
+				name     string
+				executor *task.Executor
+			}{
+				{
+					name: "online, always download",
+					executor: &task.Executor{
+						Dir:        dir,
+						Entrypoint: tc.rootTaskfile,
+						Timeout:    time.Minute,
+						Insecure:   true,
+						Verbose:    true,
+
+						// Without caching
+						AssumeYes: true,
+						Download:  true,
+						Offline:   false,
+					},
+				},
+				{
+					name: "offline, use-cache",
+					executor: &task.Executor{
+						Dir:        dir,
+						Entrypoint: tc.rootTaskfile,
+						Timeout:    time.Minute,
+						Insecure:   true,
+						Verbose:    true,
+
+						// With caching
+						AssumeYes: false,
+						Download:  false,
+						Offline:   true,
+					},
+				},
+			}
+
+			for j, e := range executors {
+				t.Run(fmt.Sprint(j), func(t *testing.T) {
+					var buff SyncBuffer
+					defer func() { t.Log("\noutput:\n", buff.buf.String()) }()
+
+					e.executor.Stderr = &buff
+					e.executor.Stdout = &buff
+					e.executor.Logger = &logger.Logger{Stderr: &buff, Stdout: &buff, Verbose: true}
+
+					require.NoError(t, e.executor.Setup())
+
+					for k, task := range tasks {
+						t.Run(task, func(t *testing.T) {
+							expectedContent := fmt.Sprint(random.Int63())
+							t.Setenv("CONTENT", expectedContent)
+
+							outputFile := fmt.Sprintf("%d.%d.txt", i, k)
+							t.Setenv("OUTPUT_FILE", outputFile)
+
+							path := filepath.Join(dir, outputFile)
+							require.NoError(t, os.RemoveAll(path))
+
+							require.NoError(t, e.executor.Run(context.Background(), &ast.Call{Task: task}))
+
+							actualContent, err := os.ReadFile(path)
+							require.NoError(t, err)
+							assert.Equal(t, expectedContent, strings.TrimSpace(string(actualContent)))
+						})
+					}
+
+					for _, task := range tc.extraTasks {
+						t.Run(task, func(t *testing.T) {
+							require.NoError(t, e.executor.Run(context.Background(), &ast.Call{Task: task}))
+						})
+					}
+				})
+			}
+		})
+	}
+}
+
+func createZipFileOfDir(t *testing.T, zipFilePath string, dir string) {
+	f, err := os.OpenFile(zipFilePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o644)
+	require.NoError(t, err)
+	defer f.Close()
+
+	w := zip.NewWriter(f)
+	err = w.AddFS(os.DirFS(dir))
+	require.NoError(t, err)
+	w.Close()
+}
+
+func getGitRemoteURL(t *testing.T, path string) string {
+	repoRoot, err := exec.Command("git", "rev-parse", "--show-toplevel").Output()
+	require.NoError(t, err)
+
+	// This is to support Github Workflows on PRs where we are in a detached HEAD state.
+	branch := os.Getenv("GITHUB_REF")
+	if branch == "" {
+		b, err := exec.Command("git", "branch", "--show-current").Output()
+		require.NoError(t, err)
+		branch = string(b)
+	}
+
+	return fmt.Sprintf("git::file://%s//%s?ref=%s&depth=1",
+		strings.TrimSpace(string(repoRoot)),
+		path,
+		strings.TrimSpace(string(branch)),
+	)
+}
+
 func TestIncludeCycle(t *testing.T) {
 	const dir = "testdata/includes_cycle"
 

taskfile/ast/location.go

@@ -1,18 +1,20 @@
 package ast
 
 type Location struct {
-	Line     int
-	Column   int
-	Taskfile string
+	Line        int
+	Column      int
+	Taskfile    string
+	TaskfileDir string
 }
 
 func (l *Location) DeepCopy() *Location {
 	if l == nil {
 		return nil
 	}
 	return &Location{
-		Line:     l.Line,
-		Column:   l.Column,
-		Taskfile: l.Taskfile,
+		Line:        l.Line,
+		Column:      l.Column,
+		Taskfile:    l.Taskfile,
+		TaskfileDir: l.TaskfileDir,
 	}
 }