Skip to content

Commit 86f986f

Browse files
pbittypbitty
committed
Add caching support for remote directories
1 parent fa07a9b commit 86f986f

File tree

7 files changed

+327
-72
lines changed

7 files changed

+327
-72
lines changed

‎task_test.go‎

Lines changed: 63 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package task_test
22

33
import (
4+
"archive/zip"
45
"bytes"
56
"context"
67
"fmt"
@@ -1053,20 +1054,32 @@ func TestIncludesMultiLevel(t *testing.T) {
10531054
func TestIncludesRemote(t *testing.T) {
10541055
dir := "testdata/includes_remote"
10551056

1057+
os.RemoveAll(filepath.Join(dir, ".task"))
1058+
10561059
srv := httptest.NewServer(http.FileServer(http.Dir(dir)))
10571060
defer srv.Close()
10581061

1062+
createZipFileOfDir(t, filepath.Join(dir, "tasks-root.zip"), dir)
1063+
createZipFileOfDir(t, filepath.Join(dir, "tasks-first.zip"), filepath.Join(dir, "first"))
1064+
10591065
tcs := []struct {
10601066
rootTaskfile string
10611067
firstRemote string
10621068
secondRemote string
10631069
extraTasks []string
10641070
}{
1071+
//
10651072
// NOTE: When adding content for tests that use `getGitRemoteURL`,
10661073
// you must commit the test data for the tests to be able to find it.
10671074
//
10681075
// These tests will not see data in the working tree because they clone
10691076
// this repo.
1077+
//
1078+
{
1079+
// Ensure non-remote includes still work
1080+
firstRemote: "./first/Taskfile.yml",
1081+
secondRemote: "./second/Taskfile.yml",
1082+
},
10701083
{
10711084
firstRemote: srv.URL + "/first/Taskfile.yml",
10721085
secondRemote: srv.URL + "/first/second/Taskfile.yml",
@@ -1108,14 +1121,36 @@ func TestIncludesRemote(t *testing.T) {
11081121
},
11091122
},
11101123
{
1111-
firstRemote: srv.URL + "/tasks.zip",
1124+
firstRemote: srv.URL + "/tasks-first.zip",
11121125
secondRemote: "./second/Taskfile.yml",
1126+
extraTasks: []string{
1127+
"first:check-if-neighbor-file-exists",
1128+
"first:second:check-if-neighbor-file-exists",
1129+
},
11131130
},
11141131
{
11151132
rootTaskfile: srv.URL + "/Taskfile.yml",
11161133
firstRemote: "./first/Taskfile.yml",
11171134
secondRemote: "./second/Taskfile.yml",
11181135
},
1136+
{
1137+
rootTaskfile: getGitRemoteURL(t, dir),
1138+
firstRemote: "./first/Taskfile.yml",
1139+
secondRemote: "./second/Taskfile.yml",
1140+
extraTasks: []string{
1141+
"first:check-if-neighbor-file-exists",
1142+
"first:second:check-if-neighbor-file-exists",
1143+
},
1144+
},
1145+
{
1146+
rootTaskfile: srv.URL + "/tasks-root.zip",
1147+
firstRemote: "./first/Taskfile.yml",
1148+
secondRemote: "./second/Taskfile.yml",
1149+
extraTasks: []string{
1150+
"first:check-if-neighbor-file-exists",
1151+
"first:second:check-if-neighbor-file-exists",
1152+
},
1153+
},
11191154
}
11201155

11211156
tasks := []string{
@@ -1144,25 +1179,24 @@ func TestIncludesRemote(t *testing.T) {
11441179
// Without caching
11451180
AssumeYes: true,
11461181
Download: true,
1182+
Offline: false,
1183+
},
1184+
},
1185+
{
1186+
name: "offline, use-cache",
1187+
executor: &task.Executor{
1188+
Dir: dir,
1189+
Entrypoint: tc.rootTaskfile,
1190+
Timeout: time.Minute,
1191+
Insecure: true,
1192+
Verbose: true,
1193+
1194+
// With caching
1195+
AssumeYes: false,
1196+
Download: false,
1197+
Offline: true,
11471198
},
11481199
},
1149-
// Disabled until we add caching support for directories
1150-
//
1151-
// {
1152-
// name: "offline, use-cache",
1153-
// executor: &task.Executor{
1154-
// Dir: dir,
1155-
// Entrypoint: tc.rootTaskfile,
1156-
// Timeout: time.Minute,
1157-
// Insecure: true,
1158-
// Verbose: true,
1159-
//
1160-
// // With caching
1161-
// AssumeYes: false,
1162-
// Download: false,
1163-
// Offline: true,
1164-
// },
1165-
// },
11661200
}
11671201

11681202
for j, e := range executors {
@@ -1206,6 +1240,17 @@ func TestIncludesRemote(t *testing.T) {
12061240
}
12071241
}
12081242

1243+
func createZipFileOfDir(t *testing.T, zipFilePath string, dir string) {
1244+
f, err := os.OpenFile(zipFilePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o644)
1245+
require.NoError(t, err)
1246+
defer f.Close()
1247+
1248+
w := zip.NewWriter(f)
1249+
err = w.AddFS(os.DirFS(dir))
1250+
require.NoError(t, err)
1251+
w.Close()
1252+
}
1253+
12091254
func getGitRemoteURL(t *testing.T, path string) string {
12101255
repoRoot, err := exec.Command("git", "rev-parse", "--show-toplevel").Output()
12111256
require.NoError(t, err)

‎taskfile/cache.go‎

Lines changed: 188 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,21 @@ import (
66
"os"
77
"path/filepath"
88
"strings"
9+
10+
"gopkg.in/yaml.v3"
11+
12+
"github.com/go-task/task/v3/errors"
913
)
1014

1115
type Cache struct {
1216
dir string
1317
}
1418

19+
type metadata struct {
20+
Checksum string
21+
TaskfileName string
22+
}
23+
1524
func NewCache(dir string) (*Cache, error) {
1625
dir = filepath.Join(dir, "remote")
1726
if err := os.MkdirAll(dir, 0o755); err != nil {
@@ -25,46 +34,207 @@ func NewCache(dir string) (*Cache, error) {
2534
func checksum(b []byte) string {
2635
h := sha256.New()
2736
h.Write(b)
28-
return fmt.Sprintf("%x", h.Sum(nil))
37+
return fmt.Sprintf("%x", h.Sum(nil))[:16]
2938
}
3039

31-
func (c *Cache) write(node Node, b []byte) error {
32-
return os.WriteFile(c.cacheFilePath(node), b, 0o644)
40+
func checksumSource(s source) (string, error) {
41+
h := sha256.New()
42+
43+
entries, err := os.ReadDir(s.FileDirectory)
44+
if err != nil {
45+
return "", fmt.Errorf("could not list files at %s: %w", s.FileDirectory, err)
46+
}
47+
48+
for _, e := range entries {
49+
if e.Type().IsRegular() {
50+
path := filepath.Join(s.FileDirectory, e.Name())
51+
f, err := os.Open(path)
52+
if err != nil {
53+
return "", fmt.Errorf("error opening file %s for checksumming: %w", path, err)
54+
}
55+
if _, err := f.WriteTo(h); err != nil {
56+
f.Close()
57+
return "", fmt.Errorf("error reading file %s for checksumming: %w", path, err)
58+
}
59+
f.Close()
60+
}
61+
}
62+
return fmt.Sprintf("%x", h.Sum(nil))[:16], nil
3363
}
3464

35-
func (c *Cache) read(node Node) ([]byte, error) {
36-
return os.ReadFile(c.cacheFilePath(node))
65+
func (c *Cache) write(node Node, src source) (*source, error) {
66+
// Clear metadata file so that if the rest of the operations fail part-way we don't
67+
// end up in an inconsistent state where we've written the contents but have old metadata
68+
if err := c.clearMetadata(node); err != nil {
69+
return nil, err
70+
}
71+
72+
p, err := c.contentsPath(node)
73+
if err != nil {
74+
return nil, err
75+
}
76+
77+
switch fi, err := os.Stat(p); {
78+
case errors.Is(err, os.ErrNotExist):
79+
// Nothign to clear, do nothing
80+
81+
case !fi.IsDir():
82+
return nil, fmt.Errorf("error writing to contents path %s: not a directory", p)
83+
84+
case err != nil:
85+
return nil, fmt.Errorf("error cheacking for previous contents path %s: %w", p, err)
86+
87+
default:
88+
err := os.RemoveAll(p)
89+
if err != nil {
90+
return nil, fmt.Errorf("error clearing contents directory: %s", err)
91+
}
92+
}
93+
94+
if err := os.Rename(src.FileDirectory, p); err != nil {
95+
return nil, err
96+
}
97+
98+
// TODO Clean up
99+
src.FileDirectory = p
100+
101+
cs, err := checksumSource(src)
102+
if err != nil {
103+
return nil, err
104+
}
105+
106+
m := metadata{
107+
Checksum: cs,
108+
TaskfileName: src.Filename,
109+
}
110+
111+
if err := c.storeMetadata(node, m); err != nil {
112+
return nil, fmt.Errorf("error storing metadata for node %s: %w", node.Location(), err)
113+
}
114+
115+
return &src, nil
37116
}
38117

39-
func (c *Cache) writeChecksum(node Node, checksum string) error {
40-
return os.WriteFile(c.checksumFilePath(node), []byte(checksum), 0o644)
118+
func (c *Cache) read(node Node) (*source, error) {
119+
path, err := c.contentsPath(node)
120+
if err != nil {
121+
return nil, err
122+
}
123+
124+
m, err := c.readMetadata(node)
125+
if err != nil {
126+
return nil, err
127+
}
128+
129+
taskfileName := m.TaskfileName
130+
131+
content, err := os.ReadFile(filepath.Join(path, m.TaskfileName))
132+
if err != nil {
133+
return nil, err
134+
}
135+
136+
return &source{
137+
FileContent: content,
138+
FileDirectory: path,
139+
Filename: taskfileName,
140+
}, nil
41141
}
42142

43143
func (c *Cache) readChecksum(node Node) string {
44-
b, _ := os.ReadFile(c.checksumFilePath(node))
45-
return string(b)
144+
m, err := c.readMetadata(node)
145+
if err != nil {
146+
return ""
147+
}
148+
return m.Checksum
149+
}
150+
151+
func (c *Cache) clearMetadata(node Node) error {
152+
path, err := c.metadataFilePath(node)
153+
if err != nil {
154+
return fmt.Errorf("error clearing metadata file at %s: %w", path, err)
155+
}
156+
157+
fi, err := os.Stat(path)
158+
if errors.Is(err, os.ErrNotExist) {
159+
return nil
160+
}
161+
162+
if !fi.Mode().IsRegular() {
163+
return fmt.Errorf("path is not a real file when trying to delete metadata file: %s", path)
164+
}
165+
166+
// if err := os.Remove(path)
167+
if err := os.Remove(path); err != nil {
168+
return fmt.Errorf("error removing metadata file %s: %w", path, err)
169+
}
170+
171+
return nil
172+
}
173+
174+
func (c *Cache) storeMetadata(node Node, m metadata) error {
175+
path, err := c.metadataFilePath(node)
176+
if err != nil {
177+
return err
178+
}
179+
180+
f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o644)
181+
if err != nil {
182+
return fmt.Errorf("error creating metadata file %s: %w", path, err)
183+
}
184+
defer f.Close()
185+
186+
if err := yaml.NewEncoder(f).Encode(m); err != nil {
187+
return fmt.Errorf("error writing metadata into %s: %w", path, err)
188+
}
189+
190+
return nil
191+
}
192+
193+
func (c *Cache) readMetadata(node Node) (*metadata, error) {
194+
path, err := c.metadataFilePath(node)
195+
if err != nil {
196+
return nil, err
197+
}
198+
199+
f, err := os.Open(path)
200+
if err != nil {
201+
return nil, fmt.Errorf("error opening metadata file %s: %w", path, err)
202+
}
203+
defer f.Close()
204+
205+
var m *metadata
206+
if err := yaml.NewDecoder(f).Decode(&m); err != nil {
207+
return nil, fmt.Errorf("error reading metadata file %s: %w", path, err)
208+
}
209+
210+
return m, nil
46211
}
47212

48213
func (c *Cache) key(node Node) string {
49214
return strings.TrimRight(checksum([]byte(node.Location())), "=")
50215
}
51216

52-
func (c *Cache) cacheFilePath(node Node) string {
53-
return c.filePath(node, "yaml")
217+
func (c *Cache) contentsPath(node Node) (string, error) {
218+
return c.cacheFilePath(node, "contents")
54219
}
55220

56-
func (c *Cache) checksumFilePath(node Node) string {
57-
return c.filePath(node, "checksum")
221+
func (c *Cache) metadataFilePath(node Node) (string, error) {
222+
return c.cacheFilePath(node, "metadata.yaml")
58223
}
59224

60-
func (c *Cache) filePath(node Node, suffix string) string {
61-
lastDir, filename := node.FilenameAndLastDir()
62-
prefix := filename
225+
func (c *Cache) cacheFilePath(node Node, filename string) (string, error) {
226+
lastDir, prefix := node.FilenameAndLastDir()
63227
// Means it's not "", nor "." nor "/", so it's a valid directory
64228
if len(lastDir) > 1 {
65-
prefix = fmt.Sprintf("%s-%s", lastDir, filename)
229+
prefix = fmt.Sprintf("%s-%s", lastDir, prefix)
66230
}
67-
return filepath.Join(c.dir, fmt.Sprintf("%s.%s.%s", prefix, c.key(node), suffix))
231+
232+
dir := filepath.Join(c.dir, fmt.Sprintf("%s.%s", prefix, c.key(node)))
233+
if err := os.MkdirAll(dir, 0o755); err != nil {
234+
return "", fmt.Errorf("error creating cache dir %s: %w", dir, err)
235+
}
236+
237+
return filepath.Join(dir, filename), nil
68238
}
69239

70240
func (c *Cache) Clear() error {

0 commit comments

Comments
 (0)