Skip to content

Commit aa0446a

Browse files
jbowensjbowens
committed
objstorageprovider: support blob files
Extend the objstorageprovider implementation to support files of type FileTypeBlob. Close #4582.
1 parent 243d3ba commit aa0446a

File tree

7 files changed

+234
-132
lines changed

7 files changed

+234
-132
lines changed

‎objstorage/objstorageprovider/remote_backing_test.go

+135-118
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,100 @@ import (
1515
"github.com/stretchr/testify/require"
1616
)
1717

18+
var supportedFileTypes = []base.FileType{
19+
base.FileTypeTable,
20+
base.FileTypeBlob,
21+
}
22+
1823
func TestSharedObjectBacking(t *testing.T) {
1924
for _, cleanup := range []objstorage.SharedCleanupMethod{objstorage.SharedRefTracking, objstorage.SharedNoCleanup} {
2025
name := "ref-tracking"
2126
if cleanup == objstorage.SharedNoCleanup {
2227
name = "no-cleanup"
2328
}
24-
t.Run(name, func(t *testing.T) {
29+
for _, fileType := range supportedFileTypes {
30+
t.Run(fileType.String(), func(t *testing.T) {
31+
t.Run(name, func(t *testing.T) {
32+
st := DefaultSettings(vfs.NewMem(), "")
33+
sharedStorage := remote.NewInMem()
34+
st.Remote.StorageFactory = remote.MakeSimpleFactory(map[remote.Locator]remote.Storage{
35+
"foo": sharedStorage,
36+
})
37+
p, err := Open(st)
38+
require.NoError(t, err)
39+
defer p.Close()
40+
41+
const creatorID = objstorage.CreatorID(99)
42+
require.NoError(t, p.SetCreatorID(creatorID))
43+
meta := objstorage.ObjectMetadata{
44+
DiskFileNum: base.DiskFileNum(1),
45+
FileType: fileType,
46+
}
47+
meta.Remote.CreatorID = 100
48+
meta.Remote.CreatorFileNum = base.DiskFileNum(200)
49+
meta.Remote.CleanupMethod = cleanup
50+
meta.Remote.Locator = "foo"
51+
meta.Remote.CustomObjectName = "obj-name"
52+
meta.Remote.Storage = sharedStorage
53+
54+
h, err := p.RemoteObjectBacking(&meta)
55+
require.NoError(t, err)
56+
buf, err := h.Get()
57+
require.NoError(t, err)
58+
h.Close()
59+
_, err = h.Get()
60+
require.Error(t, err)
61+
62+
d1, err := decodeRemoteObjectBacking(fileType, base.DiskFileNum(100), buf)
63+
require.NoError(t, err)
64+
require.Equal(t, uint64(100), uint64(d1.meta.DiskFileNum))
65+
require.Equal(t, fileType, d1.meta.FileType)
66+
d1.meta.Remote.Storage = sharedStorage
67+
require.Equal(t, meta.Remote, d1.meta.Remote)
68+
if cleanup == objstorage.SharedRefTracking {
69+
require.Equal(t, creatorID, d1.refToCheck.creatorID)
70+
require.Equal(t, base.DiskFileNum(1), d1.refToCheck.fileNum)
71+
} else {
72+
require.Equal(t, objstorage.CreatorID(0), d1.refToCheck.creatorID)
73+
require.Equal(t, base.DiskFileNum(0), d1.refToCheck.fileNum)
74+
}
75+
76+
t.Run("unknown-tags", func(t *testing.T) {
77+
// Append a tag that is safe to ignore.
78+
buf2 := buf
79+
buf2 = binary.AppendUvarint(buf2, 13)
80+
buf2 = binary.AppendUvarint(buf2, 2)
81+
buf2 = append(buf2, 1, 1)
82+
83+
d2, err := decodeRemoteObjectBacking(fileType, base.DiskFileNum(100), buf2)
84+
require.NoError(t, err)
85+
require.Equal(t, uint64(100), uint64(d2.meta.DiskFileNum))
86+
require.Equal(t, fileType, d2.meta.FileType)
87+
d2.meta.Remote.Storage = sharedStorage
88+
require.Equal(t, meta.Remote, d2.meta.Remote)
89+
if cleanup == objstorage.SharedRefTracking {
90+
require.Equal(t, creatorID, d2.refToCheck.creatorID)
91+
require.Equal(t, base.DiskFileNum(1), d2.refToCheck.fileNum)
92+
} else {
93+
require.Equal(t, objstorage.CreatorID(0), d2.refToCheck.creatorID)
94+
require.Equal(t, base.DiskFileNum(0), d2.refToCheck.fileNum)
95+
}
96+
97+
buf3 := buf2
98+
buf3 = binary.AppendUvarint(buf3, tagNotSafeToIgnoreMask+5)
99+
_, err = decodeRemoteObjectBacking(meta.FileType, meta.DiskFileNum, buf3)
100+
require.Error(t, err)
101+
require.Contains(t, err.Error(), "unknown tag")
102+
})
103+
})
104+
})
105+
}
106+
}
107+
}
108+
109+
func TestCreateSharedObjectBacking(t *testing.T) {
110+
for _, fileType := range supportedFileTypes {
111+
t.Run(fileType.String(), func(t *testing.T) {
25112
st := DefaultSettings(vfs.NewMem(), "")
26113
sharedStorage := remote.NewInMem()
27114
st.Remote.StorageFactory = remote.MakeSimpleFactory(map[remote.Locator]remote.Storage{
@@ -31,128 +118,58 @@ func TestSharedObjectBacking(t *testing.T) {
31118
require.NoError(t, err)
32119
defer p.Close()
33120

34-
const creatorID = objstorage.CreatorID(99)
35-
require.NoError(t, p.SetCreatorID(creatorID))
36-
meta := objstorage.ObjectMetadata{
37-
DiskFileNum: base.DiskFileNum(1),
38-
FileType: base.FileTypeTable,
39-
}
40-
meta.Remote.CreatorID = 100
41-
meta.Remote.CreatorFileNum = base.DiskFileNum(200)
42-
meta.Remote.CleanupMethod = cleanup
43-
meta.Remote.Locator = "foo"
44-
meta.Remote.CustomObjectName = "obj-name"
45-
meta.Remote.Storage = sharedStorage
46-
47-
h, err := p.RemoteObjectBacking(&meta)
48-
require.NoError(t, err)
49-
buf, err := h.Get()
50-
require.NoError(t, err)
51-
h.Close()
52-
_, err = h.Get()
53-
require.Error(t, err)
121+
require.NoError(t, p.SetCreatorID(1))
54122

55-
d1, err := decodeRemoteObjectBacking(base.FileTypeTable, base.DiskFileNum(100), buf)
123+
backing, err := p.CreateExternalObjectBacking("foo", "custom-obj-name")
56124
require.NoError(t, err)
57-
require.Equal(t, uint64(100), uint64(d1.meta.DiskFileNum))
58-
require.Equal(t, base.FileTypeTable, d1.meta.FileType)
59-
d1.meta.Remote.Storage = sharedStorage
60-
require.Equal(t, meta.Remote, d1.meta.Remote)
61-
if cleanup == objstorage.SharedRefTracking {
62-
require.Equal(t, creatorID, d1.refToCheck.creatorID)
63-
require.Equal(t, base.DiskFileNum(1), d1.refToCheck.fileNum)
64-
} else {
65-
require.Equal(t, objstorage.CreatorID(0), d1.refToCheck.creatorID)
66-
require.Equal(t, base.DiskFileNum(0), d1.refToCheck.fileNum)
67-
}
68-
69-
t.Run("unknown-tags", func(t *testing.T) {
70-
// Append a tag that is safe to ignore.
71-
buf2 := buf
72-
buf2 = binary.AppendUvarint(buf2, 13)
73-
buf2 = binary.AppendUvarint(buf2, 2)
74-
buf2 = append(buf2, 1, 1)
75-
76-
d2, err := decodeRemoteObjectBacking(base.FileTypeTable, base.DiskFileNum(100), buf2)
77-
require.NoError(t, err)
78-
require.Equal(t, uint64(100), uint64(d2.meta.DiskFileNum))
79-
require.Equal(t, base.FileTypeTable, d2.meta.FileType)
80-
d2.meta.Remote.Storage = sharedStorage
81-
require.Equal(t, meta.Remote, d2.meta.Remote)
82-
if cleanup == objstorage.SharedRefTracking {
83-
require.Equal(t, creatorID, d2.refToCheck.creatorID)
84-
require.Equal(t, base.DiskFileNum(1), d2.refToCheck.fileNum)
85-
} else {
86-
require.Equal(t, objstorage.CreatorID(0), d2.refToCheck.creatorID)
87-
require.Equal(t, base.DiskFileNum(0), d2.refToCheck.fileNum)
88-
}
89-
90-
buf3 := buf2
91-
buf3 = binary.AppendUvarint(buf3, tagNotSafeToIgnoreMask+5)
92-
_, err = decodeRemoteObjectBacking(meta.FileType, meta.DiskFileNum, buf3)
93-
require.Error(t, err)
94-
require.Contains(t, err.Error(), "unknown tag")
95-
})
125+
d, err := decodeRemoteObjectBacking(fileType, base.DiskFileNum(100), backing)
126+
require.NoError(t, err)
127+
require.Equal(t, uint64(100), uint64(d.meta.DiskFileNum))
128+
require.Equal(t, fileType, d.meta.FileType)
129+
require.Equal(t, remote.Locator("foo"), d.meta.Remote.Locator)
130+
require.Equal(t, "custom-obj-name", d.meta.Remote.CustomObjectName)
131+
require.Equal(t, objstorage.SharedNoCleanup, d.meta.Remote.CleanupMethod)
96132
})
97133
}
98134
}
99135

100-
func TestCreateSharedObjectBacking(t *testing.T) {
101-
st := DefaultSettings(vfs.NewMem(), "")
102-
sharedStorage := remote.NewInMem()
103-
st.Remote.StorageFactory = remote.MakeSimpleFactory(map[remote.Locator]remote.Storage{
104-
"foo": sharedStorage,
105-
})
106-
p, err := Open(st)
107-
require.NoError(t, err)
108-
defer p.Close()
109-
110-
require.NoError(t, p.SetCreatorID(1))
111-
112-
backing, err := p.CreateExternalObjectBacking("foo", "custom-obj-name")
113-
require.NoError(t, err)
114-
d, err := decodeRemoteObjectBacking(base.FileTypeTable, base.DiskFileNum(100), backing)
115-
require.NoError(t, err)
116-
require.Equal(t, uint64(100), uint64(d.meta.DiskFileNum))
117-
require.Equal(t, base.FileTypeTable, d.meta.FileType)
118-
require.Equal(t, remote.Locator("foo"), d.meta.Remote.Locator)
119-
require.Equal(t, "custom-obj-name", d.meta.Remote.CustomObjectName)
120-
require.Equal(t, objstorage.SharedNoCleanup, d.meta.Remote.CleanupMethod)
121-
}
122-
123136
func TestAttachRemoteObjects(t *testing.T) {
124-
st := DefaultSettings(vfs.NewMem(), "")
125-
sharedStorage := remote.NewInMem()
126-
st.Remote.StorageFactory = remote.MakeSimpleFactory(map[remote.Locator]remote.Storage{
127-
"foo": sharedStorage,
128-
})
129-
p, err := Open(st)
130-
require.NoError(t, err)
131-
defer p.Close()
132-
require.NoError(t, p.SetCreatorID(1))
133-
backing, err := p.CreateExternalObjectBacking("foo", "custom-obj-name")
134-
require.NoError(t, err)
135-
_, err = p.AttachRemoteObjects([]objstorage.RemoteObjectToAttach{{
136-
FileType: base.FileTypeTable,
137-
FileNum: 100,
138-
Backing: backing,
139-
}})
140-
require.NoError(t, err)
141-
142-
// Sync, close, and reopen the provider and expect that we see
143-
// our object.
144-
require.NoError(t, p.Sync())
145-
require.NoError(t, p.Close())
146-
147-
p, err = Open(st)
148-
require.NoError(t, err)
149-
defer p.Close()
150-
require.NoError(t, p.SetCreatorID(1))
151-
objs := p.List()
152-
require.Len(t, objs, 1)
153-
o := objs[0]
154-
require.Equal(t, remote.Locator("foo"), o.Remote.Locator)
155-
require.Equal(t, "custom-obj-name", o.Remote.CustomObjectName)
156-
require.Equal(t, uint64(100), uint64(o.DiskFileNum))
157-
require.Equal(t, base.FileTypeTable, o.FileType)
137+
for _, fileType := range supportedFileTypes {
138+
t.Run(fileType.String(), func(t *testing.T) {
139+
st := DefaultSettings(vfs.NewMem(), "")
140+
sharedStorage := remote.NewInMem()
141+
st.Remote.StorageFactory = remote.MakeSimpleFactory(map[remote.Locator]remote.Storage{
142+
"foo": sharedStorage,
143+
})
144+
p, err := Open(st)
145+
require.NoError(t, err)
146+
defer p.Close()
147+
require.NoError(t, p.SetCreatorID(1))
148+
backing, err := p.CreateExternalObjectBacking("foo", "custom-obj-name")
149+
require.NoError(t, err)
150+
_, err = p.AttachRemoteObjects([]objstorage.RemoteObjectToAttach{{
151+
FileType: fileType,
152+
FileNum: 100,
153+
Backing: backing,
154+
}})
155+
require.NoError(t, err)
156+
157+
// Sync, close, and reopen the provider and expect that we see
158+
// our object.
159+
require.NoError(t, p.Sync())
160+
require.NoError(t, p.Close())
161+
162+
p, err = Open(st)
163+
require.NoError(t, err)
164+
defer p.Close()
165+
require.NoError(t, p.SetCreatorID(1))
166+
objs := p.List()
167+
require.Len(t, objs, 1)
168+
o := objs[0]
169+
require.Equal(t, remote.Locator("foo"), o.Remote.Locator)
170+
require.Equal(t, "custom-obj-name", o.Remote.CustomObjectName)
171+
require.Equal(t, uint64(100), uint64(o.DiskFileNum))
172+
require.Equal(t, fileType, o.FileType)
173+
})
174+
}
158175
}

‎objstorage/objstorageprovider/remote_obj_name.go

+15
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,11 @@ func remoteObjectName(meta objstorage.ObjectMetadata) string {
2525
"%04x-%d-%06d.sst",
2626
objHash(meta), meta.Remote.CreatorID, meta.Remote.CreatorFileNum,
2727
)
28+
case base.FileTypeBlob:
29+
return fmt.Sprintf(
30+
"%04x-%d-%06d.blob",
31+
objHash(meta), meta.Remote.CreatorID, meta.Remote.CreatorFileNum,
32+
)
2833
}
2934
panic("unknown FileType")
3035
}
@@ -51,6 +56,11 @@ func sharedObjectRefName(
5156
"%04x-%d-%06d.sst.ref.%d.%06d",
5257
objHash(meta), meta.Remote.CreatorID, meta.Remote.CreatorFileNum, refCreatorID, refFileNum,
5358
)
59+
case base.FileTypeBlob:
60+
return fmt.Sprintf(
61+
"%04x-%d-%06d.blob.ref.%d.%06d",
62+
objHash(meta), meta.Remote.CreatorID, meta.Remote.CreatorFileNum, refCreatorID, refFileNum,
63+
)
5464
}
5565
panic("unknown FileType")
5666
}
@@ -65,6 +75,11 @@ func sharedObjectRefPrefix(meta objstorage.ObjectMetadata) string {
6575
"%04x-%d-%06d.sst.ref.",
6676
objHash(meta), meta.Remote.CreatorID, meta.Remote.CreatorFileNum,
6777
)
78+
case base.FileTypeBlob:
79+
return fmt.Sprintf(
80+
"%04x-%d-%06d.blob.ref.",
81+
objHash(meta), meta.Remote.CreatorID, meta.Remote.CreatorFileNum,
82+
)
6883
}
6984
panic("unknown FileType")
7085
}

‎objstorage/objstorageprovider/remote_obj_name_test.go

+15-3
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,6 @@ import (
1616

1717
func TestSharedObjectNames(t *testing.T) {
1818
t.Run("crosscheck", func(t *testing.T) {
19-
supportedFileTypes := []base.FileType{
20-
base.FileTypeTable,
21-
}
2219
for it := 0; it < 100; it++ {
2320
var meta objstorage.ObjectMetadata
2421
meta.DiskFileNum = base.DiskFileNum(rand.IntN(100000))
@@ -61,4 +58,19 @@ func TestSharedObjectNames(t *testing.T) {
6158
"0e17-456-000789.sst.ref.101112.000123",
6259
)
6360
})
61+
t.Run("example-blobfile", func(t *testing.T) {
62+
var meta objstorage.ObjectMetadata
63+
meta.DiskFileNum = base.DiskFileNum(123)
64+
meta.FileType = base.FileTypeBlob
65+
meta.Remote.CreatorID = objstorage.CreatorID(456)
66+
meta.Remote.CreatorFileNum = base.DiskFileNum(789)
67+
require.Equal(t, remoteObjectName(meta), "0e17-456-000789.blob")
68+
require.Equal(t, sharedObjectRefPrefix(meta), "0e17-456-000789.blob.ref.")
69+
70+
refCreatorID := objstorage.CreatorID(101112)
71+
require.Equal(
72+
t, sharedObjectRefName(meta, refCreatorID, meta.DiskFileNum),
73+
"0e17-456-000789.blob.ref.101112.000123",
74+
)
75+
})
6476
}

‎objstorage/objstorageprovider/remoteobjcat/catalog_test.go

+18-9
Original file line numberDiff line numberDiff line change
@@ -42,14 +42,18 @@ func TestCatalog(t *testing.T) {
4242

4343
parseAdd := func(args []string) remoteobjcat.RemoteObjectMetadata {
4444
t.Helper()
45-
if len(args) != 3 {
46-
td.Fatalf(t, "add <file-num> <creator-id> <creator-file-num>")
45+
if len(args) != 3 && len(args) != 4 {
46+
td.Fatalf(t, "add <file-num> <creator-id> <creator-file-num> [sstable|blob]")
4747
}
48-
vals := toUInt64(args...)
48+
fileType := base.FileTypeTable
49+
if len(args) == 4 {
50+
fileType = base.FileTypeFromName(args[3])
51+
}
52+
53+
vals := toUInt64(args[:min(3, len(args))]...)
4954
return remoteobjcat.RemoteObjectMetadata{
50-
FileNum: base.DiskFileNum(vals[0]),
51-
// When we support other file types, we should let the test determine this.
52-
FileType: base.FileTypeTable,
55+
FileNum: base.DiskFileNum(vals[0]),
56+
FileType: fileType,
5357
CreatorID: objstorage.CreatorID(vals[1]),
5458
CreatorFileNum: base.DiskFileNum(vals[2]),
5559
}
@@ -145,10 +149,10 @@ func TestCatalog(t *testing.T) {
145149
var b remoteobjcat.Batch
146150
for batchIdx := 0; batchIdx < n; batchIdx++ {
147151
for i := 0; i < size; i++ {
152+
148153
b.AddObject(remoteobjcat.RemoteObjectMetadata{
149-
FileNum: base.DiskFileNum(rand.Uint64()),
150-
// When we support other file types, we should let the test determine this.
151-
FileType: base.FileTypeTable,
154+
FileNum: base.DiskFileNum(rand.Uint64()),
155+
FileType: supportedFileTypes[rand.IntN(len(supportedFileTypes))],
152156
CreatorID: objstorage.CreatorID(rand.Uint64()),
153157
CreatorFileNum: base.DiskFileNum(rand.Uint64()),
154158
})
@@ -192,3 +196,8 @@ func TestCatalog(t *testing.T) {
192196
}
193197
})
194198
}
199+
200+
var supportedFileTypes = []base.FileType{
201+
base.FileTypeTable,
202+
base.FileTypeBlob,
203+
}

0 commit comments

Comments
 (0)