Skip to content

Commit

Permalink
Used CachedAssetStore for metadata.
Browse files Browse the repository at this point in the history
  • Loading branch information
@wbxyz
wbxyz committed Dec 3, 2024
1 parent 0fc2111 commit 7c5a81f
Show file tree
Hide file tree
Showing 5 changed files with 54 additions and 60 deletions.
17 changes: 11 additions & 6 deletions cmd/api/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,16 +137,21 @@ func RebuildPackageInit(ctx context.Context) (*apiservice.RebuildPackageDeps, er
if err != nil {
return nil, errors.Wrap(err, "creating attestation uploader")
}
d.LocalMetadataStore = rebuild.NewFilesystemAssetStore(memfs.New())
// TODO: This can be optional once LocalMetadata and DebugStore are combined into a cached store.
if *debugStorage == "" {
return nil, errors.New("debug-store must be set")
}
d.DebugStoreBuilder = func(ctx context.Context) (rebuild.AssetStore, error) {
d.MetadataStoreBuilder = func(ctx context.Context) (rebuild.AssetStore, error) {
return rebuild.NewFilesystemAssetStore(memfs.New()), nil
}
} else {
if ctx.Value(rebuild.RunID) == nil {
return nil, errors.New("RunID must be set in the context")
}
return rebuild.DebugStoreFromContext(context.WithValue(ctx, rebuild.UploadArtifactsPathID, *debugStorage))
d.MetadataStoreBuilder = func(ctx context.Context) (rebuild.AssetStore, error) {
debug, err := rebuild.DebugStoreFromContext(context.WithValue(ctx, rebuild.UploadArtifactsPathID, *debugStorage))
if err != nil {
return nil, errors.Wrap(err, "creating metadata store")
}
return rebuild.NewCachedAssetStore(debug), nil
}
}
d.RemoteMetadataStoreBuilder = func(ctx context.Context, uuid string) (rebuild.LocatableAssetStore, error) {
return rebuild.NewGCSStore(context.WithValue(ctx, rebuild.RunID, uuid), "gs://"+*metadataBucket)
Expand Down
47 changes: 22 additions & 25 deletions internal/api/apiservice/rebuild.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,19 +120,17 @@ func populateArtifact(ctx context.Context, t *rebuild.Target, mux rebuild.Regist
}

type RebuildPackageDeps struct {
HTTPClient httpx.BasicClient
FirestoreClient *firestore.Client
Signer *dsse.EnvelopeSigner
GCBClient gcb.Client
BuildProject string
BuildServiceAccount string
UtilPrebuildBucket string
BuildLogsBucket string
BuildDefRepo rebuild.Location
AttestationStore rebuild.AssetStore
// TODO: The local store and the debug store can be combined into a layered (cached) asset store.
LocalMetadataStore rebuild.AssetStore
DebugStoreBuilder func(ctx context.Context) (rebuild.AssetStore, error)
HTTPClient httpx.BasicClient
FirestoreClient *firestore.Client
Signer *dsse.EnvelopeSigner
GCBClient gcb.Client
BuildProject string
BuildServiceAccount string
UtilPrebuildBucket string
BuildLogsBucket string
BuildDefRepo rebuild.Location
AttestationStore rebuild.AssetStore
MetadataStoreBuilder func(ctx context.Context) (rebuild.AssetStore, error)
RemoteMetadataStoreBuilder func(ctx context.Context, uuid string) (rebuild.LocatableAssetStore, error)
OverwriteAttestations bool
InferStub api.StubT[schema.InferenceRequest, schema.StrategyOneOf]
Expand Down Expand Up @@ -201,11 +199,7 @@ func getStrategy(ctx context.Context, deps *RebuildPackageDeps, t rebuild.Target
return strategy, entry, nil
}

func buildAndAttest(ctx context.Context, deps *RebuildPackageDeps, mux rebuild.RegistryMux, a verifier.Attestor, t rebuild.Target, strategy rebuild.Strategy, entry *repoEntry, useProxy bool, useSyscallMonitor bool) (err error) {
debugStore, err := deps.DebugStoreBuilder(ctx)
if err != nil {
return errors.Wrap(err, "creating debug store")
}
func buildAndAttest(ctx context.Context, deps *RebuildPackageDeps, metadata rebuild.AssetStore, mux rebuild.RegistryMux, a verifier.Attestor, t rebuild.Target, strategy rebuild.Strategy, entry *repoEntry, useProxy bool, useSyscallMonitor bool) (err error) {
id := uuid.New().String()
remoteMetadata, err := deps.RemoteMetadataStoreBuilder(ctx, id)
if err != nil {
Expand All @@ -218,8 +212,7 @@ func buildAndAttest(ctx context.Context, deps *RebuildPackageDeps, mux rebuild.R
BuildServiceAccount: deps.BuildServiceAccount,
UtilPrebuildBucket: deps.UtilPrebuildBucket,
LogsBucket: deps.BuildLogsBucket,
LocalMetadataStore: deps.LocalMetadataStore,
DebugStore: debugStore,
MetadataStore: metadata,
RemoteMetadataStore: remoteMetadata,
UseSyscallMonitor: useSyscallMonitor,
UseNetworkProxy: useProxy,
Expand Down Expand Up @@ -256,7 +249,7 @@ func buildAndAttest(ctx context.Context, deps *RebuildPackageDeps, mux rebuild.R
input.Strategy = entry.Strategy
loc = entry.BuildDefLoc
}
eqStmt, buildStmt, err := verifier.CreateAttestations(ctx, input, strategy, id, rb, up, deps.LocalMetadataStore, loc)
eqStmt, buildStmt, err := verifier.CreateAttestations(ctx, input, strategy, id, rb, up, metadata, loc)
if err != nil {
return errors.Wrap(err, "creating attestations")
}
Expand All @@ -266,7 +259,7 @@ func buildAndAttest(ctx context.Context, deps *RebuildPackageDeps, mux rebuild.R
return nil
}

func rebuildPackage(ctx context.Context, req schema.RebuildPackageRequest, deps *RebuildPackageDeps) (*schema.Verdict, error) {
func rebuildPackage(ctx context.Context, req schema.RebuildPackageRequest, deps *RebuildPackageDeps, metadata rebuild.AssetStore) (*schema.Verdict, error) {
t := rebuild.Target{Ecosystem: req.Ecosystem, Package: req.Package, Version: req.Version, Artifact: req.Artifact}
if req.Ecosystem == rebuild.Debian && strings.TrimSpace(req.Artifact) == "" {
return nil, api.AsStatus(codes.InvalidArgument, errors.New("debian requires artifact"))
Expand Down Expand Up @@ -306,7 +299,7 @@ func rebuildPackage(ctx context.Context, req schema.RebuildPackageRequest, deps
if strategy != nil {
v.StrategyOneof = schema.NewStrategyOneOf(strategy)
}
err = buildAndAttest(ctx, deps, mux, a, t, strategy, entry, req.UseNetworkProxy, req.UseSyscallMonitor)
err = buildAndAttest(ctx, deps, metadata, mux, a, t, strategy, entry, req.UseNetworkProxy, req.UseSyscallMonitor)
if err != nil {
v.Message = errors.Wrap(err, "executing rebuild").Error()
return &v, nil
Expand All @@ -316,12 +309,16 @@ func rebuildPackage(ctx context.Context, req schema.RebuildPackageRequest, deps

func RebuildPackage(ctx context.Context, req schema.RebuildPackageRequest, deps *RebuildPackageDeps) (*schema.Verdict, error) {
ctx = context.WithValue(ctx, rebuild.RunID, req.ID)
v, err := rebuildPackage(ctx, req, deps)
metadata, err := deps.MetadataStoreBuilder(ctx)
if err != nil {
return nil, errors.Wrap(err, "creating metadata store")
}
v, err := rebuildPackage(ctx, req, deps, metadata)
if err != nil {
return nil, err
}
var dockerfile string
w, err := deps.LocalMetadataStore.Reader(ctx, rebuild.Asset{Target: v.Target, Type: rebuild.DockerfileAsset})
w, err := metadata.Reader(ctx, rebuild.Asset{Target: v.Target, Type: rebuild.DockerfileAsset})
if err == nil {
if b, err := io.ReadAll(w); err == nil {
dockerfile = string(b)
Expand Down
12 changes: 6 additions & 6 deletions internal/api/apiservice/rebuild_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -304,14 +304,14 @@ RLpmHHG1JOVdOA==
fs := memfs.New()
afs := must(fs.Chroot("attestations"))
d.AttestationStore = rebuild.NewFilesystemAssetStore(afs)
d.DebugStoreBuilder = func(ctx context.Context) (rebuild.AssetStore, error) {
return rebuild.NewFilesystemAssetStore(must(fs.Chroot("debug-metadata"))), nil
metadata := rebuild.NewFilesystemAssetStore(must(fs.Chroot("debug-metadata")))
d.MetadataStoreBuilder = func(ctx context.Context) (rebuild.AssetStore, error) {
return metadata, nil
}
remoteMetadata := rebuild.NewFilesystemAssetStore(must(fs.Chroot("remote-metadata")))
d.RemoteMetadataStoreBuilder = func(ctx context.Context, id string) (rebuild.LocatableAssetStore, error) {
return remoteMetadata, nil
}
d.LocalMetadataStore = rebuild.NewFilesystemAssetStore(must(fs.Chroot("local-metadata")))
buildSteps := []*cloudbuild.BuildStep{
{Name: "gcr.io/foo/bar", Script: "./bar"},
}
Expand Down Expand Up @@ -360,7 +360,7 @@ RLpmHHG1JOVdOA==
return &oneof, nil
}

verdict, err := rebuildPackage(ctx, schema.RebuildPackageRequest{Ecosystem: tc.target.Ecosystem, Package: tc.target.Package, Version: tc.target.Version, Artifact: tc.target.Artifact}, &d)
verdict, err := rebuildPackage(ctx, schema.RebuildPackageRequest{Ecosystem: tc.target.Ecosystem, Package: tc.target.Package, Version: tc.target.Version, Artifact: tc.target.Artifact}, &d, metadata)
if err != nil {
t.Fatalf("RebuildPackage(): %v", err)
}
Expand All @@ -374,11 +374,11 @@ RLpmHHG1JOVdOA==
t.Fatalf("RebuildPackage() verdict: %v", verdict.Message)
}

dockerfile := must(d.LocalMetadataStore.Reader(ctx, rebuild.Asset{Type: rebuild.DockerfileAsset, Target: tc.target}))
dockerfile := must(metadata.Reader(ctx, rebuild.Asset{Type: rebuild.DockerfileAsset, Target: tc.target}))
if len(must(io.ReadAll(dockerfile))) == 0 {
t.Error("Dockerfile empty")
}
buildinfo := must(d.LocalMetadataStore.Reader(ctx, rebuild.Asset{Type: rebuild.BuildInfoAsset, Target: tc.target}))
buildinfo := must(metadata.Reader(ctx, rebuild.Asset{Type: rebuild.BuildInfoAsset, Target: tc.target}))
diff := cmp.Diff(
rebuild.BuildInfo{
Target: tc.target,
Expand Down
36 changes: 14 additions & 22 deletions pkg/rebuild/rebuild/rebuildremote.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,9 @@ type RemoteOptions struct {
Project string
BuildServiceAccount string
LogsBucket string
// LocalMetadataStore stores the dockerfile and build info. Cloud build does not need access to this.
LocalMetadataStore AssetStore
// DebugStore is the durable storage of the dockerfile and build info. Cloud build does not need access to this. It should be keyed by RunID to allow programatic access.
DebugStore AssetStore
// RemoteMetadataStore stores the rebuilt artifact. Cloud build needs access to upload assets here. It should be keyed by the unguessable UUID to sandbox each build.
// MetadataStore stores the dockerfile and build info. Cloud build does not need access to this.
MetadataStore AssetStore
// RemoteMetadataStore stores the rebuilt artifacts. Cloud build needs access to upload assets here. It should be keyed by the unguessable UUID to sandbox each build.
RemoteMetadataStore LocatableAssetStore
UtilPrebuildBucket string
// TODO: Consider moving these to Strategy.
Expand Down Expand Up @@ -481,19 +479,16 @@ func RebuildRemote(ctx context.Context, input Input, id string, opts RemoteOptio
return errors.Wrap(err, "creating dockerfile")
}
{
lw, err := opts.LocalMetadataStore.Writer(ctx, Asset{Target: t, Type: DockerfileAsset})
w, err := opts.MetadataStore.Writer(ctx, Asset{Target: t, Type: DockerfileAsset})
if err != nil {
return errors.Wrap(err, "creating writer for Dockerfile")
}
defer lw.Close()
rw, err := opts.DebugStore.Writer(ctx, Asset{Target: t, Type: DockerfileAsset})
if err != nil {
return errors.Wrap(err, "creating remote writer for Dockerfile")
}
defer rw.Close()
if _, err := io.WriteString(io.MultiWriter(lw, rw), dockerfile); err != nil {
if _, err := io.WriteString(w, dockerfile); err != nil {
return errors.Wrap(err, "writing Dockerfile")
}
if err = w.Close(); err != nil {
return errors.Wrap(err, "closing writer for Dockerfile")
}
}
build, err := makeBuild(t, dockerfile, opts)
if err != nil {
Expand All @@ -503,19 +498,16 @@ func RebuildRemote(ctx context.Context, input Input, id string, opts RemoteOptio
return errors.Wrap(err, "performing build")
}
{
lw, err := opts.LocalMetadataStore.Writer(ctx, Asset{Target: t, Type: BuildInfoAsset})
w, err := opts.MetadataStore.Writer(ctx, Asset{Target: t, Type: BuildInfoAsset})
if err != nil {
return errors.Wrap(err, "creating writer for build info")
return errors.Wrap(err, "creating writer for BuildInfo")
}
defer lw.Close()
rw, err := opts.DebugStore.Writer(ctx, Asset{Target: t, Type: BuildInfoAsset})
if err != nil {
return errors.Wrap(err, "creating remote writer for build info")
}
defer rw.Close()
if err := json.NewEncoder(io.MultiWriter(lw, rw)).Encode(bi); err != nil {
if err := json.NewEncoder(w).Encode(bi); err != nil {
return errors.Wrap(err, "marshalling and writing build info")
}
if err = w.Close(); err != nil {
return errors.Wrap(err, "closing writer for BuildInfo")
}
}
return nil
}
2 changes: 1 addition & 1 deletion pkg/rebuild/rebuild/storage.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ func AssetCopy(ctx context.Context, to, from AssetStore, a Asset) error {
}

// DebugStoreFromContext constructs a DebugStorer using values from the given context.
func DebugStoreFromContext(ctx context.Context) (AssetStore, error) {
func DebugStoreFromContext(ctx context.Context) (LocatableAssetStore, error) {
if uploadpath, ok := ctx.Value(UploadArtifactsPathID).(string); ok {
u, err := url.Parse(uploadpath)
if err != nil {
Expand Down

0 comments on commit 7c5a81f

Please sign in to comment.