Skip to content

Commit 4260607

Browse files
hsanjuanhsanjuan
authored
Merge pull request #10746 from ipfs/fix/10596-buffered-reprovides
provider: buffer pin providers
2 parents 3e1fb7e + 9de9384 commit 4260607

File tree

8 files changed

+32
-20
lines changed

8 files changed

+32
-20
lines changed

‎core/node/provider.go

+10-3
Original file line numberDiff line numberDiff line change
@@ -167,16 +167,23 @@ func newProvidingStrategy(onlyPinned, onlyRoots bool) interface{} {
167167
IPLDFetcher fetcher.Factory `name:"ipldFetcher"`
168168
}
169169
return func(in input) provider.KeyChanFunc {
170+
// Pinner-related CIDs will be buffered in memory to avoid
171+
// deadlocking the pinner when the providing process is slow.
172+
170173
if onlyRoots {
171-
return provider.NewPinnedProvider(true, in.Pinner, in.IPLDFetcher)
174+
return provider.NewBufferedProvider(
175+
provider.NewPinnedProvider(true, in.Pinner, in.IPLDFetcher),
176+
)
172177
}
173178

174179
if onlyPinned {
175-
return provider.NewPinnedProvider(false, in.Pinner, in.IPLDFetcher)
180+
return provider.NewBufferedProvider(
181+
provider.NewPinnedProvider(false, in.Pinner, in.IPLDFetcher),
182+
)
176183
}
177184

178185
return provider.NewPrioritizedProvider(
179-
provider.NewPinnedProvider(true, in.Pinner, in.IPLDFetcher),
186+
provider.NewBufferedProvider(provider.NewPinnedProvider(true, in.Pinner, in.IPLDFetcher)),
180187
provider.NewBlockstoreProvider(in.Blockstore),
181188
)
182189
}

‎docs/changelogs/v0.34.md

+5
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
- [Badger datastore update](#badger-datastore-update)
1515
- [Datastore Implementation Updates](#datastore-implementation-updates)
1616
- [One Multi-error Package](#one-multi-error-package)
17+
- [Fix hanging pinset operations during reprovides](#fix-hanging-pinset-operations-during-reprovides)
1718
- [📦️ Important dependency updates](#-important-dependency-updates)
1819
- [👨‍👩‍👧‍👦 Contributors](#-contributors)
1920

@@ -74,6 +75,10 @@ The go-ds-xxx datastore implementations have been updated to support the updated
7475

7576
Kubo previously depended on multiple multi-error packages, `github.com/hashicorp/go-multierror` and `go.uber.org/multierr`. These have nearly identical functionality so there was no need to use both. Therefore, `go.uber.org/multierr` was selected as the package to depend on. Any future code needing multi-error functionality should use `go.uber.org/multierr` to avoid introducing unneeded dependencies.
7677

78+
#### Fix hanging pinset operations during reprovides
79+
80+
The reprovide process can be quite slow. In default settings, the reprovide process will start reading CIDs that belong to the pinset. During this operation, starvation can occur for other operations that need pinset access (see https://github.com/ipfs/kubo/issue/10596). We have now switch to buffering pinset-related cids that are going to be reprovided in memory, so that we can free pinset mutexes as soon as possible so that pinset-writes and subsequent read operations can proceed. The downside is larger pinsets will need some extra memory, with an estimation of ~1GiB of RAM memory-use per 20 million items to be reprovided.
81+
7782
#### 📦️ Important dependency updates
7883

7984
- update `go-libp2p` to [v0.41.0](https://github.com/libp2p/go-libp2p/releases/tag/v0.41.0) (incl. [v0.40.0](https://github.com/libp2p/go-libp2p/releases/tag/v0.40.0))

‎docs/examples/kubo-as-a-library/go.mod

+2-2
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ go 1.24
77
replace github.com/ipfs/kubo => ./../../..
88

99
require (
10-
github.com/ipfs/boxo v0.29.0
10+
github.com/ipfs/boxo v0.29.1-0.20250307073615-93ea580a3bb0
1111
github.com/ipfs/kubo v0.0.0-00010101000000-000000000000
1212
github.com/libp2p/go-libp2p v0.41.0
1313
github.com/multiformats/go-multiaddr v0.15.0
@@ -52,7 +52,7 @@ require (
5252
github.com/francoispqt/gojay v1.2.13 // indirect
5353
github.com/fsnotify/fsnotify v1.7.0 // indirect
5454
github.com/gabriel-vasile/mimetype v1.4.6 // indirect
55-
github.com/gammazero/chanqueue v1.0.0 // indirect
55+
github.com/gammazero/chanqueue v1.1.0 // indirect
5656
github.com/gammazero/deque v1.0.0 // indirect
5757
github.com/getsentry/sentry-go v0.27.0 // indirect
5858
github.com/go-jose/go-jose/v4 v4.0.5 // indirect

‎docs/examples/kubo-as-a-library/go.sum

+4-4
Original file line numberDiff line numberDiff line change
@@ -166,8 +166,8 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos
166166
github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
167167
github.com/gabriel-vasile/mimetype v1.4.6 h1:3+PzJTKLkvgjeTbts6msPJt4DixhT4YtFNf1gtGe3zc=
168168
github.com/gabriel-vasile/mimetype v1.4.6/go.mod h1:JX1qVKqZd40hUPpAfiNTe0Sne7hdfKSbOqqmkq8GCXc=
169-
github.com/gammazero/chanqueue v1.0.0 h1:FER/sMailGFA3DDvFooEkipAMU+3c9Bg3bheloPSz6o=
170-
github.com/gammazero/chanqueue v1.0.0/go.mod h1:fMwpwEiuUgpab0sH4VHiVcEoji1pSi+EIzeG4TPeKPc=
169+
github.com/gammazero/chanqueue v1.1.0 h1:yiwtloc1azhgGLFo2gMloJtQvkYD936Ai7tBfa+rYJw=
170+
github.com/gammazero/chanqueue v1.1.0/go.mod h1:fMwpwEiuUgpab0sH4VHiVcEoji1pSi+EIzeG4TPeKPc=
171171
github.com/gammazero/deque v1.0.0 h1:LTmimT8H7bXkkCy6gZX7zNLtkbz4NdS2z8LZuor3j34=
172172
github.com/gammazero/deque v1.0.0/go.mod h1:iflpYvtGfM3U8S8j+sZEKIak3SAKYpA5/SQewgfXDKo=
173173
github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps=
@@ -298,8 +298,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd
298298
github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU=
299299
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
300300
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
301-
github.com/ipfs/boxo v0.29.0 h1:clzd7PglUcE+Ufq1KucS3aKID7pzGVaSgcdRsW395t4=
302-
github.com/ipfs/boxo v0.29.0/go.mod h1:c3R52nMlgMsN1tADffYcogKoVRsX1RJE1TMYSpJ4uVs=
301+
github.com/ipfs/boxo v0.29.1-0.20250307073615-93ea580a3bb0 h1:wkFvXf7SqRiAu5ezj6lsieE2PwDk3nUkFOs9/+5lWdo=
302+
github.com/ipfs/boxo v0.29.1-0.20250307073615-93ea580a3bb0/go.mod h1:MkDJStXiJS9U99cbAijHdcmwNfVn5DKYBmQCOgjY2NU=
303303
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
304304
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
305305
github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ=

‎go.mod

+2-2
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ require (
2121
github.com/hashicorp/go-version v1.7.0
2222
github.com/ipfs-shipyard/nopfs v0.0.14
2323
github.com/ipfs-shipyard/nopfs/ipfs v0.25.0
24-
github.com/ipfs/boxo v0.29.0
24+
github.com/ipfs/boxo v0.29.1-0.20250307073615-93ea580a3bb0
2525
github.com/ipfs/go-block-format v0.2.0
2626
github.com/ipfs/go-cid v0.5.0
2727
github.com/ipfs/go-cidutil v0.1.0
@@ -125,7 +125,7 @@ require (
125125
github.com/flynn/noise v1.1.0 // indirect
126126
github.com/francoispqt/gojay v1.2.13 // indirect
127127
github.com/gabriel-vasile/mimetype v1.4.6 // indirect
128-
github.com/gammazero/chanqueue v1.0.0 // indirect
128+
github.com/gammazero/chanqueue v1.1.0 // indirect
129129
github.com/gammazero/deque v1.0.0 // indirect
130130
github.com/getsentry/sentry-go v0.27.0 // indirect
131131
github.com/go-jose/go-jose/v4 v4.0.5 // indirect

‎go.sum

+4-4
Original file line numberDiff line numberDiff line change
@@ -200,8 +200,8 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos
200200
github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
201201
github.com/gabriel-vasile/mimetype v1.4.6 h1:3+PzJTKLkvgjeTbts6msPJt4DixhT4YtFNf1gtGe3zc=
202202
github.com/gabriel-vasile/mimetype v1.4.6/go.mod h1:JX1qVKqZd40hUPpAfiNTe0Sne7hdfKSbOqqmkq8GCXc=
203-
github.com/gammazero/chanqueue v1.0.0 h1:FER/sMailGFA3DDvFooEkipAMU+3c9Bg3bheloPSz6o=
204-
github.com/gammazero/chanqueue v1.0.0/go.mod h1:fMwpwEiuUgpab0sH4VHiVcEoji1pSi+EIzeG4TPeKPc=
203+
github.com/gammazero/chanqueue v1.1.0 h1:yiwtloc1azhgGLFo2gMloJtQvkYD936Ai7tBfa+rYJw=
204+
github.com/gammazero/chanqueue v1.1.0/go.mod h1:fMwpwEiuUgpab0sH4VHiVcEoji1pSi+EIzeG4TPeKPc=
205205
github.com/gammazero/deque v1.0.0 h1:LTmimT8H7bXkkCy6gZX7zNLtkbz4NdS2z8LZuor3j34=
206206
github.com/gammazero/deque v1.0.0/go.mod h1:iflpYvtGfM3U8S8j+sZEKIak3SAKYpA5/SQewgfXDKo=
207207
github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps=
@@ -362,8 +362,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd
362362
github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU=
363363
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
364364
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
365-
github.com/ipfs/boxo v0.29.0 h1:clzd7PglUcE+Ufq1KucS3aKID7pzGVaSgcdRsW395t4=
366-
github.com/ipfs/boxo v0.29.0/go.mod h1:c3R52nMlgMsN1tADffYcogKoVRsX1RJE1TMYSpJ4uVs=
365+
github.com/ipfs/boxo v0.29.1-0.20250307073615-93ea580a3bb0 h1:wkFvXf7SqRiAu5ezj6lsieE2PwDk3nUkFOs9/+5lWdo=
366+
github.com/ipfs/boxo v0.29.1-0.20250307073615-93ea580a3bb0/go.mod h1:MkDJStXiJS9U99cbAijHdcmwNfVn5DKYBmQCOgjY2NU=
367367
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
368368
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
369369
github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ=

‎test/dependencies/go.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@ require (
116116
github.com/huin/goupnp v1.3.0 // indirect
117117
github.com/inconshreveable/mousetrap v1.1.0 // indirect
118118
github.com/ipfs/bbloom v0.0.4 // indirect
119-
github.com/ipfs/boxo v0.29.0 // indirect
119+
github.com/ipfs/boxo v0.29.1-0.20250307073615-93ea580a3bb0 // indirect
120120
github.com/ipfs/go-block-format v0.2.0 // indirect
121121
github.com/ipfs/go-cid v0.5.0 // indirect
122122
github.com/ipfs/go-datastore v0.8.2 // indirect

‎test/dependencies/go.sum

+4-4
Original file line numberDiff line numberDiff line change
@@ -156,8 +156,8 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos
156156
github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
157157
github.com/fzipp/gocyclo v0.6.0 h1:lsblElZG7d3ALtGMx9fmxeTKZaLLpU8mET09yN4BBLo=
158158
github.com/fzipp/gocyclo v0.6.0/go.mod h1:rXPyn8fnlpa0R2csP/31uerbiVBugk5whMdlyaLkLoA=
159-
github.com/gammazero/chanqueue v1.0.0 h1:FER/sMailGFA3DDvFooEkipAMU+3c9Bg3bheloPSz6o=
160-
github.com/gammazero/chanqueue v1.0.0/go.mod h1:fMwpwEiuUgpab0sH4VHiVcEoji1pSi+EIzeG4TPeKPc=
159+
github.com/gammazero/chanqueue v1.1.0 h1:yiwtloc1azhgGLFo2gMloJtQvkYD936Ai7tBfa+rYJw=
160+
github.com/gammazero/chanqueue v1.1.0/go.mod h1:fMwpwEiuUgpab0sH4VHiVcEoji1pSi+EIzeG4TPeKPc=
161161
github.com/gammazero/deque v1.0.0 h1:LTmimT8H7bXkkCy6gZX7zNLtkbz4NdS2z8LZuor3j34=
162162
github.com/gammazero/deque v1.0.0/go.mod h1:iflpYvtGfM3U8S8j+sZEKIak3SAKYpA5/SQewgfXDKo=
163163
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
@@ -294,8 +294,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
294294
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
295295
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
296296
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
297-
github.com/ipfs/boxo v0.29.0 h1:clzd7PglUcE+Ufq1KucS3aKID7pzGVaSgcdRsW395t4=
298-
github.com/ipfs/boxo v0.29.0/go.mod h1:c3R52nMlgMsN1tADffYcogKoVRsX1RJE1TMYSpJ4uVs=
297+
github.com/ipfs/boxo v0.29.1-0.20250307073615-93ea580a3bb0 h1:wkFvXf7SqRiAu5ezj6lsieE2PwDk3nUkFOs9/+5lWdo=
298+
github.com/ipfs/boxo v0.29.1-0.20250307073615-93ea580a3bb0/go.mod h1:MkDJStXiJS9U99cbAijHdcmwNfVn5DKYBmQCOgjY2NU=
299299
github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs=
300300
github.com/ipfs/go-block-format v0.2.0/go.mod h1:+jpL11nFx5A/SPpsoBn6Bzkra/zaArfSmsknbPMYgzM=
301301
github.com/ipfs/go-cid v0.5.0 h1:goEKKhaGm0ul11IHA7I6p1GmKz8kEYniqFopaB5Otwg=

0 commit comments

Comments
 (0)