Skip to content

download can be done'ed or destroyed #389

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 38 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -305,9 +305,20 @@ Stream a blob into the drive at `path`.
}
```

#### `await drive.download(folder, [options])`
#### `const download = await drive.download(folder, [options])`

Downloads the blobs corresponding to all entries in the drive at paths prefixed with `folder`.
Downloads the blobs corresponding to all entries in the drive at paths prefixed with `folder`. Returns a `Download` object that resolves once all data has been downloaded:

```js
const download = await drive.download(key)
await download.done()
```

You can also cancel an ongoing download using `destroy()`.

```js
download.destroy()
```

`options` are the same as those for `drive.list(folder, [options])`.

Expand All @@ -331,13 +342,36 @@ If an entry exists in `drive.version` of the `folder` but not in `version`, then

#### `await drive.downloadDiff(version, folder, [options])`

Downloads all the blobs in `folder` corresponding to entries in `drive.checkout(version)` that are not in `drive.version`.
Downloads all the blobs in `folder` corresponding to entries in `drive.checkout(version)` that are not in `drive.version`. Returns a `Download` object that resolves once all data has been downloaded:

```js
const download = await drive.downloadDiff(version, folder)
await download.done()
```

You can also cancel an ongoing download using `destroy()`.

```js
download.destroy()
```

In other words, downloads all the blobs added to `folder` up to `version` of the drive.

#### `await drive.downloadRange(dbRanges, blobRanges)`

Downloads the entries and blobs stored in the [ranges][core-range-docs] `dbRanges` and `blobRanges`.
Downloads the entries and blobs stored in the [ranges][core-range-docs] `dbRanges` and `blobRanges`. Returns a `Download` object that resolves once all data has been downloaded:


```js
const download = await drive.downloadRange(dbRanges, blobRanges)
await download.done()
```

You can also cancel an ongoing download using `destroy()`.

```js
download.destroy()
```

#### `await drive.has(path)`

Expand Down
20 changes: 6 additions & 14 deletions index.js
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ const crypto = require('hypercore-crypto')
const Hypercore = require('hypercore')
const { BLOCK_NOT_AVAILABLE, BAD_ARGUMENT } = require('hypercore-errors')
const Monitor = require('./lib/monitor')
const Download = require('./lib/download')

const keyEncoding = new SubEncoder('files', 'utf-8')

Expand Down Expand Up @@ -410,10 +411,7 @@ module.exports = class Hyperdrive extends ReadyResource {
dls.push(blobs.core.download({ start: b.blockOffset, length: b.blockLength }))
}

const proms = []
for (const r of dls) proms.push(r.downloaded())

await Promise.allSettled(proms)
return new Download(dls)
}

async downloadRange (dbRanges, blobRanges) {
Expand All @@ -431,10 +429,7 @@ module.exports = class Hyperdrive extends ReadyResource {
dls.push(blobs.core.download(range))
}

const proms = []
for (const r of dls) proms.push(r.downloaded())

await Promise.allSettled(proms)
return new Download(dls)
}

entries (range, opts) {
Expand All @@ -453,8 +448,8 @@ module.exports = class Hyperdrive extends ReadyResource {
const b = entry.value.blob
if (!b) return
const blobs = await this.getBlobs()
await blobs.core.download({ start: b.blockOffset, length: b.blockLength }).downloaded()
return
const download = await blobs.core.download({ start: b.blockOffset, length: b.blockLength })
return new Download(download)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

make it an array here so its always an array

}

// first preload the list so we can use the full power afterwards to actually preload everything
Expand All @@ -471,10 +466,7 @@ module.exports = class Hyperdrive extends ReadyResource {
dls.push(blobs.core.download({ start: b.blockOffset, length: b.blockLength }))
}

const proms = []
for (const r of dls) proms.push(r.downloaded())

await Promise.allSettled(proms)
return new Download(dls)
}

async has (path) {
Expand Down
13 changes: 13 additions & 0 deletions lib/download.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
module.exports = class Download {
constructor (coreDownloads = []) {
this._coreDownloads = Array.isArray(coreDownloads) ? coreDownloads : [coreDownloads]
}

destroy () {
this._coreDownloads.forEach((d) => d.destroy())
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

do a normal loop

}

async done () {
await Promise.allSettled(this._coreDownloads.map((d) => d.done()))
}
}
38 changes: 33 additions & 5 deletions test.js
Original file line number Diff line number Diff line change
Expand Up @@ -683,7 +683,8 @@ test('drive.download(folder, [options])', async (t) => {
await drive.put('/parent/sibling/grandchild1', nil)

t.is(count, 0)
await mirror.drive.download('/parent/child')
const download = await mirror.drive.download('/parent/child')
await download.done()
t.is(max, l - 1)
const _count = count
t.ok(await mirror.drive.get('/parent/child/grandchild1'))
Expand Down Expand Up @@ -714,7 +715,8 @@ test('drive.download(filename, [options])', async (t) => {
await eventFlush()

await mirror.drive.getBlobs()
await mirror.drive.download('/file')
const download = await mirror.drive.download('/file')
await download.done()

t.ok(await mirror.drive.get('/file', { wait: false }))

Expand Down Expand Up @@ -779,7 +781,8 @@ test.skip('drive.downloadDiff(version, folder, [options])', async (t) => {
const filestelem = downloadShark(mirror.drive.core)
const blobstelem = downloadShark((await mirror.drive.getBlobs()).core)

await mirror.drive.downloadDiff(version, '/parent/child')
const downloadDiff = await mirror.drive.downloadDiff(version, '/parent/child')
await downloadDiff.done()

let filescount = filestelem.count
let blobscount = blobstelem.count
Expand Down Expand Up @@ -827,14 +830,16 @@ test('drive.has(path)', async (t) => {

await drive.put('/parent/sibling/grandchild1', nil)

await mirror.drive.download('/parent/child/')
const downloadChild = await mirror.drive.download('/parent/child/')
await downloadChild.done()

await eventFlush()

t.ok(await mirror.drive.has('/parent/child/'))
t.absent(await mirror.drive.has('/parent/'))

await mirror.drive.download('/parent/sibling/')
const downloadSibling = await mirror.drive.download('/parent/sibling/')
await downloadSibling.done()

await eventFlush()
t.ok(await mirror.drive.has('/parent/'))
Expand Down Expand Up @@ -1690,6 +1695,29 @@ test('drive.list ignore and unignore', async (t) => {
t.alike(entries, expectedEntries)
})

test('download can be destroyed', async (t) => {
t.plan(1)
const { corestore, drive, swarm, mirror } = await testenv(t)
swarm.on('connection', (conn) => corestore.replicate(conn))
swarm.join(drive.discoveryKey, { server: true, client: false })
await swarm.flush()

mirror.swarm.on('connection', (conn) => mirror.corestore.replicate(conn))
mirror.swarm.join(drive.discoveryKey, { server: false, client: true })
await mirror.swarm.flush()

await drive.put('/file', b4a.allocUnsafe(1024 * 1024 * 10))

await eventFlush()

const download = await mirror.drive.download('/file')
download.destroy()

await eventFlush()

t.is((await mirror.drive.blobs.core.info()).contiguousLength, 0)
})

// VERY TIMING DEPENDENT, NEEDS FIX
test.skip('upload/download can be monitored', async (t) => {
t.plan(27)
Expand Down