feat(files): CAR Import (#2323)

* feat: add dropdown menu for import car

* feat: add modal for import car

* locale: add en locale

* refactor,feat: change to functional component, add check file, rename file and validate file name

* feat: wiring handling uploading car dag file

* locale: add placeholder

* feat: add import car file

* refactor: clear unused params

* update dependency

* refactor: cleanup modal

* feat: update types

* feat: wip on dag import

* feat: add util

* feat: fix parse file for dag import

* docs: Add AddByCarModal as storybook

* fix: fix typecheck error

* fix: revert back

* feat: update types of IPFSService

* feat: revert types to use modified type

* refactor: remove unused

* refactor: move definition from imports to local

* refactor: change to pass stream to dag import

* refactor: remove unused util

* typo: Update wording

* typo: Update wording in select file button

* type: Update i18n, use wording for title in i18n

* fix: fix logic on file name

* fix: fix import to relative path within mfs

* fix: icon, label, and styling

making  UI match bulk import feature

* docs: cli tutor for cat import

+ unifying import/export language

* chore: throw helpful error if name exists

* fix: dont throw error when importing car if destination doesnt exist

* fix: swallow error when file doesn't exist

* chore: remove unnecessary changes

* chore: use kubo-rpc-client types for IPFSService

* chore: minor ux fixes

* fix: dont error if path doesn't exist

* fix: remove duplicate catch block

---------

Co-authored-by: Marcin Rataj <[email protected]>
Co-authored-by: Daniel N <[email protected]>
Co-authored-by: Russell Dempsey <[email protected]>

Author: 4 people

@types/ipfs/index.d.ts

@@ -2,13 +2,15 @@ declare module 'ipfs' {
   import type { CID } from 'multiformats/cid'
   import type { Multiaddr } from '@multiformats/multiaddr'
   import type { Buffer } from 'buffer'
+  import type { KuboRPCClient } from 'kubo-rpc-client'
 
   declare export interface IPFSService extends CoreService {
     pin: PinService;
     files: FileService;
     name: NameService;
     object: ObjectService;
     config: ConfigService;
+    dag: KuboRPCClient['dag'];
 
     stop(options?: TimeoutOptions): Promise<void>
   }

package-lock.json

@@ -73,7 +73,7 @@
     "it-first": "^1.0.6",
     "it-last": "^1.0.5",
     "it-map": "^1.0.5",
-    "kubo-rpc-client": "^4.1.1",
+    "kubo-rpc-client": "^5.0.2",
     "milliseconds": "^1.0.3",
     "money-clip": "^3.0.5",
     "multiformats": "^13.0.1",

package.json

@@ -30,7 +30,7 @@
     "unselectAll": "Unselect all",
     "generate": "Generate",
     "publish": "Publish",
-    "downloadCar": "Download as CAR",
+    "downloadCar": "Export CAR",
     "done": "Done"
   },
   "cliModal": {

public/locales/en/app.json

@@ -14,6 +14,7 @@
   "openWithLocalAndPublicGateway": "Try opening it instead with your <1>local gateway</1> or <3>public gateway</3>.",
   "cantBePreviewed": "Sorry, this file can’t be previewed",
   "addByPath": "From IPFS",
+  "addByCar": "From CAR",
   "bulkImport": "Bulk import",
   "newFolder": "New folder",
   "generating": "Generating…",
@@ -60,6 +61,13 @@
     "namePlaceholder": "Name (optional)",
     "examples": "Examples:"
   },
+  "addByCarModal": {
+    "title": "CAR Import",
+    "description": "Choose a CAR file to import a DAG from and specify a name for it to be imported into the current directory.",
+    "selectCARButtonText": "Select CAR…",
+    "namePlaceholder": "Name",
+    "renameImportPath": "Rename import"
+  },
   "bulkImportModal": {
     "title": "Bulk import with text file",
     "description": "Upload a text file with a list of CIDs (names are optional). Example:",

public/locales/en/files.json

@@ -399,10 +399,56 @@ const actions = () => ({
   }),
 
   /**
- * Reads a text file containing CIDs and adds each one to IPFS at the given root path.
- * @param {FileStream[]} source - The text file containing CIDs
- * @param {string} root - Destination directory in IPFS
- */
+   * Adds CAR file. On completion will trigger `doFilesFetch` to update the state.
+   * @param {string} root
+   * @param {FileStream} carFile
+   * @param {string} name
+   */
+  doAddCarFile: (root, carFile, name = '') => perform(ACTIONS.ADD_CAR_FILE, async (/** @type {IPFSService} */ ipfs, { store }) => {
+    ensureMFS(store)
+
+    const stream = carFile.content.stream()
+    try {
+      // @ts-expect-error - https://github.com/ipfs/js-kubo-rpc-client/issues/278
+      const result = await all(ipfs.dag.import(stream, {
+        pinRoots: true
+      }))
+      const cid = result[0].root.cid
+      const src = `/ipfs/${cid}`
+      const dst = realMfsPath(join(root, name))
+      let dstExists = false
+
+      // Check if destination path already exists
+      await ipfs.files.stat(dst).then(() => {
+        dstExists = true
+      }).catch(() => {
+        // Swallow error. We can add the file to the dst path
+      })
+
+      if (dstExists) {
+        throw new Error(`The name "${name}" already exists in the current directory. Try importing with a different name.`)
+      }
+
+      try {
+        await ipfs.files.cp(src, dst)
+      } catch (/** @type {any} */ err) {
+        // TODO: Not sure why we do this. Perhaps a generic error is used
+        // to avoid leaking private information via Countly?
+        throw Object.assign(new Error('ipfs.files.cp call failed'), {
+          code: 'ERR_FILES_CP_FAILED'
+        })
+      }
+      return carFile
+    } finally {
+      await store.doFilesFetch()
+    }
+  }),
+
+  /**
+   * Reads a text file containing CIDs and adds each one to IPFS at the given root path.
+   * @param {FileStream[]} source - The text file containing CIDs
+   * @param {string} root - Destination directory in IPFS
+   */
   doFilesBulkCidImport: (source, root) => perform(ACTIONS.BULK_CID_IMPORT, async function (ipfs, { store }) {
     ensureMFS(store)
 

src/bundles/files/actions.js

@@ -23,6 +23,8 @@ export const ACTIONS = {
   SHARE_LINK: ('FILES_SHARE_LINK'),
   /** @type {'FILES_ADDBYPATH'} */
   ADD_BY_PATH: ('FILES_ADDBYPATH'),
+  /** @type {'FILES_ADD_CAR'} */
+  ADD_CAR_FILE: ('FILES_ADD_CAR'),
   /** @type {'FILES_BULK_CID_IMPORT'} */
   BULK_CID_IMPORT: ('FILES_BULK_CID_IMPORT'),
   /** @type {'FILES_PIN_ADD'} */
@@ -80,6 +82,7 @@ export const cliCmdKeys = {
   ADD_DIRECTORY: 'addNewDirectory',
   CREATE_NEW_DIRECTORY: 'createNewDirectory',
   FROM_IPFS: 'fromIpfs',
+  FROM_CAR: 'fromCar',
   ADD_NEW_PEER: 'addNewPeer',
   PUBLISH_WITH_IPNS: 'publishWithIPNS',
   DOWNLOAD_CAR_COMMAND: 'downloadCarCommand'
@@ -128,6 +131,10 @@ export const cliCommandList = {
    * @param {string} path
    */
   [cliCmdKeys.FROM_IPFS]: (path) => `ipfs files cp /ipfs/<cid> "${path}/<dest-name>"`,
+  /**
+   * @param {string} path
+   */
+  [cliCmdKeys.FROM_CAR]: (path) => `ipfs dag import file.car && ipfs files cp /ipfs/<imported-cid> "${path}/<dest-name>"`,
   [cliCmdKeys.ADD_NEW_PEER]: () => 'ipfs swarm connect <peer-multiaddr>',
   /**
    * @param {string} ipfsPath

src/bundles/files/consts.js

@@ -4,6 +4,7 @@ import * as Task from '../task.js'
 
 /**
  * @typedef {import('ipfs').IPFSService} IPFSService
+ * @typedef {import('../../lib/files').FileStream} FileStream
  * @typedef {import('./actions').Ext} Ext
  * @typedef {import('./actions').Extra} Extra
  */

src/bundles/files/utils.js

@@ -15,13 +15,13 @@ import FilesList from './files-list/FilesList.js'
 import { getJoyrideLocales } from '../helpers/i8n.js'
 
 // Icons
-import Modals, { DELETE, NEW_FOLDER, SHARE, RENAME, ADD_BY_PATH, BULK_CID_IMPORT, CLI_TUTOR_MODE, PINNING, PUBLISH } from './modals/Modals.js'
+import Modals, { DELETE, NEW_FOLDER, ADD_BY_CAR, SHARE, RENAME, ADD_BY_PATH, BULK_CID_IMPORT, CLI_TUTOR_MODE, PINNING, PUBLISH } from './modals/Modals.js'
 import Header from './header/Header.js'
 import FileImportStatus from './file-import-status/FileImportStatus.js'
 import { useExplore } from 'ipld-explorer-components/providers'
 
 const FilesPage = ({
-  doFetchPinningServices, doFilesFetch, doPinsFetch, doFilesSizeGet, doFilesDownloadLink, doFilesDownloadCarLink, doFilesWrite, doFilesBulkCidImport, doFilesAddPath, doUpdateHash,
+  doFetchPinningServices, doFilesFetch, doPinsFetch, doFilesSizeGet, doFilesDownloadLink, doFilesDownloadCarLink, doFilesWrite, doAddCarFile, doFilesBulkCidImport, doFilesAddPath, doUpdateHash,
   doFilesUpdateSorting, doFilesNavigateTo, doFilesMove, doSetCliOptions, doFetchRemotePins, remotePins, pendingPins, failedPins,
   ipfsProvider, ipfsConnected, doFilesMakeDir, doFilesShareLink, doFilesDelete, doSetPinning, onRemotePinClick, doPublishIpnsKey,
   files, filesPathInfo, pinningServices, toursEnabled, handleJoyrideCallback, isCliTutorModeEnabled, cliOptions, t
@@ -79,6 +79,14 @@ const FilesPage = ({
   }
 
   const onAddByPath = (path, name) => doFilesAddPath(files.path, path, name)
+  /**
+   *
+   * @param {File} file
+   * @param {string} name
+   */
+  const onAddByCar = (file, name) => {
+    doAddCarFile(files.path, file, name)
+  }
   const onInspect = (cid) => doUpdateHash(`/explore/${cid}`)
   const showModal = (modal, files = null) => setModals({ show: modal, files })
   const hideModal = () => setModals({})
@@ -212,6 +220,7 @@ const FilesPage = ({
         onAddFiles={onAddFiles}
         onMove={doFilesMove}
         onAddByPath={(files) => showModal(ADD_BY_PATH, files)}
+        onAddByCar={(files) => showModal(ADD_BY_CAR, files)}
         onBulkCidImport={(files) => showModal(BULK_CID_IMPORT, files)}
         onNewFolder={(files) => showModal(NEW_FOLDER, files)}
         onCliTutorMode={() => showModal(CLI_TUTOR_MODE)}
@@ -233,6 +242,7 @@ const FilesPage = ({
         onShareLink={doFilesShareLink}
         onRemove={doFilesDelete}
         onAddByPath={onAddByPath}
+        onAddByCar={onAddByCar}
         onBulkCidImport={onBulkCidImport}
         onPinningSet={doSetPinning}
         onPublish={doPublishIpnsKey}
@@ -280,6 +290,7 @@ export default connect(
   'doFilesShareLink',
   'doFilesDelete',
   'doFilesAddPath',
+  'doAddCarFile',
   'doFilesNavigateTo',
   'doFilesUpdateSorting',
   'selectFilesSorting',

src/files/FilesPage.js

@@ -8,6 +8,7 @@ import DocumentIcon from '../../icons/StrokeDocument.js'
 import FolderIcon from '../../icons/StrokeFolder.js'
 import NewFolderIcon from '../../icons/StrokeNewFolder.js'
 import DecentralizationIcon from '../../icons/StrokeDecentralization.js'
+import DataIcon from '../../icons/StrokeData.js'
 // Components
 import { Dropdown, DropdownMenu, Option } from '../dropdown/Dropdown.js'
 import Button from '../../components/button/button.tsx'
@@ -50,6 +51,11 @@ class FileInput extends React.Component {
     this.toggleDropdown()
   }
 
+  onAddByCar = () => {
+    this.props.onAddByCar()
+    this.toggleDropdown()
+  }
+
   onBulkCidImport = () => {
     this.props.onBulkCidImport()
     this.toggleDropdown()
@@ -87,15 +93,20 @@ class FileInput extends React.Component {
               <FolderIcon className='fill-aqua w2 mr1' />
               {t('app:terms.folder')}
             </Option>
+            <Option onClick={this.onNewFolder} id='add-new-folder' onCliTutorMode={() => this.onCliTutorMode(cliCmdKeys.CREATE_NEW_DIRECTORY)}
+              isCliTutorModeEnabled={isCliTutorModeEnabled}>
+              <NewFolderIcon className='fill-aqua w2 h2 mr1' />
+              {t('newFolder')}
+            </Option>
             <Option onClick={this.onAddByPath} id='add-by-path' onCliTutorMode={() => this.onCliTutorMode(cliCmdKeys.FROM_IPFS)}
               isCliTutorModeEnabled={isCliTutorModeEnabled}>
               <DecentralizationIcon className='fill-aqua w2 mr1' />
               {t('addByPath')}
             </Option>
-            <Option onClick={this.onNewFolder} id='add-new-folder' onCliTutorMode={() => this.onCliTutorMode(cliCmdKeys.CREATE_NEW_DIRECTORY)}
+            <Option onClick={this.onAddByCar} id='add-by-car' onCliTutorMode={() => this.onCliTutorMode(cliCmdKeys.FROM_CAR)}
               isCliTutorModeEnabled={isCliTutorModeEnabled}>
-              <NewFolderIcon className='fill-aqua w2 h2 mr1' />
-              {t('newFolder')}
+              <DataIcon className='fill-aqua w2 mr1' />
+              {t('addByCar')}
             </Option>
             <Option onClick={this.onBulkCidImport} id='bulk-cid-import'>
               <DocumentIcon className='fill-aqua w2 mr1' />
@@ -129,8 +140,9 @@ FileInput.propTypes = {
   t: PropTypes.func.isRequired,
   onAddFiles: PropTypes.func.isRequired,
   onAddByPath: PropTypes.func.isRequired,
-  onNewFolder: PropTypes.func.isRequired,
-  onBulkCidImport: PropTypes.func.isRequired
+  onAddByCar: PropTypes.func.isRequired,
+  onBulkCidImport: PropTypes.func.isRequired,
+  onNewFolder: PropTypes.func.isRequired
 }
 
 export default connect(