Add Backfilled data script

.github/workflows/test.yml

@@ -20,23 +20,20 @@ jobs:
         with:
           node-version: 14
       - name: Install dependencies
-        run: npm run install:functions
+        run: yarn install:functions
       - name: Run style check
-        run: npm run lint
+        run: yarn lint
 
   integration-tests:
+    name: integration-tests
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        node: ["12", "14"]
-    name: integration-tests node.js_${{ matrix.node }}_test
     steps:
       - uses: actions/checkout@v2
       - name: Setup node
         uses: actions/setup-node@v1
         with:
-          node-version: ${{ matrix.node }}
+          node-version: 14
       - name: Install dependencies
-        run: npm run install:functions
+        run: yarn install:functions
       - name: Run tests with coverage
-        run: npm run test-coverage
+        run: yarn test:coverage

CONTRIBUTING.md

@@ -38,9 +38,13 @@ To run this project, you will need:
 - Npm >= v7
 - A google account
 - Latest version of `firebase-tools` the Firebase CLI:
-``` bash
-npm install -g firebase-tools
-```
+  ``` bash
+  yarn global add firebase-tools
+  ```
+  Add the directory for the commands of the packages installed globally in yarn, to access of firebase binary:
+  ``` bash
+  export PATH="$(yarn global bin):$PATH"
+  ```
 
 ### Setup <!-- omit in toc -->
 
@@ -54,21 +58,25 @@ firebase --open-sesame extdev
 ```
 Install dependencies:
 ``` bash
-npm run install:functions
+yarn install:functions
 ```
 Build the project:
 ``` bash
-npm run build
+yarn build
 ```
 Launch MeiliSearch instance:
 ``` bash
-docker pull getmeili/meilisearch:latest # Fetch the latest version of MeiliSearch image from Docker Hub
-docker run -p 7700:7700 getmeili/meilisearch:latest ./meilisearch --master-key=masterKey --no-analytics=true
+curl -L https://install.meilisearch.com | sh # download MeiliSearch
+./meilisearch --master-key=masterKey --no-analytics=true # run MeiliSearch
 ```
 Launch emulator:
 ``` bash
 firebase ext:dev:emulators:start --test-params=test-params-example.env --import=dataset --project=name-of-the-project
 ```
+or just
+``` bash
+yarn emulator
+```
 The emulator runs with environment parameters found in `test-params-example.env` and with a provided dataset found in `/dataset`.
 
 Once it is running, open the emulator in a browser at the following address: http://localhost:4000
@@ -80,20 +88,55 @@ NB: If you want to change your MeiliSearch credentials or the plugins options yo
 Each PR should pass the tests and the linter to be accepted.
 
 ```bash
+curl -L https://install.meilisearch.com | sh # download MeiliSearch
+./meilisearch --master-key=masterKey --no-analytics=true # run MeiliSearch
+
 # Tests
-npm run test
+yarn test
 
 # Tests in watch mode
-npm run test:watch
+yarn test:watch
 
 # Linter
-npm run lint
+yarn lint
 
 # Linter with fixing
-npm run lint:fix
+yarn lint:fix
 
 # Build the project
-npm run build
+yarn build
+```
+
+### Run the backfilled-data script
+
+Run the import script using [npx](https://www.npmjs.com/package/npx) (the Node Package Runner) via npm (the Node Package Manager).
+- Set up credentials. The import script uses Application Default Credentials to communicate with Firebase. Please follow the instructions [generate a private key file for your service account](https://firebase.google.com/docs/admin/setup#initialize-sdk).
+
+- Run the import script interactively via `npx` by running ONE of the following commands:
+  - Run interactively:
+    ```bash
+    npx firestore-meilisearch
+    ```
+
+  - Or run non-interactively with paramaters:
+    ```bash
+    npx firestore-meilisearch --project <project_id> --source-collection-path <collection_name> --index <index_uid> --batch-size <100/default=300> --non-interactive --host <host_adress> --api-key <api_key>
+    ```
+    **Note**: The `--batch-size` argument is optional. To see its usage, run the above command with --help.
+
+- Run the project for development:
+Launch MeiliSearch instance:
+``` bash
+curl -L https://install.meilisearch.com | sh # download MeiliSearch
+./meilisearch --master-key=masterKey --no-analytics=true # run MeiliSearch
+```
+Launch the watcher on the project:
+``` bash
+yarn watch
+```
+Launch the watcher on the script. You must first have modified this command in the `package.json` file with your own data:
+``` bash
+yarn playground
 ```
 
 ## Git Guidelines
@@ -139,7 +182,26 @@ _[Read more about this](https://github.com/meilisearch/integration-guides/blob/m
 
 ### How to Publish the Release <!-- omit in toc -->
 
-WIP
+⚠️ Before doing anything, make sure you got through the guide about [Releasing an Integration](https://github.com/meilisearch/integration-guides/blob/main/resources/integration-release.md).
+
+1. Make a PR modifying the files [`package.json`](/package.json), [`package.json` in functions directory](/functions/package.json) and the [`extension.yaml`](/extension.yaml) with the right version.
+
+```javascript
+"version": "X.X.X",
+```
+
+2. Test the extension  by installing it on Firestore database:
+```bash
+firebase ext:install . --project=meilisearch-e2fe1
+```
+
+3. Publish the extension by run the following command in the root of the extension directory:
+```bash
+firebase ext:dev:publish meilisearch/firestore-meilisearch --project=meilisearch-e2fe1
+```
+**Note**: `meilisearch` is the publisher id and `meilisearch-e2fe1` is the publish project id for this extension.
+
+Once the changes are merged on `main`, you can publish the current draft release via the [GitHub interface](https://github.com/meilisearch/meilisearch-go/releases): on this page, click on `Edit` (related to the draft release) > update the description (be sure you apply [these recommandations](https://github.com/meilisearch/integration-guides/blob/main/resources/integration-release.md#writting-the-release-description)) > when you are ready, click on `Publish release`.
 
 <hr>
 

extension.yaml

@@ -1,4 +1,4 @@
-# Copyright 2021 Meilisearch
+# Copyright 2021 MeiliSearch
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -26,7 +26,7 @@ displayName: Search in Firestore with MeiliSearch
 
 # Brief description of the task your extension performs (~1 sentence)
 description: >-
-  Fulltext Search on Firebase with Meilisearch
+  Fulltext Search on Firebase with MeiliSearch
 
 license: Apache-2.0  # https://spdx.org/licenses/
 
@@ -42,7 +42,7 @@ author:
 
 # Specify whether a paid-tier billing plan is required to use your extension.
 # Learn more in the docs: https://firebase.google.com/docs/extensions/alpha/ref-extension-yaml#billing-required-field
-billingRequired: false
+billingRequired: true
 
 # In an `apis` field, list any Google APIs (like Cloud Translation, BigQuery, etc.)
 # required for your extension to operate.
@@ -57,7 +57,7 @@ resources:
   - name: indexingWorker
     type: firebaseextensions.v1beta.function
     description: >-
-      Cloud function triggered by document modification in Firestore to import changes into Meilisearch.
+      Cloud function triggered by document modification in Firestore to import changes into MeiliSearch.
     properties:
       # LOCATION is a user-configured parameter value specified by the user
       # during installation.
@@ -118,14 +118,13 @@ params:
         value: southamerica-east1
       - label: Sydney (australia-southeast1)
         value: australia-southeast1
-    default: europe-west1
     required: true
     immutable: true
 
   - param: COLLECTION_PATH
     label: Collection path
     description: >-
-      What is the path of the collection you would like to export into Meilisearch?
+      What is the path of the collection you would like to export into MeiliSearch?
     example: movies
     type: string
     validationRegex: "^[^/]+(/[^/]+/[^/]+)*$"
@@ -138,11 +137,11 @@ params:
       What are the names of the fields you want to be indexed in MeiliSearch?
       Create a comma-separated list of the fields name, or leave blank to take all fields.
       The id field is always indexed even when omitted in the list.
+    default: ''
     example: title,overview,release_date
     validationRegex: "^[^,]?[a-zA-Z-_0-9,]*[^,]$"
     validationErrorMessage:
       Fields must be given through a comma-separated list.
-    default: ''
     required: false
 
   - param: SEARCHABLE_FIELDS
@@ -152,15 +151,15 @@ params:
       This features is optional. See [the documentation about it](https://docs.meilisearch.com/reference/features/field_properties.html#searchable-fields)
       for more information.
       Create a comma-separated list of fields, or leave blank to include all fields.
+    default: ''
     example: title,overview,release_date
     validationRegex: "^[^,]?[a-zA-Z-_0-9,]*[^,]$"
     validationErrorMessage:
       Fields must be given through a comma-separated list.
-    default: ''
     required: false
 
   - param: MEILISEARCH_INDEX_NAME
-    label: Meilisearch Index Name
+    label: MeiliSearch Index Name
     description: >-
       What is the name of the index you want your data to be indexed in?
     type: string
@@ -172,15 +171,16 @@ params:
     required: true
 
   - param: MEILISEARCH_HOST
-    label: Meilisearch host
+    label: MeiliSearch host
     description: >-
-      What is the url of the host of your Meilisearch database?
+      What is the url of the host of your MeiliSearch database?
+      Make sure your url starts with `http://` or `https://`
     type: string
     example: 'http://127.0.0.1:7700'
     required: true
 
   - param: MEILISEARCH_API_KEY
-    label: Meilisearch API key
+    label: MeiliSearch API key
     description: >-
       What is your MeiliSearch API key with permission to perform actions on indexes?
       Both the private key and the master key are valid choices but we strongly recommend using the private key for security purposes.

functions/__tests__/adapter.test.ts

@@ -18,12 +18,26 @@ describe('extensions process', () => {
     test('adaptDocument with fields not set', () => {
       mockedGetFieldsToIndex.mockReturnValueOnce([])
       const snapshot = firebaseMock.firestore.makeDocumentSnapshot(
-        defaultDocument,
-        'docs/1'
+        defaultDocument.document,
+        `docs/${defaultDocument.id}`
       )
-      expect(adaptDocument(defaultDocument.id, snapshot)).toStrictEqual(
-        defaultDocument
+      expect(adaptDocument(defaultDocument.id, snapshot)).toStrictEqual({
+        _firestore_id: defaultDocument.id,
+        ...defaultDocument.document,
+      })
+    })
+
+    test('adaptDocument with fields not set and with id field in document', () => {
+      mockedGetFieldsToIndex.mockReturnValueOnce([])
+      const snapshot = firebaseMock.firestore.makeDocumentSnapshot(
+        { id: '12345', ...defaultDocument.document },
+        `docs/${defaultDocument.id}`
       )
+      expect(adaptDocument(defaultDocument.id, snapshot)).toStrictEqual({
+        _firestore_id: defaultDocument.id,
+        id: '12345',
+        ...defaultDocument.document,
+      })
     })
 
     test('adaptDocument with fields set', () => {
@@ -33,14 +47,14 @@ describe('extensions process', () => {
         'release_date',
       ])
       const snapshot = firebaseMock.firestore.makeDocumentSnapshot(
-        defaultDocument,
-        'docs/1'
+        defaultDocument.document,
+        `docs/${defaultDocument.id}`
       )
       expect(adaptDocument(defaultDocument.id, snapshot)).toStrictEqual({
-        id: defaultDocument.id,
-        title: defaultDocument.title,
-        overview: defaultDocument.overview,
-        release_date: defaultDocument.release_date,
+        _firestore_id: defaultDocument.id,
+        title: defaultDocument.document.title,
+        overview: defaultDocument.document.overview,
+        release_date: defaultDocument.document.release_date,
       })
     })
   })

functions/__tests__/config.test.ts

@@ -20,19 +20,21 @@ describe('extensions config', () => {
   beforeEach(() => {
     jest.resetModules()
     restoreEnv = mockedEnv(defaultEnvironment)
-    config = require('../src/config').default
+    config = require('../src/config').config
   })
   afterEach(() => restoreEnv())
 
   test('config loaded from environment variables', () => {
     const testConfig = {
       location: defaultEnvironment.LOCATION,
       collectionPath: defaultEnvironment.COLLECTION_PATH,
-      fieldsToIndex: defaultEnvironment.FIELDS_TO_INDEX,
-      searchableFields: defaultEnvironment.SEARCHABLE_FIELDS,
-      meilisearchIndex: defaultEnvironment.MEILISEARCH_INDEX_NAME,
-      meilisearchHost: defaultEnvironment.MEILISEARCH_HOST,
-      meilisearchApiKey: defaultEnvironment.MEILISEARCH_API_KEY,
+      meilisearch: {
+        host: defaultEnvironment.MEILISEARCH_HOST,
+        apiKey: defaultEnvironment.MEILISEARCH_API_KEY,
+        indexUid: defaultEnvironment.MEILISEARCH_INDEX_NAME,
+        fieldsToIndex: defaultEnvironment.FIELDS_TO_INDEX,
+        searchableFields: defaultEnvironment.SEARCHABLE_FIELDS,
+      },
     }
 
     expect(config).toStrictEqual(testConfig)

functions/__tests__/data/document.ts

@@ -1,9 +1,11 @@
 export default {
   id: '0jKUH3XUNXFyujm3rZAa',
-  title: 'The Lord of the Rings: The Fellowship of the Ring',
-  genre: ['Adventure', 'Fantasy', 'Action'],
-  overview:
-    'Young hobbit Frodo Baggins, after inheriting a mysterious ring from his uncle Bilbo, must leave his home in order to keep it from falling into the hands of its evil creator. Along the way, a fellowship is formed to protect the ringbearer and make sure that the ring arrives at its final destination: Mt. Doom, the only place where it can be destroyed.',
-  poster: 'https://image.tmdb.org/t/p/w500/6oom5QYQ2yQTMJIbnvbkBL9cHo6.jpg',
-  release_date: 1008633600,
+  document: {
+    title: 'The Lord of the Rings: The Fellowship of the Ring',
+    genre: ['Adventure', 'Fantasy', 'Action'],
+    overview:
+      'Young hobbit Frodo Baggins, after inheriting a mysterious ring from his uncle Bilbo, must leave his home in order to keep it from falling into the hands of its evil creator. Along the way, a fellowship is formed to protect the ringbearer and make sure that the ring arrives at its final destination: Mt. Doom, the only place where it can be destroyed.',
+    poster: 'https://image.tmdb.org/t/p/w500/6oom5QYQ2yQTMJIbnvbkBL9cHo6.jpg',
+    release_date: 1008633600,
+  },
 }