Write correctly formatted ndjson bodies in the curl exporter

Author: miguelgrinberg

src/cli.ts

@@ -16,6 +16,7 @@ async function main() {
       "output complete code that includes the creation of the client",
       false,
     )
+    .option("--windows", "output a Windows command (curl format only)", false)
     .option(
       "--elasticsearch-url",
       "Elasticsearch endpoint URL. Only needed when --complete is given.",
@@ -34,6 +35,7 @@ async function main() {
 
   const code = (await convertRequests(data, opts.format, {
     complete: opts.complete,
+    windows: opts.windows,
     elasticsearchUrl: opts.elasticsearchUrl,
     debug: opts.debug,
     printResponse: opts.printResponse,

src/exporters/curl.ts

@@ -12,19 +12,50 @@ export class CurlExporter implements FormatExporter {
     options: ConvertOptions,
   ): Promise<string> {
     const escapedSingleQuote = options.windows ? "''" : "'\"'\"'";
+    const escapedDoubleQuote = options.windows ? '""' : '\\"';
     const envPrefix = options.windows ? "$Env:" : "$";
+    const newLineInCmd = options.windows ? "`n" : "\\n";
     const auth = ` -H "Authorization: ApiKey ${envPrefix}ELASTIC_API_KEY"`;
     const otherUrls = (options.otherUrls as Record<string, string>) ?? {};
     let output = "";
     for (const request of requests) {
       let headers = auth;
       let body = "";
       if (request.body) {
-        headers += ' -H "Content-Type: application/json"';
-        body =
-          " -d '" +
-          JSON.stringify(request.body).replaceAll("'", escapedSingleQuote) +
-          "'";
+        if (
+          Array.isArray(request.body) &&
+          request.mediaTypes?.includes("application/x-ndjson")
+        ) {
+          // this is a bulk request with ndjson payload
+          headers += ' -H "Content-Type: application/x-ndjson"';
+          if (!options.windows) {
+            body =
+              " -d $'" +
+              request.body
+                .map((line) =>
+                  JSON.stringify(line).replaceAll("'", escapedSingleQuote),
+                )
+                .join(newLineInCmd) +
+              newLineInCmd +
+              "'";
+          } else {
+            body =
+              ' -d "' +
+              request.body
+                .map((line) =>
+                  JSON.stringify(line).replaceAll('"', escapedDoubleQuote),
+                )
+                .join(newLineInCmd) +
+              newLineInCmd +
+              '"';
+          }
+        } else {
+          headers += ' -H "Content-Type: application/json"';
+          body =
+            " -d '" +
+            JSON.stringify(request.body).replaceAll("'", escapedSingleQuote) +
+            "'";
+        }
       }
       const method =
         request.method != "HEAD" ? `-X ${request.method}` : "--head";

src/parse.ts

@@ -26,6 +26,8 @@ export type ParsedRequest = {
   api?: string;
   /** The availability definition from the Elasticsearch specification. */
   availability?: Availabilities;
+  /** The media types that are accepted for the request body. */
+  mediaTypes?: string[];
   /** The request definition from the Elasticsearch specification that applies to this request. */
   request?: Request;
   /** The dynamic parameters that are part of the request's URL. */
@@ -49,6 +51,7 @@ type ESRoute = {
   name: string;
   availability: Availabilities;
   request: Request;
+  mediaTypes?: string[];
 };
 
 let router = Router.make<ESRoute>({
@@ -296,6 +299,7 @@ export async function loadSchema(filename_or_object: string | object) {
       // find the request in the spec
       try {
         let req: Request | undefined;
+        let mt: string[] | undefined;
         for (const type of spec.types) {
           if (
             type.name.namespace == endpoint.request?.namespace &&
@@ -308,13 +312,15 @@ export async function loadSchema(filename_or_object: string | object) {
               );
             }
             req = type as Request;
+            mt = endpoint.requestMediaType;
             break;
           }
         }
         const r = {
           name: endpoint.name,
           availability: endpoint.availability,
           request: req as Request,
+          mediaTypes: mt,
         };
         router.on(methods, formattedPath as Router.PathInput, r);
       } catch (err) {
@@ -365,6 +371,7 @@ export async function parseRequest(
       req.api = route.handler.name;
       req.availability = route.handler.availability;
       req.request = route.handler.request;
+      req.mediaTypes = route.handler.mediaTypes;
       if (Object.keys(route.params).length > 0) {
         req.params = route.params;
       }

tests/convert.test.ts

@@ -23,6 +23,15 @@ POST /my-index/_search?from=40&size=20
   }
 }`;
 
+const devConsoleBulkScript = `POST _bulk
+{ "index" : { "_index" : "test", "_id" : "1" } }
+{ "field1" : "value1" }
+{ "delete" : { "_index" : "test", "_id" : "2" } }
+{ "create" : { "_index" : "test", "_id" : "3" } }
+{ "field1" : "value3" }
+{ "update" : {"_id" : "1", "_index" : "test"} }
+{ "doc" : {"field2" : "value2"} }`;
+
 const kibanaScript = `GET /
 
 GET kbn:/api/saved_objects/_find?type=dashboard`;
@@ -139,6 +148,27 @@ describe("convert", () => {
     );
   });
 
+  it("converts a bulk request to curl", async () => {
+    expect(
+      await convertRequests(devConsoleBulkScript, "curl", {
+        elasticsearchUrl: "http://localhost:9876",
+      }),
+    ).toEqual(
+      `curl -X POST -H "Authorization: ApiKey $ELASTIC_API_KEY" -H "Content-Type: application/x-ndjson" -d $'{"index":{"_index":"test","_id":"1"}}\\n{"field1":"value1"}\\n{"delete":{"_index":"test","_id":"2"}}\\n{"create":{"_index":"test","_id":"3"}}\\n{"field1":"value3"}\\n{"update":{"_id":"1","_index":"test"}}\\n{"doc":{"field2":"value2"}}\\n' "http://localhost:9876/_bulk"\n`,
+    );
+  });
+
+  it("converts a bulk request to curl on Windows", async () => {
+    expect(
+      await convertRequests(devConsoleBulkScript, "curl", {
+        elasticsearchUrl: "http://localhost:9876",
+        windows: true,
+      }),
+    ).toEqual(
+      `curl -X POST -H "Authorization: ApiKey $Env:ELASTIC_API_KEY" -H "Content-Type: application/x-ndjson" -d "{""index"":{""_index"":""test"",""_id"":""1""}}\`n{""field1"":""value1""}\`n{""delete"":{""_index"":""test"",""_id"":""2""}}\`n{""create"":{""_index"":""test"",""_id"":""3""}}\`n{""field1"":""value3""}\`n{""update"":{""_id"":""1"",""_index"":""test""}}\`n{""doc"":{""field2"":""value2""}}\`n" "http://localhost:9876/_bulk"\n`,
+    );
+  });
+
   it("converts Kibana to curl", async () => {
     expect(
       await convertRequests(kibanaScript, "curl", {

tests/parse.test.ts

@@ -169,6 +169,7 @@ POST\n_ml/anomaly_detectors/it_ops_new_logs/model_snapshots/1491852978/_update\n
       rawPath: "/_bulk",
       query: { foo: "bar" },
       body: [{ name: "John Doe" }, { name: "John Doe" }, { name: "John Doe" }],
+      mediaTypes: ["application/x-ndjson"],
     });
     expect(reqs[6]).toMatchObject({
       source:
@@ -186,6 +187,7 @@ POST\n_ml/anomaly_detectors/it_ops_new_logs/model_snapshots/1491852978/_update\n
         { name: "John\nDoe" },
         { name: "John\nDoe" },
       ],
+      mediaTypes: ["application/x-ndjson"],
     });
     expect(reqs[7]).toMatchObject({
       source: "GET /{customer}/_doc/1",