[CI] Storybook parallel build

.buildkite/package.json

@@ -15,6 +15,7 @@
     "js-yaml": "^4.1.0",
     "minimatch": "^5.0.1",
     "minimist": "^1.2.8",
+    "p-limit": "^3.1.0",
     "tslib": "*"
   },
   "devDependencies": {
@@ -23,7 +24,7 @@
     "@types/jscodeshift": "^0.12.0",
     "@types/minimatch": "^3.0.5",
     "@types/minimist": "^1.2.5",
-    "@types/node": "^15.12.2",
+    "@types/node": "^22.17.1",
     "jest": "^30.0.3",
     "jscodeshift": "^17.1.2",
     "nock": "^12.0.2",

.buildkite/scripts/steps/storybooks/build_and_upload.ts

@@ -7,9 +7,11 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { execSync } from 'child_process';
+import { execSync, spawn } from 'child_process';
 import fs from 'fs';
 import path from 'path';
+import os from 'os';
+import pLimit from 'p-limit';
 import { storybookAliases } from '../../../../src/dev/storybook/aliases';
 import { getKibanaDir } from '#pipeline-utils';
 
@@ -25,6 +27,42 @@ const STORYBOOK_BASE_URL = `${STORYBOOK_BUCKET_URL}`;
 
 const exec = (...args: string[]) => execSync(args.join(' '), { stdio: 'inherit' });
 
+const buildStorybook = (storybook: string): Promise<{ logs: string }> => {
+  return new Promise((resolve, reject) => {
+    const logsBuffer: string[] = [];
+    const handleBufferChunk = (chunk: Buffer) => {
+      logsBuffer.push(chunk.toString());
+    };
+
+    const child = spawn('yarn', ['storybook', '--site', storybook], {
+      stdio: 'pipe',
+      env: {
+        ...process.env,
+        STORYBOOK_BASE_URL,
+        NODE_OPTIONS: '--max-old-space-size=6144',
+      },
+    });
+
+    child.stdout?.on('data', handleBufferChunk);
+    child.stderr?.on('data', handleBufferChunk);
+
+    child.on('close', (code) => {
+      if (code === 0) {
+        logsBuffer.unshift(`--- ✅ ${storybook} storybook\n`);
+        resolve({ logs: logsBuffer.join('') });
+      } else {
+        logsBuffer.unshift(`--- ❌ ${storybook} storybook\n`);
+        reject(new Error(logsBuffer.join('')));
+      }
+    });
+
+    child.on('error', () => {
+      logsBuffer.unshift(`--- ❌ ${storybook} storybook\n`);
+      reject(new Error(logsBuffer.join('')));
+    });
+  });
+};
+
 const ghStatus = (state: string, description: string) =>
   exec(
     `gh api "repos/elastic/kibana/statuses/${process.env.BUILDKITE_COMMIT}"`,
@@ -35,15 +73,23 @@ const ghStatus = (state: string, description: string) =>
     `--silent`
   );
 
-const build = () => {
+const build = async () => {
   console.log('--- Building Storybooks');
 
-  for (const storybook of Object.keys(storybookAliases)) {
-    exec(
-      `STORYBOOK_BASE_URL=${STORYBOOK_BASE_URL}`,
-      `NODE_OPTIONS=--max-old-space-size=6144`,
-      `yarn storybook --site ${storybook}`
+  const limit = pLimit(os.availableParallelism());
+  const storybooks = Object.keys(storybookAliases);
+
+  try {
+    const results = await Promise.all(
+      storybooks.map((storybook) => limit(() => buildStorybook(storybook)))
     );
+
+    results.forEach(({ logs }) => {
+      console.log(logs);
+    });
+  } catch (error) {
+    console.error(error);
+    throw error;
   }
 };
 
@@ -84,8 +130,8 @@ const upload = () => {
     );
     exec(`
       ${activateScript} gs://ci-artifacts.kibana.dev
-      gsutil -h "Cache-Control:no-cache, max-age=0, no-transform" -q -m cp -r -z js,css,html,json,map,txt,svg '*' 'gs://${STORYBOOK_BUCKET}/${STORYBOOK_DIRECTORY}/'
-      gsutil -h "Cache-Control:no-cache, max-age=0, no-transform" cp -z html 'index.html' 'gs://${STORYBOOK_BUCKET}/${STORYBOOK_DIRECTORY}/latest/'
+      gcloud storage cp --cache-control="no-cache, max-age=0, no-transform" --gzip-local=js,css,html,json,map,txt,svg --recursive --no-user-output-enabled '*' 'gs://${STORYBOOK_BUCKET}/${STORYBOOK_DIRECTORY}/'
+      gcloud storage cp --cache-control="no-cache, max-age=0, no-transform" --gzip-local=html --no-user-output-enabled 'index.html' 'gs://${STORYBOOK_BUCKET}/${STORYBOOK_DIRECTORY}/latest/'
     `);
 
     if (process.env.BUILDKITE_PULL_REQUEST && process.env.BUILDKITE_PULL_REQUEST !== 'false') {
@@ -98,12 +144,14 @@ const upload = () => {
   }
 };
 
-try {
-  ghStatus('pending', 'Building Storybooks');
-  build();
-  upload();
-  ghStatus('success', 'Storybooks built');
-} catch (error) {
-  ghStatus('error', 'Building Storybooks failed');
-  throw error;
-}
+(async () => {
+  try {
+    ghStatus('pending', 'Building Storybooks');
+    await build();
+    upload();
+    ghStatus('success', 'Storybooks built');
+  } catch (error) {
+    ghStatus('error', 'Building Storybooks failed');
+    throw error;
+  }
+})();