feat: initial VMAP API implementation

jest.config.js

@@ -1,4 +1,5 @@
 module.exports = {
   preset: 'ts-jest',
-  testEnvironment: 'node'
+  testEnvironment: 'node',
+  verbose: true
 };

readme.md

@@ -1,9 +1,13 @@
 # Ad Normalizer
 
-A Proxy put in fron of an ad server that dispatches transcoding and packaging of VAST creatives.
+A Proxy put in front of an ad server that dispatches transcoding and packaging of VAST and VMAP creatives.
 
 [![Badge OSC](https://img.shields.io/badge/Evaluate-24243B?style=for-the-badge&logo=data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMjQiIGhlaWdodD0iMjQiIHZpZXdCb3g9IjAgMCAyNCAyNCIgZmlsbD0ibm9uZSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPGNpcmNsZSBjeD0iMTIiIGN5PSIxMiIgcj0iMTIiIGZpbGw9InVybCgjcGFpbnQwX2xpbmVhcl8yODIxXzMxNjcyKSIvPgo8Y2lyY2xlIGN4PSIxMiIgY3k9IjEyIiByPSI3IiBzdHJva2U9ImJsYWNrIiBzdHJva2Utd2lkdGg9IjIiLz4KPGRlZnM%2BCjxsaW5lYXJHcmFkaWVudCBpZD0icGFpbnQwX2xpbmVhcl8yODIxXzMxNjcyIiB4MT0iMTIiIHkxPSIwIiB4Mj0iMTIiIHkyPSIyNCIgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiPgo8c3RvcCBzdG9wLWNvbG9yPSIjQzE4M0ZGIi8%2BCjxzdG9wIG9mZnNldD0iMSIgc3RvcC1jb2xvcj0iIzREQzlGRiIvPgo8L2xpbmVhckdyYWRpZW50Pgo8L2RlZnM%2BCjwvc3ZnPgo%3D)](https://app.osaas.io/browse/eyevinn-ad-normalizer)
 
+The service provides two main endpoints:
+
+### VAST Endpoint
+
 The service accepts requests to the endpoint `api/v1/vast`, and returns a JSON array with the following structure if no conent type is requested:
 
 ```
@@ -18,7 +22,7 @@ The service accepts requests to the endpoint `api/v1/vast`, and returns a JSON a
       "masterPlaylistUrl": "https://your-minio-endpoint/creativeId/substring/index.m3u8"
     }
   ],
-  "vastXml": "<VAST...>"
+  "xml": "<VAST...>"
 }
 ```
 
@@ -47,6 +51,34 @@ results in:
 }
 ```
 
+### VMAP Endpoint
+
+The service also accepts requests to the endpoint `api/v1/vmap`, which handles VMAP (Video Multiple Ad Playlist) documents. The endpoint returns XML with transcoded assets:
+
+```
+% curl -v "http://localhost:8000/api/v1/vmap"
+```
+
+```json
+{
+  "assets": [
+    {
+      "creativeId": "abcd1234",
+      "masterPlaylistUrl": "https://your-minio-endpoint/creativeId/substring/index.m3u8"
+    }
+  ],
+  "xml": "<vmap:VMAP...>"
+}
+```
+
+For XML response:
+
+```
+% curl -v -H 'accept: application/xml' "http://localhost:8000/api/v1/vmap"
+```
+
+The VMAP endpoint processes all VAST ads within the VMAP document, ensuring that all video assets are properly transcoded and available in HLS format.
+
 The service uses redis to keep track of transcoded creatives, and returns the master playlist URL if one is found; if the service does not know of any packaged assets for a creative, it creates a transcoding and packaging pipeline, and monitors the provided minio bucket for asset uploads. Once the assets are in place, the master playlist URL is added to the redis cache. Redis is also used as a distributed lock to avoid multiple jobs being created for the same creative.
 
 ## Requirements

src/api.ts

@@ -6,6 +6,7 @@ import { TypeBoxTypeProvider } from '@fastify/type-provider-typebox';
 import { Static, Type } from '@sinclair/typebox';
 import { FastifyPluginCallback } from 'fastify';
 import { ManifestAsset, vastApi } from './vast/vastApi';
+import { vmapApi } from './vmap/vmapApi';
 import getConfiguration from './config/config';
 import { IN_PROGRESS, DEFAULT_TTL, RedisClient } from './redis/redisclient';
 import logger from './util/logger';
@@ -130,5 +131,29 @@ export default (opts: ApiOptions) => {
       );
     }
   });
+
+  api.register(vmapApi, {
+    adServerUrl: config.adServerUrl,
+    assetServerUrl: `https://${config.s3Endpoint}/${config.bucket}/`,
+    lookUpAsset: async (mediaFile: string) => redisclient.get(mediaFile),
+    onMissingAsset: async (asset: ManifestAsset) => {
+      saveToRedis(
+        asset.creativeId,
+        IN_PROGRESS,
+        config.inFlightTtl ? config.inFlightTtl : DEFAULT_TTL
+      );
+      return encoreClient.createEncoreJob(asset);
+    },
+    setupNotification: (asset: ManifestAsset) => {
+      logger.debug('Setting up notification for asset', { asset });
+      minioClient.listenForNotifications(
+        config.bucket,
+        asset.creativeId + '/',
+        'index.m3u8',
+        async (notification: MinioNotification) =>
+          await saveToRedis(asset.creativeId, notification.s3.object.key, 0)
+      );
+    }
+  });
   return api;
 };

src/minio/minio.test.ts

@@ -1,4 +1,4 @@
-import { MinioClient, MinioNotification } from './minio';
+import { MinioClient } from './minio';
 import * as Minio from 'minio';
 import logger from '../util/logger';
 

src/minio/minio.ts

@@ -37,7 +37,7 @@ export class MinioClient {
     bucketName: string,
     assetId: string,
     masterPlaylistName: string,
-    onNotification: (r: any) => Promise<void>
+    onNotification: (r: MinioNotification) => Promise<void>
   ) => {
     logger.debug('Listening for notifications', {
       bucketName,
@@ -56,7 +56,7 @@ export class MinioClient {
     if (poller == undefined) {
       logger.error('Failed to create poller');
     }
-    poller?.on('notification', (record) => {
+    poller?.on('notification', (record: unknown) => {
       logger.debug('Received notification', record);
       onNotification(record as MinioNotification);
       poller.stop();

src/redis/redisclient.test.ts

@@ -1,7 +1,16 @@
 import { RedisClient } from './redisclient';
-import { createClient } from 'redis';
+import { createClient, RedisClientType } from 'redis';
 import logger from '../util/logger';
 
+// Define a type for our mocked Redis client
+type MockRedisClient = {
+  get: jest.Mock;
+  set: jest.Mock;
+  expire: jest.Mock;
+  expireTime?: jest.Mock;
+  persist?: jest.Mock;
+};
+
 jest.mock('redis', () => ({
   createClient: jest.fn(() => ({
     on: jest.fn().mockReturnThis(),
@@ -42,7 +51,7 @@ describe('RedisClient', () => {
     const mockValue = 'testValue';
     redisClient['client'] = {
       get: jest.fn().mockResolvedValue(mockValue)
-    } as any;
+    } as unknown as RedisClientType;
 
     const value = await redisClient.get(mockKey);
     expect(redisClient['client']?.get).toHaveBeenCalledWith(mockKey);
@@ -55,7 +64,7 @@ describe('RedisClient', () => {
     redisClient['client'] = {
       set: jest.fn(),
       expire: jest.fn()
-    } as any;
+    } as unknown as RedisClientType;
 
     await redisClient.set(mockKey, mockValue, 10);
     expect(redisClient['client']?.set).toHaveBeenCalledWith(mockKey, mockValue);
@@ -65,32 +74,37 @@ describe('RedisClient', () => {
   it('should set a value in Redis and persist if it has an expire time', async () => {
     const mockKey = 'testKey';
     const mockValue = 'testValue';
-    redisClient['client'] = {
+    const mockClient: MockRedisClient = {
+      get: jest.fn(),
       set: jest.fn(),
       expire: jest.fn(),
       expireTime: jest.fn().mockResolvedValue(10),
       persist: jest.fn()
-    } as any;
+    };
+    redisClient['client'] = mockClient as unknown as RedisClientType;
 
     await redisClient.set(mockKey, mockValue, 0);
-    expect(redisClient['client']?.set).toHaveBeenCalledWith(mockKey, mockValue);
-    expect(redisClient['client']?.expireTime).toHaveBeenCalledWith(mockKey);
-    expect(redisClient['client']?.persist).toHaveBeenCalledWith(mockKey);
+    expect(mockClient.set).toHaveBeenCalledWith(mockKey, mockValue);
+    expect(mockClient.expireTime).toHaveBeenCalledWith(mockKey);
+    expect(mockClient.persist).toHaveBeenCalledWith(mockKey);
   });
 
   it('should set a value in Redis and not persist if expire time is -1', async () => {
     const mockKey = 'testKey';
     const mockValue = 'testValue';
-    redisClient['client'] = {
+    const mockClient: MockRedisClient = {
+      get: jest.fn(),
       set: jest.fn(),
       expire: jest.fn(),
       expireTime: jest.fn().mockResolvedValue(-1),
       persist: jest.fn()
-    } as any;
+    };
+    redisClient['client'] = mockClient as unknown as RedisClientType;
 
     await redisClient.set(mockKey, mockValue, 0);
-    expect(redisClient['client']?.set).toHaveBeenCalledWith(mockKey, mockValue);
-    expect(redisClient['client']?.expireTime).toHaveBeenCalledWith(mockKey);
+    expect(mockClient.set).toHaveBeenCalledWith(mockKey, mockValue);
+    expect(mockClient.expireTime).toHaveBeenCalledWith(mockKey);
+    expect(mockClient.persist).not.toHaveBeenCalled();
   });
 
   it('should log an error if client is not connected when getting a key', async () => {