Merge branch 'master' into chore/eslint-v9-bump

Author: barjin

CHANGELOG.md

@@ -3,6 +3,17 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+# [3.4.0](https://github.com/apify/apify-sdk-js/compare/[email protected]@3.4.0) (2025-04-01)
+
+
+### Features
+
+* sign record's public url ([#358](https://github.com/apify/apify-sdk-js/issues/358)) ([6274cc0](https://github.com/apify/apify-sdk-js/commit/6274cc018ab3b02787c234eacbb025d4d973a95d))
+
+
+
+
+
 ## [3.3.2](https://github.com/apify/apify-sdk-js/compare/[email protected]@3.3.2) (2025-02-20)
 
 ### Bug Fixes

package-lock.json

@@ -67,7 +67,7 @@
         "@types/tough-cookie": "^4.0.5",
         "@types/ws": "^8.5.12",
         "commitlint": "^19.3.0",
-        "crawlee": "^3.11.5",
+        "crawlee": "^3.13.0",
         "eslint": "^9.23.0",
         "eslint-config-prettier": "^10.1.1",
         "fs-extra": "^11.2.0",
@@ -80,7 +80,7 @@
         "puppeteer": "^24.0.0",
         "rimraf": "^6.0.1",
         "tsx": "^4.16.5",
-        "turbo": "2.4.4",
+        "turbo": "2.5.0",
         "typescript": "~5.8.0",
         "typescript-eslint": "^8.28.0",
         "vite-tsconfig-paths": "^4.3.2",

package.json

@@ -255,7 +255,7 @@ export class CrawlerSetup implements CrawlerSetupOptions {
             launchContext: {
                 useChrome: this.input.useChrome,
                 launchOptions: {
-                    ignoreHTTPSErrors: this.input.ignoreSslErrors,
+                    acceptInsecureCerts: this.input.ignoreSslErrors,
                     defaultViewport: DEFAULT_VIEWPORT,
                     devtools: this.devtools,
                     args,

packages/actor-scraper/puppeteer-scraper/src/internals/crawler_setup.ts

@@ -300,7 +300,7 @@ export class CrawlerSetup implements CrawlerSetupOptions {
             launchContext: {
                 useChrome: this.input.useChrome,
                 launchOptions: {
-                    ignoreHTTPSErrors: this.input.ignoreSslErrors,
+                    acceptInsecureCerts: this.input.ignoreSslErrors,
                     defaultViewport: DEFAULT_VIEWPORT,
                     args,
                     headless: this.input.headless,

packages/actor-scraper/web-scraper/src/internals/crawler_setup.ts

@@ -1,6 +1,6 @@
 {
     "name": "apify",
-    "version": "3.3.2",
+    "version": "3.4.0",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=16.0.0"
@@ -58,11 +58,11 @@
         "@apify/input_secrets": "^1.1.40",
         "@apify/log": "^2.4.3",
         "@apify/timeout": "^0.3.0",
-        "@apify/utilities": "^2.9.3",
-        "@crawlee/core": "^3.9.0",
-        "@crawlee/types": "^3.9.0",
-        "@crawlee/utils": "^3.9.0",
-        "apify-client": "^2.12.0",
+        "@apify/utilities": "^2.13.0",
+        "@crawlee/core": "^3.13.0",
+        "@crawlee/types": "^3.13.0",
+        "@crawlee/utils": "^3.13.0",
+        "apify-client": "^2.12.1",
         "fs-extra": "^11.2.0",
         "ow": "^0.28.2",
         "semver": "^7.5.4",

packages/apify/package.json

@@ -17,6 +17,9 @@ export interface ConfigurationOptions extends CoreConfigurationOptions {
     actorRunId?: string;
     actorTaskId?: string;
     apiBaseUrl?: string;
+    // apiBaseUrl is the internal API URL, accessible only within the platform(private network),
+    // while apiPublicBaseUrl is the public API URL, available externally(through internet).
+    apiPublicBaseUrl?: string;
     containerPort?: number;
     containerUrl?: string;
     proxyHostname?: string;
@@ -141,6 +144,7 @@ export class Configuration extends CoreConfiguration {
         APIFY_ACTOR_EVENTS_WS_URL: 'actorEventsWsUrl',
         APIFY_ACTOR_ID: 'actorId',
         APIFY_API_BASE_URL: 'apiBaseUrl',
+        APIFY_API_PUBLIC_BASE_URL: 'apiPublicBaseUrl',
         APIFY_IS_AT_HOME: 'isAtHome',
         APIFY_ACTOR_RUN_ID: 'actorRunId',
         APIFY_ACTOR_TASK_ID: 'actorTaskId',
@@ -200,6 +204,7 @@ export class Configuration extends CoreConfiguration {
             LOCAL_ACTOR_ENV_VARS[ACTOR_ENV_VARS.DEFAULT_REQUEST_QUEUE_ID],
         inputKey: 'INPUT',
         apiBaseUrl: 'https://api.apify.com',
+        apiPublicBaseUrl: 'https://api.apify.com',
         proxyStatusUrl: 'http://proxy.apify.com',
         proxyHostname: LOCAL_APIFY_ENV_VARS[APIFY_ENV_VARS.PROXY_HOSTNAME],
         proxyPort: +LOCAL_APIFY_ENV_VARS[APIFY_ENV_VARS.PROXY_PORT],

packages/apify/src/configuration.ts

@@ -1,3 +1,4 @@
+import { createHmacSignature } from '@apify/utilities';
 import type { StorageManagerOptions } from '@crawlee/core';
 import { KeyValueStore as CoreKeyValueStore } from '@crawlee/core';
 
@@ -15,11 +16,18 @@ export class KeyValueStore extends CoreKeyValueStore {
      * access the value in the remote key-value store.
      */
     override getPublicUrl(key: string): string {
-        if (!(this.config as Configuration).get('isAtHome') && getPublicUrl) {
+        const config = this.config as Configuration;
+        if (!config.get('isAtHome') && getPublicUrl) {
             return getPublicUrl.call(this, key);
         }
 
-        return `https://api.apify.com/v2/key-value-stores/${this.id}/records/${key}`;
+        const publicUrl = new URL(`${config.get('apiPublicBaseUrl')}/v2/key-value-stores/${this.id}/records/${key}`);
+
+        if (this.storageObject?.urlSigningSecretKey) {
+            publicUrl.searchParams.append('signature', createHmacSignature(this.storageObject.urlSigningSecretKey as string, key));
+        }
+
+        return publicUrl.toString();
     }
 
     /**

packages/apify/src/key_value_store.ts

@@ -356,7 +356,7 @@ export class ProxyConfiguration extends CoreProxyConfiguration {
             return this._handleTieredUrl(
                 sessionId ?? cryptoRandomObjectId(6),
                 options,
-            ).proxyUrl;
+            ).proxyUrl ?? undefined;
         }
 
         return this.composeDefaultUrl(sessionId);

packages/apify/src/proxy_configuration.ts

@@ -9,11 +9,13 @@ import { ACTOR_ENV_VARS, APIFY_ENV_VARS } from '@apify/consts';
 describe('events', () => {
     let wss: WebSocketServer = null;
     const config = Configuration.getGlobalConfig();
-    const events = new PlatformEventManager(config);
-    config.useEventManager(events);
+    let events: PlatformEventManager | null = null;
 
     beforeEach(() => {
         wss = new WebSocketServer({ port: 9099 });
+        events = new PlatformEventManager(config);
+        config.useEventManager(events);
+
         vitest.useFakeTimers();
         process.env[ACTOR_ENV_VARS.EVENTS_WEBSOCKET_URL] =
             'ws://localhost:9099/someRunId';
@@ -128,11 +130,13 @@ describe('events', () => {
 
     test('should send persist state events in regular interval', async () => {
         const eventsReceived = [];
+        const interval = config.get('persistStateIntervalMillis');
+
         events.on(EventType.PERSIST_STATE, (data) => eventsReceived.push(data));
         await events.init();
-        vitest.advanceTimersByTime(60001);
-        vitest.advanceTimersByTime(60001);
-        vitest.advanceTimersByTime(60001);
+        await vitest.advanceTimersByTimeAsync(1.1 * interval);
+        await vitest.advanceTimersByTimeAsync(1.1 * interval);
+        await vitest.advanceTimersByTimeAsync(1.1 * interval);
         await events.close();
         expect(eventsReceived.length).toBe(5);
     });

test/apify/events.test.ts

@@ -0,0 +1,19 @@
+import { Actor, log } from 'apify';
+
+await Actor.init();
+
+const { data } = await Actor.getInput();
+
+await Actor.setValue('public-record-key', JSON.stringify(data), { contentType: `application/json` });
+
+const defaultKeyValueStore = await Actor.openKeyValueStore();
+const publicUrl = defaultKeyValueStore.getPublicUrl('public-record-key');
+
+// Here we store the url itself
+await Actor.setValue('urlToPublicData', publicUrl);
+
+log.info('Generated public url', { publicUrl });
+
+await Actor.pushData({ publicUrl });
+
+await Actor.exit();

test/e2e/sdk/publicUrl/src/main.mjs

@@ -0,0 +1,19 @@
+import assert from 'node:assert/strict';
+
+import { ApifyClient } from 'apify';
+
+const PUBLIC_DATA = { exposedData: 'test' };
+
+const client = new ApifyClient({
+    token: process.env.APIFY_TOKEN,
+});
+
+const actor = client.actor(process.argv[2]);
+
+const run = await actor.call({ data: PUBLIC_DATA }, { waitSecs: 15 });
+assert.equal(run.exitCode, 0);
+
+const publicUrl = await client.keyValueStore(run.defaultKeyValueStoreId).getRecord('urlToPublicData');
+const data = await fetch(publicUrl.value).then((res) => res.json());
+
+assert.deepEqual(data, PUBLIC_DATA);

test/e2e/sdk/publicUrl/test.mjs

@@ -28,7 +28,7 @@
     },
     "dependencies": {
         "@apify/docs-theme": "^1.0.131",
-        "@apify/docusaurus-plugin-typedoc-api": "^4.3.12",
+        "@apify/docusaurus-plugin-typedoc-api": "^4.4.3",
         "@docusaurus/core": "^3.5.2",
         "@docusaurus/plugin-client-redirects": "^3.5.2",
         "@docusaurus/preset-classic": "^3.5.2",

website/package-lock.json

@@ -0,0 +1 @@
+[{"entryPoints":{"index":{"label":"Index","path":"src/index.ts"}},"packageRoot":"../packages/apify","packagePath":"packages/apify","packageSlug":"apify","packageName":"apify","packageVersion":"3.4.0"}]