From b1d5e8f92e69e36baf207af51ad48e6de674e664 Mon Sep 17 00:00:00 2001 From: Nate Date: Sun, 19 Jan 2025 09:30:02 -0800 Subject: [PATCH 01/11] config-yaml rewrite for finalized secret resolution scheme --- packages/config-yaml/src/README.md | 17 ++ packages/config-yaml/src/converter.ts | 1 - packages/config-yaml/src/index.ts | 116 +----------- .../src/interfaces/SecretResult.test.ts | 41 +++++ .../src/interfaces/SecretResult.ts | 92 ++++++++++ packages/config-yaml/src/interfaces/index.ts | 87 +++++++++ .../config-yaml/src/interfaces/slugs.test.ts | 82 +++++++++ packages/config-yaml/src/interfaces/slugs.ts | 68 +++++++ packages/config-yaml/src/load/clientRender.ts | 59 ++++++ packages/config-yaml/src/load/merge.ts | 18 ++ .../src/load/proxySecretResolution.ts | 32 ++++ packages/config-yaml/src/load/unroll.ts | 172 ++++++++++++++++++ .../config-yaml/src/resolveSecretsOnClient.ts | 64 ------- packages/config-yaml/src/schemas/context.ts | 6 - packages/config-yaml/src/schemas/data.ts | 5 - packages/config-yaml/src/schemas/index.ts | 21 ++- packages/config-yaml/src/schemas/models.ts | 1 - packages/config-yaml/test/index.test.ts | 106 +++++++++++ .../test/packages/test-org/assistant.yaml | 18 ++ .../test/packages/test-org/docs.yaml | 7 + .../test/packages/test-org/models.yaml | 13 ++ .../test/packages/test-org/rules.yaml | 6 + 22 files changed, 841 insertions(+), 191 deletions(-) create mode 100644 packages/config-yaml/src/README.md create mode 100644 packages/config-yaml/src/interfaces/SecretResult.test.ts create mode 100644 packages/config-yaml/src/interfaces/SecretResult.ts create mode 100644 packages/config-yaml/src/interfaces/index.ts create mode 100644 packages/config-yaml/src/interfaces/slugs.test.ts create mode 100644 packages/config-yaml/src/interfaces/slugs.ts create mode 100644 packages/config-yaml/src/load/clientRender.ts create mode 100644 packages/config-yaml/src/load/merge.ts create mode 100644 packages/config-yaml/src/load/proxySecretResolution.ts create mode 100644 packages/config-yaml/src/load/unroll.ts delete mode 100644 packages/config-yaml/src/resolveSecretsOnClient.ts delete mode 100644 packages/config-yaml/src/schemas/context.ts delete mode 100644 packages/config-yaml/src/schemas/data.ts create mode 100644 packages/config-yaml/test/index.test.ts create mode 100644 packages/config-yaml/test/packages/test-org/assistant.yaml create mode 100644 packages/config-yaml/test/packages/test-org/docs.yaml create mode 100644 packages/config-yaml/test/packages/test-org/models.yaml create mode 100644 packages/config-yaml/test/packages/test-org/rules.yaml diff --git a/packages/config-yaml/src/README.md b/packages/config-yaml/src/README.md new file mode 100644 index 0000000000..4ef80cb49a --- /dev/null +++ b/packages/config-yaml/src/README.md @@ -0,0 +1,17 @@ +# config.yaml specification + +This specification is a work in progress and subject to change. + +## Loading a config.yaml file + +config.yaml is loaded in the following steps + +## Unrolling + +A "source" config.yaml is "unrolled" so that its packages all get merged into a single config.yaml. This is done by recursively loading all packages and merging them into the config.yaml. + +This happens on the server, unless using local mode. + +## Client rendering + +The unrolled config.yaml is then rendered on the client. This is done by replacing all user secret template variables with their values and replacing all other secrets with secret locations. diff --git a/packages/config-yaml/src/converter.ts b/packages/config-yaml/src/converter.ts index 485d469a9e..56d5326d2a 100644 --- a/packages/config-yaml/src/converter.ts +++ b/packages/config-yaml/src/converter.ts @@ -65,7 +65,6 @@ function convertCustomCommand( name: cmd.name, description: cmd.description, prompt: (cmd as any).prompt, // The type is wrong in @continuedev/config-types - type: "slash-command", }; } diff --git a/packages/config-yaml/src/index.ts b/packages/config-yaml/src/index.ts index c3be3e7133..9ebf6cb320 100644 --- a/packages/config-yaml/src/index.ts +++ b/packages/config-yaml/src/index.ts @@ -1,107 +1,11 @@ -import * as YAML from "yaml"; -import { ConfigYaml, configYamlSchema } from "./schemas/index.js"; - -const REGISTRY_URL = "https://registry.continue.dev"; -const LATEST = "latest"; - -function parseUses(uses: string): { - owner: string; - packageName: string; - version: string; -} { - const [owner, packageNameAndVersion] = uses.split("/"); - const [packageName, version] = packageNameAndVersion.split("@"); - return { - owner, - packageName, - version: version ?? LATEST, - }; -} - -export function extendConfig(config: ConfigYaml, pkg: ConfigYaml): ConfigYaml { - return { - ...config, - models: [...(config.models ?? []), ...(pkg.models ?? [])], - context: [...(config.context ?? []), ...(pkg.context ?? [])], - tools: [...(config.tools ?? []), ...(pkg.tools ?? [])], - data: [...(config.data ?? []), ...(pkg.data ?? [])], - mcpServers: [...(config.mcpServers ?? []), ...(pkg.mcpServers ?? [])], - }; -} - -export async function resolvePackages( - configYaml: ConfigYaml, -): Promise { - if (!configYaml.packages) return configYaml; - - for (const pkgDesc of configYaml.packages) { - const { owner, packageName, version } = parseUses(pkgDesc.uses); - const downloadUrl = new URL( - `/${owner}/${packageName}/${version}`, - REGISTRY_URL, - ); - const resp = await fetch(downloadUrl); - if (!resp.ok) { - throw new Error( - `Failed to fetch package ${pkgDesc.uses} from registry: ${resp.statusText}`, - ); - } - const downloadBuf = await resp.arrayBuffer(); - const downloadStr = new TextDecoder().decode(downloadBuf); - const pkg = YAML.parse(downloadStr); - - const validatedPkg = configYamlSchema.parse(pkg); - configYaml = extendConfig(configYaml, validatedPkg); - } - return configYaml; -} - -export function renderConfigYaml(configYaml: string): ConfigYaml { - try { - const parsed = YAML.parse(configYaml); - const result = configYamlSchema.parse(parsed); - return result; - } catch (e: any) { - throw new Error(`Failed to parse config yaml: ${e.message}`); - } -} - -const TEMPLATE_VAR_REGEX = /\${{[\s]*([^}\s]+)[\s]*}}/g; - -export function getTemplateVariables(templatedYaml: string): string[] { - const variables = new Set(); - const matches = templatedYaml.matchAll(TEMPLATE_VAR_REGEX); - for (const match of matches) { - variables.add(match[1]); - } - return Array.from(variables); -} - -export function fillTemplateVariables( - templatedYaml: string, - data: { [key: string]: string }, -): string { - return templatedYaml.replace(TEMPLATE_VAR_REGEX, (match, variableName) => { - // Inject data - if (variableName in data) { - return data[variableName]; - } - // If variable doesn't exist, return the original expression - return match; - }); -} - -export { convertJsonToYamlConfig } from "./converter.js"; -export { resolveSecretsOnClient } from "./resolveSecretsOnClient.js"; -export { - ClientConfigYaml, - clientConfigYamlSchema, - ConfigYaml, - configYamlSchema, -} from "./schemas/index.js"; +export * from "./converter.js"; +export * from "./interfaces/index.js"; +export * from "./interfaces/SecretResult.js"; +export * from "./interfaces/slugs.js"; +export * from "./load/clientRender.js"; +export * from "./load/merge.js"; +export * from "./load/proxySecretResolution.js"; +export * from "./load/unroll.js"; +export * from "./schemas/index.js"; export type { ModelConfig } from "./schemas/models.js"; -export { - ConfigResult, - ConfigValidationError, - validateConfigYaml, -} from "./validation.js"; +export * from "./validation.js"; diff --git a/packages/config-yaml/src/interfaces/SecretResult.test.ts b/packages/config-yaml/src/interfaces/SecretResult.test.ts new file mode 100644 index 0000000000..2828f015cf --- /dev/null +++ b/packages/config-yaml/src/interfaces/SecretResult.test.ts @@ -0,0 +1,41 @@ +import { + SecretType, + decodeSecretLocation, + encodeSecretLocation, +} from "./SecretResult.js"; +import { PackageSlug } from "./slugs.js"; + +describe("SecretLocation encoding/decoding", () => { + it("encodes/decodes organization secret location", () => { + const orgSecretLocation = { + secretType: SecretType.Organization as const, + orgSlug: "test-org", + secretName: "secret1", + }; + + const encoded = encodeSecretLocation(orgSecretLocation); + expect(encoded).toBe("organization:test-org/secret1"); + + const decoded = decodeSecretLocation(encoded); + expect(decoded).toEqual(orgSecretLocation); + }); + + it("encodes/decodes package secret location", () => { + const packageSlug: PackageSlug = { + ownerSlug: "test-org", + packageSlug: "test-package", + }; + + const packageSecretLocation = { + secretType: SecretType.Package as const, + packageSlug, + secretName: "secret1", + }; + + const encoded = encodeSecretLocation(packageSecretLocation); + expect(encoded).toBe("package:test-org/test-package/secret1"); + + const decoded = decodeSecretLocation(encoded); + expect(decoded).toEqual(packageSecretLocation); + }); +}); diff --git a/packages/config-yaml/src/interfaces/SecretResult.ts b/packages/config-yaml/src/interfaces/SecretResult.ts new file mode 100644 index 0000000000..987b336763 --- /dev/null +++ b/packages/config-yaml/src/interfaces/SecretResult.ts @@ -0,0 +1,92 @@ +import { FQSN, PackageSlug, encodePackageSlug } from "./slugs.js"; + +export enum SecretType { + User = "user", + Package = "package", + Organization = "organization", +} + +export interface OrgSecretLocation { + secretType: SecretType.Organization; + orgSlug: string; + secretName: string; +} + +export interface PackageSecretLocation { + secretType: SecretType.Package; + packageSlug: PackageSlug; + secretName: string; +} + +export interface UserSecretLocation { + secretType: SecretType.User; + userSlug: string; + secretName: string; +} + +export type SecretLocation = + | OrgSecretLocation + | PackageSecretLocation + | UserSecretLocation; + +export function encodeSecretLocation(secretLocation: SecretLocation): string { + if (secretLocation.secretType === SecretType.Organization) { + return `${SecretType.Organization}:${secretLocation.orgSlug}/${secretLocation.secretName}`; + } else if (secretLocation.secretType === SecretType.User) { + return `${SecretType.User}:${secretLocation.userSlug}/${secretLocation.secretName}`; + } else { + return `${SecretType.Package}:${encodePackageSlug(secretLocation.packageSlug)}/${secretLocation.secretName}`; + } +} + +export function decodeSecretLocation(secretLocation: string): SecretLocation { + const [secretType, rest] = secretLocation.split(":"); + const parts = rest.split("/"); + const secretName = parts[parts.length - 1]; + + switch (secretType) { + case SecretType.Organization: + return { + secretType: SecretType.Organization, + orgSlug: parts[0], + secretName, + }; + case SecretType.User: + return { + secretType: SecretType.User, + userSlug: parts[0], + secretName, + }; + case SecretType.Package: + return { + secretType: SecretType.Package, + packageSlug: { ownerSlug: parts[0], packageSlug: parts[1] }, + secretName, + }; + default: + throw new Error(`Invalid secret type: ${secretType}`); + } +} + +export interface NotFoundSecretResult { + found: false; + fqsn: FQSN; +} + +export interface FoundSecretResult { + found: true; + secretLocation: OrgSecretLocation | PackageSecretLocation; + fqsn: FQSN; +} + +export interface FoundUserSecretResult { + found: true; + secretLocation: UserSecretLocation; + value: string; + fqsn: FQSN; +} + +export type SecretResult = + | FoundSecretResult + | FoundUserSecretResult + | NotFoundSecretResult; diff --git a/packages/config-yaml/src/interfaces/index.ts b/packages/config-yaml/src/interfaces/index.ts new file mode 100644 index 0000000000..b65a334d67 --- /dev/null +++ b/packages/config-yaml/src/interfaces/index.ts @@ -0,0 +1,87 @@ +import { SecretLocation, SecretResult, SecretType } from "./SecretResult.js"; +import { FQSN, FullSlug } from "./slugs.js"; + +/** + * A registry stores the content of packages + */ +export interface Registry { + getContent(fullSlug: FullSlug): Promise; +} +export type SecretNamesMap = Map; + +/** + * A secret store stores secrets + */ +export interface SecretStore { + get(secretName: string): Promise; + set(secretName: string, secretValue: string): Promise; +} + +export interface PlatformClient { + resolveFQSNs(fqsns: FQSN[]): Promise<(SecretResult | undefined)[]>; +} + +export interface PlatformSecretStore { + getSecretFromSecretLocation( + secretLocation: SecretLocation, + ): Promise; +} + +export async function resolveFQSN( + currentUserSlug: string, + currentUserOrgSlug: string, + fqsn: FQSN, + platformSecretStore: PlatformSecretStore, +): Promise { + // First create the list of secret locations to try in order + const reversedSlugs = [...fqsn.packageSlugs].reverse(); + + const locationsToLook: SecretLocation[] = [ + // Packages first + ...reversedSlugs.map((slug) => ({ + secretType: SecretType.Package as const, + packageSlug: slug, + secretName: fqsn.secretName, + })), + // Then user + { + secretType: SecretType.User as const, + userSlug: currentUserSlug, + secretName: fqsn.secretName, + }, + // Then organization + { + secretType: SecretType.Organization as const, + orgSlug: currentUserOrgSlug, + secretName: fqsn.secretName, + }, + ]; + + // Then try to get the secret from each location + for (const secretLocation of locationsToLook) { + const secret = + await platformSecretStore.getSecretFromSecretLocation(secretLocation); + if (secret) { + if (secretLocation.secretType === SecretType.User) { + // Only user secret values get sent back to client + return { + found: true, + fqsn, + secretLocation, + value: secret, + }; + } else { + return { + found: true, + fqsn, + secretLocation, + }; + } + } + } + + return { + found: false, + fqsn, + }; +} diff --git a/packages/config-yaml/src/interfaces/slugs.test.ts b/packages/config-yaml/src/interfaces/slugs.test.ts new file mode 100644 index 0000000000..cab57e5d24 --- /dev/null +++ b/packages/config-yaml/src/interfaces/slugs.test.ts @@ -0,0 +1,82 @@ +import { + decodeFQSN, + decodeFullSlug, + decodePackageSlug, + encodeFQSN, + encodeFullSlug, + encodePackageSlug, + VirtualTags, +} from "./slugs.js"; + +describe("PackageSlug", () => { + it("should encode/decode package slugs", () => { + const testSlug = { + ownerSlug: "test-owner", + packageSlug: "test-package", + }; + const encoded = encodePackageSlug(testSlug); + expect(encoded).toBe("test-owner/test-package"); + const decoded = decodePackageSlug(encoded); + expect(decoded).toEqual(testSlug); + }); + + it("should encode/decode full slugs", () => { + const testFullSlug = { + ownerSlug: "test-owner", + packageSlug: "test-package", + versionSlug: "1.0.0", + }; + const encoded = encodeFullSlug(testFullSlug); + expect(encoded).toBe("test-owner/test-package@1.0.0"); + const decoded = decodeFullSlug(encoded); + expect(decoded).toEqual(testFullSlug); + }); + + it("should use latest tag when no version provided", () => { + const encoded = "test-owner/test-package"; + const decoded = decodeFullSlug(encoded); + expect(decoded.versionSlug).toBe(VirtualTags.Latest); + }); + + it("should encode/decode FQSN with single package", () => { + const testFQSN = { + packageSlugs: [ + { + ownerSlug: "test-owner", + packageSlug: "test-package", + }, + ], + secretName: "test-secret", + }; + const encoded = encodeFQSN(testFQSN); + expect(encoded).toBe("test-owner/test-package/test-secret"); + const decoded = decodeFQSN(encoded); + expect(decoded).toEqual(testFQSN); + }); + + it("should encode/decode FQSN with multiple packages", () => { + const testFQSN = { + packageSlugs: [ + { + ownerSlug: "owner1", + packageSlug: "package1", + }, + { + ownerSlug: "owner2", + packageSlug: "package2", + }, + ], + secretName: "test-secret", + }; + const encoded = encodeFQSN(testFQSN); + expect(encoded).toBe("owner1/package1/owner2/package2/test-secret"); + const decoded = decodeFQSN(encoded); + expect(decoded).toEqual(testFQSN); + }); + + it("should throw error for invalid FQSN format", () => { + expect(() => decodeFQSN("owner1/package1/owner2/test-secret")).toThrow( + "Invalid FQSN format: package slug must have two parts", + ); + }); +}); diff --git a/packages/config-yaml/src/interfaces/slugs.ts b/packages/config-yaml/src/interfaces/slugs.ts new file mode 100644 index 0000000000..a6926fb5b6 --- /dev/null +++ b/packages/config-yaml/src/interfaces/slugs.ts @@ -0,0 +1,68 @@ +export interface PackageSlug { + ownerSlug: string; + packageSlug: string; +} +export interface FullSlug extends PackageSlug { + versionSlug: string; +} + +export enum VirtualTags { + Latest = "latest", +} + +export function encodePackageSlug(packageSlug: PackageSlug): string { + return `${packageSlug.ownerSlug}/${packageSlug.packageSlug}`; +} + +export function decodePackageSlug(pkgSlug: string): PackageSlug { + const [ownerSlug, packageSlug] = pkgSlug.split("/"); + return { + ownerSlug, + packageSlug, + }; +} + +export function encodeFullSlug(fullSlug: FullSlug): string { + return `${fullSlug.ownerSlug}/${fullSlug.packageSlug}@${fullSlug.versionSlug}`; +} + +export function decodeFullSlug(fullSlug: string): FullSlug { + const [ownerSlug, packageSlug, versionSlug] = fullSlug.split(/[/@]/); + return { + ownerSlug, + packageSlug, + versionSlug: versionSlug || VirtualTags.Latest, + }; +} + +/** + * FQSN = Fully Qualified Secret Name + */ +export interface FQSN { + packageSlugs: PackageSlug[]; + secretName: string; +} + +export function encodeFQSN(fqsn: FQSN): string { + const parts = [...fqsn.packageSlugs.map(encodePackageSlug), fqsn.secretName]; + return parts.join("/"); +} + +export function decodeFQSN(fqsn: string): FQSN { + const parts = fqsn.split("/"); + const secretName = parts.pop()!; + const packageSlugs: PackageSlug[] = []; + + // Process parts two at a time to decode package slugs + for (let i = 0; i < parts.length; i += 2) { + if (i + 1 >= parts.length) { + throw new Error("Invalid FQSN format: package slug must have two parts"); + } + packageSlugs.push({ + ownerSlug: parts[i], + packageSlug: parts[i + 1], + }); + } + + return { packageSlugs, secretName }; +} diff --git a/packages/config-yaml/src/load/clientRender.ts b/packages/config-yaml/src/load/clientRender.ts new file mode 100644 index 0000000000..b58aca05f6 --- /dev/null +++ b/packages/config-yaml/src/load/clientRender.ts @@ -0,0 +1,59 @@ +import * as YAML from "yaml"; +import { PlatformClient, SecretStore } from "../interfaces/index.js"; +import { encodeSecretLocation } from "../interfaces/SecretResult.js"; +import { FQSN, decodeFQSN, encodeFQSN } from "../interfaces/slugs.js"; +import { ConfigYaml } from "../schemas/index.js"; +import { + fillTemplateVariables, + getTemplateVariables, + parseConfigYaml, +} from "./unroll.js"; + +export async function clientRender( + unrolledConfig: ConfigYaml, + secretStore: SecretStore, + platformClient: PlatformClient, +): Promise { + const rawYaml = YAML.stringify(unrolledConfig); + + // 1. First we need to get a list of all the FQSNs that are required to render the config + const secrets = getTemplateVariables(rawYaml); + + // 2. Then, we will check which of the secrets are found in the local personal secret store. Here we’re checking for anything that matches the last part of the FQSN, not worrying about the owner/package/owner/package slugs + const secretsTemplateData: Record = {}; + + const unresolvedFQSNs: FQSN[] = []; + for (const secret of secrets) { + const fqsn = decodeFQSN(secret.replace("secrets.", "")); + const secretValue = await secretStore.get(fqsn.secretName); + if (secretValue) { + secretsTemplateData[secret] = secretValue; + } else { + unresolvedFQSNs.push(fqsn); + } + } + + // 3. For any secrets not found, we send the FQSNs to the Continue Platform at the `/ide/sync-secrets` endpoint. This endpoint replies for each of the FQSNs with the following information (`SecretResult`): `foundAt`: tells which secret store it was found in (this is “user”, “org”, “package” or null if not found anywhere). If it’s found in an org or a package, it tells us the `secretLocation`, which is either just an org slug, or is a full org/package slug. If it’s found in “user” secrets, we send back the `value`. Full definition of `SecretResult` at [2]. The method of resolving an FQSN to a `SecretResult` is detailed at [3] + const secretResults = await platformClient.resolveFQSNs(unresolvedFQSNs); + + // 4. (back to the client) Any “user” secrets that were returned back are added to the local secret store so we don’t have to request them again + for (const secretResult of secretResults) { + if (!secretResult?.found) continue; + + if ("value" in secretResult) { + secretStore.set(secretResult.fqsn.secretName, secretResult.value); + } + + secretsTemplateData["secrets." + encodeFQSN(secretResult.fqsn)] = + "value" in secretResult + ? secretResult.value + : `\${{ secrets.${encodeSecretLocation(secretResult.secretLocation)} }}`; + } + + // 5. User secrets are rendered in place of the template variable. Others remain templated, but replaced with the specific location where they are to be found (`${{ secrets. }}` instead of `${{ secrets. }}`) + const renderedYaml = fillTemplateVariables(rawYaml, secretsTemplateData); + + // 6. The rendered YAML is parsed and validated again + const finalParsedYaml = parseConfigYaml(renderedYaml); + return finalParsedYaml; +} diff --git a/packages/config-yaml/src/load/merge.ts b/packages/config-yaml/src/load/merge.ts new file mode 100644 index 0000000000..590572edba --- /dev/null +++ b/packages/config-yaml/src/load/merge.ts @@ -0,0 +1,18 @@ +import { ConfigYaml } from "../schemas/index.js"; + +export function mergePackages( + current: ConfigYaml, + incoming: ConfigYaml, +): ConfigYaml { + return { + ...current, + models: [...(current.models ?? []), ...(incoming.models ?? [])], + context: [...(current.context ?? []), ...(incoming.context ?? [])], + data: [...(current.data ?? []), ...(incoming.data ?? [])], + tools: [...(current.tools ?? []), ...(incoming.tools ?? [])], + mcpServers: [...(current.mcpServers ?? []), ...(incoming.mcpServers ?? [])], + rules: [...(current.rules ?? []), ...(incoming.rules ?? [])], + prompts: [...(current.prompts ?? []), ...(incoming.prompts ?? [])], + docs: [...(current.docs ?? []), ...(incoming.docs ?? [])], + }; +} diff --git a/packages/config-yaml/src/load/proxySecretResolution.ts b/packages/config-yaml/src/load/proxySecretResolution.ts new file mode 100644 index 0000000000..ff3b0ec4e6 --- /dev/null +++ b/packages/config-yaml/src/load/proxySecretResolution.ts @@ -0,0 +1,32 @@ +import { PlatformSecretStore, SecretStore } from "../interfaces/index.js"; +import { + SecretLocation, + encodeSecretLocation, +} from "../interfaces/SecretResult.js"; + +export async function resolveSecretLocationInProxy( + secretLocaton: SecretLocation, + platformSecretStore: PlatformSecretStore, + environmentSecretStore?: SecretStore, +): Promise { + // 1. Check environment variables (if supported) + if (environmentSecretStore) { + const envSecretValue = await environmentSecretStore.get( + secretLocaton.secretName, + ); + if (envSecretValue) { + return envSecretValue; + } + } + + // 2. Get from secret location + const platformSecret = + await platformSecretStore.getSecretFromSecretLocation(secretLocaton); + if (platformSecret) { + return platformSecret; + } + + throw new Error( + `Could not resolve secret with location ${encodeSecretLocation(secretLocaton)}`, + ); +} diff --git a/packages/config-yaml/src/load/unroll.ts b/packages/config-yaml/src/load/unroll.ts new file mode 100644 index 0000000000..54774c5a3e --- /dev/null +++ b/packages/config-yaml/src/load/unroll.ts @@ -0,0 +1,172 @@ +import * as YAML from "yaml"; +import { Registry } from "../interfaces/index.js"; +import { + PackageSlug, + decodeFullSlug, + encodePackageSlug, +} from "../interfaces/slugs.js"; +import { ConfigYaml, configYamlSchema } from "../schemas/index.js"; +import { mergePackages } from "./merge.js"; + +export function parseConfigYaml(configYaml: string): ConfigYaml { + try { + const parsed = YAML.parse(configYaml); + const result = configYamlSchema.parse(parsed); + return result; + } catch (e: any) { + throw new Error(`Failed to parse config yaml: ${e.message}`); + } +} + +const TEMPLATE_VAR_REGEX = /\${{[\s]*([^}\s]+)[\s]*}}/g; + +export function getTemplateVariables(templatedYaml: string): string[] { + const variables = new Set(); + const matches = templatedYaml.matchAll(TEMPLATE_VAR_REGEX); + for (const match of matches) { + variables.add(match[1]); + } + return Array.from(variables); +} + +export function fillTemplateVariables( + templatedYaml: string, + data: { [key: string]: string }, +): string { + return templatedYaml.replace(TEMPLATE_VAR_REGEX, (match, variableName) => { + // Inject data + if (variableName in data) { + return data[variableName]; + } + // If variable doesn't exist, return the original expression + return match; + }); +} + +export async function unrollImportedPackage( + pkgImport: NonNullable[number], + parentPackages: PackageSlug[], + registry: Registry, +): Promise { + const { uses, with: params } = pkgImport; + + const fullSlug = decodeFullSlug(uses); + + // Request the content from the registry + const rawContent = await registry.getContent(fullSlug); + + // Convert the raw YAML to unrolled config + return await unrollPackageFromContent( + rawContent, + params, + parentPackages, + registry, + ); +} + +export interface TemplateData { + params: Record | undefined; + secrets: Record | undefined; + continue: {}; +} + +function flattenTemplateData( + templateData: TemplateData, +): Record { + const flattened: Record = {}; + + if (templateData.params) { + for (const [key, value] of Object.entries(templateData.params)) { + flattened[`params.${key}`] = value; + } + } + if (templateData.secrets) { + for (const [key, value] of Object.entries(templateData.secrets)) { + flattened[`secrets.${key}`] = value; + } + } + + return flattened; +} + +function secretToFQSNMap( + secretNames: string[], + parentPackages: PackageSlug[], +): Record { + const map: Record = {}; + for (const secret of secretNames) { + const parentSlugs = parentPackages.map(encodePackageSlug); + const parts = [...parentSlugs, secret]; + const fqsn = parts.join("/"); + map[secret] = `\${{ secrets.${fqsn} }}`; + } + + return map; +} + +function extractFQSNMap( + rawContent: string, + parentPackages: PackageSlug[], +): Record { + const templateVars = getTemplateVariables(rawContent); + const secrets = templateVars + .filter((v) => v.startsWith("secrets.")) + .map((v) => v.replace("secrets.", "")); + + return secretToFQSNMap(secrets, parentPackages); +} + +export async function unrollPackageFromContent( + rawContent: string, + params: Record | undefined, + packagePath: PackageSlug[], + registry: Registry, +): Promise { + // Collect template data + const templateData: TemplateData = { + // params are passed from the parent package + params: params, + // at this stage, secrets are mapped to a (still templated) FQSN + secrets: extractFQSNMap(rawContent, packagePath), + // Built-in variables + continue: {}, + }; + + const templatedYaml = fillTemplateVariables( + rawContent, + flattenTemplateData(templateData), + ); + + let parsedYaml = parseConfigYaml(templatedYaml); + + const unrolledChildPackages = await Promise.all( + parsedYaml.packages?.map((pkg) => { + const pkgSlug = decodeFullSlug(pkg.uses); + return unrollImportedPackage(pkg, [...packagePath, pkgSlug], registry); + }) ?? [], + ); + + delete parsedYaml.packages; + for (const childPkg of unrolledChildPackages) { + parsedYaml = mergePackages(parsedYaml, childPkg); + } + + return parsedYaml; +} + +/** + * Loading an assistant is equivalent to loading a package without params + */ +export async function unrollAssistant( + fullSlug: string, + registry: Registry, +): Promise { + const packageSlug = decodeFullSlug(fullSlug); + return await unrollImportedPackage( + { + uses: fullSlug, + }, + [packageSlug], + registry, + ); +} diff --git a/packages/config-yaml/src/resolveSecretsOnClient.ts b/packages/config-yaml/src/resolveSecretsOnClient.ts deleted file mode 100644 index c3029fbda2..0000000000 --- a/packages/config-yaml/src/resolveSecretsOnClient.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { ClientConfigYaml, ConfigYaml } from "./schemas/index.js"; -type SecretProvider = ( - secretNames: string[], -) => Promise<{ [key: string]: string }>; - -/** - * Take a ConfigYaml with apiKeySecrets, and look to fill in these secrets - * with whatever secret store exists in the client. - */ -export async function resolveSecretsOnClient( - configYaml: ClientConfigYaml, - getSecretsFromClientStore: SecretProvider, - getSecretsFromServer: SecretProvider, -): Promise { - const requiredSecrets = getRequiredSecretsInClientConfig(configYaml); - - const secretsFoundOnClient = await getSecretsFromClientStore(requiredSecrets); - - const secretsNotFoundOnClient = requiredSecrets.filter( - (secret) => !secretsFoundOnClient[secret], - ); - - let secretsFoundOnServer = {}; - if (secretsNotFoundOnClient.length > 0) { - secretsFoundOnServer = await getSecretsFromServer(secretsNotFoundOnClient); - } - - const clientSecrets = { - ...secretsFoundOnClient, - ...secretsFoundOnServer, - }; - - const finalConfigYaml = injectClientSecrets(configYaml, clientSecrets); - - // Anything with an apiKeySecret left over must use proxy - return finalConfigYaml; -} - -function getRequiredSecretsInClientConfig( - configYaml: ClientConfigYaml, -): string[] { - const secrets = new Set(); - for (const model of configYaml.models ?? []) { - if (model.apiKeySecret) { - secrets.add(model.apiKeySecret); - } - } - return Array.from(secrets); -} - -function injectClientSecrets( - configYaml: ClientConfigYaml, - clientSecrets: Record, -): ConfigYaml { - for (const model of configYaml.models ?? []) { - if (model.apiKeySecret && clientSecrets[model.apiKeySecret]) { - // Remove apiKeySecret and place the client secret in apiKey - model.apiKey = clientSecrets[model.apiKeySecret]; - delete model.apiKeySecret; - } - } - - return configYaml; -} diff --git a/packages/config-yaml/src/schemas/context.ts b/packages/config-yaml/src/schemas/context.ts deleted file mode 100644 index 4441bd5a87..0000000000 --- a/packages/config-yaml/src/schemas/context.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { z } from "zod"; - -export const contextSchema = z.object({ - uses: z.string(), - with: z.any().optional(), -}); diff --git a/packages/config-yaml/src/schemas/data.ts b/packages/config-yaml/src/schemas/data.ts deleted file mode 100644 index 573da2448a..0000000000 --- a/packages/config-yaml/src/schemas/data.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { z } from "zod"; - -export const dataSchema = z.object({ - provider: z.string(), -}); diff --git a/packages/config-yaml/src/schemas/index.ts b/packages/config-yaml/src/schemas/index.ts index f763c7be56..394c1dd410 100644 --- a/packages/config-yaml/src/schemas/index.ts +++ b/packages/config-yaml/src/schemas/index.ts @@ -1,18 +1,23 @@ import * as z from "zod"; -import { contextSchema } from "./context.js"; -import { dataSchema } from "./data.js"; import { modelSchema } from "./models.js"; const packageSchema = z.object({ uses: z.string(), with: z.any().optional(), - secrets: z.array(z.string()).optional(), +}); + +export const dataSchema = z.object({ + provider: z.string(), +}); + +export const contextSchema = z.object({ + uses: z.string(), + with: z.any().optional(), }); const toolSchema = z.object({ name: z.string(), description: z.string(), - policy: z.enum(["automatic", "allowed", "disabled"]).optional(), url: z.string(), apiKey: z.string().optional(), }); @@ -21,14 +26,12 @@ const mcpServerSchema = z.object({ name: z.string(), command: z.string(), args: z.array(z.string()).optional(), - env: z.record(z.string()).optional(), }); const promptSchema = z.object({ name: z.string(), description: z.string().optional(), - type: z.enum(["slash-command", "context-provider"]).optional(), prompt: z.string(), }); @@ -55,6 +58,8 @@ export const configYamlSchema = z.object({ export type ConfigYaml = z.infer; -export const clientConfigYamlSchema = configYamlSchema.omit({ packages: true }); +export const unrolledConfigYamlSchema = configYamlSchema.omit({ + packages: true, +}); -export type ClientConfigYaml = z.infer; +export type UnrolledConfigYaml = z.infer; diff --git a/packages/config-yaml/src/schemas/models.ts b/packages/config-yaml/src/schemas/models.ts index 89b740e573..8a4198947a 100644 --- a/packages/config-yaml/src/schemas/models.ts +++ b/packages/config-yaml/src/schemas/models.ts @@ -48,7 +48,6 @@ export const modelSchema = z.object({ model: z.string(), apiKey: z.string().optional(), apiBase: z.string().optional(), - apiKeySecret: z.string().optional(), roles: modelRolesSchema.array().optional(), defaultCompletionOptions: completionOptionsSchema.optional(), requestOptions: requestOptionsSchema.optional(), diff --git a/packages/config-yaml/test/index.test.ts b/packages/config-yaml/test/index.test.ts new file mode 100644 index 0000000000..289052c5e9 --- /dev/null +++ b/packages/config-yaml/test/index.test.ts @@ -0,0 +1,106 @@ +import * as fs from "fs"; +import { + clientRender, + FQSN, + FullSlug, + PlatformClient, + PlatformSecretStore, + Registry, + resolveFQSN, + SecretLocation, + SecretResult, + SecretStore, + SecretType, + unrollAssistant, +} from "../src"; + +// Test e2e flows from raw yaml -> unroll -> client render -> resolve secrets on proxy +describe("E2E Scenarios", () => { + const userSecrets: Record = { + OPENAI_API_KEY: "sk-123", + }; + + const packageSecrets: Record = { + ANTHROPIC_API_KEY: "sk-ant", + }; + + const localUserSecretStore: SecretStore = { + get: async function (secretName: string): Promise { + return userSecrets[secretName]; + }, + set: function (secretName: string, secretValue: string): Promise { + throw new Error("Function not implemented."); + }, + }; + + const platformClient: PlatformClient = { + resolveFQSNs: async function ( + fqsns: FQSN[], + ): Promise<(SecretResult | undefined)[]> { + return await Promise.all( + fqsns.map((fqsn) => + resolveFQSN("test-user", "test-org", fqsn, platformSecretStore), + ), + ); + }, + }; + + const platformSecretStore: PlatformSecretStore = { + getSecretFromSecretLocation: async function ( + secretLocation: SecretLocation, + ): Promise { + if (secretLocation.secretType === SecretType.Package) { + return packageSecrets[secretLocation.secretName]; + } else if (secretLocation.secretType === SecretType.User) { + return userSecrets[secretLocation.secretName]; + } else { + return undefined; + } + }, + }; + + const registry: Registry = { + getContent: async function (fullSlug: FullSlug): Promise { + return fs + .readFileSync( + `./test/packages/${fullSlug.ownerSlug}/${fullSlug.packageSlug}.yaml`, + ) + .toString(); + }, + }; + + it("should correctly unroll assistant", async () => { + const unrolledConfig = await unrollAssistant( + "test-org/assistant", + registry, + ); + + // Test that packages were correctly unrolled and params replaced + expect(unrolledConfig.models?.length).toBe(3); + expect(unrolledConfig.models?.[0].apiKey).toBe( + "${{ secrets.test-org/assistant/OPENAI_API_KEY }}", + ); + expect(unrolledConfig.models?.[1].apiKey).toBe("sk-456"); + expect(unrolledConfig.models?.[2].apiKey).toBe( + "${{ secrets.test-org/assistant/test-org/models/ANTHROPIC_API_KEY }}", + ); + + expect(unrolledConfig.rules?.length).toBe(3); + expect(unrolledConfig.docs?.[0].startUrl).toBe( + "https://docs.python.org/release/3.13.1", + ); + + const clientRendered = await clientRender( + unrolledConfig, + localUserSecretStore, + platformClient, + ); + + // Test that user secrets were injected and other secrets remain templated + expect(clientRendered.models?.[0].apiKey).toBe("sk-123"); + expect(clientRendered.models?.[1].apiKey).toBe("sk-456"); + expect(clientRendered.models?.[2].apiKey).toBe( + "${{ secrets.package:test-org/models/ANTHROPIC_API_KEY }}", + ); + }); +}); diff --git a/packages/config-yaml/test/packages/test-org/assistant.yaml b/packages/config-yaml/test/packages/test-org/assistant.yaml new file mode 100644 index 0000000000..74fefc5815 --- /dev/null +++ b/packages/config-yaml/test/packages/test-org/assistant.yaml @@ -0,0 +1,18 @@ +name: Assistant +version: 0.0.1 + +packages: + - uses: test-org/models + - uses: test-org/docs + with: + version: 3.13.1 + - uses: test-org/rules + +models: + - name: gpt-5 + provider: openai + model: gpt-5 + apiKey: ${{ secrets.OPENAI_API_KEY }} + +rules: + - Use KaTeX for math diff --git a/packages/config-yaml/test/packages/test-org/docs.yaml b/packages/config-yaml/test/packages/test-org/docs.yaml new file mode 100644 index 0000000000..1b540d2275 --- /dev/null +++ b/packages/config-yaml/test/packages/test-org/docs.yaml @@ -0,0 +1,7 @@ +name: Docs +version: 0.0.1 + +docs: + - name: Python + startUrl: https://docs.python.org/release/${{ params.version }} + rootUrl: https://docs.python.org/release/${{ params.version }} diff --git a/packages/config-yaml/test/packages/test-org/models.yaml b/packages/config-yaml/test/packages/test-org/models.yaml new file mode 100644 index 0000000000..26091aa4b3 --- /dev/null +++ b/packages/config-yaml/test/packages/test-org/models.yaml @@ -0,0 +1,13 @@ +name: Models +version: 0.0.1 + +models: + - name: gpt-4 + provider: openai + model: gpt-4 + apiKey: sk-456 + + - name: claude-3-5-sonnet-latest + provider: anthropic + model: claude-3-5-sonnet-latest + apiKey: ${{ secrets.ANTHROPIC_API_KEY }} diff --git a/packages/config-yaml/test/packages/test-org/rules.yaml b/packages/config-yaml/test/packages/test-org/rules.yaml new file mode 100644 index 0000000000..2a4f36a9e1 --- /dev/null +++ b/packages/config-yaml/test/packages/test-org/rules.yaml @@ -0,0 +1,6 @@ +name: Rules +version: 0.0.1 + +rules: + - Be kind + - Be concise From 760dd220a0b1b036d0f6c02eec767be8d47f4e1c Mon Sep 17 00:00:00 2001 From: Nate Date: Sun, 19 Jan 2025 10:02:16 -0800 Subject: [PATCH 02/11] use proxy provider and test proxy secret injection --- packages/config-yaml/src/load/clientRender.ts | 50 +++++++++++++++++-- packages/config-yaml/src/schemas/models.ts | 17 +++++-- packages/config-yaml/test/index.test.ts | 43 ++++++++++++++-- 3 files changed, 100 insertions(+), 10 deletions(-) diff --git a/packages/config-yaml/src/load/clientRender.ts b/packages/config-yaml/src/load/clientRender.ts index b58aca05f6..a849be67c8 100644 --- a/packages/config-yaml/src/load/clientRender.ts +++ b/packages/config-yaml/src/load/clientRender.ts @@ -1,7 +1,11 @@ import * as YAML from "yaml"; import { PlatformClient, SecretStore } from "../interfaces/index.js"; -import { encodeSecretLocation } from "../interfaces/SecretResult.js"; -import { FQSN, decodeFQSN, encodeFQSN } from "../interfaces/slugs.js"; +import { + decodeSecretLocation, + encodeSecretLocation, + SecretLocation, +} from "../interfaces/SecretResult.js"; +import { decodeFQSN, encodeFQSN, FQSN } from "../interfaces/slugs.js"; import { ConfigYaml } from "../schemas/index.js"; import { fillTemplateVariables, @@ -54,6 +58,44 @@ export async function clientRender( const renderedYaml = fillTemplateVariables(rawYaml, secretsTemplateData); // 6. The rendered YAML is parsed and validated again - const finalParsedYaml = parseConfigYaml(renderedYaml); - return finalParsedYaml; + const parsedYaml = parseConfigYaml(renderedYaml); + + // 7. We update any of the items with the proxy version if there are un-rendered secrets + const finalConfig = useProxyForUnrenderedSecrets(parsedYaml); + return finalConfig; +} + +function getUnrenderedSecretLocation( + value: string | undefined, +): SecretLocation | undefined { + if (!value) return undefined; + + const templateVars = getTemplateVariables(value); + if (templateVars.length === 1) { + const secretLocationEncoded = templateVars[0].split("secrets.")[1]; + const secretLocation = decodeSecretLocation(secretLocationEncoded); + return secretLocation; + } + + return undefined; +} + +function useProxyForUnrenderedSecrets(config: ConfigYaml): ConfigYaml { + if (config.models) { + for (let i = 0; i < config.models.length; i++) { + const apiKeyLocation = getUnrenderedSecretLocation( + config.models[i].apiKey, + ); + if (apiKeyLocation) { + config.models[i] = { + ...config.models[i], + provider: "continue-proxy", + apiKeyLocation: encodeSecretLocation(apiKeyLocation), + apiKey: undefined, + }; + } + } + } + + return config; } diff --git a/packages/config-yaml/src/schemas/models.ts b/packages/config-yaml/src/schemas/models.ts index 8a4198947a..713d2cc2d9 100644 --- a/packages/config-yaml/src/schemas/models.ts +++ b/packages/config-yaml/src/schemas/models.ts @@ -42,15 +42,26 @@ export const completionOptionsSchema = z.object({ }); export type CompletionOptions = z.infer; -export const modelSchema = z.object({ +const baseModelFields = { name: z.string(), - provider: z.string(), model: z.string(), apiKey: z.string().optional(), apiBase: z.string().optional(), roles: modelRolesSchema.array().optional(), defaultCompletionOptions: completionOptionsSchema.optional(), requestOptions: requestOptionsSchema.optional(), -}); +}; + +export const modelSchema = z.union([ + z.object({ + ...baseModelFields, + provider: z.literal("continue-proxy"), + apiKeyLocation: z.string(), + }), + z.object({ + ...baseModelFields, + provider: z.string().refine((val) => val !== "continue-proxy"), + }), +]); export type ModelConfig = z.infer; diff --git a/packages/config-yaml/test/index.test.ts b/packages/config-yaml/test/index.test.ts index 289052c5e9..a17790d658 100644 --- a/packages/config-yaml/test/index.test.ts +++ b/packages/config-yaml/test/index.test.ts @@ -1,4 +1,5 @@ import * as fs from "fs"; +import { decodeSecretLocation, resolveSecretLocationInProxy } from "../dist"; import { clientRender, FQSN, @@ -13,6 +14,7 @@ import { SecretType, unrollAssistant, } from "../src"; +import exp = require("constants"); // Test e2e flows from raw yaml -> unroll -> client render -> resolve secrets on proxy describe("E2E Scenarios", () => { @@ -24,6 +26,10 @@ describe("E2E Scenarios", () => { ANTHROPIC_API_KEY: "sk-ant", }; + const proxyEnvSecrets: Record = { + ANTHROPIC_API_KEY: "sk-ant-env", + }; + const localUserSecretStore: SecretStore = { get: async function (secretName: string): Promise { return userSecrets[secretName]; @@ -45,6 +51,15 @@ describe("E2E Scenarios", () => { }, }; + const environmentSecretStore: SecretStore = { + get: async function (secretName: string): Promise { + return proxyEnvSecrets[secretName]; + }, + set: function (secretName: string, secretValue: string): Promise { + throw new Error("Function not implemented."); + }, + }; + const platformSecretStore: PlatformSecretStore = { getSecretFromSecretLocation: async function ( secretLocation: SecretLocation, @@ -96,11 +111,33 @@ describe("E2E Scenarios", () => { platformClient, ); - // Test that user secrets were injected and other secrets remain templated + // Test that user secrets were injected and others were changed to use proxy + const anthropicSecretLocation = "package:test-org/models/ANTHROPIC_API_KEY"; expect(clientRendered.models?.[0].apiKey).toBe("sk-123"); expect(clientRendered.models?.[1].apiKey).toBe("sk-456"); - expect(clientRendered.models?.[2].apiKey).toBe( - "${{ secrets.package:test-org/models/ANTHROPIC_API_KEY }}", + expect(clientRendered.models?.[2].provider).toBe("continue-proxy"); + expect((clientRendered.models?.[2] as any).apiKeyLocation).toBe( + anthropicSecretLocation, + ); + expect(clientRendered.models?.[2].apiKey).toBeUndefined(); + + // Test that proxy can correctly resolve secrets + const secretLocation = decodeSecretLocation(anthropicSecretLocation); + + // With environment + const secretValue = await resolveSecretLocationInProxy( + secretLocation, + platformSecretStore, + environmentSecretStore, + ); + expect(secretValue).toBe("sk-ant-env"); + + // Without environment + const secretValue2 = await resolveSecretLocationInProxy( + secretLocation, + platformSecretStore, + undefined, ); + expect(secretValue2).toBe("sk-ant"); }); }); From 40700c40be5d4fd5907a2c3090d3877427b90d13 Mon Sep 17 00:00:00 2001 From: Nate Date: Sun, 19 Jan 2025 14:22:03 -0800 Subject: [PATCH 03/11] update config-yaml --- packages/config-yaml/package.json | 2 +- packages/config-yaml/src/interfaces/index.ts | 7 +++---- packages/config-yaml/src/load/clientRender.ts | 12 ++++++------ packages/config-yaml/test/index.test.ts | 5 +++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/packages/config-yaml/package.json b/packages/config-yaml/package.json index 1e77d7d228..bbcfbda46c 100644 --- a/packages/config-yaml/package.json +++ b/packages/config-yaml/package.json @@ -1,6 +1,6 @@ { "name": "@continuedev/config-yaml", - "version": "1.0.11", + "version": "1.0.13", "description": "", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/packages/config-yaml/src/interfaces/index.ts b/packages/config-yaml/src/interfaces/index.ts index b65a334d67..8ad1d9c38f 100644 --- a/packages/config-yaml/src/interfaces/index.ts +++ b/packages/config-yaml/src/interfaces/index.ts @@ -29,7 +29,6 @@ export interface PlatformSecretStore { export async function resolveFQSN( currentUserSlug: string, - currentUserOrgSlug: string, fqsn: FQSN, platformSecretStore: PlatformSecretStore, ): Promise { @@ -50,11 +49,11 @@ export async function resolveFQSN( secretName: fqsn.secretName, }, // Then organization - { + ...reversedSlugs.map((slug) => ({ secretType: SecretType.Organization as const, - orgSlug: currentUserOrgSlug, + orgSlug: slug.ownerSlug, secretName: fqsn.secretName, - }, + })), ]; // Then try to get the secret from each location diff --git a/packages/config-yaml/src/load/clientRender.ts b/packages/config-yaml/src/load/clientRender.ts index a849be67c8..f5cc238d9c 100644 --- a/packages/config-yaml/src/load/clientRender.ts +++ b/packages/config-yaml/src/load/clientRender.ts @@ -1,4 +1,3 @@ -import * as YAML from "yaml"; import { PlatformClient, SecretStore } from "../interfaces/index.js"; import { decodeSecretLocation, @@ -14,14 +13,12 @@ import { } from "./unroll.js"; export async function clientRender( - unrolledConfig: ConfigYaml, + unrolledConfigContent: string, secretStore: SecretStore, platformClient: PlatformClient, ): Promise { - const rawYaml = YAML.stringify(unrolledConfig); - // 1. First we need to get a list of all the FQSNs that are required to render the config - const secrets = getTemplateVariables(rawYaml); + const secrets = getTemplateVariables(unrolledConfigContent); // 2. Then, we will check which of the secrets are found in the local personal secret store. Here we’re checking for anything that matches the last part of the FQSN, not worrying about the owner/package/owner/package slugs const secretsTemplateData: Record = {}; @@ -55,7 +52,10 @@ export async function clientRender( } // 5. User secrets are rendered in place of the template variable. Others remain templated, but replaced with the specific location where they are to be found (`${{ secrets. }}` instead of `${{ secrets. }}`) - const renderedYaml = fillTemplateVariables(rawYaml, secretsTemplateData); + const renderedYaml = fillTemplateVariables( + unrolledConfigContent, + secretsTemplateData, + ); // 6. The rendered YAML is parsed and validated again const parsedYaml = parseConfigYaml(renderedYaml); diff --git a/packages/config-yaml/test/index.test.ts b/packages/config-yaml/test/index.test.ts index a17790d658..20a1656a61 100644 --- a/packages/config-yaml/test/index.test.ts +++ b/packages/config-yaml/test/index.test.ts @@ -1,4 +1,5 @@ import * as fs from "fs"; +import * as YAML from "yaml"; import { decodeSecretLocation, resolveSecretLocationInProxy } from "../dist"; import { clientRender, @@ -45,7 +46,7 @@ describe("E2E Scenarios", () => { ): Promise<(SecretResult | undefined)[]> { return await Promise.all( fqsns.map((fqsn) => - resolveFQSN("test-user", "test-org", fqsn, platformSecretStore), + resolveFQSN("test-user", fqsn, platformSecretStore), ), ); }, @@ -106,7 +107,7 @@ describe("E2E Scenarios", () => { ); const clientRendered = await clientRender( - unrolledConfig, + YAML.stringify(unrolledConfig), localUserSecretStore, platformClient, ); From 2d865425ce55d2d4a03eb47120748e0139e3700a Mon Sep 17 00:00:00 2001 From: Nate Date: Sun, 19 Jan 2025 16:43:18 -0800 Subject: [PATCH 04/11] updates to yaml loading --- core/config/profile/PlatformProfileLoader.ts | 18 +++++- core/config/profile/doLoadConfig.ts | 9 ++- core/config/yaml/clientRender.ts | 35 ++++++++++++ core/config/yaml/loadYaml.ts | 60 +++++++------------- core/config/yaml/models.ts | 16 +----- core/control-plane/client.ts | 19 ++++++- core/package-lock.json | 8 +-- core/package.json | 2 +- extensions/vscode/package-lock.json | 2 +- gui/package-lock.json | 16 +++--- gui/package.json | 2 +- 11 files changed, 108 insertions(+), 79 deletions(-) create mode 100644 core/config/yaml/clientRender.ts diff --git a/core/config/profile/PlatformProfileLoader.ts b/core/config/profile/PlatformProfileLoader.ts index d0f7d6f802..5319097b36 100644 --- a/core/config/profile/PlatformProfileLoader.ts +++ b/core/config/profile/PlatformProfileLoader.ts @@ -1,10 +1,12 @@ -import { ClientConfigYaml } from "@continuedev/config-yaml/dist/schemas/index.js"; +import { ConfigYaml } from "@continuedev/config-yaml/dist/schemas/index.js"; +import * as YAML from "yaml"; import { ControlPlaneClient } from "../../control-plane/client.js"; import { ContinueConfig, IDE, IdeSettings } from "../../index.js"; import { ConfigResult } from "@continuedev/config-yaml"; import { ProfileDescription } from "../ProfileLifecycleManager.js"; +import { clientRenderHelper } from "../yaml/clientRender.js"; import doLoadConfig from "./doLoadConfig.js"; import { IProfileLoader } from "./IProfileLoader.js"; @@ -24,7 +26,7 @@ export default class PlatformProfileLoader implements IProfileLoader { description: ProfileDescription; constructor( - private configResult: ConfigResult, + private configResult: ConfigResult, private readonly ownerSlug: string, private readonly packageSlug: string, private readonly controlPlaneClient: ControlPlaneClient, @@ -49,8 +51,18 @@ export default class PlatformProfileLoader implements IProfileLoader { if (!newConfigResult) { return; } + + let renderedConfig: ConfigYaml | undefined = undefined; + if (newConfigResult.config) { + renderedConfig = await clientRenderHelper( + YAML.stringify(newConfigResult.config), + this.ide, + this.controlPlaneClient, + ); + } + this.configResult = { - config: newConfigResult.config, + config: renderedConfig, errors: newConfigResult.errors, configLoadInterrupted: false, }; diff --git a/core/config/profile/doLoadConfig.ts b/core/config/profile/doLoadConfig.ts index 5f20ac7e29..8e2326e3db 100644 --- a/core/config/profile/doLoadConfig.ts +++ b/core/config/profile/doLoadConfig.ts @@ -1,7 +1,10 @@ import fs from "fs"; -import { ConfigResult, ConfigValidationError } from "@continuedev/config-yaml"; -import { ClientConfigYaml } from "@continuedev/config-yaml/dist/schemas"; +import { + ConfigResult, + ConfigValidationError, + ConfigYaml, +} from "@continuedev/config-yaml"; import { ContinueConfig, ContinueRcJson, @@ -27,7 +30,7 @@ export default async function doLoadConfig( controlPlaneClient: ControlPlaneClient, writeLog: (message: string) => Promise, overrideConfigJson: SerializedContinueConfig | undefined, - overrideConfigYaml: ClientConfigYaml | undefined, + overrideConfigYaml: ConfigYaml | undefined, platformConfigMetadata: PlatformConfigMetadata | undefined, workspaceId?: string, ): Promise> { diff --git a/core/config/yaml/clientRender.ts b/core/config/yaml/clientRender.ts new file mode 100644 index 0000000000..7e96890464 --- /dev/null +++ b/core/config/yaml/clientRender.ts @@ -0,0 +1,35 @@ +import { + clientRender, + PlatformClient, + SecretStore, +} from "@continuedev/config-yaml"; + +import { IDE } from "../.."; +import { ControlPlaneClient } from "../../control-plane/client"; + +export function clientRenderHelper( + unrolledAssistant: string, + ide: IDE, + controlPlaneClient: ControlPlaneClient, +) { + const ideSecretStore: SecretStore = { + get: async function (secretName: string): Promise { + const results = await ide.readSecrets([secretName]); + return results[secretName]; + }, + set: async function ( + secretName: string, + secretValue: string, + ): Promise { + return await ide.writeSecrets({ + [secretName]: secretValue, + }); + }, + }; + + const platformClient: PlatformClient = { + resolveFQSNs: controlPlaneClient.resolveFQSNs, + }; + + return clientRender(unrolledAssistant, ideSecretStore, platformClient); +} diff --git a/core/config/yaml/loadYaml.ts b/core/config/yaml/loadYaml.ts index 5e0161ad80..a6fcd3f2d5 100644 --- a/core/config/yaml/loadYaml.ts +++ b/core/config/yaml/loadYaml.ts @@ -2,13 +2,10 @@ import fs from "node:fs"; import { ConfigResult, - fillTemplateVariables, - resolveSecretsOnClient, + ConfigYaml, validateConfigYaml, } from "@continuedev/config-yaml"; -import { ClientConfigYaml } from "@continuedev/config-yaml/dist/schemas"; import { fetchwithRequestOptions } from "@continuedev/fetch"; -import * as YAML from "yaml"; import { BrowserSerializedContinueConfig, @@ -21,44 +18,33 @@ import { } from "../.."; import { AllRerankers } from "../../context/allRerankers"; import { MCPManagerSingleton } from "../../context/mcp"; +import CodebaseContextProvider from "../../context/providers/CodebaseContextProvider"; +import FileContextProvider from "../../context/providers/FileContextProvider"; import { contextProviderClassFromName } from "../../context/providers/index"; +import PromptFilesContextProvider from "../../context/providers/PromptFilesContextProvider"; +import { ControlPlaneClient } from "../../control-plane/client"; import { allEmbeddingsProviders } from "../../indexing/allEmbeddingsProviders"; import FreeTrial from "../../llm/llms/FreeTrial"; import TransformersJsEmbeddingsProvider from "../../llm/llms/TransformersJsEmbeddingsProvider"; import { slashCommandFromPromptFileV1 } from "../../promptFiles/v1/slashCommandFromPromptFile"; import { getAllPromptFiles } from "../../promptFiles/v2/getPromptFiles"; -import { getConfigYamlPath, getContinueDotEnv } from "../../util/paths"; +import { getConfigYamlPath } from "../../util/paths"; import { getSystemPromptDotFile } from "../getSystemPromptDotFile"; import { PlatformConfigMetadata } from "../profile/PlatformProfileLoader"; -import CodebaseContextProvider from "../../context/providers/CodebaseContextProvider"; -import FileContextProvider from "../../context/providers/FileContextProvider"; -import PromptFilesContextProvider from "../../context/providers/PromptFilesContextProvider"; -import { ControlPlaneClient } from "../../control-plane/client"; +import { clientRenderHelper } from "./clientRender"; import { llmsFromModelConfig } from "./models"; -function renderTemplateVars(configYaml: string): string { - const data: Record = {}; - - // env.* - const envVars = getContinueDotEnv(); - Object.entries(envVars).forEach(([key, value]) => { - data[`env.${key}`] = value; - }); - - // secrets.* not filled in - - return fillTemplateVariables(configYaml, data); -} - -function loadConfigYaml( +async function loadConfigYaml( workspaceConfigs: string[], rawYaml: string, - overrideConfigYaml: ClientConfigYaml | undefined, -): ConfigResult { + overrideConfigYaml: ConfigYaml | undefined, + ide: IDE, + controlPlaneClient: ControlPlaneClient, +): Promise> { let config = overrideConfigYaml ?? - (YAML.parse(renderTemplateVars(rawYaml)) as ClientConfigYaml); + (await clientRenderHelper(rawYaml, ide, controlPlaneClient)); const errors = validateConfigYaml(config); if (errors?.some((error) => error.fatal)) { @@ -98,7 +84,7 @@ async function slashCommandsFromV1PromptFiles( } async function configYamlToContinueConfig( - config: ClientConfigYaml, + config: ConfigYaml, ide: IDE, ideSettings: IdeSettings, uniqueId: string, @@ -304,7 +290,7 @@ export async function loadContinueConfigFromYaml( uniqueId: string, writeLog: (log: string) => Promise, workOsAccessToken: string | undefined, - overrideConfigYaml: ClientConfigYaml | undefined, + overrideConfigYaml: ConfigYaml | undefined, platformConfigMetadata: PlatformConfigMetadata | undefined, controlPlaneClient: ControlPlaneClient, ): Promise> { @@ -314,10 +300,12 @@ export async function loadContinueConfigFromYaml( ? fs.readFileSync(configYamlPath, "utf-8") : ""; - const configYamlResult = loadConfigYaml( + const configYamlResult = await loadConfigYaml( workspaceConfigs, rawYaml, overrideConfigYaml, + ide, + controlPlaneClient, ); if (!configYamlResult.config || configYamlResult.configLoadInterrupted) { @@ -328,18 +316,8 @@ export async function loadContinueConfigFromYaml( }; } - const configYaml = await resolveSecretsOnClient( - configYamlResult.config, - ide.readSecrets.bind(ide), - async (secretNames: string[]) => { - const secretValues = await controlPlaneClient.syncSecrets(secretNames); - await ide.writeSecrets(secretValues); - return secretValues; - }, - ); - const continueConfig = await configYamlToContinueConfig( - configYaml, + configYamlResult.config, ide, ideSettings, uniqueId, diff --git a/core/config/yaml/models.ts b/core/config/yaml/models.ts index 309e061d19..8fe34677f4 100644 --- a/core/config/yaml/models.ts +++ b/core/config/yaml/models.ts @@ -3,25 +3,13 @@ import { ModelConfig } from "@continuedev/config-yaml"; import { IDE, IdeSettings, LLMOptions } from "../.."; import { BaseLLM } from "../../llm"; import { LLMClasses } from "../../llm/llms"; -import ContinueProxy from "../../llm/llms/stubs/ContinueProxy"; import { PlatformConfigMetadata } from "../profile/PlatformProfileLoader"; const AUTODETECT = "AUTODETECT"; -function useContinueProxy( - model: ModelConfig, - platformConfigMetadata: PlatformConfigMetadata | undefined, -): boolean { - return !!platformConfigMetadata && model.apiKeySecret !== undefined; -} - function getModelClass( model: ModelConfig, - platformConfigMetadata: PlatformConfigMetadata | undefined, ): (typeof LLMClasses)[number] | undefined { - if (useContinueProxy(model, platformConfigMetadata)) { - return ContinueProxy; - } return LLMClasses.find((llm) => llm.providerName === model.provider); } @@ -41,13 +29,13 @@ async function modelConfigToBaseLLM( platformConfigMetadata: PlatformConfigMetadata | undefined, systemMessage: string | undefined, ): Promise { - const cls = getModelClass(model, platformConfigMetadata); + const cls = getModelClass(model); if (!cls) { return undefined; } - const usingContinueProxy = useContinueProxy(model, platformConfigMetadata); + const usingContinueProxy = model.provider === "continue-proxy"; const modelName = usingContinueProxy ? getContinueProxyModelName( platformConfigMetadata!.ownerSlug, diff --git a/core/control-plane/client.ts b/core/control-plane/client.ts index 3a29b3fee9..bb7661ee83 100644 --- a/core/control-plane/client.ts +++ b/core/control-plane/client.ts @@ -1,10 +1,10 @@ import { ConfigJson } from "@continuedev/config-types"; -import { ClientConfigYaml } from "@continuedev/config-yaml/dist/schemas/index.js"; +import { ConfigYaml } from "@continuedev/config-yaml/dist/schemas/index.js"; import fetch, { RequestInit, Response } from "node-fetch"; import { ModelDescription } from "../index.js"; -import { ConfigResult } from "@continuedev/config-yaml"; +import { ConfigResult, FQSN, SecretResult } from "@continuedev/config-yaml"; import { controlPlaneEnv } from "./env.js"; export interface ControlPlaneSessionInfo { @@ -38,6 +38,19 @@ export class ControlPlaneClient { >, ) {} + async resolveFQSNs(fqsns: FQSN[]): Promise<(SecretResult | undefined)[]> { + const userId = await this.userId; + if (!userId) { + throw new Error("No user id"); + } + + const resp = await this.request("ide/sync-secrets", { + method: "POST", + body: JSON.stringify({ fqsns }), + }); + return (await resp.json()) as any; + } + get userId(): Promise { return this.sessionInfoPromise.then( (sessionInfo) => sessionInfo?.account.id, @@ -89,7 +102,7 @@ export class ControlPlaneClient { public async listAssistants(): Promise< { - configResult: ConfigResult; + configResult: ConfigResult; ownerSlug: string; packageSlug: string; iconUrl: string; diff --git a/core/package-lock.json b/core/package-lock.json index 350607e4d2..d020deb94a 100644 --- a/core/package-lock.json +++ b/core/package-lock.json @@ -13,7 +13,7 @@ "@aws-sdk/client-sagemaker-runtime": "^3.621.0", "@aws-sdk/credential-providers": "^3.620.1", "@continuedev/config-types": "^1.0.13", - "@continuedev/config-yaml": "^1.0.11", + "@continuedev/config-yaml": "^1.0.13", "@continuedev/fetch": "^1.0.4", "@continuedev/llm-info": "^1.0.2", "@continuedev/openai-adapters": "^1.0.10", @@ -3038,9 +3038,9 @@ } }, "node_modules/@continuedev/config-yaml": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/@continuedev/config-yaml/-/config-yaml-1.0.11.tgz", - "integrity": "sha512-E3RBQfNEPBGBmlnAbCXgeAasDzTjo4ON/HH0hr5g292i+WdAN3i/omjQ6Iusx00L1Fz7klZGJePZ3GVQKOGEUg==", + "version": "1.0.13", + "resolved": "https://registry.npmjs.org/@continuedev/config-yaml/-/config-yaml-1.0.13.tgz", + "integrity": "sha512-mBet+NPqQHDvIx+CKLaR3dCSlvhrVaJSZ9DAMuZ2bqTNLUGN6I7GikJi0CmhiiD8qU2dlrfNfdkLDfnoCwpaOA==", "dependencies": { "@continuedev/config-types": "^1.0.14", "yaml": "^2.6.1", diff --git a/core/package.json b/core/package.json index 0e107f9916..f34b077beb 100644 --- a/core/package.json +++ b/core/package.json @@ -46,7 +46,7 @@ "@aws-sdk/client-sagemaker-runtime": "^3.621.0", "@aws-sdk/credential-providers": "^3.620.1", "@continuedev/config-types": "^1.0.13", - "@continuedev/config-yaml": "^1.0.11", + "@continuedev/config-yaml": "^1.0.13", "@continuedev/fetch": "^1.0.4", "@continuedev/llm-info": "^1.0.2", "@continuedev/openai-adapters": "^1.0.10", diff --git a/extensions/vscode/package-lock.json b/extensions/vscode/package-lock.json index 31f06cbb14..84b56c12a4 100644 --- a/extensions/vscode/package-lock.json +++ b/extensions/vscode/package-lock.json @@ -106,7 +106,7 @@ "@aws-sdk/client-sagemaker-runtime": "^3.621.0", "@aws-sdk/credential-providers": "^3.620.1", "@continuedev/config-types": "^1.0.13", - "@continuedev/config-yaml": "^1.0.11", + "@continuedev/config-yaml": "^1.0.13", "@continuedev/fetch": "^1.0.4", "@continuedev/llm-info": "^1.0.2", "@continuedev/openai-adapters": "^1.0.10", diff --git a/gui/package-lock.json b/gui/package-lock.json index d99d5ebab7..342824f2d4 100644 --- a/gui/package-lock.json +++ b/gui/package-lock.json @@ -7,7 +7,7 @@ "name": "gui", "license": "Apache-2.0", "dependencies": { - "@continuedev/config-yaml": "^1.0.11", + "@continuedev/config-yaml": "^1.0.13", "@headlessui/react": "^1.7.17", "@heroicons/react": "^2.0.18", "@reduxjs/toolkit": "^2.3.0", @@ -108,7 +108,7 @@ "@aws-sdk/client-sagemaker-runtime": "^3.621.0", "@aws-sdk/credential-providers": "^3.620.1", "@continuedev/config-types": "^1.0.13", - "@continuedev/config-yaml": "^1.0.11", + "@continuedev/config-yaml": "^1.0.13", "@continuedev/fetch": "^1.0.4", "@continuedev/llm-info": "^1.0.2", "@continuedev/openai-adapters": "^1.0.10", @@ -556,9 +556,9 @@ } }, "node_modules/@continuedev/config-yaml": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/@continuedev/config-yaml/-/config-yaml-1.0.11.tgz", - "integrity": "sha512-E3RBQfNEPBGBmlnAbCXgeAasDzTjo4ON/HH0hr5g292i+WdAN3i/omjQ6Iusx00L1Fz7klZGJePZ3GVQKOGEUg==", + "version": "1.0.13", + "resolved": "https://registry.npmjs.org/@continuedev/config-yaml/-/config-yaml-1.0.13.tgz", + "integrity": "sha512-mBet+NPqQHDvIx+CKLaR3dCSlvhrVaJSZ9DAMuZ2bqTNLUGN6I7GikJi0CmhiiD8qU2dlrfNfdkLDfnoCwpaOA==", "dependencies": { "@continuedev/config-types": "^1.0.14", "yaml": "^2.6.1", @@ -13748,9 +13748,9 @@ } }, "node_modules/zod": { - "version": "3.23.8", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.23.8.tgz", - "integrity": "sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==", + "version": "3.24.1", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.24.1.tgz", + "integrity": "sha512-muH7gBL9sI1nciMZV67X5fTKKBLtwpZ5VBp1vsOQzj1MhrBZ4wlVCm3gedKZWLp0Oyel8sIGfeiz54Su+OVT+A==", "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/gui/package.json b/gui/package.json index bd58519fb0..6bd3d9374e 100644 --- a/gui/package.json +++ b/gui/package.json @@ -15,7 +15,7 @@ "test:watch": "vitest" }, "dependencies": { - "@continuedev/config-yaml": "^1.0.11", + "@continuedev/config-yaml": "^1.0.13", "@headlessui/react": "^1.7.17", "@heroicons/react": "^2.0.18", "@reduxjs/toolkit": "^2.3.0", From cbdc79e60133327b0fa91d0063e96def38cddfcc Mon Sep 17 00:00:00 2001 From: Nate Date: Sun, 19 Jan 2025 18:27:49 -0800 Subject: [PATCH 05/11] clean up config loading bugs --- core/config/ConfigHandler.ts | 43 ++++++++++++------- core/config/yaml/clientRender.ts | 2 +- core/package-lock.json | 8 ++-- core/package.json | 2 +- extensions/vscode/package-lock.json | 2 +- gui/package-lock.json | 10 ++--- gui/package.json | 2 +- packages/config-yaml/package.json | 2 +- packages/config-yaml/src/load/clientRender.ts | 22 ++++++++-- 9 files changed, 61 insertions(+), 32 deletions(-) diff --git a/core/config/ConfigHandler.ts b/core/config/ConfigHandler.ts index 626e237dbf..fbac40e6e2 100644 --- a/core/config/ConfigHandler.ts +++ b/core/config/ConfigHandler.ts @@ -14,9 +14,11 @@ import Ollama from "../llm/llms/Ollama.js"; import { GlobalContext } from "../util/GlobalContext.js"; import { getConfigJsonPath } from "../util/paths.js"; -import { ConfigResult } from "@continuedev/config-yaml"; +import { ConfigResult, ConfigYaml } from "@continuedev/config-yaml"; +import * as YAML from "yaml"; import { controlPlaneEnv } from "../control-plane/env.js"; import { usePlatform } from "../control-plane/flags.js"; +import { localPathToUri } from "../util/pathToUri.js"; import { LOCAL_ONBOARDING_CHAT_MODEL, ONBOARDING_LOCAL_MODEL_TITLE, @@ -28,7 +30,7 @@ import { ProfileDescription, ProfileLifecycleManager, } from "./ProfileLifecycleManager.js"; -import { localPathToUri } from "../util/pathToUri.js"; +import { clientRenderHelper } from "./yaml/clientRender.js"; export type { ProfileDescription }; @@ -109,19 +111,30 @@ export class ConfigHandler { this.profiles = this.profiles.filter( (profile) => profile.profileDescription.id === "local", ); - assistants.forEach((assistant) => { - const profileLoader = new PlatformProfileLoader( - assistant.configResult, - assistant.ownerSlug, - assistant.packageSlug, - this.controlPlaneClient, - this.ide, - this.ideSettingsPromise, - this.writeLog, - this.reloadConfig.bind(this), - ); - this.profiles.push(new ProfileLifecycleManager(profileLoader)); - }); + await Promise.all( + assistants.map(async (assistant) => { + let renderedConfig: ConfigYaml | undefined = undefined; + if (assistant.configResult.config) { + renderedConfig = await clientRenderHelper( + YAML.stringify(assistant.configResult.config), + this.ide, + this.controlPlaneClient, + ); + } + + const profileLoader = new PlatformProfileLoader( + { ...assistant.configResult, config: renderedConfig }, + assistant.ownerSlug, + assistant.packageSlug, + this.controlPlaneClient, + this.ide, + this.ideSettingsPromise, + this.writeLog, + this.reloadConfig.bind(this), + ); + this.profiles.push(new ProfileLifecycleManager(profileLoader)); + }), + ); this.notifyProfileListeners( this.profiles.map((profile) => profile.profileDescription), diff --git a/core/config/yaml/clientRender.ts b/core/config/yaml/clientRender.ts index 7e96890464..1f9e2f98fa 100644 --- a/core/config/yaml/clientRender.ts +++ b/core/config/yaml/clientRender.ts @@ -28,7 +28,7 @@ export function clientRenderHelper( }; const platformClient: PlatformClient = { - resolveFQSNs: controlPlaneClient.resolveFQSNs, + resolveFQSNs: controlPlaneClient.resolveFQSNs.bind(controlPlaneClient), }; return clientRender(unrolledAssistant, ideSecretStore, platformClient); diff --git a/core/package-lock.json b/core/package-lock.json index d020deb94a..82e73913f9 100644 --- a/core/package-lock.json +++ b/core/package-lock.json @@ -13,7 +13,7 @@ "@aws-sdk/client-sagemaker-runtime": "^3.621.0", "@aws-sdk/credential-providers": "^3.620.1", "@continuedev/config-types": "^1.0.13", - "@continuedev/config-yaml": "^1.0.13", + "@continuedev/config-yaml": "^1.0.15", "@continuedev/fetch": "^1.0.4", "@continuedev/llm-info": "^1.0.2", "@continuedev/openai-adapters": "^1.0.10", @@ -3038,9 +3038,9 @@ } }, "node_modules/@continuedev/config-yaml": { - "version": "1.0.13", - "resolved": "https://registry.npmjs.org/@continuedev/config-yaml/-/config-yaml-1.0.13.tgz", - "integrity": "sha512-mBet+NPqQHDvIx+CKLaR3dCSlvhrVaJSZ9DAMuZ2bqTNLUGN6I7GikJi0CmhiiD8qU2dlrfNfdkLDfnoCwpaOA==", + "version": "1.0.15", + "resolved": "https://registry.npmjs.org/@continuedev/config-yaml/-/config-yaml-1.0.15.tgz", + "integrity": "sha512-00c1pz+FqAVEWLo8t71OJzm1KBAAJCXqIjd917Vy8lr7xO0Eoj42lo2GFlVRnqQ0hQvhFQ0s9onHhRag/phHZA==", "dependencies": { "@continuedev/config-types": "^1.0.14", "yaml": "^2.6.1", diff --git a/core/package.json b/core/package.json index f34b077beb..3f5391b289 100644 --- a/core/package.json +++ b/core/package.json @@ -46,7 +46,7 @@ "@aws-sdk/client-sagemaker-runtime": "^3.621.0", "@aws-sdk/credential-providers": "^3.620.1", "@continuedev/config-types": "^1.0.13", - "@continuedev/config-yaml": "^1.0.13", + "@continuedev/config-yaml": "^1.0.15", "@continuedev/fetch": "^1.0.4", "@continuedev/llm-info": "^1.0.2", "@continuedev/openai-adapters": "^1.0.10", diff --git a/extensions/vscode/package-lock.json b/extensions/vscode/package-lock.json index 84b56c12a4..c81130d17f 100644 --- a/extensions/vscode/package-lock.json +++ b/extensions/vscode/package-lock.json @@ -106,7 +106,7 @@ "@aws-sdk/client-sagemaker-runtime": "^3.621.0", "@aws-sdk/credential-providers": "^3.620.1", "@continuedev/config-types": "^1.0.13", - "@continuedev/config-yaml": "^1.0.13", + "@continuedev/config-yaml": "^1.0.15", "@continuedev/fetch": "^1.0.4", "@continuedev/llm-info": "^1.0.2", "@continuedev/openai-adapters": "^1.0.10", diff --git a/gui/package-lock.json b/gui/package-lock.json index 342824f2d4..612acedbd7 100644 --- a/gui/package-lock.json +++ b/gui/package-lock.json @@ -7,7 +7,7 @@ "name": "gui", "license": "Apache-2.0", "dependencies": { - "@continuedev/config-yaml": "^1.0.13", + "@continuedev/config-yaml": "^1.0.15", "@headlessui/react": "^1.7.17", "@heroicons/react": "^2.0.18", "@reduxjs/toolkit": "^2.3.0", @@ -108,7 +108,7 @@ "@aws-sdk/client-sagemaker-runtime": "^3.621.0", "@aws-sdk/credential-providers": "^3.620.1", "@continuedev/config-types": "^1.0.13", - "@continuedev/config-yaml": "^1.0.13", + "@continuedev/config-yaml": "^1.0.15", "@continuedev/fetch": "^1.0.4", "@continuedev/llm-info": "^1.0.2", "@continuedev/openai-adapters": "^1.0.10", @@ -556,9 +556,9 @@ } }, "node_modules/@continuedev/config-yaml": { - "version": "1.0.13", - "resolved": "https://registry.npmjs.org/@continuedev/config-yaml/-/config-yaml-1.0.13.tgz", - "integrity": "sha512-mBet+NPqQHDvIx+CKLaR3dCSlvhrVaJSZ9DAMuZ2bqTNLUGN6I7GikJi0CmhiiD8qU2dlrfNfdkLDfnoCwpaOA==", + "version": "1.0.15", + "resolved": "https://registry.npmjs.org/@continuedev/config-yaml/-/config-yaml-1.0.15.tgz", + "integrity": "sha512-00c1pz+FqAVEWLo8t71OJzm1KBAAJCXqIjd917Vy8lr7xO0Eoj42lo2GFlVRnqQ0hQvhFQ0s9onHhRag/phHZA==", "dependencies": { "@continuedev/config-types": "^1.0.14", "yaml": "^2.6.1", diff --git a/gui/package.json b/gui/package.json index 6bd3d9374e..5bed963ddc 100644 --- a/gui/package.json +++ b/gui/package.json @@ -15,7 +15,7 @@ "test:watch": "vitest" }, "dependencies": { - "@continuedev/config-yaml": "^1.0.13", + "@continuedev/config-yaml": "^1.0.15", "@headlessui/react": "^1.7.17", "@heroicons/react": "^2.0.18", "@reduxjs/toolkit": "^2.3.0", diff --git a/packages/config-yaml/package.json b/packages/config-yaml/package.json index bbcfbda46c..6fb90abbf9 100644 --- a/packages/config-yaml/package.json +++ b/packages/config-yaml/package.json @@ -1,6 +1,6 @@ { "name": "@continuedev/config-yaml", - "version": "1.0.13", + "version": "1.0.15", "description": "", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/packages/config-yaml/src/load/clientRender.ts b/packages/config-yaml/src/load/clientRender.ts index f5cc238d9c..dfb1c41815 100644 --- a/packages/config-yaml/src/load/clientRender.ts +++ b/packages/config-yaml/src/load/clientRender.ts @@ -39,7 +39,17 @@ export async function clientRender( // 4. (back to the client) Any “user” secrets that were returned back are added to the local secret store so we don’t have to request them again for (const secretResult of secretResults) { - if (!secretResult?.found) continue; + if (!secretResult) { + continue; + } + + if (!secretResult.found) { + // When a secret isn't found anywhere, we keep it templated as just the secret name + // in case it can be found in the on-prem proxy's env + secretsTemplateData["secrets." + encodeFQSN(secretResult.fqsn)] = + `\${{ secrets.${secretResult.fqsn.secretName} }}`; + continue; + } if ("value" in secretResult) { secretStore.set(secretResult.fqsn.secretName, secretResult.value); @@ -73,8 +83,14 @@ function getUnrenderedSecretLocation( const templateVars = getTemplateVariables(value); if (templateVars.length === 1) { const secretLocationEncoded = templateVars[0].split("secrets.")[1]; - const secretLocation = decodeSecretLocation(secretLocationEncoded); - return secretLocation; + try { + const secretLocation = decodeSecretLocation(secretLocationEncoded); + return secretLocation; + } catch (e) { + // If it's templated but not a valid secret location, leave it be + // in case on-prem proxy has the secret in an env variable + return undefined; + } } return undefined; From a219ab58ef9edd5009146b867067b12e57ea1dbd Mon Sep 17 00:00:00 2001 From: Nate Date: Mon, 20 Jan 2025 14:00:15 -0800 Subject: [PATCH 06/11] adjust for on-prem proxy --- core/index.d.ts | 4 +- core/llm/index.ts | 4 +- core/llm/llms/stubs/ContinueProxy.ts | 6 +- packages/config-yaml/src/load/clientRender.ts | 5 +- packages/config-yaml/test/index.test.ts | 58 +++++++++++++++---- .../test/packages/test-org/models.yaml | 5 ++ 6 files changed, 61 insertions(+), 21 deletions(-) diff --git a/core/index.d.ts b/core/index.d.ts index 518e301d96..2f096dc493 100644 --- a/core/index.d.ts +++ b/core/index.d.ts @@ -460,7 +460,7 @@ export interface LLMOptions { writeLog?: (str: string) => Promise; llmRequestHook?: (model: string, prompt: string) => any; apiKey?: string; - apiKeySecret?: string; + apiKeyLocation?: string; aiGatewaySlug?: string; apiBase?: string; cacheBehavior?: CacheBehavior; @@ -896,7 +896,7 @@ export interface ModelDescription { provider: string; model: string; apiKey?: string; - apiKeySecret?: string; + apiKeyLocation?: string; apiBase?: string; contextLength?: number; maxStopWords?: number; diff --git a/core/llm/index.ts b/core/llm/index.ts index 5ecfa33ffa..70565b5ee5 100644 --- a/core/llm/index.ts +++ b/core/llm/index.ts @@ -117,7 +117,7 @@ export abstract class BaseLLM implements ILLM { writeLog?: (str: string) => Promise; llmRequestHook?: (model: string, prompt: string) => any; apiKey?: string; - apiKeySecret?: string; + apiKeyLocation?: string; apiBase?: string; cacheBehavior?: CacheBehavior; capabilities?: ModelCapability; @@ -195,7 +195,7 @@ export abstract class BaseLLM implements ILLM { this.writeLog = options.writeLog; this.llmRequestHook = options.llmRequestHook; this.apiKey = options.apiKey; - this.apiKeySecret = options.apiKeySecret; + this.apiKeyLocation = options.apiKeyLocation; this.aiGatewaySlug = options.aiGatewaySlug; this.apiBase = options.apiBase; this.cacheBehavior = options.cacheBehavior; diff --git a/core/llm/llms/stubs/ContinueProxy.ts b/core/llm/llms/stubs/ContinueProxy.ts index 59d05b115a..848b3fff72 100644 --- a/core/llm/llms/stubs/ContinueProxy.ts +++ b/core/llm/llms/stubs/ContinueProxy.ts @@ -14,12 +14,11 @@ class ContinueProxy extends OpenAI { // but we need to keep track of the actual values that the proxy will use // to call whatever LLM API is chosen private actualApiBase?: string; - private actualApiKey?: string; constructor(options: LLMOptions) { super(options); this.actualApiBase = options.apiBase; - this.actualApiKey = options.apiKey; + this.apiKeyLocation = options.apiKeyLocation; } static providerName = "continue-proxy"; @@ -30,9 +29,8 @@ class ContinueProxy extends OpenAI { protected extraBodyProperties(): Record { return { continueProperties: { - apiKey: this.actualApiKey, + apiKeyLocation: this.apiKeyLocation, apiBase: this.actualApiBase, - apiKeySecret: this.apiKeySecret, }, }; } diff --git a/packages/config-yaml/src/load/clientRender.ts b/packages/config-yaml/src/load/clientRender.ts index dfb1c41815..52e89cbfce 100644 --- a/packages/config-yaml/src/load/clientRender.ts +++ b/packages/config-yaml/src/load/clientRender.ts @@ -87,8 +87,11 @@ function getUnrenderedSecretLocation( const secretLocation = decodeSecretLocation(secretLocationEncoded); return secretLocation; } catch (e) { - // If it's templated but not a valid secret location, leave it be + // If it's a templated secret but not a valid secret location, leave it be // in case on-prem proxy has the secret in an env variable + if (templateVars[0].startsWith("secrets.")) { + return undefined; // TODO + } return undefined; } } diff --git a/packages/config-yaml/test/index.test.ts b/packages/config-yaml/test/index.test.ts index 20a1656a61..3f75ff0922 100644 --- a/packages/config-yaml/test/index.test.ts +++ b/packages/config-yaml/test/index.test.ts @@ -1,6 +1,10 @@ import * as fs from "fs"; import * as YAML from "yaml"; -import { decodeSecretLocation, resolveSecretLocationInProxy } from "../dist"; +import { + decodeSecretLocation, + encodeSecretLocation, + resolveSecretLocationInProxy, +} from "../dist"; import { clientRender, FQSN, @@ -24,11 +28,13 @@ describe("E2E Scenarios", () => { }; const packageSecrets: Record = { - ANTHROPIC_API_KEY: "sk-ant", + "test-org/assistant/ANTHROPIC_API_KEY": "sk-ant", + "test-org/models/GEMINI_API_KEY": "gemini-api-key", }; const proxyEnvSecrets: Record = { ANTHROPIC_API_KEY: "sk-ant-env", + GEMINI_API_KEY: "gemini-api-key-env", }; const localUserSecretStore: SecretStore = { @@ -66,7 +72,9 @@ describe("E2E Scenarios", () => { secretLocation: SecretLocation, ): Promise { if (secretLocation.secretType === SecretType.Package) { - return packageSecrets[secretLocation.secretName]; + return packageSecrets[ + encodeSecretLocation(secretLocation).split(":")[1] + ]; } else if (secretLocation.secretType === SecretType.User) { return userSecrets[secretLocation.secretName]; } else { @@ -92,7 +100,7 @@ describe("E2E Scenarios", () => { ); // Test that packages were correctly unrolled and params replaced - expect(unrolledConfig.models?.length).toBe(3); + expect(unrolledConfig.models?.length).toBe(4); expect(unrolledConfig.models?.[0].apiKey).toBe( "${{ secrets.test-org/assistant/OPENAI_API_KEY }}", ); @@ -100,6 +108,9 @@ describe("E2E Scenarios", () => { expect(unrolledConfig.models?.[2].apiKey).toBe( "${{ secrets.test-org/assistant/test-org/models/ANTHROPIC_API_KEY }}", ); + expect(unrolledConfig.models?.[3].apiKey).toBe( + "${{ secrets.test-org/assistant/test-org/models/GEMINI_API_KEY }}", + ); expect(unrolledConfig.rules?.length).toBe(3); expect(unrolledConfig.docs?.[0].startUrl).toBe( @@ -113,7 +124,9 @@ describe("E2E Scenarios", () => { ); // Test that user secrets were injected and others were changed to use proxy - const anthropicSecretLocation = "package:test-org/models/ANTHROPIC_API_KEY"; + const anthropicSecretLocation = + "package:test-org/assistant/ANTHROPIC_API_KEY"; + const geminiSecretLocation = "package:test-org/models/GEMINI_API_KEY"; expect(clientRendered.models?.[0].apiKey).toBe("sk-123"); expect(clientRendered.models?.[1].apiKey).toBe("sk-456"); expect(clientRendered.models?.[2].provider).toBe("continue-proxy"); @@ -121,24 +134,45 @@ describe("E2E Scenarios", () => { anthropicSecretLocation, ); expect(clientRendered.models?.[2].apiKey).toBeUndefined(); + expect(clientRendered.models?.[3].provider).toBe("continue-proxy"); + expect((clientRendered.models?.[3] as any).apiKeyLocation).toBe( + geminiSecretLocation, + ); + expect(clientRendered.models?.[3].apiKey).toBeUndefined(); // Test that proxy can correctly resolve secrets - const secretLocation = decodeSecretLocation(anthropicSecretLocation); + const decodedAnthropicSecretLocation = decodeSecretLocation( + anthropicSecretLocation, + ); + const decodedGeminiSecretLocation = + decodeSecretLocation(geminiSecretLocation); // With environment - const secretValue = await resolveSecretLocationInProxy( - secretLocation, + const antSecretValue = await resolveSecretLocationInProxy( + decodedAnthropicSecretLocation, platformSecretStore, environmentSecretStore, ); - expect(secretValue).toBe("sk-ant-env"); + expect(antSecretValue).toBe("sk-ant-env"); + const geminiSecretValue = await resolveSecretLocationInProxy( + decodedGeminiSecretLocation, + platformSecretStore, + environmentSecretStore, + ); + expect(geminiSecretValue).toBe("gemini-api-key-env"); // Without environment - const secretValue2 = await resolveSecretLocationInProxy( - secretLocation, + const antSecretValue2 = await resolveSecretLocationInProxy( + decodedAnthropicSecretLocation, + platformSecretStore, + undefined, + ); + expect(antSecretValue2).toBe("sk-ant"); + const geminiSecretValue2 = await resolveSecretLocationInProxy( + decodedGeminiSecretLocation, platformSecretStore, undefined, ); - expect(secretValue2).toBe("sk-ant"); + expect(geminiSecretValue2).toBe("gemini-api-key"); }); }); diff --git a/packages/config-yaml/test/packages/test-org/models.yaml b/packages/config-yaml/test/packages/test-org/models.yaml index 26091aa4b3..f47831662d 100644 --- a/packages/config-yaml/test/packages/test-org/models.yaml +++ b/packages/config-yaml/test/packages/test-org/models.yaml @@ -11,3 +11,8 @@ models: provider: anthropic model: claude-3-5-sonnet-latest apiKey: ${{ secrets.ANTHROPIC_API_KEY }} + + - name: gemini + provider: gemini + model: gemini + apiKey: ${{ secrets.GEMINI_API_KEY }} From 914042af870049b613d04031ac0affe44a806749 Mon Sep 17 00:00:00 2001 From: Nate Date: Mon, 20 Jan 2025 14:01:36 -0800 Subject: [PATCH 07/11] bump package.json --- extensions/vscode/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/vscode/package.json b/extensions/vscode/package.json index f6520de7d3..23c151699b 100644 --- a/extensions/vscode/package.json +++ b/extensions/vscode/package.json @@ -2,7 +2,7 @@ "name": "continue", "icon": "media/icon.png", "author": "Continue Dev, Inc", - "version": "0.9.253", + "version": "0.9.254", "repository": { "type": "git", "url": "https://github.com/continuedev/continue" From 81a4a5152de5c0cc658b1df37755cb453e1f3044 Mon Sep 17 00:00:00 2001 From: Nate Date: Mon, 20 Jan 2025 14:16:44 -0800 Subject: [PATCH 08/11] fix file links --- extensions/vscode/package-lock.json | 4 ++-- gui/src/components/markdown/FilenameLink.tsx | 20 +++++++++++++------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/extensions/vscode/package-lock.json b/extensions/vscode/package-lock.json index c81130d17f..c85a40fe4e 100644 --- a/extensions/vscode/package-lock.json +++ b/extensions/vscode/package-lock.json @@ -1,12 +1,12 @@ { "name": "continue", - "version": "0.9.252", + "version": "0.9.254", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "continue", - "version": "0.9.252", + "version": "0.9.254", "license": "Apache-2.0", "dependencies": { "@continuedev/fetch": "^1.0.3", diff --git a/gui/src/components/markdown/FilenameLink.tsx b/gui/src/components/markdown/FilenameLink.tsx index 6ff704cc05..80a206a1d8 100644 --- a/gui/src/components/markdown/FilenameLink.tsx +++ b/gui/src/components/markdown/FilenameLink.tsx @@ -1,10 +1,10 @@ import { RangeInFile } from "core"; +import { findUriInDirs, getUriPathBasename } from "core/util/uri"; import { useContext } from "react"; +import { v4 as uuidv4 } from "uuid"; import { IdeMessengerContext } from "../../context/IdeMessenger"; import FileIcon from "../FileIcon"; -import { findUriInDirs, getUriPathBasename } from "core/util/uri"; import { ToolTip } from "../gui/Tooltip"; -import { v4 as uuidv4 } from "uuid"; interface FilenameLinkProps { rif: RangeInFile; @@ -23,10 +23,16 @@ function FilenameLink({ rif }: FilenameLinkProps) { const id = uuidv4(); - const { relativePathOrBasename } = findUriInDirs( - rif.filepath, - window.workspacePaths ?? [], - ); + let relPathOrBasename = ""; + try { + const { relativePathOrBasename } = findUriInDirs( + rif.filepath, + window.workspacePaths ?? [], + ); + relPathOrBasename = relativePathOrBasename; + } catch (e) { + return getUriPathBasename(rif.filepath); + } return ( <> @@ -42,7 +48,7 @@ function FilenameLink({ rif }: FilenameLinkProps) { - {"/" + relativePathOrBasename} + {"/" + relPathOrBasename} ); From 7c1f54389187fcc4f07f61102c02ef383ced31fa Mon Sep 17 00:00:00 2001 From: Nate Date: Mon, 20 Jan 2025 22:17:02 -0800 Subject: [PATCH 09/11] fix e2e tests by not calling platform on local mode --- core/config/yaml/clientRender.ts | 9 +++- core/package-lock.json | 8 ++-- core/package.json | 2 +- extensions/vscode/package-lock.json | 2 +- gui/package-lock.json | 2 +- gui/src/components/markdown/FilenameLink.tsx | 2 +- packages/config-yaml/package.json | 2 +- packages/config-yaml/src/load/clientRender.ts | 47 ++++++++++--------- 8 files changed, 41 insertions(+), 33 deletions(-) diff --git a/core/config/yaml/clientRender.ts b/core/config/yaml/clientRender.ts index 1f9e2f98fa..8cb727456f 100644 --- a/core/config/yaml/clientRender.ts +++ b/core/config/yaml/clientRender.ts @@ -7,7 +7,7 @@ import { import { IDE } from "../.."; import { ControlPlaneClient } from "../../control-plane/client"; -export function clientRenderHelper( +export async function clientRenderHelper( unrolledAssistant: string, ide: IDE, controlPlaneClient: ControlPlaneClient, @@ -31,5 +31,10 @@ export function clientRenderHelper( resolveFQSNs: controlPlaneClient.resolveFQSNs.bind(controlPlaneClient), }; - return clientRender(unrolledAssistant, ideSecretStore, platformClient); + const userId = await controlPlaneClient.userId; + return await clientRender( + unrolledAssistant, + ideSecretStore, + userId ? platformClient : undefined, + ); } diff --git a/core/package-lock.json b/core/package-lock.json index 82e73913f9..ec82ab0cdf 100644 --- a/core/package-lock.json +++ b/core/package-lock.json @@ -13,7 +13,7 @@ "@aws-sdk/client-sagemaker-runtime": "^3.621.0", "@aws-sdk/credential-providers": "^3.620.1", "@continuedev/config-types": "^1.0.13", - "@continuedev/config-yaml": "^1.0.15", + "@continuedev/config-yaml": "^1.0.16", "@continuedev/fetch": "^1.0.4", "@continuedev/llm-info": "^1.0.2", "@continuedev/openai-adapters": "^1.0.10", @@ -3038,9 +3038,9 @@ } }, "node_modules/@continuedev/config-yaml": { - "version": "1.0.15", - "resolved": "https://registry.npmjs.org/@continuedev/config-yaml/-/config-yaml-1.0.15.tgz", - "integrity": "sha512-00c1pz+FqAVEWLo8t71OJzm1KBAAJCXqIjd917Vy8lr7xO0Eoj42lo2GFlVRnqQ0hQvhFQ0s9onHhRag/phHZA==", + "version": "1.0.16", + "resolved": "https://registry.npmjs.org/@continuedev/config-yaml/-/config-yaml-1.0.16.tgz", + "integrity": "sha512-r3dzTvL8aQ5RrzM+FX6fF+U12/crU4wpPXgDcDloSsxiOQfu3k/shLCbNzFPW57XhLUQAmUJYSZsJbIvYrNPNA==", "dependencies": { "@continuedev/config-types": "^1.0.14", "yaml": "^2.6.1", diff --git a/core/package.json b/core/package.json index 3f5391b289..477290c9bc 100644 --- a/core/package.json +++ b/core/package.json @@ -46,7 +46,7 @@ "@aws-sdk/client-sagemaker-runtime": "^3.621.0", "@aws-sdk/credential-providers": "^3.620.1", "@continuedev/config-types": "^1.0.13", - "@continuedev/config-yaml": "^1.0.15", + "@continuedev/config-yaml": "^1.0.16", "@continuedev/fetch": "^1.0.4", "@continuedev/llm-info": "^1.0.2", "@continuedev/openai-adapters": "^1.0.10", diff --git a/extensions/vscode/package-lock.json b/extensions/vscode/package-lock.json index c85a40fe4e..38d0db7c80 100644 --- a/extensions/vscode/package-lock.json +++ b/extensions/vscode/package-lock.json @@ -106,7 +106,7 @@ "@aws-sdk/client-sagemaker-runtime": "^3.621.0", "@aws-sdk/credential-providers": "^3.620.1", "@continuedev/config-types": "^1.0.13", - "@continuedev/config-yaml": "^1.0.15", + "@continuedev/config-yaml": "^1.0.16", "@continuedev/fetch": "^1.0.4", "@continuedev/llm-info": "^1.0.2", "@continuedev/openai-adapters": "^1.0.10", diff --git a/gui/package-lock.json b/gui/package-lock.json index 612acedbd7..922ac8e10c 100644 --- a/gui/package-lock.json +++ b/gui/package-lock.json @@ -108,7 +108,7 @@ "@aws-sdk/client-sagemaker-runtime": "^3.621.0", "@aws-sdk/credential-providers": "^3.620.1", "@continuedev/config-types": "^1.0.13", - "@continuedev/config-yaml": "^1.0.15", + "@continuedev/config-yaml": "^1.0.16", "@continuedev/fetch": "^1.0.4", "@continuedev/llm-info": "^1.0.2", "@continuedev/openai-adapters": "^1.0.10", diff --git a/gui/src/components/markdown/FilenameLink.tsx b/gui/src/components/markdown/FilenameLink.tsx index 80a206a1d8..baa0d3a90d 100644 --- a/gui/src/components/markdown/FilenameLink.tsx +++ b/gui/src/components/markdown/FilenameLink.tsx @@ -31,7 +31,7 @@ function FilenameLink({ rif }: FilenameLinkProps) { ); relPathOrBasename = relativePathOrBasename; } catch (e) { - return getUriPathBasename(rif.filepath); + return {getUriPathBasename(rif.filepath)}; } return ( diff --git a/packages/config-yaml/package.json b/packages/config-yaml/package.json index 6fb90abbf9..85474ad170 100644 --- a/packages/config-yaml/package.json +++ b/packages/config-yaml/package.json @@ -1,6 +1,6 @@ { "name": "@continuedev/config-yaml", - "version": "1.0.15", + "version": "1.0.16", "description": "", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/packages/config-yaml/src/load/clientRender.ts b/packages/config-yaml/src/load/clientRender.ts index 52e89cbfce..cf9b6ba918 100644 --- a/packages/config-yaml/src/load/clientRender.ts +++ b/packages/config-yaml/src/load/clientRender.ts @@ -15,7 +15,7 @@ import { export async function clientRender( unrolledConfigContent: string, secretStore: SecretStore, - platformClient: PlatformClient, + platformClient?: PlatformClient, ): Promise { // 1. First we need to get a list of all the FQSNs that are required to render the config const secrets = getTemplateVariables(unrolledConfigContent); @@ -34,31 +34,34 @@ export async function clientRender( } } - // 3. For any secrets not found, we send the FQSNs to the Continue Platform at the `/ide/sync-secrets` endpoint. This endpoint replies for each of the FQSNs with the following information (`SecretResult`): `foundAt`: tells which secret store it was found in (this is “user”, “org”, “package” or null if not found anywhere). If it’s found in an org or a package, it tells us the `secretLocation`, which is either just an org slug, or is a full org/package slug. If it’s found in “user” secrets, we send back the `value`. Full definition of `SecretResult` at [2]. The method of resolving an FQSN to a `SecretResult` is detailed at [3] - const secretResults = await platformClient.resolveFQSNs(unresolvedFQSNs); + // Don't use platform client in local mode + if (platformClient) { + // 3. For any secrets not found, we send the FQSNs to the Continue Platform at the `/ide/sync-secrets` endpoint. This endpoint replies for each of the FQSNs with the following information (`SecretResult`): `foundAt`: tells which secret store it was found in (this is “user”, “org”, “package” or null if not found anywhere). If it’s found in an org or a package, it tells us the `secretLocation`, which is either just an org slug, or is a full org/package slug. If it’s found in “user” secrets, we send back the `value`. Full definition of `SecretResult` at [2]. The method of resolving an FQSN to a `SecretResult` is detailed at [3] + const secretResults = await platformClient.resolveFQSNs(unresolvedFQSNs); - // 4. (back to the client) Any “user” secrets that were returned back are added to the local secret store so we don’t have to request them again - for (const secretResult of secretResults) { - if (!secretResult) { - continue; - } + // 4. (back to the client) Any “user” secrets that were returned back are added to the local secret store so we don’t have to request them again + for (const secretResult of secretResults) { + if (!secretResult) { + continue; + } - if (!secretResult.found) { - // When a secret isn't found anywhere, we keep it templated as just the secret name - // in case it can be found in the on-prem proxy's env - secretsTemplateData["secrets." + encodeFQSN(secretResult.fqsn)] = - `\${{ secrets.${secretResult.fqsn.secretName} }}`; - continue; - } + if (!secretResult.found) { + // When a secret isn't found anywhere, we keep it templated as just the secret name + // in case it can be found in the on-prem proxy's env + secretsTemplateData["secrets." + encodeFQSN(secretResult.fqsn)] = + `\${{ secrets.${secretResult.fqsn.secretName} }}`; + continue; + } - if ("value" in secretResult) { - secretStore.set(secretResult.fqsn.secretName, secretResult.value); - } + if ("value" in secretResult) { + secretStore.set(secretResult.fqsn.secretName, secretResult.value); + } - secretsTemplateData["secrets." + encodeFQSN(secretResult.fqsn)] = - "value" in secretResult - ? secretResult.value - : `\${{ secrets.${encodeSecretLocation(secretResult.secretLocation)} }}`; + secretsTemplateData["secrets." + encodeFQSN(secretResult.fqsn)] = + "value" in secretResult + ? secretResult.value + : `\${{ secrets.${encodeSecretLocation(secretResult.secretLocation)} }}`; + } } // 5. User secrets are rendered in place of the template variable. Others remain templated, but replaced with the specific location where they are to be found (`${{ secrets. }}` instead of `${{ secrets. }}`) From 2ff56ee8908b43b9fe1dd91c2423158406a5b98e Mon Sep 17 00:00:00 2001 From: Nate Date: Tue, 21 Jan 2025 07:55:06 -0800 Subject: [PATCH 10/11] load tools with yaml --- core/config/yaml/loadYaml.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/config/yaml/loadYaml.ts b/core/config/yaml/loadYaml.ts index a6fcd3f2d5..09003f6eb4 100644 --- a/core/config/yaml/loadYaml.ts +++ b/core/config/yaml/loadYaml.ts @@ -32,6 +32,7 @@ import { getConfigYamlPath } from "../../util/paths"; import { getSystemPromptDotFile } from "../getSystemPromptDotFile"; import { PlatformConfigMetadata } from "../profile/PlatformProfileLoader"; +import { allTools } from "../../tools"; import { clientRenderHelper } from "./clientRender"; import { llmsFromModelConfig } from "./models"; @@ -97,7 +98,7 @@ async function configYamlToContinueConfig( slashCommands: await slashCommandsFromV1PromptFiles(ide), models: [], tabAutocompleteModels: [], - tools: [], + tools: allTools, systemMessage: config.rules?.join("\n"), embeddingsProvider: new TransformersJsEmbeddingsProvider(), experimental: { From 7c22b6fb27210153e8f8e6383304c089cb1b603c Mon Sep 17 00:00:00 2001 From: Nate Date: Tue, 21 Jan 2025 09:00:20 -0800 Subject: [PATCH 11/11] fix duplicate profile loading --- core/config/ConfigHandler.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/core/config/ConfigHandler.ts b/core/config/ConfigHandler.ts index fbac40e6e2..24987528a3 100644 --- a/core/config/ConfigHandler.ts +++ b/core/config/ConfigHandler.ts @@ -132,7 +132,12 @@ export class ConfigHandler { this.writeLog, this.reloadConfig.bind(this), ); - this.profiles.push(new ProfileLifecycleManager(profileLoader)); + this.profiles = [ + ...this.profiles.filter( + (profile) => profile.profileDescription.id === "local", + ), + new ProfileLifecycleManager(profileLoader), + ]; }), );