From c352347aeb1e0f31985dd107c8f5d4f110273d0b Mon Sep 17 00:00:00 2001 From: Kravets <57632712+kravetsone@users.noreply.github.com> Date: Thu, 1 Aug 2024 17:44:37 +0300 Subject: [PATCH 01/18] Replace `zx` with native `child_process` --- drizzle-kit/src/utils/certs.ts | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/drizzle-kit/src/utils/certs.ts b/drizzle-kit/src/utils/certs.ts index b9a6d4de8..a3c80ab61 100644 --- a/drizzle-kit/src/utils/certs.ts +++ b/drizzle-kit/src/utils/certs.ts @@ -1,19 +1,25 @@ import envPaths from 'env-paths'; import { mkdirSync } from 'fs'; import { access, readFile } from 'fs/promises'; +import { exec, ExecOptions } from 'node:child_process'; import { join } from 'path'; -import { $ } from 'zx'; const p = envPaths('drizzle-studio', { suffix: '', }); -$.verbose = false; -$.cwd = p.data; mkdirSync(p.data, { recursive: true }); +export function runCommand(command: string, options: ExecOptions = {}) { + return new Promise<{ exitCode: number }>((resolve, reject) => { + exec(command, options, (error, stdout, stderr) => { + return resolve({ exitCode: error?.code ?? 0 }); + }); + }); +} + export const certs = async () => { - const res = await $`mkcert --help`.nothrow(); + const res = await runCommand(`mkcert --help`, { cwd: p.data }); // ~/.local/share/drizzle-studio const keyPath = join(p.data, 'localhost-key.pem'); @@ -23,7 +29,7 @@ export const certs = async () => { try { await Promise.all([access(keyPath), access(certPath)]); } catch (e) { - await $`mkcert localhost`.nothrow(); + await runCommand(`mkcert localhost`, { cwd: p.data }); } const [key, cert] = await Promise.all([ readFile(keyPath, { encoding: 'utf-8' }), From fc2c675376b7c8cc091dfdff5fca79b85612cee8 Mon Sep 17 00:00:00 2001 From: Maciej Zieniuk Date: Sun, 29 Dec 2024 14:06:04 +0000 Subject: [PATCH 02/18] DROP INDEX does not include PG Schema prefix In PostgreSQL created indexes will belong to the same schema, as the table, to which the index belongs to. When generating SQL for migration an index name does not currently contain the schema prefix. This is incorrect, since there is no guarantee that the selected schema of the connection, during the SQL migration, is the same as the schema of the index, causing the SQL migration to fail. Therefore, all index names needs to be prefixed with the schema name. --- drizzle-kit/src/sqlgenerator.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drizzle-kit/src/sqlgenerator.ts b/drizzle-kit/src/sqlgenerator.ts index 26adaf531..6d3034b61 100644 --- a/drizzle-kit/src/sqlgenerator.ts +++ b/drizzle-kit/src/sqlgenerator.ts @@ -3557,8 +3557,12 @@ class PgDropIndexConvertor extends Convertor { } convert(statement: JsonDropIndexStatement): string { + const { schema } = statement; const { name } = PgSquasher.unsquashIdx(statement.data); - return `DROP INDEX "${name}";`; + + const indexNameWithSchema = schema ? `"${schema}"."${name}"` : `"${name}"`; + + return `DROP INDEX ${indexNameWithSchema};`; } } From 32456cbbe666779b023bb8358939f69021b6931a Mon Sep 17 00:00:00 2001 From: Stephan de Vries Date: Sun, 5 Jan 2025 01:16:58 +0100 Subject: [PATCH 03/18] =?UTF-8?q?Fix=20minor=20typo=20(bacth=20=E2=86=92?= =?UTF-8?q?=20batch)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- changelogs/drizzle-orm/0.29.5.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelogs/drizzle-orm/0.29.5.md b/changelogs/drizzle-orm/0.29.5.md index 9ef656b57..74748caad 100644 --- a/changelogs/drizzle-orm/0.29.5.md +++ b/changelogs/drizzle-orm/0.29.5.md @@ -67,7 +67,7 @@ await migrate(db, { }); ``` -### 🎉 SQLite Proxy bacth and Relational Queries support +### 🎉 SQLite Proxy batch and Relational Queries support - You can now use `.query.findFirst` and `.query.findMany` syntax with sqlite proxy driver From 93e8b5415a02f7416ee82e71ac23e08882ac4f94 Mon Sep 17 00:00:00 2001 From: Roman Date: Mon, 6 Jan 2025 19:04:50 +0200 Subject: [PATCH 04/18] fix: Fix certs util --- drizzle-kit/src/utils/certs.ts | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/drizzle-kit/src/utils/certs.ts b/drizzle-kit/src/utils/certs.ts index b9a6d4de8..e4fd72486 100644 --- a/drizzle-kit/src/utils/certs.ts +++ b/drizzle-kit/src/utils/certs.ts @@ -4,25 +4,28 @@ import { access, readFile } from 'fs/promises'; import { join } from 'path'; import { $ } from 'zx'; -const p = envPaths('drizzle-studio', { - suffix: '', -}); - -$.verbose = false; -$.cwd = p.data; -mkdirSync(p.data, { recursive: true }); - export const certs = async () => { const res = await $`mkcert --help`.nothrow(); - // ~/.local/share/drizzle-studio - const keyPath = join(p.data, 'localhost-key.pem'); - const certPath = join(p.data, 'localhost.pem'); - if (res.exitCode === 0) { + const p = envPaths('drizzle-studio', { + suffix: '', + }); + + $.verbose = false; + $.cwd = p.data; + + // create ~/.local/share/drizzle-studio + mkdirSync(p.data, { recursive: true }); + + const keyPath = join(p.data, 'localhost-key.pem'); + const certPath = join(p.data, 'localhost.pem'); + try { + // check if the files exist await Promise.all([access(keyPath), access(certPath)]); } catch (e) { + // if not create them await $`mkcert localhost`.nothrow(); } const [key, cert] = await Promise.all([ @@ -33,5 +36,3 @@ export const certs = async () => { } return null; }; - -certs(); From 2329e17b98dc344956f21bc26d6d86ba86532c4e Mon Sep 17 00:00:00 2001 From: Roman Date: Mon, 6 Jan 2025 19:21:46 +0200 Subject: [PATCH 05/18] dprint --- drizzle-kit/src/utils/certs.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drizzle-kit/src/utils/certs.ts b/drizzle-kit/src/utils/certs.ts index e4fd72486..6c3560aa6 100644 --- a/drizzle-kit/src/utils/certs.ts +++ b/drizzle-kit/src/utils/certs.ts @@ -11,7 +11,7 @@ export const certs = async () => { const p = envPaths('drizzle-studio', { suffix: '', }); - + $.verbose = false; $.cwd = p.data; From c4ae5d802d26f667ce34ec65a2134a8ba11161aa Mon Sep 17 00:00:00 2001 From: Mitchell Adair Date: Mon, 16 Dec 2024 18:38:49 -0500 Subject: [PATCH 06/18] implement vector type, start tests --- drizzle-kit/src/introspect-singlestore.ts | 11 +++ .../src/serializer/singlestoreSerializer.ts | 2 +- drizzle-kit/tests/push/singlestore.test.ts | 8 ++ .../src/singlestore-core/columns/all.ts | 2 + .../src/singlestore-core/columns/index.ts | 1 + .../src/singlestore-core/columns/vector.ts | 80 +++++++++++++++++++ .../src/singlestore-core/expressions.ts | 9 +++ drizzle-orm/type-tests/singlestore/tables.ts | 5 ++ .../tests/singlestore/singlestore-common.ts | 43 ++++++++++ 9 files changed, 160 insertions(+), 1 deletion(-) create mode 100644 drizzle-orm/src/singlestore-core/columns/vector.ts diff --git a/drizzle-kit/src/introspect-singlestore.ts b/drizzle-kit/src/introspect-singlestore.ts index 09c2feec0..8f9c98acd 100644 --- a/drizzle-kit/src/introspect-singlestore.ts +++ b/drizzle-kit/src/introspect-singlestore.ts @@ -49,6 +49,7 @@ const singlestoreImportsList = new Set([ 'tinyint', 'varbinary', 'varchar', + 'vector', 'year', 'enum', ]); @@ -789,6 +790,16 @@ const column = ( return out; } + if (lowered.startsWith('vector')) { + const [dimensions, elementType] = lowered.substring('vector'.length + 1, lowered.length - 1).split(','); + let out = `${casing(name)}: vector(${ + dbColumnName({ name, casing: rawCasing, withMode: true }) + }{ dimensions: ${dimensions}${elementType ? `, elementType: ${elementType}` : ''} })`; + + out += defaultValue ? `.default(${mapColumnDefault(defaultValue, isExpression)})` : ''; + return out; + } + console.log('uknown', type); return `// Warning: Can't parse ${type} from database\n\t// ${type}Type: ${type}("${name}")`; }; diff --git a/drizzle-kit/src/serializer/singlestoreSerializer.ts b/drizzle-kit/src/serializer/singlestoreSerializer.ts index e8c89f1d1..e65f53d25 100644 --- a/drizzle-kit/src/serializer/singlestoreSerializer.ts +++ b/drizzle-kit/src/serializer/singlestoreSerializer.ts @@ -130,7 +130,7 @@ export const generateSingleStoreSnapshot = ( if (typeof column.default === 'string') { columnToSet.default = `'${column.default}'`; } else { - if (sqlTypeLowered === 'json') { + if (sqlTypeLowered === 'json' || Array.isArray(column.default)) { columnToSet.default = `'${JSON.stringify(column.default)}'`; } else if (column.default instanceof Date) { if (sqlTypeLowered === 'date') { diff --git a/drizzle-kit/tests/push/singlestore.test.ts b/drizzle-kit/tests/push/singlestore.test.ts index 82c72063c..dea28759c 100644 --- a/drizzle-kit/tests/push/singlestore.test.ts +++ b/drizzle-kit/tests/push/singlestore.test.ts @@ -23,6 +23,7 @@ import { tinyint, varbinary, varchar, + vector, year, } from 'drizzle-orm/singlestore-core'; import getPort from 'get-port'; @@ -249,6 +250,13 @@ const singlestoreSuite: DialectSuite = { columnNotNull: binary('column_not_null', { length: 1 }).notNull(), columnDefault: binary('column_default', { length: 12 }), }), + + allVectors: singlestoreTable('all_vectors', { + vectorSimple: vector('vector_simple', { dimensions: 1 }), + vectorElementType: vector('vector_element_type', { dimensions: 1, elementType: 'I8' }), + vectorNotNull: vector('vector_not_null', { dimensions: 1 }).notNull(), + vectorDefault: vector('vector_default', { dimensions: 1 }).default([1]), + }), }; const { statements } = await diffTestSchemasPushSingleStore( diff --git a/drizzle-orm/src/singlestore-core/columns/all.ts b/drizzle-orm/src/singlestore-core/columns/all.ts index 450417060..7bb704a8b 100644 --- a/drizzle-orm/src/singlestore-core/columns/all.ts +++ b/drizzle-orm/src/singlestore-core/columns/all.ts @@ -21,6 +21,7 @@ import { timestamp } from './timestamp.ts'; import { tinyint } from './tinyint.ts'; import { varbinary } from './varbinary.ts'; import { varchar } from './varchar.ts'; +import { vector } from './vector.ts'; import { year } from './year.ts'; export function getSingleStoreColumnBuilders() { @@ -51,6 +52,7 @@ export function getSingleStoreColumnBuilders() { tinyint, varbinary, varchar, + vector, year, }; } diff --git a/drizzle-orm/src/singlestore-core/columns/index.ts b/drizzle-orm/src/singlestore-core/columns/index.ts index b51f0fac4..ec17fa21a 100644 --- a/drizzle-orm/src/singlestore-core/columns/index.ts +++ b/drizzle-orm/src/singlestore-core/columns/index.ts @@ -22,4 +22,5 @@ export * from './timestamp.ts'; export * from './tinyint.ts'; export * from './varbinary.ts'; export * from './varchar.ts'; +export * from './vector.ts'; export * from './year.ts'; diff --git a/drizzle-orm/src/singlestore-core/columns/vector.ts b/drizzle-orm/src/singlestore-core/columns/vector.ts new file mode 100644 index 000000000..dee6ba9e9 --- /dev/null +++ b/drizzle-orm/src/singlestore-core/columns/vector.ts @@ -0,0 +1,80 @@ +import type { ColumnBaseConfig } from '~/column'; +import type { ColumnBuilderBaseConfig, ColumnBuilderRuntimeConfig, MakeColumnConfig } from '~/column-builder'; +import { entityKind } from '~/entity.ts'; +import type { AnySingleStoreTable } from '~/singlestore-core/table.ts'; +import { getColumnNameAndConfig } from '~/utils.ts'; +import { SingleStoreColumn, SingleStoreColumnBuilder } from './common.ts'; + +export type SingleStoreVectorBuilderInitial = SingleStoreVectorBuilder<{ + name: TName; + dataType: 'array'; + columnType: 'SingleStoreVector'; + data: Array; + driverParam: Array; + enumValues: undefined; + generated: undefined; +}>; + +export class SingleStoreVectorBuilder> + extends SingleStoreColumnBuilder +{ + static override readonly [entityKind]: string = 'SingleStoreVectorBuilder'; + + constructor(name: T['name'], config: SingleStoreVectorConfig) { + super(name, 'array', 'SingleStoreVector'); + this.config.dimensions = config.dimensions; + this.config.elementType = config.elementType; + } + + /** @internal */ + override build( + table: AnySingleStoreTable<{ name: TTableName }>, + ): SingleStoreVector> { + return new SingleStoreVector(table, this.config as ColumnBuilderRuntimeConfig); + } +} + +export class SingleStoreVector> extends SingleStoreColumn { + static override readonly [entityKind]: string = 'SingleStoreVector'; + + readonly dimensions: number; + readonly elementType: ElementType | undefined; + + constructor(table: AnySingleStoreTable<{ name: T['tableName'] }>, config: SingleStoreVectorBuilder['config']) { + super(table, config); + this.dimensions = config.dimensions; + this.elementType = config.elementType; + } + + getSQLType(): string { + const et = this.elementType === undefined ? '' : `, ${this.elementType}`; + return `vector(${this.dimensions}${et})`; + } + + override mapToDriverValue(value: Array) { + return JSON.stringify(value); + } + + override mapFromDriverValue(value: string): Array { + return JSON.parse(value); + } +} + +type ElementType = 'I8' | 'I16' | 'I32' | 'I64' | 'F32' | 'F64'; + +export interface SingleStoreVectorConfig { + dimensions: number; + elementType?: ElementType; +} + +export function vector( + config: SingleStoreVectorConfig, +): SingleStoreVectorBuilderInitial<''>; +export function vector( + name: TName, + config: SingleStoreVectorConfig, +): SingleStoreVectorBuilderInitial; +export function vector(a: string | SingleStoreVectorConfig, b?: SingleStoreVectorConfig) { + const { name, config } = getColumnNameAndConfig(a, b); + return new SingleStoreVectorBuilder(name, config); +} diff --git a/drizzle-orm/src/singlestore-core/expressions.ts b/drizzle-orm/src/singlestore-core/expressions.ts index 6d4284d18..397e87392 100644 --- a/drizzle-orm/src/singlestore-core/expressions.ts +++ b/drizzle-orm/src/singlestore-core/expressions.ts @@ -23,3 +23,12 @@ export function substring( chunks.push(sql`)`); return sql.join(chunks); } + +// Vectors +export function dotProduct(column: SingleStoreColumn | SQL.Aliased, value: Array) { + return sql`${column} <*> ${JSON.stringify(value)}`; +} + +export function euclideanDistance(column: SingleStoreColumn | SQL.Aliased, value: Array) { + return sql`${column} <-> ${JSON.stringify(value)}`; +} diff --git a/drizzle-orm/type-tests/singlestore/tables.ts b/drizzle-orm/type-tests/singlestore/tables.ts index 73d9c6993..fb02eb774 100644 --- a/drizzle-orm/type-tests/singlestore/tables.ts +++ b/drizzle-orm/type-tests/singlestore/tables.ts @@ -34,6 +34,7 @@ import { uniqueIndex, varbinary, varchar, + vector, year, } from '~/singlestore-core/index.ts'; import { singlestoreSchema } from '~/singlestore-core/schema.ts'; @@ -917,6 +918,8 @@ Expect< varchar: varchar('varchar', { length: 1 }), varchar2: varchar('varchar2', { length: 1, enum: ['a', 'b', 'c'] }), varchardef: varchar('varchardef', { length: 1 }).default(''), + vector: vector('vector', { dimensions: 1 }), + vector2: vector('vector2', { dimensions: 1, elementType: 'I8' }), year: year('year'), yeardef: year('yeardef').default(0), }); @@ -1015,6 +1018,8 @@ Expect< varchar: varchar({ length: 1 }), varchar2: varchar({ length: 1, enum: ['a', 'b', 'c'] }), varchardef: varchar({ length: 1 }).default(''), + vector: vector({ dimensions: 1 }), + vector2: vector({ dimensions: 1, elementType: 'I8' }), year: year(), yeardef: year().default(0), }); diff --git a/integration-tests/tests/singlestore/singlestore-common.ts b/integration-tests/tests/singlestore/singlestore-common.ts index fe7c2afb4..5c5d357bf 100644 --- a/integration-tests/tests/singlestore/singlestore-common.ts +++ b/integration-tests/tests/singlestore/singlestore-common.ts @@ -58,8 +58,10 @@ import { uniqueIndex, uniqueKeyName, varchar, + vector, year, } from 'drizzle-orm/singlestore-core'; +import { euclideanDistance, dotProduct } from 'drizzle-orm/singlestore-core/expressions'; import { migrate } from 'drizzle-orm/singlestore/migrator'; import getPort from 'get-port'; import { v4 as uuid } from 'uuid'; @@ -156,6 +158,12 @@ const aggregateTable = singlestoreTable('aggregate_table', { nullOnly: int('null_only'), }); +const vectorSearchTable = singlestoreTable('vector_search', { + id: serial('id').notNull(), + text: text('text').notNull(), + embedding: vector('embedding', { dimensions: 10 }), +}); + // To test another schema and multischema const mySchema = singlestoreSchema(`mySchema`); @@ -366,6 +374,23 @@ export function tests(driver?: string) { ]); } + async function setupVectorSearchTest(db: TestSingleStoreDB) { + await db.execute(sql`drop table if exists \`vector_search\``); + await db.execute( + sql` + create table \`vector_search\` ( + \`id\` integer primary key auto_increment not null, + \`text\` text not null, + \`embedding\` vector(10) not null + ) + `, + ) + await db.insert(vectorSearchTable).values([ + { id: 1, text: "I like dogs", embedding: [0.6119,0.1395,0.2921,0.3664,0.4561,0.7852,0.1997,0.5142,0.5924,0.0465] }, + { id: 2, text: "I like cats", embedding: [0.6075,0.1705,0.0651,0.9489,0.9656,0.8084,0.3046,0.0977,0.6842,0.4402] } + ]) + } + test('table config: unsigned ints', async () => { const unsignedInts = singlestoreTable('cities1', { bigint: bigint('bigint', { mode: 'number', unsigned: true }), @@ -2907,6 +2932,24 @@ export function tests(driver?: string) { expect(result2[0]?.value).toBe(null); }); + test('simple vector search', async (ctx) => { + const { db } = ctx.singlestore; + const table = vectorSearchTable; + const embedding = [0.42,0.93,0.88,0.57,0.32,0.64,0.76,0.52,0.19,0.81]; // ChatGPT's 10 dimension embedding for "dogs are cool" + await setupVectorSearchTest(db); + + const withRankEuclidean = db.select({ id: table.id, text: table.text, rank: sql`row_number() over (order by ${euclideanDistance(table.embedding, embedding)})`.as('rank') }).from(table).as('with_rank') + const withRankDotProduct = db.select({ id: table.id, text: table.text, rank: sql`row_number() over (order by ${dotProduct(table.embedding, embedding)})`.as('rank') }).from(table).as('with_rank') + const result1 = await db.select({ id: withRankEuclidean.id, text: withRankEuclidean.text }).from(withRankEuclidean).where(eq(withRankEuclidean.rank, 1)); + const result2 = await db.select({ id: withRankDotProduct.id, text: withRankDotProduct.text }).from(withRankDotProduct).where(eq(withRankDotProduct.rank, 1)); + + expect(result1.length).toEqual(1) + expect(result1[0]).toEqual({ id: 1, text: "I like dogs" }); + + expect(result2.length).toEqual(1) + expect(result2[0]).toEqual({ id: 1, text: "I like dogs" }); + }) + test('test $onUpdateFn and $onUpdate works as $default', async (ctx) => { const { db } = ctx.singlestore; From fc20e3529efc967fddd15a5021f6bf5e818c3f13 Mon Sep 17 00:00:00 2001 From: Mitchell Adair Date: Tue, 17 Dec 2024 13:49:09 -0500 Subject: [PATCH 07/18] lint fix, fix test/introspection --- drizzle-kit/src/introspect-singlestore.ts | 2 +- .../src/singlestore-core/columns/vector.ts | 3 +- .../tests/singlestore/singlestore-common.ts | 54 +++++++++++++------ 3 files changed, 39 insertions(+), 20 deletions(-) diff --git a/drizzle-kit/src/introspect-singlestore.ts b/drizzle-kit/src/introspect-singlestore.ts index 8f9c98acd..ee0ae5e0d 100644 --- a/drizzle-kit/src/introspect-singlestore.ts +++ b/drizzle-kit/src/introspect-singlestore.ts @@ -794,7 +794,7 @@ const column = ( const [dimensions, elementType] = lowered.substring('vector'.length + 1, lowered.length - 1).split(','); let out = `${casing(name)}: vector(${ dbColumnName({ name, casing: rawCasing, withMode: true }) - }{ dimensions: ${dimensions}${elementType ? `, elementType: ${elementType}` : ''} })`; + }{ dimensions: ${dimensions}, elementType: ${elementType} })`; out += defaultValue ? `.default(${mapColumnDefault(defaultValue, isExpression)})` : ''; return out; diff --git a/drizzle-orm/src/singlestore-core/columns/vector.ts b/drizzle-orm/src/singlestore-core/columns/vector.ts index dee6ba9e9..b70e0f1d1 100644 --- a/drizzle-orm/src/singlestore-core/columns/vector.ts +++ b/drizzle-orm/src/singlestore-core/columns/vector.ts @@ -47,8 +47,7 @@ export class SingleStoreVector) { diff --git a/integration-tests/tests/singlestore/singlestore-common.ts b/integration-tests/tests/singlestore/singlestore-common.ts index 5c5d357bf..b8fe39608 100644 --- a/integration-tests/tests/singlestore/singlestore-common.ts +++ b/integration-tests/tests/singlestore/singlestore-common.ts @@ -61,7 +61,7 @@ import { vector, year, } from 'drizzle-orm/singlestore-core'; -import { euclideanDistance, dotProduct } from 'drizzle-orm/singlestore-core/expressions'; +import { dotProduct, euclideanDistance } from 'drizzle-orm/singlestore-core/expressions'; import { migrate } from 'drizzle-orm/singlestore/migrator'; import getPort from 'get-port'; import { v4 as uuid } from 'uuid'; @@ -384,11 +384,19 @@ export function tests(driver?: string) { \`embedding\` vector(10) not null ) `, - ) + ); await db.insert(vectorSearchTable).values([ - { id: 1, text: "I like dogs", embedding: [0.6119,0.1395,0.2921,0.3664,0.4561,0.7852,0.1997,0.5142,0.5924,0.0465] }, - { id: 2, text: "I like cats", embedding: [0.6075,0.1705,0.0651,0.9489,0.9656,0.8084,0.3046,0.0977,0.6842,0.4402] } - ]) + { + id: 1, + text: 'I like dogs', + embedding: [0.6119, 0.1395, 0.2921, 0.3664, 0.4561, 0.7852, 0.1997, 0.5142, 0.5924, 0.0465], + }, + { + id: 2, + text: 'I like cats', + embedding: [0.6075, 0.1705, 0.0651, 0.9489, 0.9656, 0.8084, 0.3046, 0.0977, 0.6842, 0.4402], + }, + ]); } test('table config: unsigned ints', async () => { @@ -2935,20 +2943,32 @@ export function tests(driver?: string) { test('simple vector search', async (ctx) => { const { db } = ctx.singlestore; const table = vectorSearchTable; - const embedding = [0.42,0.93,0.88,0.57,0.32,0.64,0.76,0.52,0.19,0.81]; // ChatGPT's 10 dimension embedding for "dogs are cool" + const embedding = [0.42, 0.93, 0.88, 0.57, 0.32, 0.64, 0.76, 0.52, 0.19, 0.81]; // ChatGPT's 10 dimension embedding for "dogs are cool" not sure how accurate but it works await setupVectorSearchTest(db); - const withRankEuclidean = db.select({ id: table.id, text: table.text, rank: sql`row_number() over (order by ${euclideanDistance(table.embedding, embedding)})`.as('rank') }).from(table).as('with_rank') - const withRankDotProduct = db.select({ id: table.id, text: table.text, rank: sql`row_number() over (order by ${dotProduct(table.embedding, embedding)})`.as('rank') }).from(table).as('with_rank') - const result1 = await db.select({ id: withRankEuclidean.id, text: withRankEuclidean.text }).from(withRankEuclidean).where(eq(withRankEuclidean.rank, 1)); - const result2 = await db.select({ id: withRankDotProduct.id, text: withRankDotProduct.text }).from(withRankDotProduct).where(eq(withRankDotProduct.rank, 1)); - - expect(result1.length).toEqual(1) - expect(result1[0]).toEqual({ id: 1, text: "I like dogs" }); - - expect(result2.length).toEqual(1) - expect(result2[0]).toEqual({ id: 1, text: "I like dogs" }); - }) + const withRankEuclidean = db.select({ + id: table.id, + text: table.text, + rank: sql`row_number() over (order by ${euclideanDistance(table.embedding, embedding)})`.as('rank'), + }).from(table).as('with_rank'); + const withRankDotProduct = db.select({ + id: table.id, + text: table.text, + rank: sql`row_number() over (order by ${dotProduct(table.embedding, embedding)})`.as('rank'), + }).from(table).as('with_rank'); + const result1 = await db.select({ id: withRankEuclidean.id, text: withRankEuclidean.text }).from( + withRankEuclidean, + ).where(eq(withRankEuclidean.rank, 1)); + const result2 = await db.select({ id: withRankDotProduct.id, text: withRankDotProduct.text }).from( + withRankDotProduct, + ).where(eq(withRankDotProduct.rank, 1)); + + expect(result1.length).toEqual(1); + expect(result1[0]).toEqual({ id: 1, text: 'I like dogs' }); + + expect(result2.length).toEqual(1); + expect(result2[0]).toEqual({ id: 1, text: 'I like dogs' }); + }); test('test $onUpdateFn and $onUpdate works as $default', async (ctx) => { const { db } = ctx.singlestore; From c37de3adbbdda3b72c6a8949474c3c078ef9e2a3 Mon Sep 17 00:00:00 2001 From: Mitchell Adair Date: Wed, 18 Dec 2024 11:33:21 -0500 Subject: [PATCH 08/18] fix types --- drizzle-orm/src/singlestore-core/columns/vector.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drizzle-orm/src/singlestore-core/columns/vector.ts b/drizzle-orm/src/singlestore-core/columns/vector.ts index b70e0f1d1..285f4342d 100644 --- a/drizzle-orm/src/singlestore-core/columns/vector.ts +++ b/drizzle-orm/src/singlestore-core/columns/vector.ts @@ -66,7 +66,7 @@ export interface SingleStoreVectorConfig { elementType?: ElementType; } -export function vector( +export function vector( config: SingleStoreVectorConfig, ): SingleStoreVectorBuilderInitial<''>; export function vector( From 54f4fe62c8b74f84dcbed1623775b493f4f82f32 Mon Sep 17 00:00:00 2001 From: Mitchell Adair Date: Wed, 18 Dec 2024 13:56:57 -0500 Subject: [PATCH 09/18] more type changes, test not actually fixed :/ --- drizzle-orm/src/singlestore-core/columns/vector.ts | 3 +-- drizzle-orm/src/singlestore-core/expressions.ts | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/drizzle-orm/src/singlestore-core/columns/vector.ts b/drizzle-orm/src/singlestore-core/columns/vector.ts index 285f4342d..c8f05a63f 100644 --- a/drizzle-orm/src/singlestore-core/columns/vector.ts +++ b/drizzle-orm/src/singlestore-core/columns/vector.ts @@ -10,9 +10,8 @@ export type SingleStoreVectorBuilderInitial = SingleStoreV dataType: 'array'; columnType: 'SingleStoreVector'; data: Array; - driverParam: Array; + driverParam: string; enumValues: undefined; - generated: undefined; }>; export class SingleStoreVectorBuilder> diff --git a/drizzle-orm/src/singlestore-core/expressions.ts b/drizzle-orm/src/singlestore-core/expressions.ts index 397e87392..4e382e238 100644 --- a/drizzle-orm/src/singlestore-core/expressions.ts +++ b/drizzle-orm/src/singlestore-core/expressions.ts @@ -25,10 +25,10 @@ export function substring( } // Vectors -export function dotProduct(column: SingleStoreColumn | SQL.Aliased, value: Array) { +export function dotProduct(column: SingleStoreColumn | SQL.Aliased, value: Array): SQL { return sql`${column} <*> ${JSON.stringify(value)}`; } -export function euclideanDistance(column: SingleStoreColumn | SQL.Aliased, value: Array) { +export function euclideanDistance(column: SingleStoreColumn | SQL.Aliased, value: Array): SQL { return sql`${column} <-> ${JSON.stringify(value)}`; } From cb41edfde6b340b71ced217c1e91fe660f96a05f Mon Sep 17 00:00:00 2001 From: Mitchell Adair Date: Wed, 18 Dec 2024 15:27:55 -0500 Subject: [PATCH 10/18] simplify vector class --- .../src/singlestore-core/columns/vector.ts | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/drizzle-orm/src/singlestore-core/columns/vector.ts b/drizzle-orm/src/singlestore-core/columns/vector.ts index c8f05a63f..7bfc950de 100644 --- a/drizzle-orm/src/singlestore-core/columns/vector.ts +++ b/drizzle-orm/src/singlestore-core/columns/vector.ts @@ -29,21 +29,20 @@ export class SingleStoreVectorBuilder( table: AnySingleStoreTable<{ name: TTableName }>, ): SingleStoreVector> { - return new SingleStoreVector(table, this.config as ColumnBuilderRuntimeConfig); + return new SingleStoreVector>( + table, + this.config as ColumnBuilderRuntimeConfig, + ); } } -export class SingleStoreVector> extends SingleStoreColumn { +export class SingleStoreVector> + extends SingleStoreColumn +{ static override readonly [entityKind]: string = 'SingleStoreVector'; - readonly dimensions: number; - readonly elementType: ElementType | undefined; - - constructor(table: AnySingleStoreTable<{ name: T['tableName'] }>, config: SingleStoreVectorBuilder['config']) { - super(table, config); - this.dimensions = config.dimensions; - this.elementType = config.elementType; - } + dimensions: number = this.config.dimensions; + elementType: ElementType | undefined = this.config.elementType; getSQLType(): string { return `vector(${this.dimensions}, ${this.elementType || 'F32'})`; From 8a13fab5ad8b8fd5dc056b01c0033166e64fe4f8 Mon Sep 17 00:00:00 2001 From: Mitchell Adair Date: Mon, 6 Jan 2025 14:14:31 -0500 Subject: [PATCH 11/18] fix attw --- drizzle-orm/src/singlestore-core/columns/vector.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/drizzle-orm/src/singlestore-core/columns/vector.ts b/drizzle-orm/src/singlestore-core/columns/vector.ts index 7bfc950de..d33b819b9 100644 --- a/drizzle-orm/src/singlestore-core/columns/vector.ts +++ b/drizzle-orm/src/singlestore-core/columns/vector.ts @@ -1,9 +1,10 @@ -import type { ColumnBaseConfig } from '~/column'; -import type { ColumnBuilderBaseConfig, ColumnBuilderRuntimeConfig, MakeColumnConfig } from '~/column-builder'; +import type { ColumnBuilderBaseConfig, ColumnBuilderRuntimeConfig, MakeColumnConfig } from '~/column-builder.ts'; +import type { ColumnBaseConfig } from '~/column.ts'; import { entityKind } from '~/entity.ts'; import type { AnySingleStoreTable } from '~/singlestore-core/table.ts'; +import { SQL } from '~/sql/index.ts'; import { getColumnNameAndConfig } from '~/utils.ts'; -import { SingleStoreColumn, SingleStoreColumnBuilder } from './common.ts'; +import { SingleStoreColumn, SingleStoreColumnBuilder, SingleStoreGeneratedColumnConfig } from './common.ts'; export type SingleStoreVectorBuilderInitial = SingleStoreVectorBuilder<{ name: TName; @@ -34,6 +35,11 @@ export class SingleStoreVectorBuilder, ); } + + /** @internal */ + override generatedAlwaysAs(as: SQL | (() => SQL) | T['data'], config?: SingleStoreGeneratedColumnConfig) { + throw new Error('not implemented'); + } } export class SingleStoreVector> From 9d1aac11e394e16cd7f949dfb303e7825d6514fb Mon Sep 17 00:00:00 2001 From: Roman Date: Tue, 7 Jan 2025 15:52:38 +0200 Subject: [PATCH 12/18] disable console verbosity --- drizzle-kit/src/utils/certs.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drizzle-kit/src/utils/certs.ts b/drizzle-kit/src/utils/certs.ts index 6c3560aa6..74bd77e71 100644 --- a/drizzle-kit/src/utils/certs.ts +++ b/drizzle-kit/src/utils/certs.ts @@ -5,6 +5,8 @@ import { join } from 'path'; import { $ } from 'zx'; export const certs = async () => { + $.verbose = false; + const res = await $`mkcert --help`.nothrow(); if (res.exitCode === 0) { @@ -12,7 +14,6 @@ export const certs = async () => { suffix: '', }); - $.verbose = false; $.cwd = p.data; // create ~/.local/share/drizzle-studio From de3c537eb26b9462d59c35d0b456083d7392923d Mon Sep 17 00:00:00 2001 From: Roman Date: Tue, 7 Jan 2025 15:57:08 +0200 Subject: [PATCH 13/18] up version --- changelogs/drizzle-kit/0.30.2.md | 1 + drizzle-kit/package.json | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelogs/drizzle-kit/0.30.2.md diff --git a/changelogs/drizzle-kit/0.30.2.md b/changelogs/drizzle-kit/0.30.2.md new file mode 100644 index 000000000..8db1c961a --- /dev/null +++ b/changelogs/drizzle-kit/0.30.2.md @@ -0,0 +1 @@ +- Fix certificates generation utility for Drizzle Studio; [[BUG]: [drizzle-kit]: drizzle-kit dependency on drizzle-studio perms error](https://github.com/drizzle-team/drizzle-orm/issues/3729) \ No newline at end of file diff --git a/drizzle-kit/package.json b/drizzle-kit/package.json index 32a2c48de..6e2fec181 100644 --- a/drizzle-kit/package.json +++ b/drizzle-kit/package.json @@ -1,6 +1,6 @@ { "name": "drizzle-kit", - "version": "0.30.1", + "version": "0.30.2", "homepage": "https://orm.drizzle.team", "keywords": [ "drizzle", From 570c50621d407d22953ce55397214bed2e0773a0 Mon Sep 17 00:00:00 2001 From: Roman Date: Thu, 16 Jan 2025 12:47:05 +0200 Subject: [PATCH 14/18] fix: Fix certs util --- drizzle-kit/src/utils/certs.ts | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/drizzle-kit/src/utils/certs.ts b/drizzle-kit/src/utils/certs.ts index a3c80ab61..873b4e665 100644 --- a/drizzle-kit/src/utils/certs.ts +++ b/drizzle-kit/src/utils/certs.ts @@ -4,31 +4,34 @@ import { access, readFile } from 'fs/promises'; import { exec, ExecOptions } from 'node:child_process'; import { join } from 'path'; -const p = envPaths('drizzle-studio', { - suffix: '', -}); - -mkdirSync(p.data, { recursive: true }); - export function runCommand(command: string, options: ExecOptions = {}) { - return new Promise<{ exitCode: number }>((resolve, reject) => { - exec(command, options, (error, stdout, stderr) => { + return new Promise<{ exitCode: number }>((resolve) => { + exec(command, options, (error) => { return resolve({ exitCode: error?.code ?? 0 }); }); }); } export const certs = async () => { - const res = await runCommand(`mkcert --help`, { cwd: p.data }); - - // ~/.local/share/drizzle-studio - const keyPath = join(p.data, 'localhost-key.pem'); - const certPath = join(p.data, 'localhost.pem'); + const res = await runCommand('mkcert --help'); if (res.exitCode === 0) { + const p = envPaths('drizzle-studio', { + suffix: '', + }); + + // create ~/.local/share/drizzle-studio + mkdirSync(p.data, { recursive: true }); + + // ~/.local/share/drizzle-studio + const keyPath = join(p.data, 'localhost-key.pem'); + const certPath = join(p.data, 'localhost.pem'); + try { + // check if the files exist await Promise.all([access(keyPath), access(certPath)]); } catch (e) { + // if not create them await runCommand(`mkcert localhost`, { cwd: p.data }); } const [key, cert] = await Promise.all([ @@ -39,5 +42,3 @@ export const certs = async () => { } return null; }; - -certs(); From 15cd9985eab6c6ee9d59382b07d8f6ea26eb0615 Mon Sep 17 00:00:00 2001 From: AndriiSherman Date: Thu, 16 Jan 2025 13:52:38 +0200 Subject: [PATCH 15/18] Up orm version --- changelogs/drizzle-orm/0.38.4.md | 3 +++ drizzle-orm/package.json | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 changelogs/drizzle-orm/0.38.4.md diff --git a/changelogs/drizzle-orm/0.38.4.md b/changelogs/drizzle-orm/0.38.4.md new file mode 100644 index 000000000..037d7809d --- /dev/null +++ b/changelogs/drizzle-orm/0.38.4.md @@ -0,0 +1,3 @@ +- New SingleStore type `vector` - thanks @mitchwadair +- Fix wrong DROP INDEX statement generation, [#3866](https://github.com/drizzle-team/drizzle-orm/pull/3866) - thanks @WaciX +- Typo fixes - thanks @stephan281094 \ No newline at end of file diff --git a/drizzle-orm/package.json b/drizzle-orm/package.json index 5ab4de9b2..7e49ec522 100644 --- a/drizzle-orm/package.json +++ b/drizzle-orm/package.json @@ -1,6 +1,6 @@ { "name": "drizzle-orm", - "version": "0.38.3", + "version": "0.38.4", "description": "Drizzle ORM package for SQL databases", "type": "module", "scripts": { From 0350a59aec7c3329221917119c026f63ce865817 Mon Sep 17 00:00:00 2001 From: AndriiSherman Date: Thu, 16 Jan 2025 14:03:48 +0200 Subject: [PATCH 16/18] switch off PS tests for now --- integration-tests/vitest.config.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/integration-tests/vitest.config.ts b/integration-tests/vitest.config.ts index a04281018..21aa98be8 100644 --- a/integration-tests/vitest.config.ts +++ b/integration-tests/vitest.config.ts @@ -61,6 +61,9 @@ export default defineConfig({ 'tests/mysql/tidb-serverless.test.ts', // waiting for json_array from singlestore team 'tests/relational/singlestore.test.ts', + // get back when planetscale will open free tier for our CI/CD + 'tests/mysql/mysql-planetscale.test.ts', + 'tests/relational/mysql.planetscale.test.ts', ], typecheck: { tsconfig: 'tsconfig.json', From 471d797dd81db7b2b7338b115be050e7ce929400 Mon Sep 17 00:00:00 2001 From: AndriiSherman Date: Thu, 16 Jan 2025 14:57:20 +0200 Subject: [PATCH 17/18] remove js-tests for PS as well --- integration-tests/vitest.config.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/integration-tests/vitest.config.ts b/integration-tests/vitest.config.ts index 21aa98be8..0028b7aac 100644 --- a/integration-tests/vitest.config.ts +++ b/integration-tests/vitest.config.ts @@ -64,6 +64,8 @@ export default defineConfig({ // get back when planetscale will open free tier for our CI/CD 'tests/mysql/mysql-planetscale.test.ts', 'tests/relational/mysql.planetscale.test.ts', + 'js-tests/driver-init/module/planetscale.test.mjs', + 'js-tests/driver-init/module/planetscale.test.cjs', ], typecheck: { tsconfig: 'tsconfig.json', From 9927cf17aa8fd37614b991fd8c091061778ba8b5 Mon Sep 17 00:00:00 2001 From: AndriiSherman Date: Thu, 16 Jan 2025 15:33:36 +0200 Subject: [PATCH 18/18] forgot 1 more PS test --- integration-tests/vitest.config.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/integration-tests/vitest.config.ts b/integration-tests/vitest.config.ts index 0028b7aac..e8a8be220 100644 --- a/integration-tests/vitest.config.ts +++ b/integration-tests/vitest.config.ts @@ -66,6 +66,7 @@ export default defineConfig({ 'tests/relational/mysql.planetscale.test.ts', 'js-tests/driver-init/module/planetscale.test.mjs', 'js-tests/driver-init/module/planetscale.test.cjs', + 'js-tests/driver-init/commonjs/planetscale.test.cjs', ], typecheck: { tsconfig: 'tsconfig.json',