From f5f7c20eed01072fbb1960c2002906e28faca908 Mon Sep 17 00:00:00 2001 From: Angelelz Date: Wed, 10 Jan 2024 19:25:21 -0500 Subject: [PATCH 1/7] Postgres-js: Added json and jsonb to the list of bypassed types on postgres.js driver --- drizzle-orm/src/postgres-js/driver.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drizzle-orm/src/postgres-js/driver.ts b/drizzle-orm/src/postgres-js/driver.ts index ae1b48a21..2b2523e25 100644 --- a/drizzle-orm/src/postgres-js/driver.ts +++ b/drizzle-orm/src/postgres-js/driver.ts @@ -23,7 +23,7 @@ export function drizzle = Record val; // Override postgres.js default date parsers: https://github.com/porsager/postgres/discussions/761 - for (const type of ['1184', '1082', '1083', '1114']) { + for (const type of ['1184', '1082', '1083', '1114', '114', '3802']) { client.options.parsers[type as any] = transparentParser; client.options.serializers[type as any] = transparentParser; } From fcc8be7d8b27ea9a9299c3bdc24d31c53968c2e9 Mon Sep 17 00:00:00 2001 From: Angelelz Date: Wed, 10 Jan 2024 19:26:29 -0500 Subject: [PATCH 2/7] [Pg] Added simple tests to pg and postgres-js integration tests for json and jsonb columns --- integration-tests/tests/pg.test.ts | 37 +++++++++++++++++++++ integration-tests/tests/postgres.js.test.ts | 37 +++++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/integration-tests/tests/pg.test.ts b/integration-tests/tests/pg.test.ts index 3b31d7d60..e88a01833 100644 --- a/integration-tests/tests/pg.test.ts +++ b/integration-tests/tests/pg.test.ts @@ -49,6 +49,7 @@ import { intersect, intersectAll, interval, + json, jsonb, macaddr, macaddr8, @@ -2839,6 +2840,42 @@ test.serial('test mode string for timestamp with timezone in different timezone' await db.execute(sql`drop table if exists ${table}`); }); +test.serial('proper json and jsonb handling', async (t) => { + const { db } = t.context; + + const jsonTable = pgTable('json_table', { + json: json('json').$type<{ name: string; age: number }>(), + jsonb: jsonb('jsonb').$type<{ name: string; age: number }>(), + }); + + await db.execute(sql`drop table if exists ${jsonTable}`); + + db.execute(sql`create table ${jsonTable} (json json, jsonb jsonb)`); + + await db.insert(jsonTable).values({ json: { name: 'Tom', age: 75 }, jsonb: { name: 'Pete', age: 23 } }); + + const result = await db.select().from(jsonTable); + + const justNames = await db.select({ + name1: sql`${jsonTable.json}->>'name'`.as('name1'), + name2: sql`${jsonTable.jsonb}->>'name'`.as('name2'), + }).from(jsonTable); + + t.deepEqual(result, [ + { + json: { name: 'Tom', age: 75 }, + jsonb: { name: 'Pete', age: 23 }, + }, + ]); + + t.deepEqual(justNames, [ + { + name1: 'Tom', + name2: 'Pete', + }, + ]); +}); + test.serial('orderBy with aliased column', (t) => { const { db } = t.context; diff --git a/integration-tests/tests/postgres.js.test.ts b/integration-tests/tests/postgres.js.test.ts index 0fd0c45ea..d23b294b4 100644 --- a/integration-tests/tests/postgres.js.test.ts +++ b/integration-tests/tests/postgres.js.test.ts @@ -31,6 +31,7 @@ import { getViewConfig, integer, interval, + json, jsonb, type PgColumn, pgEnum, @@ -1812,6 +1813,42 @@ test.serial('select from enum', async (t) => { await db.execute(sql`drop type ${name(categoryEnum.enumName)}`); }); +test.serial('proper json and jsonb handling', async (t) => { + const { db } = t.context; + + const jsonTable = pgTable('json_table', { + json: json('json').$type<{ name: string; age: number }>(), + jsonb: jsonb('jsonb').$type<{ name: string; age: number }>(), + }); + + await db.execute(sql`drop table if exists ${jsonTable}`); + + db.execute(sql`create table ${jsonTable} (json json, jsonb jsonb)`); + + await db.insert(jsonTable).values({ json: { name: 'Tom', age: 75 }, jsonb: { name: 'Pete', age: 23 } }); + + const result = await db.select().from(jsonTable); + + const justNames = await db.select({ + name1: sql`${jsonTable.json}->>'name'`.as('name1'), + name2: sql`${jsonTable.jsonb}->>'name'`.as('name2'), + }).from(jsonTable); + + t.deepEqual(result, [ + { + json: { name: 'Tom', age: 75 }, + jsonb: { name: 'Pete', age: 23 }, + }, + ]); + + t.deepEqual(justNames, [ + { + name1: 'Tom', + name2: 'Pete', + }, + ]); +}); + test.serial('orderBy with aliased column', (t) => { const { db } = t.context; From dd5835868cf88f0e9c38359e186d1584a00aa7da Mon Sep 17 00:00:00 2001 From: Angelelz Date: Wed, 10 Jan 2024 21:27:40 -0500 Subject: [PATCH 3/7] fix: bypassing the tranformation is only needed in the parser, not the serializer --- drizzle-orm/src/postgres-js/driver.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drizzle-orm/src/postgres-js/driver.ts b/drizzle-orm/src/postgres-js/driver.ts index 2b2523e25..2c4031c83 100644 --- a/drizzle-orm/src/postgres-js/driver.ts +++ b/drizzle-orm/src/postgres-js/driver.ts @@ -23,10 +23,12 @@ export function drizzle = Record val; // Override postgres.js default date parsers: https://github.com/porsager/postgres/discussions/761 - for (const type of ['1184', '1082', '1083', '1114', '114', '3802']) { - client.options.parsers[type as any] = transparentParser; + for (const type of ['1184', '1082', '1083', '1114']) { + if (type !== '114' && type !== '3802') client.options.parsers[type as any] = transparentParser; client.options.serializers[type as any] = transparentParser; } + client.options.parsers['114'] = transparentParser; + client.options.parsers['3802'] = transparentParser; const dialect = new PgDialect(); let logger; From ca792625fc64ca696cacafce1da32b0902b843ad Mon Sep 17 00:00:00 2001 From: Angelelz Date: Wed, 10 Jan 2024 21:32:52 -0500 Subject: [PATCH 4/7] Added additional tests to postgres-js integration tests --- integration-tests/tests/postgres.js.test.ts | 127 ++++++++++++++++++++ 1 file changed, 127 insertions(+) diff --git a/integration-tests/tests/postgres.js.test.ts b/integration-tests/tests/postgres.js.test.ts index d23b294b4..3382f24e2 100644 --- a/integration-tests/tests/postgres.js.test.ts +++ b/integration-tests/tests/postgres.js.test.ts @@ -94,6 +94,12 @@ const orders = pgTable('orders', { quantity: integer('quantity').notNull(), }); +const jsonTestTable = pgTable('jsontest', { + id: serial('id').primaryKey(), + json: json('json').$type<{ string: string; number: number }>(), + jsonb: jsonb('jsonb').$type<{ string: string; number: number }>(), +}); + const usersMigratorTable = pgTable('users12', { id: serial('id').primaryKey(), name: text('name').notNull(), @@ -234,6 +240,15 @@ test.beforeEach(async (t) => { ) `, ); + await ctx.db.execute( + sql` + create table jsontest ( + id serial primary key, + json json, + jsonb jsonb + ) + `, + ); }); test.serial('select all fields', async (t) => { @@ -422,6 +437,118 @@ test.serial('json insert', async (t) => { t.deepEqual(result, [{ id: 1, name: 'John', jsonb: ['foo', 'bar'] }]); }); +test.serial('set json/jsonb fields with objects and retrieve with the ->> operator', async (t) => { + const { db } = t.context; + + const obj = { string: 'test', number: 123 }; + const { string: testString, number: testNumber } = obj; + + await db.insert(jsonTestTable).values({ + json: obj, + jsonb: obj, + }); + + const result = await db.select({ + jsonStringField: sql`${jsonTestTable.json}->>'string'`, + jsonNumberField: sql`${jsonTestTable.json}->>'number'`, + jsonbStringField: sql`${jsonTestTable.jsonb}->>'string'`, + jsonbNumberField: sql`${jsonTestTable.jsonb}->>'number'`, + }).from(jsonTestTable); + + t.deepEqual(result, [{ + jsonStringField: testString, + jsonNumberField: String(testNumber), + jsonbStringField: testString, + jsonbNumberField: String(testNumber), + }]); + + await db.execute(sql`drop table ${jsonTestTable}`); +}); + +test.serial('set json/jsonb fields with strings and retrieve with the ->> operator', async (t) => { + const { db } = t.context; + + const obj = { string: 'test', number: 123 }; + const { string: testString, number: testNumber } = obj; + + await db.insert(jsonTestTable).values({ + json: sql`${JSON.stringify(obj)}`, + jsonb: sql`${JSON.stringify(obj)}`, + }); + + const result = await db.select({ + jsonStringField: sql`${jsonTestTable.json}->>'string'`, + jsonNumberField: sql`${jsonTestTable.json}->>'number'`, + jsonbStringField: sql`${jsonTestTable.jsonb}->>'string'`, + jsonbNumberField: sql`${jsonTestTable.jsonb}->>'number'`, + }).from(jsonTestTable); + + t.deepEqual(result, [{ + jsonStringField: testString, + jsonNumberField: String(testNumber), + jsonbStringField: testString, + jsonbNumberField: String(testNumber), + }]); + + await db.execute(sql`drop table ${jsonTestTable}`); +}); + +test.serial('set json/jsonb fields with objects and retrieve with the -> operator', async (t) => { + const { db } = t.context; + + const obj = { string: 'test', number: 123 }; + const { string: testString, number: testNumber } = obj; + + await db.insert(jsonTestTable).values({ + json: obj, + jsonb: obj, + }); + + const result = await db.select({ + jsonStringField: sql`${jsonTestTable.json}->'string'`, + jsonNumberField: sql`${jsonTestTable.json}->'number'`, + jsonbStringField: sql`${jsonTestTable.jsonb}->'string'`, + jsonbNumberField: sql`${jsonTestTable.jsonb}->'number'`, + }).from(jsonTestTable); + + t.deepEqual(result, [{ + jsonStringField: testString, + jsonNumberField: testNumber, + jsonbStringField: testString, + jsonbNumberField: testNumber, + }]); + + await db.execute(sql`drop table ${jsonTestTable}`); +}); + +test.serial('set json/jsonb fields with strings and retrieve with the -> operator', async (t) => { + const { db } = t.context; + + const obj = { string: 'test', number: 123 }; + const { string: testString, number: testNumber } = obj; + + await db.insert(jsonTestTable).values({ + json: sql`${JSON.stringify(obj)}`, + jsonb: sql`${JSON.stringify(obj)}`, + }); + + const result = await db.select({ + jsonStringField: sql`${jsonTestTable.json}->'string'`, + jsonNumberField: sql`${jsonTestTable.json}->'number'`, + jsonbStringField: sql`${jsonTestTable.jsonb}->'string'`, + jsonbNumberField: sql`${jsonTestTable.jsonb}->'number'`, + }).from(jsonTestTable); + + t.deepEqual(result, [{ + jsonStringField: testString, + jsonNumberField: testNumber, + jsonbStringField: testString, + jsonbNumberField: testNumber, + }]); + + await db.execute(sql`drop table ${jsonTestTable}`); +}); + test.serial('insert with overridden default values', async (t) => { const { db } = t.context; From 562c25bb28cdd8243a28b1970b6c25b004b7d3d8 Mon Sep 17 00:00:00 2001 From: Angelelz Date: Wed, 10 Jan 2024 22:06:18 -0500 Subject: [PATCH 5/7] fixed parsing properly --- drizzle-orm/src/postgres-js/driver.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drizzle-orm/src/postgres-js/driver.ts b/drizzle-orm/src/postgres-js/driver.ts index 2c4031c83..7f44344e8 100644 --- a/drizzle-orm/src/postgres-js/driver.ts +++ b/drizzle-orm/src/postgres-js/driver.ts @@ -24,11 +24,11 @@ export function drizzle = Record Date: Tue, 6 Aug 2024 14:27:46 +0300 Subject: [PATCH 6/7] Add tests to pg-common --- integration-tests/tests/pg/pg-common.ts | 169 +++++++++++++++++++++++- 1 file changed, 162 insertions(+), 7 deletions(-) diff --git a/integration-tests/tests/pg/pg-common.ts b/integration-tests/tests/pg/pg-common.ts index fb69c5877..df1582bea 100644 --- a/integration-tests/tests/pg/pg-common.ts +++ b/integration-tests/tests/pg/pg-common.ts @@ -70,6 +70,7 @@ import { uniqueKeyName, uuid as pgUuid, varchar, + json, } from 'drizzle-orm/pg-core'; import getPort from 'get-port'; import { v4 as uuidV4 } from 'uuid'; @@ -199,6 +200,12 @@ const users2MySchemaTable = mySchema.table('users2', { cityId: integer('city_id').references(() => citiesTable.id), }); +const jsonTestTable = pgTable('jsontest', { + id: serial('id').primaryKey(), + json: json('json').$type<{ string: string; number: number }>(), + jsonb: jsonb('jsonb').$type<{ string: string; number: number }>(), +}); + let pgContainer: Docker.Container; export async function createDockerDB(): Promise<{ connectionString: string; container: Docker.Container }> { @@ -358,6 +365,16 @@ export function tests() { ) `, ); + + await db.execute( + sql` + create table jsontest ( + id serial primary key, + json json, + jsonb jsonb + ) + `, + ); }); async function setupSetOperationTest(db: PgDatabase) { @@ -2347,9 +2364,8 @@ export function tests() { await db.execute(sql`drop type if exists ${sql.identifier(categoryEnum.enumName)}`); await db.execute( - sql`create type ${ - sql.identifier(muscleEnum.enumName) - } as enum ('abdominals', 'hamstrings', 'adductors', 'quadriceps', 'biceps', 'shoulders', 'chest', 'middle_back', 'calves', 'glutes', 'lower_back', 'lats', 'triceps', 'traps', 'forearms', 'neck', 'abductors')`, + sql`create type ${sql.identifier(muscleEnum.enumName) + } as enum ('abdominals', 'hamstrings', 'adductors', 'quadriceps', 'biceps', 'shoulders', 'chest', 'middle_back', 'calves', 'glutes', 'lower_back', 'lats', 'triceps', 'traps', 'forearms', 'neck', 'abductors')`, ); await db.execute( sql`create type ${sql.identifier(forceEnum.enumName)} as enum ('isometric', 'isotonic', 'isokinetic')`, @@ -2359,9 +2375,8 @@ export function tests() { ); await db.execute(sql`create type ${sql.identifier(mechanicEnum.enumName)} as enum ('compound', 'isolation')`); await db.execute( - sql`create type ${ - sql.identifier(equipmentEnum.enumName) - } as enum ('barbell', 'dumbbell', 'bodyweight', 'machine', 'cable', 'kettlebell')`, + sql`create type ${sql.identifier(equipmentEnum.enumName) + } as enum ('barbell', 'dumbbell', 'bodyweight', 'machine', 'cable', 'kettlebell')`, ); await db.execute( sql`create type ${sql.identifier(categoryEnum.enumName)} as enum ('upper_body', 'lower_body', 'full_body')`, @@ -4481,5 +4496,145 @@ export function tests() { expect(users.length).toBeGreaterThan(0); }); + + test('proper json and jsonb handling', async (ctx) => { + const { db } = ctx.pg; + + const jsonTable = pgTable('json_table', { + json: json('json').$type<{ name: string; age: number }>(), + jsonb: jsonb('jsonb').$type<{ name: string; age: number }>(), + }); + + await db.execute(sql`drop table if exists ${jsonTable}`); + + db.execute(sql`create table ${jsonTable} (json json, jsonb jsonb)`); + + await db.insert(jsonTable).values({ json: { name: 'Tom', age: 75 }, jsonb: { name: 'Pete', age: 23 } }); + + const result = await db.select().from(jsonTable); + + const justNames = await db.select({ + name1: sql`${jsonTable.json}->>'name'`.as('name1'), + name2: sql`${jsonTable.jsonb}->>'name'`.as('name2'), + }).from(jsonTable); + + expect(result).toStrictEqual([ + { + json: { name: 'Tom', age: 75 }, + jsonb: { name: 'Pete', age: 23 }, + }, + ]); + + expect(justNames).toStrictEqual([ + { + name1: 'Tom', + name2: 'Pete', + }, + ]); + }); + + test('set json/jsonb fields with objects and retrieve with the ->> operator', async (ctx) => { + const { db } = ctx.pg; + + const obj = { string: 'test', number: 123 }; + const { string: testString, number: testNumber } = obj; + + await db.insert(jsonTestTable).values({ + json: obj, + jsonb: obj, + }); + + const result = await db.select({ + jsonStringField: sql`${jsonTestTable.json}->>'string'`, + jsonNumberField: sql`${jsonTestTable.json}->>'number'`, + jsonbStringField: sql`${jsonTestTable.jsonb}->>'string'`, + jsonbNumberField: sql`${jsonTestTable.jsonb}->>'number'`, + }).from(jsonTestTable); + + expect(result).toStrictEqual([{ + jsonStringField: testString, + jsonNumberField: String(testNumber), + jsonbStringField: testString, + jsonbNumberField: String(testNumber), + }]) + }); + + test('set json/jsonb fields with strings and retrieve with the ->> operator', async (ctx) => { + const { db } = ctx.pg; + + const obj = { string: 'test', number: 123 }; + const { string: testString, number: testNumber } = obj; + + await db.insert(jsonTestTable).values({ + json: sql`${JSON.stringify(obj)}`, + jsonb: sql`${JSON.stringify(obj)}`, + }); + + const result = await db.select({ + jsonStringField: sql`${jsonTestTable.json}->>'string'`, + jsonNumberField: sql`${jsonTestTable.json}->>'number'`, + jsonbStringField: sql`${jsonTestTable.jsonb}->>'string'`, + jsonbNumberField: sql`${jsonTestTable.jsonb}->>'number'`, + }).from(jsonTestTable); + + expect(result).toStrictEqual([{ + jsonStringField: testString, + jsonNumberField: String(testNumber), + jsonbStringField: testString, + jsonbNumberField: String(testNumber), + }]) + }); + + test('set json/jsonb fields with objects and retrieve with the -> operator', async (ctx) => { + const { db } = ctx.pg; + + const obj = { string: 'test', number: 123 }; + const { string: testString, number: testNumber } = obj; + + await db.insert(jsonTestTable).values({ + json: obj, + jsonb: obj, + }); + + const result = await db.select({ + jsonStringField: sql`${jsonTestTable.json}->'string'`, + jsonNumberField: sql`${jsonTestTable.json}->'number'`, + jsonbStringField: sql`${jsonTestTable.jsonb}->'string'`, + jsonbNumberField: sql`${jsonTestTable.jsonb}->'number'`, + }).from(jsonTestTable); + + expect(result).toStrictEqual([{ + jsonStringField: testString, + jsonNumberField: testNumber, + jsonbStringField: testString, + jsonbNumberField: testNumber, + }]) + }); + + test('set json/jsonb fields with strings and retrieve with the -> operator', async (ctx) => { + const { db } = ctx.pg; + + const obj = { string: 'test', number: 123 }; + const { string: testString, number: testNumber } = obj; + + await db.insert(jsonTestTable).values({ + json: sql`${JSON.stringify(obj)}`, + jsonb: sql`${JSON.stringify(obj)}`, + }); + + const result = await db.select({ + jsonStringField: sql`${jsonTestTable.json}->'string'`, + jsonNumberField: sql`${jsonTestTable.json}->'number'`, + jsonbStringField: sql`${jsonTestTable.jsonb}->'string'`, + jsonbNumberField: sql`${jsonTestTable.jsonb}->'number'`, + }).from(jsonTestTable); + + expect(result).toStrictEqual([{ + jsonStringField: testString, + jsonNumberField: testNumber, + jsonbStringField: testString, + jsonbNumberField: testNumber, + }]) + }); }); -} +} \ No newline at end of file From 0accd97a853294f7ed9916673aee974f06419e2e Mon Sep 17 00:00:00 2001 From: AndriiSherman Date: Tue, 6 Aug 2024 14:51:08 +0300 Subject: [PATCH 7/7] Use dprint --- integration-tests/tests/pg/pg-common.ts | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/integration-tests/tests/pg/pg-common.ts b/integration-tests/tests/pg/pg-common.ts index df1582bea..0b2b48769 100644 --- a/integration-tests/tests/pg/pg-common.ts +++ b/integration-tests/tests/pg/pg-common.ts @@ -49,6 +49,7 @@ import { intersect, intersectAll, interval, + json, jsonb, macaddr, macaddr8, @@ -70,7 +71,6 @@ import { uniqueKeyName, uuid as pgUuid, varchar, - json, } from 'drizzle-orm/pg-core'; import getPort from 'get-port'; import { v4 as uuidV4 } from 'uuid'; @@ -2364,8 +2364,9 @@ export function tests() { await db.execute(sql`drop type if exists ${sql.identifier(categoryEnum.enumName)}`); await db.execute( - sql`create type ${sql.identifier(muscleEnum.enumName) - } as enum ('abdominals', 'hamstrings', 'adductors', 'quadriceps', 'biceps', 'shoulders', 'chest', 'middle_back', 'calves', 'glutes', 'lower_back', 'lats', 'triceps', 'traps', 'forearms', 'neck', 'abductors')`, + sql`create type ${ + sql.identifier(muscleEnum.enumName) + } as enum ('abdominals', 'hamstrings', 'adductors', 'quadriceps', 'biceps', 'shoulders', 'chest', 'middle_back', 'calves', 'glutes', 'lower_back', 'lats', 'triceps', 'traps', 'forearms', 'neck', 'abductors')`, ); await db.execute( sql`create type ${sql.identifier(forceEnum.enumName)} as enum ('isometric', 'isotonic', 'isokinetic')`, @@ -2375,8 +2376,9 @@ export function tests() { ); await db.execute(sql`create type ${sql.identifier(mechanicEnum.enumName)} as enum ('compound', 'isolation')`); await db.execute( - sql`create type ${sql.identifier(equipmentEnum.enumName) - } as enum ('barbell', 'dumbbell', 'bodyweight', 'machine', 'cable', 'kettlebell')`, + sql`create type ${ + sql.identifier(equipmentEnum.enumName) + } as enum ('barbell', 'dumbbell', 'bodyweight', 'machine', 'cable', 'kettlebell')`, ); await db.execute( sql`create type ${sql.identifier(categoryEnum.enumName)} as enum ('upper_body', 'lower_body', 'full_body')`, @@ -4507,7 +4509,7 @@ export function tests() { await db.execute(sql`drop table if exists ${jsonTable}`); - db.execute(sql`create table ${jsonTable} (json json, jsonb jsonb)`); + await db.execute(sql`create table ${jsonTable} (json json, jsonb jsonb)`); await db.insert(jsonTable).values({ json: { name: 'Tom', age: 75 }, jsonb: { name: 'Pete', age: 23 } }); @@ -4556,7 +4558,7 @@ export function tests() { jsonNumberField: String(testNumber), jsonbStringField: testString, jsonbNumberField: String(testNumber), - }]) + }]); }); test('set json/jsonb fields with strings and retrieve with the ->> operator', async (ctx) => { @@ -4582,7 +4584,7 @@ export function tests() { jsonNumberField: String(testNumber), jsonbStringField: testString, jsonbNumberField: String(testNumber), - }]) + }]); }); test('set json/jsonb fields with objects and retrieve with the -> operator', async (ctx) => { @@ -4608,7 +4610,7 @@ export function tests() { jsonNumberField: testNumber, jsonbStringField: testString, jsonbNumberField: testNumber, - }]) + }]); }); test('set json/jsonb fields with strings and retrieve with the -> operator', async (ctx) => { @@ -4634,7 +4636,7 @@ export function tests() { jsonNumberField: testNumber, jsonbStringField: testString, jsonbNumberField: testNumber, - }]) + }]); }); }); -} \ No newline at end of file +}