From c33935e5e11bd0811949b744d422f4c3ae7854f4 Mon Sep 17 00:00:00 2001 From: = Date: Wed, 22 Oct 2025 19:56:42 +0530 Subject: [PATCH 1/8] feat: resolved locking of membership table --- .../db/migrations/20251018061215_sub-org.ts | 44 ++++++++++++++----- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/backend/src/db/migrations/20251018061215_sub-org.ts b/backend/src/db/migrations/20251018061215_sub-org.ts index 6378fa66a..95e96d8c8 100644 --- a/backend/src/db/migrations/20251018061215_sub-org.ts +++ b/backend/src/db/migrations/20251018061215_sub-org.ts @@ -3,6 +3,7 @@ import { Knex } from "knex"; import { dropConstraintIfExists } from "@app/db/migrations/utils/dropConstraintIfExists"; import { AccessScope, TableName } from "../schemas"; +import { chunkArray } from "@app/lib/fn"; export async function up(knex: Knex): Promise { const hasParentOrgId = await knex.schema.hasColumn(TableName.Organization, "parentOrgId"); @@ -29,19 +30,38 @@ export async function up(knex: Knex): Promise { t.foreign("orgId").references("id").inTable(TableName.Organization).onDelete("CASCADE"); }); - await knex.raw( - ` - UPDATE ?? AS identity - SET "orgId" = membership."scopeOrgId" - FROM ?? AS membership - WHERE - membership."actorIdentityId" = identity."id" - AND membership."scope" = ? -`, - [TableName.Identity, TableName.Membership, AccessScope.Organization] - ); + const identityMemberships = await knex(TableName.Membership) + .where({ + scope: AccessScope.Organization + }) + .whereNotNull("actorIdentityId") + .select("actorIdentityId", "scopeOrgId"); - await knex.raw(`DELETE FROM ?? WHERE "orgId" IS NULL`, [TableName.Identity]); + const identityToOrgMapping: Record = {}; + identityMemberships.forEach((el) => { + if (el.actorIdentityId) { + identityToOrgMapping[el.actorIdentityId] = el.scopeOrgId; + } + }); + + const batchMemberships = chunkArray(identityMemberships, 500); + for await (const membership of batchMemberships) { + const identityIds = membership.map((el) => el.actorIdentityId).filter(Boolean) as string[]; + if (identityIds.length) { + const identities = await knex(TableName.Identity).whereIn("id", identityIds).select("*"); + await knex(TableName.Identity) + .insert( + identities.map((el) => ({ + ...el, + orgId: identityToOrgMapping[el.id] + })) + ) + .onConflict("id") + .merge(); + } + } + + await knex(TableName.Identity).whereNull("orgId").delete(); await knex.schema.alterTable(TableName.Identity, (t) => { t.uuid("orgId").notNullable().alter(); From 1f33f24797397a3d266eadfd5d6a673834fde8b6 Mon Sep 17 00:00:00 2001 From: = Date: Wed, 22 Oct 2025 19:58:26 +0530 Subject: [PATCH 2/8] feat: fixed esint issue --- backend/src/db/migrations/20251018061215_sub-org.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/db/migrations/20251018061215_sub-org.ts b/backend/src/db/migrations/20251018061215_sub-org.ts index 95e96d8c8..3254a4aca 100644 --- a/backend/src/db/migrations/20251018061215_sub-org.ts +++ b/backend/src/db/migrations/20251018061215_sub-org.ts @@ -1,9 +1,9 @@ import { Knex } from "knex"; import { dropConstraintIfExists } from "@app/db/migrations/utils/dropConstraintIfExists"; +import { chunkArray } from "@app/lib/fn"; import { AccessScope, TableName } from "../schemas"; -import { chunkArray } from "@app/lib/fn"; export async function up(knex: Knex): Promise { const hasParentOrgId = await knex.schema.hasColumn(TableName.Organization, "parentOrgId"); From 619598c6c3221a6b77afe0ae53b257c63b15b360 Mon Sep 17 00:00:00 2001 From: = Date: Fri, 24 Oct 2025 20:37:52 +0530 Subject: [PATCH 3/8] feat: seperated into two migration --- .../db/migrations/20251018061215_sub-org.ts | 42 +-------------- ...0251024150349_sub-org-identity-backfill.ts | 51 +++++++++++++++++++ 2 files changed, 52 insertions(+), 41 deletions(-) create mode 100644 backend/src/db/migrations/20251024150349_sub-org-identity-backfill.ts diff --git a/backend/src/db/migrations/20251018061215_sub-org.ts b/backend/src/db/migrations/20251018061215_sub-org.ts index 3254a4aca..089aeef90 100644 --- a/backend/src/db/migrations/20251018061215_sub-org.ts +++ b/backend/src/db/migrations/20251018061215_sub-org.ts @@ -1,9 +1,8 @@ import { Knex } from "knex"; import { dropConstraintIfExists } from "@app/db/migrations/utils/dropConstraintIfExists"; -import { chunkArray } from "@app/lib/fn"; -import { AccessScope, TableName } from "../schemas"; +import { TableName } from "../schemas"; export async function up(knex: Knex): Promise { const hasParentOrgId = await knex.schema.hasColumn(TableName.Organization, "parentOrgId"); @@ -19,8 +18,6 @@ export async function up(knex: Knex): Promise { await dropConstraintIfExists(TableName.Organization, "organizations_slug_unique", knex); t.unique(["rootOrgId", "parentOrgId", "slug"]); }); - - // had to switch to raw for null not distinct } const hasIdentityOrgCol = await knex.schema.hasColumn(TableName.Identity, "orgId"); @@ -29,43 +26,6 @@ export async function up(knex: Knex): Promise { t.uuid("orgId"); t.foreign("orgId").references("id").inTable(TableName.Organization).onDelete("CASCADE"); }); - - const identityMemberships = await knex(TableName.Membership) - .where({ - scope: AccessScope.Organization - }) - .whereNotNull("actorIdentityId") - .select("actorIdentityId", "scopeOrgId"); - - const identityToOrgMapping: Record = {}; - identityMemberships.forEach((el) => { - if (el.actorIdentityId) { - identityToOrgMapping[el.actorIdentityId] = el.scopeOrgId; - } - }); - - const batchMemberships = chunkArray(identityMemberships, 500); - for await (const membership of batchMemberships) { - const identityIds = membership.map((el) => el.actorIdentityId).filter(Boolean) as string[]; - if (identityIds.length) { - const identities = await knex(TableName.Identity).whereIn("id", identityIds).select("*"); - await knex(TableName.Identity) - .insert( - identities.map((el) => ({ - ...el, - orgId: identityToOrgMapping[el.id] - })) - ) - .onConflict("id") - .merge(); - } - } - - await knex(TableName.Identity).whereNull("orgId").delete(); - - await knex.schema.alterTable(TableName.Identity, (t) => { - t.uuid("orgId").notNullable().alter(); - }); } } diff --git a/backend/src/db/migrations/20251024150349_sub-org-identity-backfill.ts b/backend/src/db/migrations/20251024150349_sub-org-identity-backfill.ts new file mode 100644 index 000000000..12aa38edc --- /dev/null +++ b/backend/src/db/migrations/20251024150349_sub-org-identity-backfill.ts @@ -0,0 +1,51 @@ +import { Knex } from "knex"; + +import { chunkArray } from "@app/lib/fn"; + +import { AccessScope, TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + const hasIdentityOrgCol = await knex.schema.hasColumn(TableName.Identity, "orgId"); + if (hasIdentityOrgCol) { + const identityMemberships = await knex(TableName.Membership) + .where({ + scope: AccessScope.Organization + }) + .whereNotNull("actorIdentityId") + .select("actorIdentityId", "scopeOrgId"); + + const identityToOrgMapping: Record = {}; + identityMemberships.forEach((el) => { + if (el.actorIdentityId) { + identityToOrgMapping[el.actorIdentityId] = el.scopeOrgId; + } + }); + + const batchMemberships = chunkArray(identityMemberships, 500); + for await (const membership of batchMemberships) { + const identityIds = membership.map((el) => el.actorIdentityId).filter(Boolean) as string[]; + if (identityIds.length) { + const identities = await knex(TableName.Identity).whereIn("id", identityIds).select("*"); + await knex(TableName.Identity) + .insert( + identities.map((el) => ({ + ...el, + orgId: identityToOrgMapping[el.id] + })) + ) + .onConflict("id") + .merge(); + } + } + + await knex(TableName.Identity).whereNull("orgId").delete(); + await knex.schema.alterTable(TableName.Identity, (t) => { + t.uuid("orgId").notNullable().alter(); + }); + } +} + +export async function down(): Promise {} + +const config = { transaction: false }; +export { config }; From 1d2708afe9dd49acd885fda02ae5e5b3aec2e559 Mon Sep 17 00:00:00 2001 From: = Date: Fri, 24 Oct 2025 20:39:13 +0530 Subject: [PATCH 4/8] feat: moved to close to sub org migration --- ...ty-backfill.ts => 20251019061215_sub-org-identity-backfill.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename backend/src/db/migrations/{20251024150349_sub-org-identity-backfill.ts => 20251019061215_sub-org-identity-backfill.ts} (100%) diff --git a/backend/src/db/migrations/20251024150349_sub-org-identity-backfill.ts b/backend/src/db/migrations/20251019061215_sub-org-identity-backfill.ts similarity index 100% rename from backend/src/db/migrations/20251024150349_sub-org-identity-backfill.ts rename to backend/src/db/migrations/20251019061215_sub-org-identity-backfill.ts From 555f900172858d5e593d11e15a6710967b71453d Mon Sep 17 00:00:00 2001 From: = Date: Fri, 24 Oct 2025 21:55:25 +0530 Subject: [PATCH 5/8] feat: added transaction --- ...0251019061215_sub-org-identity-backfill.ts | 70 ++++++++++--------- 1 file changed, 36 insertions(+), 34 deletions(-) diff --git a/backend/src/db/migrations/20251019061215_sub-org-identity-backfill.ts b/backend/src/db/migrations/20251019061215_sub-org-identity-backfill.ts index 12aa38edc..71e8924d4 100644 --- a/backend/src/db/migrations/20251019061215_sub-org-identity-backfill.ts +++ b/backend/src/db/migrations/20251019061215_sub-org-identity-backfill.ts @@ -5,44 +5,46 @@ import { chunkArray } from "@app/lib/fn"; import { AccessScope, TableName } from "../schemas"; export async function up(knex: Knex): Promise { - const hasIdentityOrgCol = await knex.schema.hasColumn(TableName.Identity, "orgId"); - if (hasIdentityOrgCol) { - const identityMemberships = await knex(TableName.Membership) - .where({ - scope: AccessScope.Organization - }) - .whereNotNull("actorIdentityId") - .select("actorIdentityId", "scopeOrgId"); + await knex.transaction(async (tx) => { + const hasIdentityOrgCol = await tx.schema.hasColumn(TableName.Identity, "orgId"); + if (hasIdentityOrgCol) { + const identityMemberships = await tx(TableName.Membership) + .where({ + scope: AccessScope.Organization + }) + .whereNotNull("actorIdentityId") + .select("actorIdentityId", "scopeOrgId"); - const identityToOrgMapping: Record = {}; - identityMemberships.forEach((el) => { - if (el.actorIdentityId) { - identityToOrgMapping[el.actorIdentityId] = el.scopeOrgId; - } - }); + const identityToOrgMapping: Record = {}; + identityMemberships.forEach((el) => { + if (el.actorIdentityId) { + identityToOrgMapping[el.actorIdentityId] = el.scopeOrgId; + } + }); - const batchMemberships = chunkArray(identityMemberships, 500); - for await (const membership of batchMemberships) { - const identityIds = membership.map((el) => el.actorIdentityId).filter(Boolean) as string[]; - if (identityIds.length) { - const identities = await knex(TableName.Identity).whereIn("id", identityIds).select("*"); - await knex(TableName.Identity) - .insert( - identities.map((el) => ({ - ...el, - orgId: identityToOrgMapping[el.id] - })) - ) - .onConflict("id") - .merge(); + const batchMemberships = chunkArray(identityMemberships, 500); + for await (const membership of batchMemberships) { + const identityIds = membership.map((el) => el.actorIdentityId).filter(Boolean) as string[]; + if (identityIds.length) { + const identities = await tx(TableName.Identity).whereIn("id", identityIds).select("*"); + await tx(TableName.Identity) + .insert( + identities.map((el) => ({ + ...el, + orgId: identityToOrgMapping[el.id] + })) + ) + .onConflict("id") + .merge(); + } } + + await tx(TableName.Identity).whereNull("orgId").delete(); + await tx.schema.alterTable(TableName.Identity, (t) => { + t.uuid("orgId").notNullable().alter(); + }); } - - await knex(TableName.Identity).whereNull("orgId").delete(); - await knex.schema.alterTable(TableName.Identity, (t) => { - t.uuid("orgId").notNullable().alter(); - }); - } + }); } export async function down(): Promise {} From 71972f9ffb6e73c3af055c314fd6e0eace5c03d2 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Wed, 29 Oct 2025 00:15:07 +0800 Subject: [PATCH 6/8] misc: separate migration steps for remvoing fk and orphaned identities --- ...0251019061215_sub-org-identity-backfill.ts | 5 ----- ...-access-token-remove-fk-for-identity-id.ts | 22 +++++++++++++++++++ ...20251028160921_delete-no-org-identities.ts | 20 +++++++++++++++++ 3 files changed, 42 insertions(+), 5 deletions(-) create mode 100644 backend/src/db/migrations/20251028155708_identity-access-token-remove-fk-for-identity-id.ts create mode 100644 backend/src/db/migrations/20251028160921_delete-no-org-identities.ts diff --git a/backend/src/db/migrations/20251019061215_sub-org-identity-backfill.ts b/backend/src/db/migrations/20251019061215_sub-org-identity-backfill.ts index 71e8924d4..9054a03ee 100644 --- a/backend/src/db/migrations/20251019061215_sub-org-identity-backfill.ts +++ b/backend/src/db/migrations/20251019061215_sub-org-identity-backfill.ts @@ -38,11 +38,6 @@ export async function up(knex: Knex): Promise { .merge(); } } - - await tx(TableName.Identity).whereNull("orgId").delete(); - await tx.schema.alterTable(TableName.Identity, (t) => { - t.uuid("orgId").notNullable().alter(); - }); } }); } diff --git a/backend/src/db/migrations/20251028155708_identity-access-token-remove-fk-for-identity-id.ts b/backend/src/db/migrations/20251028155708_identity-access-token-remove-fk-for-identity-id.ts new file mode 100644 index 000000000..0974f39c2 --- /dev/null +++ b/backend/src/db/migrations/20251028155708_identity-access-token-remove-fk-for-identity-id.ts @@ -0,0 +1,22 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + await knex.transaction(async (tx) => { + await tx.schema.alterTable(TableName.IdentityAccessToken, (table) => { + table.dropForeign("identityId"); + }); + }); +} + +export async function down(knex: Knex): Promise { + await knex.transaction(async (tx) => { + await tx.schema.alterTable(TableName.IdentityAccessToken, (table) => { + table.foreign("identityId").references("id").inTable(TableName.Identity); + }); + }); +} + +const config = { transaction: false }; +export { config }; diff --git a/backend/src/db/migrations/20251028160921_delete-no-org-identities.ts b/backend/src/db/migrations/20251028160921_delete-no-org-identities.ts new file mode 100644 index 000000000..bf4692cf0 --- /dev/null +++ b/backend/src/db/migrations/20251028160921_delete-no-org-identities.ts @@ -0,0 +1,20 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + await knex.transaction(async (tx) => { + const hasIdentityOrgCol = await tx.schema.hasColumn(TableName.Identity, "orgId"); + if (hasIdentityOrgCol) { + await tx(TableName.Identity).whereNull("orgId").delete(); + await tx.schema.alterTable(TableName.Identity, (t) => { + t.uuid("orgId").notNullable().alter(); + }); + } + }); +} + +export async function down(): Promise {} + +const config = { transaction: false }; +export { config }; From 394f1ae2bbb8c5d993ed933fa1b93f2b3fd7c1ac Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Wed, 29 Oct 2025 02:41:47 +0800 Subject: [PATCH 7/8] misc: updated statement timeout --- ...20251028160921_delete-no-org-identities.ts | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/backend/src/db/migrations/20251028160921_delete-no-org-identities.ts b/backend/src/db/migrations/20251028160921_delete-no-org-identities.ts index bf4692cf0..c38b62380 100644 --- a/backend/src/db/migrations/20251028160921_delete-no-org-identities.ts +++ b/backend/src/db/migrations/20251028160921_delete-no-org-identities.ts @@ -2,14 +2,24 @@ import { Knex } from "knex"; import { TableName } from "../schemas"; +const MIGRATION_TIMEOUT = 30 * 60 * 1000; // 30 minutes + export async function up(knex: Knex): Promise { + const result = await knex.raw("SHOW statement_timeout"); + const originalTimeout = result.rows[0].statement_timeout; + await knex.transaction(async (tx) => { - const hasIdentityOrgCol = await tx.schema.hasColumn(TableName.Identity, "orgId"); - if (hasIdentityOrgCol) { - await tx(TableName.Identity).whereNull("orgId").delete(); - await tx.schema.alterTable(TableName.Identity, (t) => { - t.uuid("orgId").notNullable().alter(); - }); + try { + await tx.raw(`SET statement_timeout = ${MIGRATION_TIMEOUT}`); + const hasIdentityOrgCol = await tx.schema.hasColumn(TableName.Identity, "orgId"); + if (hasIdentityOrgCol) { + await tx(TableName.Identity).whereNull("orgId").delete(); + await tx.schema.alterTable(TableName.Identity, (t) => { + t.uuid("orgId").notNullable().alter(); + }); + } + } finally { + await knex.raw(`SET statement_timeout = '${originalTimeout}'`); } }); } From 78510aee8ef6f851caff58c465be357646670050 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Wed, 29 Oct 2025 04:16:08 +0800 Subject: [PATCH 8/8] misc: used tx instead of knex --- .../db/migrations/20251028160921_delete-no-org-identities.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/db/migrations/20251028160921_delete-no-org-identities.ts b/backend/src/db/migrations/20251028160921_delete-no-org-identities.ts index c38b62380..0eb0e7ae0 100644 --- a/backend/src/db/migrations/20251028160921_delete-no-org-identities.ts +++ b/backend/src/db/migrations/20251028160921_delete-no-org-identities.ts @@ -19,7 +19,7 @@ export async function up(knex: Knex): Promise { }); } } finally { - await knex.raw(`SET statement_timeout = '${originalTimeout}'`); + await tx.raw(`SET statement_timeout = '${originalTimeout}'`); } }); }