From 4d184003a854ed49c98bfa7e217421a3061d48b1 Mon Sep 17 00:00:00 2001 From: Akhil Mohan Date: Fri, 26 Jan 2024 12:18:28 +0530 Subject: [PATCH] feat(infisical-pg): made identity ua client sec backward compat --- .../20231228074908_identity-universal-auth.ts | 2 +- .../20231228075011_identity-access-token.ts | 2 +- .../src/db/schemas/identity-access-tokens.ts | 2 +- .../db/schemas/identity-ua-client-secrets.ts | 2 +- pg-migrator/src/index.ts | 100 ++++++++++-------- .../src/schemas/identity-access-tokens.ts | 2 +- .../src/schemas/identity-ua-client-secrets.ts | 15 ++- 7 files changed, 72 insertions(+), 53 deletions(-) diff --git a/backend-pg/src/db/migrations/20231228074908_identity-universal-auth.ts b/backend-pg/src/db/migrations/20231228074908_identity-universal-auth.ts index d594e0ca3..76d05db91 100644 --- a/backend-pg/src/db/migrations/20231228074908_identity-universal-auth.ts +++ b/backend-pg/src/db/migrations/20231228074908_identity-universal-auth.ts @@ -20,7 +20,7 @@ export async function up(knex: Knex): Promise { } if (!(await knex.schema.hasTable(TableName.IdentityUaClientSecret))) { await knex.schema.createTable(TableName.IdentityUaClientSecret, (t) => { - t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid()); + t.string("id", 36).primary().defaultTo(knex.fn.uuid()); t.string("description").notNullable(); t.string("clientSecretPrefix").notNullable(); t.string("clientSecretHash").notNullable(); diff --git a/backend-pg/src/db/migrations/20231228075011_identity-access-token.ts b/backend-pg/src/db/migrations/20231228075011_identity-access-token.ts index cd81f2960..78512ced4 100644 --- a/backend-pg/src/db/migrations/20231228075011_identity-access-token.ts +++ b/backend-pg/src/db/migrations/20231228075011_identity-access-token.ts @@ -14,7 +14,7 @@ export async function up(knex: Knex): Promise { t.datetime("accessTokenLastUsedAt"); t.datetime("accessTokenLastRenewedAt"); t.boolean("isAccessTokenRevoked").defaultTo(false).notNullable(); - t.uuid("identityUAClientSecretId"); + t.string("identityUAClientSecretId"); t.foreign("identityUAClientSecretId") .references("id") .inTable(TableName.IdentityUaClientSecret) diff --git a/backend-pg/src/db/schemas/identity-access-tokens.ts b/backend-pg/src/db/schemas/identity-access-tokens.ts index 934941c4c..62f74f4aa 100644 --- a/backend-pg/src/db/schemas/identity-access-tokens.ts +++ b/backend-pg/src/db/schemas/identity-access-tokens.ts @@ -16,7 +16,7 @@ export const IdentityAccessTokensSchema = z.object({ accessTokenLastUsedAt: z.date().nullable().optional(), accessTokenLastRenewedAt: z.date().nullable().optional(), isAccessTokenRevoked: z.boolean().default(false), - identityUAClientSecretId: z.string().uuid().nullable().optional(), + identityUAClientSecretId: z.string().nullable().optional(), identityId: z.string().uuid(), createdAt: z.date(), updatedAt: z.date(), diff --git a/backend-pg/src/db/schemas/identity-ua-client-secrets.ts b/backend-pg/src/db/schemas/identity-ua-client-secrets.ts index fc1a724fd..f17b1ed7f 100644 --- a/backend-pg/src/db/schemas/identity-ua-client-secrets.ts +++ b/backend-pg/src/db/schemas/identity-ua-client-secrets.ts @@ -8,7 +8,7 @@ import { z } from "zod"; import { TImmutableDBKeys } from "./models"; export const IdentityUaClientSecretsSchema = z.object({ - id: z.string().uuid(), + id: z.string(), description: z.string(), clientSecretPrefix: z.string(), clientSecretHash: z.string(), diff --git a/pg-migrator/src/index.ts b/pg-migrator/src/index.ts index 1e17c5a82..df4cd927f 100644 --- a/pg-migrator/src/index.ts +++ b/pg-migrator/src/index.ts @@ -180,7 +180,9 @@ export const migrateCollection = async < console.log("Total batches", Math.ceil(totalMongoCount / 1000)); let batch = 1; - for await (const doc of mongooseCollection.find(filter || {}).cursor({ batchSize: 100 })) { + for await (const doc of mongooseCollection + .find(filter || {}) + .cursor({ batchSize: 100 })) { mongooseDoc.push(doc); const preProcessedData = await preProcessing( doc.toObject({ virtuals: true }), @@ -234,7 +236,10 @@ export const migrateCollection = async < pgDoc.splice(0, pgDoc.length); } - migrationCheckPointsKv.put(`${postgresTableName}-${mongooseCollection.modelName}`, "done"); + migrationCheckPointsKv.put( + `${postgresTableName}-${mongooseCollection.modelName}`, + "done", + ); console.log( "Finished migration of ", @@ -282,7 +287,7 @@ const main = async () => { console.log("Starting rolling back to latest, comment this out later"); await db.migrate.rollback({}, true); - await kdb.clear(); + await kdb.clear(); console.log("Rolling back completed"); console.log("Executing migration"); @@ -585,7 +590,7 @@ const main = async () => { const getEnvId = async (workspace: string, environment: string) => { const envKv = envPKv.sublevel(workspace); - return envKv.get(environment) + return envKv.get(environment); }; const getFolderKv = (workspace: string, environment: string) => { const envKv = envPKv.sublevel(workspace); @@ -643,41 +648,40 @@ const main = async () => { : null; if (!orgId) return; - let results = []; - for (const env of doc.environments) { + let results = []; + for (const env of doc.environments) { const id = uuidV4(); - + // case: we forgot to clean up folders that belong to deleted env slugs const isEnvFound = await getEnvId( doc._id.toString(), truncateAndSlugify(env.slug), ).catch(() => null); - + if (!isEnvFound) continue; - + const envId = await getEnvId( - doc._id.toString(), - truncateAndSlugify(env.slug), + doc._id.toString(), + truncateAndSlugify(env.slug), ); - + const folderKv = getFolderKv( - doc._id.toString(), - truncateAndSlugify(env.slug), + doc._id.toString(), + truncateAndSlugify(env.slug), ); - + await folderKv.put("root", id); results.push({ - id, - name: "root", - envId, - version: 1, - createdAt: new Date(), - updatedAt: new Date(), + id, + name: "root", + envId, + version: 1, + createdAt: new Date(), + updatedAt: new Date(), }); - } - - return results; + } + return results; }, }); @@ -825,9 +829,14 @@ const main = async () => { if (folder.id !== "root") { const { name, version } = folder; const id = uuidV4(); - await folderKv.put(folder.id, id); - const parentId = folder?.parentId ? await folderKv.get(folder?.parentId).catch((e) => {console.log("parent folder not found==>", folder); throw e;}) : null; - + await folderKv.put(folder.id, id); + const parentId = folder?.parentId + ? await folderKv.get(folder?.parentId).catch((e) => { + console.log("parent folder not found==>", folder); + throw e; + }) + : null; + pgFolder.push({ name, version, @@ -869,7 +878,7 @@ const main = async () => { truncateAndSlugify(doc.environment), ).catch(() => null); if (!isEnvFound) return; - + envId = await getEnvId( doc.workspace.toString(), truncateAndSlugify(doc.environment), @@ -1084,17 +1093,19 @@ const main = async () => { // Case: if folder id doesn't exist and the root of the folder also doesn;t exist, THEN put the secret at the ROOT // case: after deleting a folder, we don't clean up the secrets that link to that folder - const folderId = await folderKv.get(doc.folder || "root").catch(() => null); - - // case: when environments are renamed, there used to be a TIMEE when the related folder's slugs weren't updated with it... :( - if (!folderId) return + const folderId = await folderKv + .get(doc.folder || "root") + .catch(() => null); - // issue with personal + // case: when environments are renamed, there used to be a TIMEE when the related folder's slugs weren't updated with it... :( + if (!folderId) return; + + // issue with personal const userId = doc.user ? await userKv.get(doc.user.toString()).catch(() => null) : null; - if ( doc.type === "personal" && !userId) return; + if (doc.type === "personal" && !userId) return; const id = uuidV4(); await secKv.put(doc._id.toString(), id); @@ -1291,8 +1302,10 @@ const main = async () => { if (!projectKvRes) return; // if we try to process two bots for the same workspace, then skip - if (await projectBotKv.get(doc.workspace.toString()).catch(() => null)){ - return + if ( + await projectBotKv.get(doc.workspace.toString()).catch(() => null) + ) { + return; } await projectBotKv.put(doc.workspace.toString(), id); @@ -1304,11 +1317,11 @@ const main = async () => { }) .sort({ isActive: -1 }) .lean(); - - // case: when no bots are found for this project, skip - if (!bot) return - const botKey = await BotKey.findOne({bot: bot?._id}) + // case: when no bots are found for this project, skip + if (!bot) return; + + const botKey = await BotKey.findOne({ bot: bot?._id }); const senderId = botKey?.sender ? await userKv.get(botKey.sender.toString()).catch(() => null) @@ -1554,13 +1567,12 @@ const main = async () => { postgresTableName: TableName.IdentityUaClientSecret, returnKeys: ["id"], preProcessing: async (doc) => { - const id = uuidV4(); const identityUAId = await identityUaKv.get( doc.identityUniversalAuth.toString(), ); - await identityUaClientSecKv.put(doc._id.toString(), id); + await identityUaClientSecKv.put(doc._id.toString(), doc._id.toString()); return { - id, + id: doc._id.toString(), identityUAId, description: doc.description, clientSecretTTL: doc.clientSecretTTL, @@ -2348,7 +2360,7 @@ const main = async () => { // }, // }); - console.log("MIGRATION SCRIPT COMPLETED SUCCESSFULLY") + console.log("MIGRATION SCRIPT COMPLETED SUCCESSFULLY"); process.exit(1); } catch (error) { console.error(error); diff --git a/pg-migrator/src/schemas/identity-access-tokens.ts b/pg-migrator/src/schemas/identity-access-tokens.ts index a7a685cf9..aa86e0356 100644 --- a/pg-migrator/src/schemas/identity-access-tokens.ts +++ b/pg-migrator/src/schemas/identity-access-tokens.ts @@ -16,7 +16,7 @@ export const IdentityAccessTokensSchema = z.object({ accessTokenLastUsedAt: z.date().nullable().optional(), accessTokenLastRenewedAt: z.date().nullable().optional(), isAccessTokenRevoked: z.boolean().default(false), - identityUAClientSecretId: z.string().uuid().nullable().optional(), + identityUAClientSecretId: z.string().nullable().optional(), identityId: z.string().uuid(), createdAt: z.date(), updatedAt: z.date(), diff --git a/pg-migrator/src/schemas/identity-ua-client-secrets.ts b/pg-migrator/src/schemas/identity-ua-client-secrets.ts index 46b22c496..a7b7cb9a8 100644 --- a/pg-migrator/src/schemas/identity-ua-client-secrets.ts +++ b/pg-migrator/src/schemas/identity-ua-client-secrets.ts @@ -8,7 +8,7 @@ import { z } from "zod"; import { TImmutableDBKeys } from "./models"; export const IdentityUaClientSecretsSchema = z.object({ - id: z.string().uuid(), + id: z.string(), description: z.string(), clientSecretPrefix: z.string(), clientSecretHash: z.string(), @@ -22,6 +22,13 @@ export const IdentityUaClientSecretsSchema = z.object({ identityUAId: z.string().uuid(), }); -export type TIdentityUaClientSecrets = z.infer; -export type TIdentityUaClientSecretsInsert = Omit; -export type TIdentityUaClientSecretsUpdate = Partial>; +export type TIdentityUaClientSecrets = z.infer< + typeof IdentityUaClientSecretsSchema +>; +export type TIdentityUaClientSecretsInsert = Omit< + TIdentityUaClientSecrets, + TImmutableDBKeys +>; +export type TIdentityUaClientSecretsUpdate = Partial< + Omit +>;