From 9e314d7a09bf6badc6f7dc2e398ea27ada62fe18 Mon Sep 17 00:00:00 2001 From: = Date: Thu, 18 Jul 2024 14:19:23 +0530 Subject: [PATCH] feat: migration updated for secret v2 snapshot and secret approval --- backend/src/@types/knex.d.ts | 24 +++++++++ .../db/migrations/20240716105646_secret-v2.ts | 53 +++++++++++++++++++ backend/src/db/schemas/index.ts | 3 ++ backend/src/db/schemas/models.ts | 3 ++ .../secret-approval-request-secret-tags-v2.ts | 25 +++++++++ .../secret-approval-requests-secrets-v2.ts | 37 +++++++++++++ .../db/schemas/secret-snapshot-secrets-v2.ts | 23 ++++++++ backend/src/server/routes/index.ts | 8 +-- .../secret-v2-bridge/secret-v2-bridge-fns.ts | 2 +- .../secret-v2-bridge-service.ts | 4 +- 10 files changed, 175 insertions(+), 7 deletions(-) create mode 100644 backend/src/db/schemas/secret-approval-request-secret-tags-v2.ts create mode 100644 backend/src/db/schemas/secret-approval-requests-secrets-v2.ts create mode 100644 backend/src/db/schemas/secret-snapshot-secrets-v2.ts diff --git a/backend/src/@types/knex.d.ts b/backend/src/@types/knex.d.ts index be98cd212..6b77af1c5 100644 --- a/backend/src/@types/knex.d.ts +++ b/backend/src/@types/knex.d.ts @@ -204,6 +204,9 @@ import { TSecretApprovalRequestSecretTags, TSecretApprovalRequestSecretTagsInsert, TSecretApprovalRequestSecretTagsUpdate, + TSecretApprovalRequestSecretTagsV2, + TSecretApprovalRequestSecretTagsV2Insert, + TSecretApprovalRequestSecretTagsV2Update, TSecretApprovalRequestsInsert, TSecretApprovalRequestsReviewers, TSecretApprovalRequestsReviewersInsert, @@ -211,6 +214,9 @@ import { TSecretApprovalRequestsSecrets, TSecretApprovalRequestsSecretsInsert, TSecretApprovalRequestsSecretsUpdate, + TSecretApprovalRequestsSecretsV2, + TSecretApprovalRequestsSecretsV2Insert, + TSecretApprovalRequestsSecretsV2Update, TSecretApprovalRequestsUpdate, TSecretBlindIndexes, TSecretBlindIndexesInsert, @@ -251,6 +257,9 @@ import { TSecretSnapshotSecrets, TSecretSnapshotSecretsInsert, TSecretSnapshotSecretsUpdate, + TSecretSnapshotSecretsV2, + TSecretSnapshotSecretsV2Insert, + TSecretSnapshotSecretsV2Update, TSecretSnapshotsInsert, TSecretSnapshotsUpdate, TSecretsUpdate, @@ -694,6 +703,21 @@ declare module "knex/types/tables" { TSecretVersionV2TagJunctionInsert, TSecretVersionV2TagJunctionUpdate >; + [TableName.SnapshotSecretV2]: KnexOriginal.CompositeTableType< + TSecretSnapshotSecretsV2, + TSecretSnapshotSecretsV2Insert, + TSecretSnapshotSecretsV2Update + >; + [TableName.SecretApprovalRequestSecretV2]: KnexOriginal.CompositeTableType< + TSecretApprovalRequestsSecretsV2, + TSecretApprovalRequestsSecretsV2Insert, + TSecretApprovalRequestsSecretsV2Update + >; + [TableName.SecretApprovalRequestSecretTagV2]: KnexOriginal.CompositeTableType< + TSecretApprovalRequestSecretTagsV2, + TSecretApprovalRequestSecretTagsV2Insert, + TSecretApprovalRequestSecretTagsV2Update + >; // KMS service [TableName.KmsServerRootConfig]: KnexOriginal.CompositeTableType< TKmsRootConfig, diff --git a/backend/src/db/migrations/20240716105646_secret-v2.ts b/backend/src/db/migrations/20240716105646_secret-v2.ts index 630f4a61e..4aae70513 100644 --- a/backend/src/db/migrations/20240716105646_secret-v2.ts +++ b/backend/src/db/migrations/20240716105646_secret-v2.ts @@ -66,9 +66,62 @@ export async function up(knex: Knex): Promise { } await createJunctionTable(knex, TableName.SecretVersionV2Tag, TableName.SecretVersionV2, TableName.SecretTag); + + if (!(await knex.schema.hasTable(TableName.SecretApprovalRequestSecretV2))) { + await knex.schema.createTable(TableName.SecretApprovalRequestSecretV2, (t) => { + // everything related to secret + t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid()); + t.integer("version").defaultTo(1); + t.string("key", 500).notNullable(); + t.binary("encryptedValue"); + t.binary("encryptedComment"); + t.string("reminderNote"); + t.integer("reminderRepeatDays"); + t.boolean("skipMultilineEncoding").defaultTo(false); + t.jsonb("metadata"); + t.timestamps(true, true, true); + // commit details + t.uuid("requestId").notNullable(); + t.foreign("requestId").references("id").inTable(TableName.SecretApprovalRequest).onDelete("CASCADE"); + t.string("op").notNullable(); + t.uuid("secretId"); + t.foreign("secretId").references("id").inTable(TableName.SecretV2).onDelete("SET NULL"); + t.uuid("secretVersion"); + t.foreign("secretVersion").references("id").inTable(TableName.SecretVersionV2).onDelete("SET NULL"); + }); + } + + if (!(await knex.schema.hasTable(TableName.SecretApprovalRequestSecretTagV2))) { + await knex.schema.createTable(TableName.SecretApprovalRequestSecretTagV2, (t) => { + t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid()); + t.uuid("secretId").notNullable(); + t.foreign("secretId").references("id").inTable(TableName.SecretApprovalRequestSecretV2).onDelete("CASCADE"); + t.uuid("tagId").notNullable(); + t.foreign("tagId").references("id").inTable(TableName.SecretTag).onDelete("CASCADE"); + t.timestamps(true, true, true); + }); + } + + if (!(await knex.schema.hasTable(TableName.SnapshotSecretV2))) { + await knex.schema.createTable(TableName.SnapshotSecretV2, (t) => { + t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid()); + t.uuid("envId").notNullable(); + t.foreign("envId").references("id").inTable(TableName.Environment).onDelete("CASCADE"); + // not a relation kept like that to keep it when rolled back + t.uuid("secretVersionId").notNullable(); + t.foreign("secretVersionId").references("id").inTable(TableName.SecretVersionV2).onDelete("CASCADE"); + t.uuid("snapshotId").notNullable(); + t.foreign("snapshotId").references("id").inTable(TableName.Snapshot).onDelete("CASCADE"); + t.timestamps(true, true, true); + }); + } } export async function down(knex: Knex): Promise { + await knex.schema.dropTableIfExists(TableName.SnapshotSecretV2); + await knex.schema.dropTableIfExists(TableName.SecretApprovalRequestSecretTagV2); + await knex.schema.dropTableIfExists(TableName.SecretApprovalRequestSecretV2); + await knex.schema.dropTableIfExists(TableName.SecretV2JnTag); await knex.schema.dropTableIfExists(TableName.SecretReferenceV2); diff --git a/backend/src/db/schemas/index.ts b/backend/src/db/schemas/index.ts index f0d43b98f..d0aa56727 100644 --- a/backend/src/db/schemas/index.ts +++ b/backend/src/db/schemas/index.ts @@ -66,9 +66,11 @@ export * from "./scim-tokens"; export * from "./secret-approval-policies"; export * from "./secret-approval-policies-approvers"; export * from "./secret-approval-request-secret-tags"; +export * from "./secret-approval-request-secret-tags-v2"; export * from "./secret-approval-requests"; export * from "./secret-approval-requests-reviewers"; export * from "./secret-approval-requests-secrets"; +export * from "./secret-approval-requests-secrets-v2"; export * from "./secret-blind-indexes"; export * from "./secret-folder-versions"; export * from "./secret-folders"; @@ -81,6 +83,7 @@ export * from "./secret-scanning-git-risks"; export * from "./secret-sharing"; export * from "./secret-snapshot-folders"; export * from "./secret-snapshot-secrets"; +export * from "./secret-snapshot-secrets-v2"; export * from "./secret-snapshots"; export * from "./secret-tag-junction"; export * from "./secret-tags"; diff --git a/backend/src/db/schemas/models.ts b/backend/src/db/schemas/models.ts index 44d0cfd91..c4140eeda 100644 --- a/backend/src/db/schemas/models.ts +++ b/backend/src/db/schemas/models.ts @@ -93,6 +93,9 @@ export enum TableName { SecretV2 = "secrets_v2", SecretReferenceV2 = "secret_references_v2", SecretVersionV2 = "secret_versions_v2", + SecretApprovalRequestSecretV2 = "secret_approval_requests_secrets_v2", + SecretApprovalRequestSecretTagV2 = "secret_approval_request_secret_tags_v2", + SnapshotSecretV2 = "secret_snapshot_secrets_v2", // junction tables with tags SecretV2JnTag = "secret_v2_tag_junction", JnSecretTag = "secret_tag_junction", diff --git a/backend/src/db/schemas/secret-approval-request-secret-tags-v2.ts b/backend/src/db/schemas/secret-approval-request-secret-tags-v2.ts new file mode 100644 index 000000000..6ded05a5d --- /dev/null +++ b/backend/src/db/schemas/secret-approval-request-secret-tags-v2.ts @@ -0,0 +1,25 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const SecretApprovalRequestSecretTagsV2Schema = z.object({ + id: z.string().uuid(), + secretId: z.string().uuid(), + tagId: z.string().uuid(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TSecretApprovalRequestSecretTagsV2 = z.infer; +export type TSecretApprovalRequestSecretTagsV2Insert = Omit< + z.input, + TImmutableDBKeys +>; +export type TSecretApprovalRequestSecretTagsV2Update = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/secret-approval-requests-secrets-v2.ts b/backend/src/db/schemas/secret-approval-requests-secrets-v2.ts new file mode 100644 index 000000000..ee25ed6ef --- /dev/null +++ b/backend/src/db/schemas/secret-approval-requests-secrets-v2.ts @@ -0,0 +1,37 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { zodBuffer } from "@app/lib/zod"; + +import { TImmutableDBKeys } from "./models"; + +export const SecretApprovalRequestsSecretsV2Schema = z.object({ + id: z.string().uuid(), + version: z.number().default(1).nullable().optional(), + key: z.string(), + encryptedValue: zodBuffer.nullable().optional(), + encryptedComment: zodBuffer.nullable().optional(), + reminderNote: z.string().nullable().optional(), + reminderRepeatDays: z.number().nullable().optional(), + skipMultilineEncoding: z.boolean().default(false).nullable().optional(), + metadata: z.unknown().nullable().optional(), + createdAt: z.date(), + updatedAt: z.date(), + requestId: z.string().uuid(), + op: z.string(), + secretId: z.string().uuid().nullable().optional(), + secretVersion: z.string().uuid().nullable().optional() +}); + +export type TSecretApprovalRequestsSecretsV2 = z.infer; +export type TSecretApprovalRequestsSecretsV2Insert = Omit< + z.input, + TImmutableDBKeys +>; +export type TSecretApprovalRequestsSecretsV2Update = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/secret-snapshot-secrets-v2.ts b/backend/src/db/schemas/secret-snapshot-secrets-v2.ts new file mode 100644 index 000000000..d643f7f8c --- /dev/null +++ b/backend/src/db/schemas/secret-snapshot-secrets-v2.ts @@ -0,0 +1,23 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const SecretSnapshotSecretsV2Schema = z.object({ + id: z.string().uuid(), + envId: z.string().uuid(), + secretVersionId: z.string().uuid(), + snapshotId: z.string().uuid(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TSecretSnapshotSecretsV2 = z.infer; +export type TSecretSnapshotSecretsV2Insert = Omit, TImmutableDBKeys>; +export type TSecretSnapshotSecretsV2Update = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 625790da3..3e4f940f7 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -160,6 +160,10 @@ import { secretSharingDALFactory } from "@app/services/secret-sharing/secret-sha import { secretSharingServiceFactory } from "@app/services/secret-sharing/secret-sharing-service"; import { secretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; import { secretTagServiceFactory } from "@app/services/secret-tag/secret-tag-service"; +import { secretV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-dal"; +import { secretV2BridgeServiceFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-service"; +import { secretVersionV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-version-dal"; +import { secretVersionV2TagBridgeDALFactory } from "@app/services/secret-v2-bridge/secret-version-tag-dal"; import { serviceTokenDALFactory } from "@app/services/service-token/service-token-dal"; import { serviceTokenServiceFactory } from "@app/services/service-token/service-token-service"; import { TSmtpService } from "@app/services/smtp/smtp-service"; @@ -182,10 +186,6 @@ import { registerSecretScannerGhApp } from "../plugins/secret-scanner"; import { registerV1Routes } from "./v1"; import { registerV2Routes } from "./v2"; import { registerV3Routes } from "./v3"; -import { secretV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-dal"; -import { secretVersionV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-version-dal"; -import { secretVersionV2TagBridgeDALFactory } from "@app/services/secret-v2-bridge/secret-version-tag-dal"; -import { secretV2BridgeServiceFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-service"; export const registerRoutes = async ( server: FastifyZodProvider, diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts index 1dca5b28d..11e3c81c9 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts @@ -388,7 +388,7 @@ export const interpolateSecrets = ({ projectId, decryptSecret, secretDAL, folder const decryptedSec = secrets.reduce>((prev, secret) => { // eslint-disable-next-line - prev[secret.key] = decryptSecret(secret.encryptedValue); + prev[secret.key] = decryptSecret(secret.encryptedValue) || ""; return prev; }, {}); diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts index 4c7d18c81..793c8a4ad 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts @@ -6,12 +6,14 @@ import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services import { TSecretApprovalPolicyServiceFactory } from "@app/ee/services/secret-approval-policy/secret-approval-policy-service"; import { TSecretApprovalRequestDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-dal"; import { TSecretApprovalRequestSecretDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-secret-dal"; +import { TSecretSnapshotServiceFactory } from "@app/ee/services/secret-snapshot/secret-snapshot-service"; import { BadRequestError, NotFoundError } from "@app/lib/errors"; import { groupBy } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { ActorType } from "../auth/auth-type"; +import { TKmsServiceFactory } from "../kms/kms-service"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TSecretQueueFactory } from "../secret/secret-queue"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; @@ -44,8 +46,6 @@ import { } from "./secret-v2-bridge-types"; import { TSecretVersionV2DALFactory } from "./secret-version-dal"; import { TSecretVersionV2TagDALFactory } from "./secret-version-tag-dal"; -import { TKmsServiceFactory } from "../kms/kms-service"; -import { TSecretSnapshotServiceFactory } from "@app/ee/services/secret-snapshot/secret-snapshot-service"; type TSecretV2BridgeServiceFactoryDep = { secretDAL: TSecretV2BridgeDALFactory;