diff --git a/backend-pg/src/@types/fastify.d.ts b/backend-pg/src/@types/fastify.d.ts index b90ade2ba..37f1616a7 100644 --- a/backend-pg/src/@types/fastify.d.ts +++ b/backend-pg/src/@types/fastify.d.ts @@ -5,6 +5,7 @@ import { TPermissionServiceFactory } from "@app/ee/services/permission/permissio import { TSecretApprovalPolicyServiceFactory } from "@app/ee/services/secret-approval-policy/secret-approval-policy-service"; import { TSecretApprovalRequestServiceFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-service"; import { TSecretRotationServiceFactory } from "@app/ee/services/secret-rotation/secret-rotation-service"; +import { TSecretSnapshotServiceFactory } from "@app/ee/services/secret-snapshot/secret-snapshot-service"; import { TApiKeyServiceFactory } from "@app/services/api-key/api-key-service"; import { TAuthLoginFactory } from "@app/services/auth/auth-login-service"; import { TAuthPasswordFactory } from "@app/services/auth/auth-password-service"; @@ -96,6 +97,7 @@ declare module "fastify" { secretApprovalPolicy: TSecretApprovalPolicyServiceFactory; secretApprovalRequest: TSecretApprovalRequestServiceFactory; secretRotation: TSecretRotationServiceFactory; + snapshot: TSecretSnapshotServiceFactory; }; // this is exclusive use for middlewares in which we need to inject data diff --git a/backend-pg/src/@types/knex.d.ts b/backend-pg/src/@types/knex.d.ts index e1174b0f5..7bc7c9eb0 100644 --- a/backend-pg/src/@types/knex.d.ts +++ b/backend-pg/src/@types/knex.d.ts @@ -91,6 +91,9 @@ import { TSecretFolders, TSecretFoldersInsert, TSecretFoldersUpdate, + TSecretFolderVersions, + TSecretFolderVersionsInsert, + TSecretFolderVersionsUpdate, TSecretImports, TSecretImportsInsert, TSecretImportsUpdate, @@ -102,6 +105,15 @@ import { TSecretRotationsUpdate, TSecrets, TSecretsInsert, + TSecretSnapshotFolders, + TSecretSnapshotFoldersInsert, + TSecretSnapshotFoldersUpdate, + TSecretSnapshots, + TSecretSnapshotSecrets, + TSecretSnapshotSecretsInsert, + TSecretSnapshotSecretsUpdate, + TSecretSnapshotsInsert, + TSecretSnapshotsUpdate, TSecretsUpdate, TSecretTagJunction, TSecretTagJunctionInsert, @@ -224,6 +236,11 @@ declare module "knex/types/tables" { TSecretFoldersInsert, TSecretFoldersUpdate >; + [TableName.SecretFolderVersion]: Knex.CompositeTableType< + TSecretFolderVersions, + TSecretFolderVersionsInsert, + TSecretFolderVersionsUpdate + >; [TableName.SecretTag]: Knex.CompositeTableType< TSecretTags, TSecretTagsInsert, @@ -234,6 +251,11 @@ declare module "knex/types/tables" { TSecretImportsInsert, TSecretImportsUpdate >; + [TableName.SecretSnapshot]: Knex.CompositeTableType< + TSecretSnapshots, + TSecretSnapshotsInsert, + TSecretSnapshotsUpdate + >; [TableName.Integration]: Knex.CompositeTableType< TIntegrations, TIntegrationsInsert, @@ -320,6 +342,21 @@ declare module "knex/types/tables" { TSecretRotationOutputsInsert, TSecretRotationOutputsUpdate >; + [TableName.Snapshot]: Knex.CompositeTableType< + TSecretSnapshots, + TSecretSnapshotsInsert, + TSecretSnapshotsUpdate + >; + [TableName.SnapshotSecret]: Knex.CompositeTableType< + TSecretSnapshotSecrets, + TSecretSnapshotSecretsInsert, + TSecretSnapshotSecretsUpdate + >; + [TableName.SnapshotFolder]: Knex.CompositeTableType< + TSecretSnapshotFolders, + TSecretSnapshotFoldersInsert, + TSecretSnapshotFoldersUpdate + >; // Junction tables [TableName.JnSecretTag]: Knex.CompositeTableType< TSecretTagJunction, diff --git a/backend-pg/src/db/migrations/20231218092441_secret-folder.ts b/backend-pg/src/db/migrations/20231218092441_secret-folder.ts index bf6c28d79..ba67e295b 100644 --- a/backend-pg/src/db/migrations/20231218092441_secret-folder.ts +++ b/backend-pg/src/db/migrations/20231218092441_secret-folder.ts @@ -17,9 +17,26 @@ export async function up(knex: Knex): Promise { }); } await createOnUpdateTrigger(knex, TableName.SecretFolder); + + if (!(await knex.schema.hasTable(TableName.SecretFolderVersion))) { + await knex.schema.createTable(TableName.SecretFolderVersion, (t) => { + t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid()); + t.string("name").notNullable(); + t.integer("version").defaultTo(1); + t.timestamps(true, true, true); + t.uuid("envId").notNullable(); + t.foreign("envId").references("id").inTable(TableName.Environment).onDelete("CASCADE"); + t.uuid("folderId").notNullable(); + // t.foreign("folderId").references("id").inTable(TableName.SecretFolder).onDelete("SET NULL"); + }); + } + + await createOnUpdateTrigger(knex, TableName.SecretFolderVersion); } export async function down(knex: Knex): Promise { + await knex.schema.dropTableIfExists(TableName.SecretFolderVersion); await knex.schema.dropTableIfExists(TableName.SecretFolder); await dropOnUpdateTrigger(knex, TableName.SecretFolder); + await dropOnUpdateTrigger(knex, TableName.SecretFolderVersion); } diff --git a/backend-pg/src/db/migrations/20231218103423_secret.ts b/backend-pg/src/db/migrations/20231218103423_secret.ts index 863f17646..c5bd76b44 100644 --- a/backend-pg/src/db/migrations/20231218103423_secret.ts +++ b/backend-pg/src/db/migrations/20231218103423_secret.ts @@ -37,7 +37,7 @@ export async function up(knex: Knex): Promise { t.text("secretCommentCiphertext"); t.text("secretCommentIV"); t.text("secretCommentTag"); - t.string("secretReminderNotice"); + t.string("secretReminderNote"); t.integer("secretReminderRepeatDays"); t.boolean("skipMultilineEncoding").defaultTo(false); t.string("algorithm").notNullable().defaultTo(SecretEncryptionAlgo.AES_256_GCM); diff --git a/backend-pg/src/db/migrations/20231220052508_secret-version.ts b/backend-pg/src/db/migrations/20231220052508_secret-version.ts index de079c1e9..7e9f97adf 100644 --- a/backend-pg/src/db/migrations/20231220052508_secret-version.ts +++ b/backend-pg/src/db/migrations/20231220052508_secret-version.ts @@ -9,9 +9,6 @@ export async function up(knex: Knex): Promise { t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid()); t.integer("version").defaultTo(1); t.string("type").notNullable().defaultTo(SecretType.Shared); - // t.text("secretKeyHash").notNullable(); - // t.text("secretValueHash"); - // t.text("secretCommentHash"); t.text("secretBlindIndex").notNullable(); t.text("secretKeyCiphertext").notNullable(); t.text("secretKeyIV").notNullable(); @@ -22,18 +19,20 @@ export async function up(knex: Knex): Promise { t.text("secretCommentCiphertext"); t.text("secretCommentIV"); t.text("secretCommentTag"); - t.string("secretReminderNotice"); + t.string("secretReminderNote"); t.integer("secretReminderRepeatDays"); t.boolean("skipMultilineEncoding").defaultTo(false); t.string("algorithm").notNullable().defaultTo(SecretEncryptionAlgo.AES_256_GCM); t.string("keyEncoding").notNullable().defaultTo(SecretKeyEncoding.UTF8); t.jsonb("metadata"); - t.uuid("secretId"); - t.foreign("secretId").references("id").inTable(TableName.Secret).onDelete("SET NULL"); + // to avoid orphan rows + t.uuid("envId"); + t.foreign("envId").references("id").inTable(TableName.Environment).onDelete("CASCADE"); + t.uuid("secretId").notNullable(); + t.uuid("folderId").notNullable(); + // t.foreign("secretId").references("id").inTable(TableName.Secret).onDelete("SET NULL"); t.uuid("userId"); t.foreign("userId").references("id").inTable(TableName.Users).onDelete("CASCADE"); - t.uuid("folderId").notNullable(); - t.foreign("folderId").references("id").inTable(TableName.SecretFolder).onDelete("CASCADE"); t.timestamps(true, true, true); }); } diff --git a/backend-pg/src/db/migrations/20240101104907_secret-approval-request.ts b/backend-pg/src/db/migrations/20240101104907_secret-approval-request.ts index d2eae9416..5fce143c3 100644 --- a/backend-pg/src/db/migrations/20240101104907_secret-approval-request.ts +++ b/backend-pg/src/db/migrations/20240101104907_secret-approval-request.ts @@ -64,7 +64,7 @@ export async function up(knex: Knex): Promise { t.text("secretCommentCiphertext"); t.text("secretCommentIV"); t.text("secretCommentTag"); - t.string("secretReminderNotice"); + t.string("secretReminderNote"); t.integer("secretReminderRepeatDays"); t.boolean("skipMultilineEncoding").defaultTo(false); t.string("algorithm").notNullable().defaultTo(SecretEncryptionAlgo.AES_256_GCM); @@ -103,7 +103,7 @@ export async function up(knex: Knex): Promise { } export async function down(knex: Knex): Promise { - await knex.schema.dropTableIfExists(TableName.SecretTag); + await knex.schema.dropTableIfExists(TableName.SarSecretTag); await knex.schema.dropTableIfExists(TableName.SarSecret); await knex.schema.dropTableIfExists(TableName.SarReviewer); await knex.schema.dropTableIfExists(TableName.SecretApprovalRequest); diff --git a/backend-pg/src/db/migrations/20240104140641_secret-snapshot.ts b/backend-pg/src/db/migrations/20240104140641_secret-snapshot.ts new file mode 100644 index 000000000..01b1e24fb --- /dev/null +++ b/backend-pg/src/db/migrations/20240104140641_secret-snapshot.ts @@ -0,0 +1,61 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; +import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils"; + +export async function up(knex: Knex): Promise { + if (!(await knex.schema.hasTable(TableName.Snapshot))) { + await knex.schema.createTable(TableName.Snapshot, (t) => { + t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid()); + t.uuid("envId").notNullable(); + t.foreign("envId").references("id").inTable(TableName.Environment).onDelete("CASCADE"); + // this is not a relation kept like that + // this ensure snapshot are not lost when folder gets deleted and rolled back + t.uuid("folderId").notNullable(); + t.uuid("parentFolderId"); + t.timestamps(true, true, true); + }); + } + await createOnUpdateTrigger(knex, TableName.Snapshot); + + if (!(await knex.schema.hasTable(TableName.SnapshotSecret))) { + await knex.schema.createTable(TableName.SnapshotSecret, (t) => { + t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid()); + t.uuid("envId").notNullable(); + t.foreign("envId").references("id").inTable(TableName.Environment).onDelete("CASCADE"); + // not a relation kept like that to keep it when rolled back + t.uuid("secretVersionId").notNullable(); + t.foreign("secretVersionId") + .references("id") + .inTable(TableName.SecretVersion) + .onDelete("CASCADE"); + t.uuid("snapshotId").notNullable(); + t.foreign("snapshotId").references("id").inTable(TableName.Snapshot).onDelete("CASCADE"); + t.timestamps(true, true, true); + }); + } + + if (!(await knex.schema.hasTable(TableName.SnapshotFolder))) { + await knex.schema.createTable(TableName.SnapshotFolder, (t) => { + t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid()); + t.uuid("envId").notNullable(); + t.foreign("envId").references("id").inTable(TableName.Environment).onDelete("CASCADE"); + // not a relation kept like that to keep it when rolled back + t.uuid("folderVersionId").notNullable(); + t.foreign("folderVersionId") + .references("id") + .inTable(TableName.SecretFolderVersion) + .onDelete("CASCADE"); + t.uuid("snapshotId").notNullable(); + t.foreign("snapshotId").references("id").inTable(TableName.Snapshot).onDelete("CASCADE"); + t.timestamps(true, true, true); + }); + } +} + +export async function down(knex: Knex): Promise { + await knex.schema.dropTableIfExists(TableName.SnapshotSecret); + await knex.schema.dropTableIfExists(TableName.SnapshotFolder); + await knex.schema.dropTableIfExists(TableName.Snapshot); + await dropOnUpdateTrigger(knex, TableName.Snapshot); +} diff --git a/backend-pg/src/db/schemas/identity-ua-client-secrets.ts b/backend-pg/src/db/schemas/identity-ua-client-secrets.ts index dc3a5e9ec..46b22c496 100644 --- a/backend-pg/src/db/schemas/identity-ua-client-secrets.ts +++ b/backend-pg/src/db/schemas/identity-ua-client-secrets.ts @@ -9,7 +9,7 @@ import { TImmutableDBKeys } from "./models"; export const IdentityUaClientSecretsSchema = z.object({ id: z.string().uuid(), - description: z.string().nullable().optional(), + description: z.string(), clientSecretPrefix: z.string(), clientSecretHash: z.string(), clientSecretLastUsedAt: z.date().nullable().optional(), @@ -19,11 +19,9 @@ export const IdentityUaClientSecretsSchema = z.object({ isClientSecretRevoked: z.boolean().default(false), createdAt: z.date(), updatedAt: z.date(), - identityUAId: z.string().uuid() + identityUAId: z.string().uuid(), }); export type TIdentityUaClientSecrets = z.infer; export type TIdentityUaClientSecretsInsert = Omit; -export type TIdentityUaClientSecretsUpdate = Partial< - Omit ->; +export type TIdentityUaClientSecretsUpdate = Partial>; diff --git a/backend-pg/src/db/schemas/identity-universal-auths.ts b/backend-pg/src/db/schemas/identity-universal-auths.ts index 41398f143..82b9fb23e 100644 --- a/backend-pg/src/db/schemas/identity-universal-auths.ts +++ b/backend-pg/src/db/schemas/identity-universal-auths.ts @@ -17,11 +17,9 @@ export const IdentityUniversalAuthsSchema = z.object({ accessTokenTrustedIps: z.unknown(), createdAt: z.date(), updatedAt: z.date(), - identityId: z.string().uuid() + identityId: z.string().uuid(), }); export type TIdentityUniversalAuths = z.infer; export type TIdentityUniversalAuthsInsert = Omit; -export type TIdentityUniversalAuthsUpdate = Partial< - Omit ->; +export type TIdentityUniversalAuthsUpdate = Partial>; diff --git a/backend-pg/src/db/schemas/index.ts b/backend-pg/src/db/schemas/index.ts index 2f04438dc..8ff1abe3f 100644 --- a/backend-pg/src/db/schemas/index.ts +++ b/backend-pg/src/db/schemas/index.ts @@ -28,10 +28,14 @@ export * from "./sar-reviewers"; export * from "./secret-approval-policies"; export * from "./secret-approval-requests"; export * from "./secret-blind-indexes"; +export * from "./secret-folder-versions"; export * from "./secret-folders"; export * from "./secret-imports"; export * from "./secret-rotation-outputs"; export * from "./secret-rotations"; +export * from "./secret-snapshot-folders"; +export * from "./secret-snapshot-secrets"; +export * from "./secret-snapshots"; export * from "./secret-tag-junction"; export * from "./secret-tags"; export * from "./secret-versions"; diff --git a/backend-pg/src/db/schemas/knex-migrations-lock.ts b/backend-pg/src/db/schemas/knex-migrations-lock.ts new file mode 100644 index 000000000..63b261fec --- /dev/null +++ b/backend-pg/src/db/schemas/knex-migrations-lock.ts @@ -0,0 +1,17 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const KnexMigrationsLockSchema = z.object({ + index: z.number(), + is_locked: z.number().nullable().optional(), +}); + +export type TKnexMigrationsLock = z.infer; +export type TKnexMigrationsLockInsert = Omit; +export type TKnexMigrationsLockUpdate = Partial>; diff --git a/backend-pg/src/db/schemas/knex-migrations.ts b/backend-pg/src/db/schemas/knex-migrations.ts new file mode 100644 index 000000000..ba15c2d38 --- /dev/null +++ b/backend-pg/src/db/schemas/knex-migrations.ts @@ -0,0 +1,19 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const KnexMigrationsSchema = z.object({ + id: z.number(), + name: z.string().nullable().optional(), + batch: z.number().nullable().optional(), + migration_time: z.date().nullable().optional(), +}); + +export type TKnexMigrations = z.infer; +export type TKnexMigrationsInsert = Omit; +export type TKnexMigrationsUpdate = Partial>; diff --git a/backend-pg/src/db/schemas/models.ts b/backend-pg/src/db/schemas/models.ts index 9c1d0cd16..f68d2eb1b 100644 --- a/backend-pg/src/db/schemas/models.ts +++ b/backend-pg/src/db/schemas/models.ts @@ -23,7 +23,11 @@ export enum TableName { SecretBlindIndex = "secret_blind_indexes", SecretVersion = "secret_versions", SecretFolder = "secret_folders", + SecretFolderVersion = "secret_folder_versions", SecretImport = "secret_imports", + Snapshot = "secret_snapshots", + SnapshotSecret = "secret_snapshot_secrets", + SnapshotFolder = "secret_snapshot_folders", SecretTag = "secret_tags", Integration = "integrations", IntegrationAuth = "integration_auths", diff --git a/backend-pg/src/db/schemas/project-bots.ts b/backend-pg/src/db/schemas/project-bots.ts index bd5e91a1c..b898be329 100644 --- a/backend-pg/src/db/schemas/project-bots.ts +++ b/backend-pg/src/db/schemas/project-bots.ts @@ -17,12 +17,12 @@ export const ProjectBotsSchema = z.object({ tag: z.string(), algorithm: z.string(), keyEncoding: z.string(), - encryptedProjectKey: z.string().optional().nullable(), - encryptedProjectKeyNonce: z.string().optional().nullable(), + encryptedProjectKey: z.string().nullable().optional(), + encryptedProjectKeyNonce: z.string().nullable().optional(), projectId: z.string().uuid(), - senderId: z.string().uuid().optional().nullable(), + senderId: z.string().uuid().nullable().optional(), createdAt: z.date(), - updatedAt: z.date() + updatedAt: z.date(), }); export type TProjectBots = z.infer; diff --git a/backend-pg/src/db/schemas/project-keys.ts b/backend-pg/src/db/schemas/project-keys.ts index 8738ba605..30ff5ce98 100644 --- a/backend-pg/src/db/schemas/project-keys.ts +++ b/backend-pg/src/db/schemas/project-keys.ts @@ -12,7 +12,7 @@ export const ProjectKeysSchema = z.object({ encryptedKey: z.string(), nonce: z.string(), receiverId: z.string().uuid(), - senderId: z.string().uuid(), + senderId: z.string().uuid().nullable().optional(), projectId: z.string().uuid(), createdAt: z.date(), updatedAt: z.date(), diff --git a/backend-pg/src/db/schemas/sa-request-secrets.ts b/backend-pg/src/db/schemas/sa-request-secrets.ts index 6cbaeca0a..7419eefe2 100644 --- a/backend-pg/src/db/schemas/sa-request-secrets.ts +++ b/backend-pg/src/db/schemas/sa-request-secrets.ts @@ -20,7 +20,7 @@ export const SaRequestSecretsSchema = z.object({ secretCommentCiphertext: z.string().nullable().optional(), secretCommentIV: z.string().nullable().optional(), secretCommentTag: z.string().nullable().optional(), - secretReminderNotice: z.string().nullable().optional(), + secretReminderNote: z.string().nullable().optional(), secretReminderRepeatDays: z.number().nullable().optional(), skipMultilineEncoding: z.boolean().default(false).nullable().optional(), algorithm: z.string().default("aes-256-gcm"), diff --git a/backend-pg/src/db/schemas/secret-approval-requests.ts b/backend-pg/src/db/schemas/secret-approval-requests.ts index 32f390ba7..7d5f2f443 100644 --- a/backend-pg/src/db/schemas/secret-approval-requests.ts +++ b/backend-pg/src/db/schemas/secret-approval-requests.ts @@ -11,18 +11,16 @@ export const SecretApprovalRequestsSchema = z.object({ id: z.string().uuid(), policyId: z.string().uuid(), hasMerged: z.boolean().default(false), - status: z.string().default("open"), + status: z.string().default('open'), conflicts: z.unknown().nullable().optional(), slug: z.string(), folderId: z.string().uuid(), - statusChangeBy: z.string().uuid().optional().nullable(), + statusChangeBy: z.string().uuid().nullable().optional(), committerId: z.string().uuid(), createdAt: z.date(), - updatedAt: z.date() + updatedAt: z.date(), }); export type TSecretApprovalRequests = z.infer; export type TSecretApprovalRequestsInsert = Omit; -export type TSecretApprovalRequestsUpdate = Partial< - Omit ->; +export type TSecretApprovalRequestsUpdate = Partial>; diff --git a/backend-pg/src/db/schemas/secret-folder-versions.ts b/backend-pg/src/db/schemas/secret-folder-versions.ts new file mode 100644 index 000000000..895f81ebc --- /dev/null +++ b/backend-pg/src/db/schemas/secret-folder-versions.ts @@ -0,0 +1,22 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const SecretFolderVersionsSchema = z.object({ + id: z.string().uuid(), + name: z.string(), + version: z.number().default(1).nullable().optional(), + createdAt: z.date(), + updatedAt: z.date(), + envId: z.string().uuid(), + folderId: z.string().uuid(), +}); + +export type TSecretFolderVersions = z.infer; +export type TSecretFolderVersionsInsert = Omit; +export type TSecretFolderVersionsUpdate = Partial>; diff --git a/backend-pg/src/db/schemas/secret-snapshot-folders.ts b/backend-pg/src/db/schemas/secret-snapshot-folders.ts new file mode 100644 index 000000000..5f3b1a88c --- /dev/null +++ b/backend-pg/src/db/schemas/secret-snapshot-folders.ts @@ -0,0 +1,21 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const SecretSnapshotFoldersSchema = z.object({ + id: z.string().uuid(), + envId: z.string().uuid(), + folderVersionId: z.string().uuid(), + snapshotId: z.string().uuid(), + createdAt: z.date(), + updatedAt: z.date(), +}); + +export type TSecretSnapshotFolders = z.infer; +export type TSecretSnapshotFoldersInsert = Omit; +export type TSecretSnapshotFoldersUpdate = Partial>; diff --git a/backend-pg/src/db/schemas/secret-snapshot-secrets.ts b/backend-pg/src/db/schemas/secret-snapshot-secrets.ts new file mode 100644 index 000000000..f8a69a695 --- /dev/null +++ b/backend-pg/src/db/schemas/secret-snapshot-secrets.ts @@ -0,0 +1,21 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const SecretSnapshotSecretsSchema = z.object({ + id: z.string().uuid(), + envId: z.string().uuid(), + secretVersionId: z.string().uuid(), + snapshotId: z.string().uuid(), + createdAt: z.date(), + updatedAt: z.date(), +}); + +export type TSecretSnapshotSecrets = z.infer; +export type TSecretSnapshotSecretsInsert = Omit; +export type TSecretSnapshotSecretsUpdate = Partial>; diff --git a/backend-pg/src/db/schemas/secret-snapshots.ts b/backend-pg/src/db/schemas/secret-snapshots.ts new file mode 100644 index 000000000..ef9e0b7d0 --- /dev/null +++ b/backend-pg/src/db/schemas/secret-snapshots.ts @@ -0,0 +1,21 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const SecretSnapshotsSchema = z.object({ + id: z.string().uuid(), + envId: z.string().uuid(), + folderId: z.string().uuid(), + parentFolderId: z.string().uuid().nullable().optional(), + createdAt: z.date(), + updatedAt: z.date(), +}); + +export type TSecretSnapshots = z.infer; +export type TSecretSnapshotsInsert = Omit; +export type TSecretSnapshotsUpdate = Partial>; diff --git a/backend-pg/src/db/schemas/secret-version-tag-junction.ts b/backend-pg/src/db/schemas/secret-version-tag-junction.ts new file mode 100644 index 000000000..11b3f0032 --- /dev/null +++ b/backend-pg/src/db/schemas/secret-version-tag-junction.ts @@ -0,0 +1,18 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const SecretVersionTagJunctionSchema = z.object({ + id: z.string().uuid(), + secret_versionsId: z.string().uuid(), + secret_tagsId: z.string().uuid(), +}); + +export type TSecretVersionTagJunction = z.infer; +export type TSecretVersionTagJunctionInsert = Omit; +export type TSecretVersionTagJunctionUpdate = Partial>; diff --git a/backend-pg/src/db/schemas/secret-versions.ts b/backend-pg/src/db/schemas/secret-versions.ts index db9cdcb6f..ee64861fc 100644 --- a/backend-pg/src/db/schemas/secret-versions.ts +++ b/backend-pg/src/db/schemas/secret-versions.ts @@ -10,7 +10,7 @@ import { TImmutableDBKeys } from "./models"; export const SecretVersionsSchema = z.object({ id: z.string().uuid(), version: z.number().default(1).nullable().optional(), - type: z.string().default('shared'), + type: z.string().default("shared"), secretBlindIndex: z.string(), secretKeyCiphertext: z.string(), secretKeyIV: z.string(), @@ -21,17 +21,18 @@ export const SecretVersionsSchema = z.object({ secretCommentCiphertext: z.string().nullable().optional(), secretCommentIV: z.string().nullable().optional(), secretCommentTag: z.string().nullable().optional(), - secretReminderNotice: z.string().nullable().optional(), + secretReminderNote: z.string().nullable().optional(), secretReminderRepeatDays: z.number().nullable().optional(), skipMultilineEncoding: z.boolean().default(false).nullable().optional(), - algorithm: z.string().default('aes-256-gcm'), - keyEncoding: z.string().default('utf8'), + algorithm: z.string().default("aes-256-gcm"), + keyEncoding: z.string().default("utf8"), metadata: z.unknown().nullable().optional(), + envId: z.string().uuid().nullable().optional(), secretId: z.string().uuid(), - userId: z.string().uuid().nullable().optional(), folderId: z.string().uuid(), + userId: z.string().uuid().nullable().optional(), createdAt: z.date(), - updatedAt: z.date(), + updatedAt: z.date() }); export type TSecretVersions = z.infer; diff --git a/backend-pg/src/db/schemas/secrets.ts b/backend-pg/src/db/schemas/secrets.ts index 73d72fcfa..18ae17a7e 100644 --- a/backend-pg/src/db/schemas/secrets.ts +++ b/backend-pg/src/db/schemas/secrets.ts @@ -10,7 +10,7 @@ import { TImmutableDBKeys } from "./models"; export const SecretsSchema = z.object({ id: z.string().uuid(), version: z.number().default(1).nullable().optional(), - type: z.string().default('shared'), + type: z.string().default("shared"), secretBlindIndex: z.string(), secretKeyCiphertext: z.string(), secretKeyIV: z.string(), @@ -21,16 +21,16 @@ export const SecretsSchema = z.object({ secretCommentCiphertext: z.string().nullable().optional(), secretCommentIV: z.string().nullable().optional(), secretCommentTag: z.string().nullable().optional(), - secretReminderNotice: z.string().nullable().optional(), + secretReminderNote: z.string().nullable().optional(), secretReminderRepeatDays: z.number().nullable().optional(), skipMultilineEncoding: z.boolean().default(false).nullable().optional(), - algorithm: z.string().default('aes-256-gcm'), - keyEncoding: z.string().default('utf8'), + algorithm: z.string().default("aes-256-gcm"), + keyEncoding: z.string().default("utf8"), metadata: z.unknown().nullable().optional(), userId: z.string().uuid().nullable().optional(), folderId: z.string().uuid(), createdAt: z.date(), - updatedAt: z.date(), + updatedAt: z.date() }); export type TSecrets = z.infer; diff --git a/backend-pg/src/ee/routes/v1/index.ts b/backend-pg/src/ee/routes/v1/index.ts index 5944382f5..6a4b22f2a 100644 --- a/backend-pg/src/ee/routes/v1/index.ts +++ b/backend-pg/src/ee/routes/v1/index.ts @@ -1,14 +1,23 @@ import { registerOrgRoleRouter } from "./org-role-router"; import { registerProjectRoleRouter } from "./project-role-router"; +import { registerProjectRouter } from "./project-router"; import { registerSecretApprovalPolicyRouter } from "./secret-approval-policy-router"; import { registerSecretApprovalRequestRouter } from "./secret-approval-request-router"; import { registerSecretRotationProviderRouter } from "./secret-rotation-provider-router"; import { registerSecretRotationRouter } from "./secret-rotation-router"; +import { registerSnapshotRouter } from "./snapshot-router"; export const registerV1EERoutes = async (server: FastifyZodProvider) => { // org role starts with organization await server.register(registerOrgRoleRouter, { prefix: "/organization" }); - await server.register(registerProjectRoleRouter, { prefix: "/workspace" }); + await server.register( + async (projectServer) => { + projectServer.register(registerProjectRoleRouter); + projectServer.register(registerProjectRouter); + }, + { prefix: "/workspace" } + ); + await server.register(registerSnapshotRouter, { prefix: "/secret-snapshot" }); await server.register(registerSecretApprovalPolicyRouter, { prefix: "/secret-approvals" }); await server.register(registerSecretApprovalRequestRouter, { prefix: "/secret-approval-requests" diff --git a/backend-pg/src/ee/routes/v1/project-router.ts b/backend-pg/src/ee/routes/v1/project-router.ts new file mode 100644 index 000000000..bda6cfff8 --- /dev/null +++ b/backend-pg/src/ee/routes/v1/project-router.ts @@ -0,0 +1,68 @@ +import { z } from "zod"; + +import { SecretSnapshotsSchema } from "@app/db/schemas"; +import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; +import { AuthMode } from "@app/services/auth/auth-type"; + +export const registerProjectRouter = async (server: FastifyZodProvider) => { + server.route({ + method: "GET", + url: "/:workspaceId/secret-snapshots", + schema: { + params: z.object({ + workspaceId: z.string().trim() + }), + querystring: z.object({ + environment: z.string().trim(), + path: z.string().trim().default("/"), + offset: z.coerce.number().default(0), + limit: z.coerce.number().default(20) + }), + response: { + 200: z.object({ + secretSnapshots: SecretSnapshotsSchema.array() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const secretSnapshots = await server.services.snapshot.listSnapshots({ + actor: req.permission.type, + actorId: req.permission.id, + projectId: req.params.workspaceId, + ...req.query + }); + return { secretSnapshots }; + } + }); + + server.route({ + method: "GET", + url: "/:workspaceId/secret-snapshots/count", + schema: { + params: z.object({ + workspaceId: z.string().trim() + }), + querystring: z.object({ + environment: z.string().trim(), + path: z.string().trim().default("/") + }), + response: { + 200: z.object({ + count: z.number() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const count = await server.services.snapshot.projectSecretSnapshotCount({ + actor: req.permission.type, + actorId: req.permission.id, + projectId: req.params.workspaceId, + environment: req.query.environment, + path: req.query.path + }); + return { count }; + } + }); +}; diff --git a/backend-pg/src/ee/routes/v1/snapshot-router.ts b/backend-pg/src/ee/routes/v1/snapshot-router.ts new file mode 100644 index 000000000..d905c7a4a --- /dev/null +++ b/backend-pg/src/ee/routes/v1/snapshot-router.ts @@ -0,0 +1,67 @@ +import { z } from "zod"; + +import { SecretSnapshotsSchema, SecretVersionsSchema } from "@app/db/schemas"; +import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; +import { AuthMode } from "@app/services/auth/auth-type"; + +export const registerSnapshotRouter = async (server: FastifyZodProvider) => { + server.route({ + method: "GET", + url: "/:secretSnapshotId", + schema: { + params: z.object({ + secretSnapshotId: z.string().trim() + }), + response: { + 200: z.object({ + secretSnapshot: z.object({ + id: z.string().uuid(), + projectId: z.string().uuid(), + environment: z.object({ + id: z.string().uuid(), + slug: z.string(), + name: z.string() + }), + secretVersions: SecretVersionsSchema.omit({ secretBlindIndex: true }).array(), + folderVersion: z.object({ id: z.string(), name: z.string() }).array(), + createdAt: z.date(), + updatedAt: z.date() + }) + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const secretSnapshot = await server.services.snapshot.getSnapshotData({ + actor: req.permission.type, + actorId: req.permission.id, + id: req.params.secretSnapshotId + }); + return { secretSnapshot }; + } + }); + + server.route({ + method: "POST", + url: "/:secretSnapshotId/rollback", + schema: { + params: z.object({ + secretSnapshotId: z.string().trim() + }), + response: { + 200: z.object({ + secretSnapshot: SecretSnapshotsSchema + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const secretSnapshot = await server.services.snapshot.rollbackSnapshot({ + actor: req.permission.type, + actorId: req.permission.id, + id: req.params.secretSnapshotId + }); + return { secretSnapshot }; + } + }); +}; diff --git a/backend-pg/src/ee/services/secret-approval-policy/secret-approval-policy-service.ts b/backend-pg/src/ee/services/secret-approval-policy/secret-approval-policy-service.ts index a8d6c276c..b461e3144 100644 --- a/backend-pg/src/ee/services/secret-approval-policy/secret-approval-policy-service.ts +++ b/backend-pg/src/ee/services/secret-approval-policy/secret-approval-policy-service.ts @@ -134,7 +134,7 @@ export const secretApprovalPolicyServiceFactory = ({ projectId: secretApprovalPolicy.projectId, $in: { id: approvers } }, - tx + { tx } ); if (secretApprovers.length !== approvers.length) throw new BadRequestError({ message: "Approver not found in project" }); diff --git a/backend-pg/src/ee/services/secret-approval-request/secret-approval-request-service.ts b/backend-pg/src/ee/services/secret-approval-request/secret-approval-request-service.ts index e589ecd54..7bd42f80b 100644 --- a/backend-pg/src/ee/services/secret-approval-request/secret-approval-request-service.ts +++ b/backend-pg/src/ee/services/secret-approval-request/secret-approval-request-service.ts @@ -301,7 +301,7 @@ export const secretApprovalRequestServiceFactory = ({ secretCommentTag, secretCommentCiphertext, skipMultilineEncoding, - secretReminderNotice, + secretReminderNote, secretReminderRepeatDays }) => ({ secretBlindIndex, @@ -316,7 +316,7 @@ export const secretApprovalRequestServiceFactory = ({ secretCommentTag, secretCommentCiphertext, skipMultilineEncoding, - secretReminderNotice, + secretReminderNote, secretReminderRepeatDays, version: 1, folderId, @@ -346,7 +346,7 @@ export const secretApprovalRequestServiceFactory = ({ secretCommentTag, secretCommentCiphertext, skipMultilineEncoding, - secretReminderNotice, + secretReminderNote, secretReminderRepeatDays }) => ({ folderId, @@ -365,7 +365,7 @@ export const secretApprovalRequestServiceFactory = ({ secretCommentTag, secretCommentCiphertext, skipMultilineEncoding, - secretReminderNotice, + secretReminderNote, secretReminderRepeatDays }) ), @@ -661,7 +661,7 @@ export const secretApprovalRequestServiceFactory = ({ secretCommentTag, secretKeyCiphertext, secretValueCiphertext, - secretReminderNotice, + secretReminderNote, skipMultilineEncoding, secretCommentCiphertext, secretReminderRepeatDays @@ -683,7 +683,7 @@ export const secretApprovalRequestServiceFactory = ({ secretCommentTag, secretKeyCiphertext, secretValueCiphertext, - secretReminderNotice, + secretReminderNote, skipMultilineEncoding, secretCommentCiphertext, secretReminderRepeatDays diff --git a/backend-pg/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts b/backend-pg/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts index 0a0effd9e..93788a89f 100644 --- a/backend-pg/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts +++ b/backend-pg/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts @@ -20,7 +20,8 @@ import { secretRotationDbFn, secretRotationHttpFn, secretRotationHttpSetFn, - secretRotationPreSetFn} from "./secret-rotation-queue-fn"; + secretRotationPreSetFn +} from "./secret-rotation-queue-fn"; import { TSecretRotationData, TSecretRotationDbFn, @@ -34,7 +35,7 @@ type TSecretRotationQueueFactoryDep = { secretRotationDal: TSecretRotationDalFactory; projectBotService: Pick; secretDal: Pick; - secretVersionDal: Pick; + secretVersionDal: Pick; }; // These error should stop the repeatable job and ask user to reconfigure rotation diff --git a/backend-pg/src/ee/services/secret-snapshot/secret-snapshot-service.ts b/backend-pg/src/ee/services/secret-snapshot/secret-snapshot-service.ts new file mode 100644 index 000000000..25929f3a4 --- /dev/null +++ b/backend-pg/src/ee/services/secret-snapshot/secret-snapshot-service.ts @@ -0,0 +1,264 @@ +import { ForbiddenError } from "@casl/ability"; + +import { BadRequestError } from "@app/lib/errors"; +import { groupBy } from "@app/lib/fn"; +import { TSecretDalFactory } from "@app/services/secret/secret-dal"; +import { TSecretVersionDalFactory } from "@app/services/secret/secret-version-dal"; +import { TSecretFolderDalFactory } from "@app/services/secret-folder/secret-folder-dal"; +import { TSecretFolderVersionDalFactory } from "@app/services/secret-folder/secret-folder-version-dal"; + +import { TPermissionServiceFactory } from "../permission/permission-service"; +import { ProjectPermissionActions, ProjectPermissionSub } from "../permission/project-permission"; +import { + TGetSnapshotDataDTO, + TProjectSnapshotCountDTO, + TProjectSnapshotListDTO, + TRollbackSnapshotDTO +} from "./secret-snapshot-types"; +import { TSnapshotDalFactory } from "./snapshot-dal"; +import { TSnapshotFolderDalFactory } from "./snapshot-folder-dal"; +import { TSnapshotSecretDalFactory } from "./snapshot-secret-dal"; + +type TSecretSnapshotServiceFactoryDep = { + snapshotDal: TSnapshotDalFactory; + snapshotSecretDal: TSnapshotSecretDalFactory; + snapshotFolderDal: TSnapshotFolderDalFactory; + secretVersionDal: Pick; + folderVersionDal: Pick< + TSecretFolderVersionDalFactory, + "findLatestVersionByFolderId" | "insertMany" + >; + secretDal: Pick; + folderDal: Pick< + TSecretFolderDalFactory, + "findById" | "findBySecretPath" | "delete" | "insertMany" + >; + permissionService: Pick; +}; + +export type TSecretSnapshotServiceFactory = ReturnType; + +export const secretSnapshotServiceFactory = ({ + snapshotDal, + folderVersionDal, + secretVersionDal, + snapshotSecretDal, + snapshotFolderDal, + folderDal, + secretDal, + permissionService +}: TSecretSnapshotServiceFactoryDep) => { + const projectSecretSnapshotCount = async ({ + environment, + projectId, + actorId, + actor, + path + }: TProjectSnapshotCountDTO) => { + const { permission } = await permissionService.getProjectPermission(actor, actorId, projectId); + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionActions.Read, + ProjectPermissionSub.SecretRollback + ); + + const folder = await folderDal.findBySecretPath(projectId, environment, path); + if (!folder) throw new BadRequestError({ message: "Folder not found" }); + + const count = await snapshotDal.countOfSnapshotsByFolderId(folder.id); + return count; + }; + + const listSnapshots = async ({ + environment, + projectId, + actorId, + actor, + path, + limit = 20, + offset = 0 + }: TProjectSnapshotListDTO) => { + const { permission } = await permissionService.getProjectPermission(actor, actorId, projectId); + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionActions.Read, + ProjectPermissionSub.SecretRollback + ); + + const folder = await folderDal.findBySecretPath(projectId, environment, path); + if (!folder) throw new BadRequestError({ message: "Folder not found" }); + + const snapshots = await snapshotDal.find( + { folderId: folder.id }, + { limit, offset, sort: [["createdAt", "desc"]] } + ); + return snapshots; + }; + + const getSnapshotData = async ({ actorId, actor, id }: TGetSnapshotDataDTO) => { + const snapshot = await snapshotDal.findSecretSnapshotDataById(id); + if (!snapshot) throw new BadRequestError({ message: "Snapshot not found" }); + const { permission } = await permissionService.getProjectPermission( + actor, + actorId, + snapshot.projectId + ); + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionActions.Read, + ProjectPermissionSub.SecretRollback + ); + return snapshot; + }; + + const performSnapshot = async (folderId: string) => { + const snapshot = await snapshotDal.transaction(async (tx) => { + const folder = await folderDal.findById(folderId, tx); + if (!folder) throw new BadRequestError({ message: "Folder not found" }); + + const secretVersions = await secretVersionDal.findLatestVersionByFolderId(folderId, tx); + const folderVersions = await folderVersionDal.findLatestVersionByFolderId(folderId, tx); + const newSnapshot = await snapshotDal.create( + { + folderId, + envId: folder.environment.envId, + parentFolderId: folder.parentId + }, + tx + ); + const snapshotSecrets = await snapshotSecretDal.insertMany( + secretVersions.map(({ id }) => ({ + secretVersionId: id, + envId: folder.environment.envId, + snapshotId: newSnapshot.id + })), + tx + ); + const snapshotFolders = await snapshotFolderDal.insertMany( + folderVersions.map(({ id }) => ({ + folderVersionId: id, + envId: folder.environment.envId, + snapshotId: newSnapshot.id + })), + tx + ); + + return { ...newSnapshot, secrets: snapshotSecrets, folder: snapshotFolders }; + }); + + return snapshot; + }; + + const rollbackSnapshot = async ({ id: snapshotId, actor, actorId }: TRollbackSnapshotDTO) => { + const snapshot = await snapshotDal.findById(snapshotId); + if (!snapshot) throw new BadRequestError({ message: "Snapshot not found" }); + + const { permission } = await permissionService.getProjectPermission( + actor, + actorId, + snapshot.projectId + ); + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionActions.Create, + ProjectPermissionSub.SecretRollback + ); + + const rollback = await snapshotDal.transaction(async (tx) => { + const rollbackSnaps = await snapshotDal.findRecursivelySnapshots(snapshot.id, tx); + // this will remove all secrets in current folder + const deletedTopLevelSecs = await secretDal.delete({ folderId: snapshot.folderId }, tx); + const deletedTopLevelSecsGroupById = groupBy(deletedTopLevelSecs, (item) => item.id); + // this will remove all secrets and folders on child + // due to sql foreign key and link list connection removing the folders removes everything below too + const deletedFolders = await folderDal.delete({ parentId: snapshot.folderId }, tx); + const deletedTopLevelFolders = groupBy( + deletedFolders.filter(({ parentId }) => parentId === snapshot.folderId), + (item) => item.id + ); + const folders = await folderDal.insertMany( + rollbackSnaps.flatMap(({ folderVersion, folderId }) => + folderVersion.map(({ name, id, latestFolderVersion }) => ({ + envId: snapshot.envId, + id, + version: latestFolderVersion + 1, + name, + parentId: folderId + })) + ), + tx + ); + const secrets = await secretDal.insertMany( + rollbackSnaps.flatMap(({ secretVersions, folderId }) => + secretVersions.map( + ({ + latestSecretVersion, + version, + updatedAt, + createdAt, + secretId, + envId, + id, + ...el + }) => ({ + ...el, + id: secretId, + version: latestSecretVersion + 1, + folderId + }) + ) + ), + tx + ); + const folderVersions = await folderVersionDal.insertMany( + folders.map(({ version, name, id, envId }) => ({ + name, + version, + folderId: id, + envId + })), + tx + ); + const secretVersions = await secretVersionDal.insertMany( + secrets.map(({ id, updatedAt, createdAt, ...el }) => ({ ...el, secretId: id })), + tx + ); + const newSnapshot = await snapshotDal.create( + { + folderId: snapshot.folderId, + envId: snapshot.envId, + parentFolderId: snapshot.parentFolderId + }, + tx + ); + const snapshotSecrets = await snapshotSecretDal.insertMany( + secretVersions + .filter(({ secretId }) => Boolean(deletedTopLevelSecsGroupById?.[secretId])) + .map(({ id }) => ({ + secretVersionId: id, + envId: newSnapshot.envId, + snapshotId: newSnapshot.id + })), + tx + ); + const snapshotFolders = await snapshotFolderDal.insertMany( + folderVersions + .filter(({ folderId }) => Boolean(deletedTopLevelFolders?.[folderId])) + .map(({ id }) => ({ + folderVersionId: id, + envId: newSnapshot.envId, + snapshotId: newSnapshot.id + })), + tx + ); + + return { ...newSnapshot, snapshotSecrets, snapshotFolders }; + }); + + return rollback; + }; + + return { + performSnapshot, + projectSecretSnapshotCount, + listSnapshots, + getSnapshotData, + rollbackSnapshot + }; +}; diff --git a/backend-pg/src/ee/services/secret-snapshot/secret-snapshot-types.ts b/backend-pg/src/ee/services/secret-snapshot/secret-snapshot-types.ts new file mode 100644 index 000000000..b0fee1a62 --- /dev/null +++ b/backend-pg/src/ee/services/secret-snapshot/secret-snapshot-types.ts @@ -0,0 +1,21 @@ +import { TProjectPermission } from "@app/lib/types"; + +export type TProjectSnapshotCountDTO = { + environment: string; + path: string; +} & TProjectPermission; + +export type TProjectSnapshotListDTO = { + environment: string; + path: string; + offset?: number; + limit?: number; +} & TProjectPermission; + +export type TGetSnapshotDataDTO = { + id: string; +} & Omit; + +export type TRollbackSnapshotDTO = { + id: string; +} & Omit; diff --git a/backend-pg/src/ee/services/secret-snapshot/snapshot-dal.ts b/backend-pg/src/ee/services/secret-snapshot/snapshot-dal.ts new file mode 100644 index 000000000..c44a37dd3 --- /dev/null +++ b/backend-pg/src/ee/services/secret-snapshot/snapshot-dal.ts @@ -0,0 +1,298 @@ +import { Knex } from "knex"; + +import { TDbClient } from "@app/db"; +import { + SecretVersionsSchema, + TableName, + TSecretFolderVersions, + TSecretSnapshotFolders, + TSecretSnapshots, + TSecretVersions} from "@app/db/schemas"; +import { DatabaseError } from "@app/lib/errors"; +import { ormify, selectAllTableCols, sqlNestRelationships } from "@app/lib/knex"; + +export type TSnapshotDalFactory = ReturnType; + +export const snapshotDalFactory = (db: TDbClient) => { + const secretSnapshotOrm = ormify(db, TableName.Snapshot); + + const findById = async (id: string, tx?: Knex) => { + try { + const data = await (tx || db)(TableName.Snapshot) + .where(`${TableName.Snapshot}.id`, id) + .join(TableName.Environment, `${TableName.Snapshot}.envId`, `${TableName.Environment}.id`) + .select(selectAllTableCols(TableName.Snapshot)) + .select( + db.ref("id").withSchema(TableName.Environment).as("envId"), + db.ref("projectId").withSchema(TableName.Environment), + db.ref("name").withSchema(TableName.Environment).as("envName"), + db.ref("slug").withSchema(TableName.Environment).as("envSlug") + ) + .first(); + if (data) { + const { envId, envName, envSlug } = data; + return { ...data, envId, enviroment: { id: envId, name: envName, slug: envSlug } }; + } + } catch (error) { + throw new DatabaseError({ error, name: "FindById" }); + } + }; + + const countOfSnapshotsByFolderId = async (folderId: string, tx?: Knex) => { + try { + const doc = await (tx || db)(TableName.Snapshot) + .where({ folderId }) + .groupBy(["folderId"]) + .count("folderId") + .first(); + return parseInt((doc?.count as string) || "0", 10); + } catch (error) { + throw new DatabaseError({ error, name: "CountOfProjectSnapshot" }); + } + }; + + const findSecretSnapshotDataById = async (snapshotId: string, tx?: Knex) => { + try { + const data = await (tx || db)(TableName.Snapshot) + .where(`${TableName.Snapshot}.id`, snapshotId) + .join(TableName.Environment, `${TableName.Snapshot}.envId`, `${TableName.Environment}.id`) + .leftJoin( + TableName.SnapshotSecret, + `${TableName.Snapshot}.id`, + `${TableName.SnapshotSecret}.snapshotId` + ) + .leftJoin( + TableName.SecretVersion, + `${TableName.SnapshotSecret}.secretVersionId`, + `${TableName.SecretVersion}.id` + ) + .leftJoin( + TableName.SnapshotFolder, + `${TableName.SnapshotFolder}.snapshotId`, + `${TableName.Snapshot}.id` + ) + .leftJoin( + TableName.SecretFolderVersion, + `${TableName.SnapshotFolder}.folderVersionId`, + `${TableName.SecretFolderVersion}.id` + ) + .select(selectAllTableCols(TableName.SecretVersion)) + .select( + db.ref("id").withSchema(TableName.Snapshot).as("snapshotId"), + db.ref("createdAt").withSchema(TableName.Snapshot).as("snapshotCreatedAt"), + db.ref("updatedAt").withSchema(TableName.Snapshot).as("snapshotUpdatedAt"), + db.ref("id").withSchema(TableName.Environment).as("envId"), + db.ref("name").withSchema(TableName.Environment).as("envName"), + db.ref("slug").withSchema(TableName.Environment).as("envSlug"), + db.ref("projectId").withSchema(TableName.Environment), + db.ref("name").withSchema(TableName.SecretFolderVersion).as("folderVerName"), + db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("folderVerId") + ); + return sqlNestRelationships({ + data, + key: "snapshotId", + parentMapper: ({ + snapshotId: id, + projectId, + envId, + envSlug, + envName, + snapshotCreatedAt: createdAt, + snapshotUpdatedAt: updatedAt + }) => ({ + id, + projectId, + createdAt, + updatedAt, + environment: { id: envId, slug: envSlug, name: envName } + }), + childrenMapper: [ + { + key: "id", + label: "secretVersions", + mapper: (el) => SecretVersionsSchema.parse(el) + }, + { + key: "folderVerId", + label: "folderVersion", + mapper: ({ folderVerId: id, folderVerName: name }) => ({ id, name }) + } + ] as const + })?.[0]; + } catch (error) { + throw new DatabaseError({ error, name: "FindSecretSnapshotDataById" }); + } + }; + + // this is used for rollback + // from a starting snapshot it will collect all the secrets and folder of that + // then it will start go through recursively the below folders latest snapshots then their child folder snapshot until leaf node + // the recursive part find all snapshot id + // then joins with respective secrets and folder + const findRecursivelySnapshots = async (snapshotId: string, tx?: Knex) => { + try { + const data = await (tx || db) + .withRecursive("parent", (qb) => { + qb.from(TableName.Snapshot) + .leftJoin( + TableName.SnapshotFolder, + `${TableName.SnapshotFolder}.snapshotId`, + `${TableName.Snapshot}.id` + ) + .leftJoin( + TableName.SecretFolderVersion, + `${TableName.SnapshotFolder}.folderVersionId`, + `${TableName.SecretFolderVersion}.id` + ) + .select(selectAllTableCols(TableName.Snapshot)) + .select({ depth: 1 }) + .select( + db.ref("name").withSchema(TableName.SecretFolderVersion).as("folderVerName"), + db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("folderVerId") + ) + .where(`${TableName.Snapshot}.id`, snapshotId) + .union((cb) => + cb + .select(selectAllTableCols(TableName.Snapshot)) + .select({ depth: db.raw("parent.depth + 1") }) + .select( + db.ref("name").withSchema(TableName.SecretFolderVersion).as("folderVerName"), + db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("folderVerId") + ) + .from(TableName.Snapshot) + .join( + db(TableName.Snapshot) + .groupBy("folderId") + .max("createdAt") + .select("folderId") + .as("latestVersion"), + `${TableName.Snapshot}.createdAt`, + "latestVersion.max" + ) + .leftJoin( + TableName.SnapshotFolder, + `${TableName.SnapshotFolder}.snapshotId`, + `${TableName.Snapshot}.id` + ) + .leftJoin( + TableName.SecretFolderVersion, + `${TableName.SnapshotFolder}.folderVersionId`, + `${TableName.SecretFolderVersion}.id` + ) + .join("parent", "parent.folderVerId", `${TableName.Snapshot}.folderId`) + ); + }) + .orderBy("depth", "asc") + .from("parent") + .leftJoin( + TableName.SnapshotSecret, + `parent.id`, + `${TableName.SnapshotSecret}.snapshotId` + ) + .leftJoin( + TableName.SecretVersion, + `${TableName.SnapshotSecret}.secretVersionId`, + `${TableName.SecretVersion}.id` + ) + .leftJoin<{ latestSecretVersion: number }>( + (tx || db)(TableName.SecretVersion) + .groupBy("secretId") + .select("secretId") + .max("version") + .as("secGroupByMaxVersion"), + `${TableName.SecretVersion}.secretId`, + "secGroupByMaxVersion.secretId" + ) + .leftJoin<{ latestFolderVersion: number }>( + (tx || db)(TableName.SecretFolderVersion) + .groupBy("folderId") + .select("folderId") + .max("version") + .as("folderGroupByMaxVersion"), + `parent.folderId`, + "folderGroupByMaxVersion.folderId" + ) + .select(selectAllTableCols(TableName.SecretVersion)) + .select( + db.ref("id").withSchema("parent").as("snapshotId"), + db.ref("folderId").withSchema("parent").as("snapshotFolderId"), + db.ref("parentFolderId").withSchema("parent").as("snapshotParentFolderId"), + db.ref("folderVerName").withSchema("parent"), + db.ref("folderVerId").withSchema("parent"), + db.ref("max").withSchema("secGroupByMaxVersion").as("latestSecretVersion"), + db.ref("max").withSchema("folderGroupByMaxVersion").as("latestFolderVersion") + ); + const formated = sqlNestRelationships({ + data, + key: "snapshotId", + parentMapper: ({ + snapshotId: id, + snapshotFolderId: folderId, + snapshotParentFolderId: parentFolderId + }) => ({ + id, + folderId, + parentFolderId + }), + childrenMapper: [ + { + key: "id", + label: "secretVersions", + mapper: (el) => ({ + ...SecretVersionsSchema.parse(el), + latestSecretVersion: el.latestSecretVersion + }) + }, + { + key: "folderVerId", + label: "folderVersion", + mapper: ({ folderVerId: id, folderVerName: name, latestFolderVersion }) => ({ + id, + name, + latestFolderVersion + }) + } + ] as const + }); + return formated; + } catch (error) { + throw new DatabaseError({ error, name: "FindRecursivelySnapshots" }); + } + }; + + // instead of copying all child folders + // we will take the latest snapshot of those folders + // when we need to rollback we will pull from these snapshots + const findLatestSnapshotByFolderId = async (folderId: string, tx?: Knex) => { + try { + const docs = await (tx || db)(TableName.Snapshot) + .where(`${TableName.Snapshot}.folderId`, folderId) + .join( + (tx || db)(TableName.Snapshot) + .groupBy("folderId") + .max("createdAt") + .select("folderId") + .as("latestVersion"), + (bd) => { + bd.on(`${TableName.Snapshot}.folderId`, "latestVersion.folderId").andOn( + `${TableName.Snapshot}.createdAt`, + "latestVersion.max" + ); + } + ) + .first(); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindLatestVersionByFolderId" }); + } + }; + + return { + ...secretSnapshotOrm, + findById, + findLatestSnapshotByFolderId, + findRecursivelySnapshots, + countOfSnapshotsByFolderId, + findSecretSnapshotDataById + }; +}; diff --git a/backend-pg/src/ee/services/secret-snapshot/snapshot-folder-dal.ts b/backend-pg/src/ee/services/secret-snapshot/snapshot-folder-dal.ts new file mode 100644 index 000000000..3eb25a2d7 --- /dev/null +++ b/backend-pg/src/ee/services/secret-snapshot/snapshot-folder-dal.ts @@ -0,0 +1,11 @@ +import { TDbClient } from "@app/db"; +import { TableName } from "@app/db/schemas"; +import { ormify } from "@app/lib/knex"; + +export type TSnapshotFolderDalFactory = ReturnType; + +export const snapshotFolderDalFactory = (db: TDbClient) => { + const snapshotFolderOrm = ormify(db, TableName.SnapshotFolder); + + return snapshotFolderOrm; +}; diff --git a/backend-pg/src/ee/services/secret-snapshot/snapshot-secret-dal.ts b/backend-pg/src/ee/services/secret-snapshot/snapshot-secret-dal.ts new file mode 100644 index 000000000..f29a7c859 --- /dev/null +++ b/backend-pg/src/ee/services/secret-snapshot/snapshot-secret-dal.ts @@ -0,0 +1,10 @@ +import { TDbClient } from "@app/db"; +import { TableName } from "@app/db/schemas"; +import { ormify } from "@app/lib/knex"; + +export type TSnapshotSecretDalFactory = ReturnType; + +export const snapshotSecretDalFactory = (db: TDbClient) => { + const snapshotSecretOrm = ormify(db, TableName.SnapshotSecret); + return snapshotSecretOrm; +}; diff --git a/backend-pg/src/folder.ts b/backend-pg/src/folder.ts new file mode 100644 index 000000000..cc4e0bfab --- /dev/null +++ b/backend-pg/src/folder.ts @@ -0,0 +1,190 @@ +import dotenv from "dotenv"; + +import { initDbConnection } from "./db"; +import { + SecretVersionsSchema, + TableName, + TSecretFolderVersions, + TSecrets, + TSecretSnapshotFolders, + TSecretSnapshots, + TSecretVersions} from "./db/schemas"; +import { selectAllTableCols, sqlNestRelationships } from "./lib/knex"; + +dotenv.config(); +// const db = initDbConnection(process.env.DB_CONNECTION_URI); + +// const main = async () => { +// const folders = db +// .withRecursive("parent", (qb) => { +// qb.select({ +// depth: 1, +// path: db.raw("'/'") +// }) +// .select(selectAllTableCols(db, TableName.SecretFolder)) +// .from(TableName.SecretFolder) +// .join( +// TableName.Environment, +// `${TableName.SecretFolder}.envId`, +// `${TableName.Environment}.id` +// ) +// .where({ +// projectId: "01c10de1-8743-490f-9c8a-7a19c4dc72a9", +// parentId: null +// }) +// .where(`${TableName.Environment}.slug`, "dev") +// .union((qb) => +// qb +// .select({ +// depth: db.raw("parent.depth + 1"), +// path: db.raw( +// "CONCAT((CASE WHEN parent.path = '/' THEN '' ELSE parent.path END),'/', secret_folders.name)" +// ) +// }) +// .select(selectAllTableCols(db, TableName.SecretFolder)) +// .whereRaw( +// `depth = array_position(ARRAY[${[1, 2] +// .map((_) => "?") +// .join(",")}]::varchar[], secret_folders.name,depth)`, +// [...["ui", "design"]] +// ) +// .from(TableName.SecretFolder) +// .join("parent", "parent.id", `${TableName.SecretFolder}.parentId`) +// ); +// }) +// .select("*") +// .from("parent") +// .orderBy("depth", "desc") +// .first(); +// console.log(folders.toSQL()); +// console.log(JSON.stringify(await folders, null, 4)); +// process.exit(0); +// }; +// +const main = async () => { + const db = initDbConnection(process.env.DB_CONNECTION_URI); + + const folders = db + .withRecursive("parent", (qb) => { + qb.from(TableName.Snapshot) + .leftJoin( + TableName.SnapshotFolder, + `${TableName.SnapshotFolder}.snapshotId`, + `${TableName.Snapshot}.id` + ) + .leftJoin( + TableName.SecretFolderVersion, + `${TableName.SnapshotFolder}.folderVersionId`, + `${TableName.SecretFolderVersion}.id` + ) + .select(selectAllTableCols(TableName.Snapshot)) + .select({ depth: 1 }) + .select( + db.ref("name").withSchema(TableName.SecretFolderVersion).as("folderVerName"), + db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("folderVerId") + ) + .where(`${TableName.Snapshot}.id`, "abe694d3-957f-40f5-a907-952d3e5ccaf1") + .union((cb) => + cb + .select(selectAllTableCols(TableName.Snapshot)) + .select({ depth: db.raw("parent.depth + 1") }) + .select( + db.ref("name").withSchema(TableName.SecretFolderVersion).as("folderVerName"), + db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("folderVerId") + ) + .from(TableName.Snapshot) + .join( + db(TableName.Snapshot) + .groupBy("folderId") + .max("createdAt") + .select("folderId") + .as("latestVersion"), + `${TableName.Snapshot}.createdAt`, + "latestVersion.max" + ) + .leftJoin( + TableName.SnapshotFolder, + `${TableName.SnapshotFolder}.snapshotId`, + `${TableName.Snapshot}.id` + ) + .leftJoin( + TableName.SecretFolderVersion, + `${TableName.SnapshotFolder}.folderVersionId`, + `${TableName.SecretFolderVersion}.id` + ) + .join("parent", "parent.folderVerId", `${TableName.Snapshot}.folderId`) + ); + }) + .orderBy("depth", "asc") + .from("parent") + .leftJoin( + TableName.SnapshotSecret, + `parent.id`, + `${TableName.SnapshotSecret}.snapshotId` + ) + .leftJoin( + TableName.SecretVersion, + `${TableName.SnapshotSecret}.secretVersionId`, + `${TableName.SecretVersion}.id` + ) + .leftJoin<{ latestSecretVersion: number }>( + db(TableName.SecretVersion) + .groupBy("secretId") + .select("secretId") + .max("version") + .as("secGroupByMaxVersion"), + `${TableName.SecretVersion}.secretId`, + "secGroupByMaxVersion.secretId" + ) + .leftJoin<{ latestFolderVersion: number }>( + db(TableName.SecretFolderVersion) + .groupBy("folderId") + .select("folderId") + .max("version") + .as("folderGroupByMaxVersion"), + `parent.folderId`, + "folderGroupByMaxVersion.folderId" + ) + .select(selectAllTableCols(TableName.SecretVersion)) + .select( + db.ref("id").withSchema("parent").as("snapshotId"), + db.ref("folderId").withSchema("parent").as("snapshotFolderId"), + db.ref("parentFolderId").withSchema("parent").as("snapshotParentFolderId"), + db.ref("folderVerName").withSchema("parent"), + db.ref("folderVerId").withSchema("parent"), + db.ref("max").withSchema("secGroupByMaxVersion").as("latestSecretVersion"), + db.ref("max").withSchema("folderGroupByMaxVersion").as("latestFolderVersion") + ); + console.log(folders.toSQL()); + const data = await folders; + // console.log(data); + const formated = sqlNestRelationships({ + data, + key: "snapshotId", + parentMapper: ({ snapshotId: id }) => ({ + id + }), + childrenMapper: [ + { + key: "id", + label: "secretVersions", + mapper: (el) => ({ + ...SecretVersionsSchema.parse(el), + latestSecretVersion: el.latestSecretVersion + }) + }, + { + key: "folderVerId", + label: "folderVersion", + mapper: ({ folderVerId: id, folderVerName: name, latestFolderVersion }) => ({ + id, + name, + version: latestFolderVersion + }) + } + ] as const + }); + console.log(formated); + process.exit(0); +}; +main(); diff --git a/backend-pg/src/lib/fn/array.ts b/backend-pg/src/lib/fn/array.ts new file mode 100644 index 000000000..4b06774aa --- /dev/null +++ b/backend-pg/src/lib/fn/array.ts @@ -0,0 +1,18 @@ +/** + * Sorts an array of items into groups. The return value is a map where the keys are + * the group ids the given getGroupId function produced and the value is an array of + * each item in that group. + */ +export const groupBy = ( + array: readonly T[], + getGroupId: (item: T) => Key +): Record => + array.reduce( + (acc, item) => { + const groupId = getGroupId(item); + if (!acc[groupId]) acc[groupId] = []; + acc[groupId].push(item); + return acc; + }, + {} as Record + ); diff --git a/backend-pg/src/lib/fn/index.ts b/backend-pg/src/lib/fn/index.ts new file mode 100644 index 000000000..6ce0d2eb0 --- /dev/null +++ b/backend-pg/src/lib/fn/index.ts @@ -0,0 +1,5 @@ +// Some of the functions are taken from https://github.com/rayepps/radash +// Full credits goes to https://github.com/rayapps to those functions +// Code taken to keep in in house and to adjust somethings for our needs +export * from "./array"; +export * from "./object"; diff --git a/backend-pg/src/lib/fn/object.ts b/backend-pg/src/lib/fn/object.ts new file mode 100644 index 000000000..34784766e --- /dev/null +++ b/backend-pg/src/lib/fn/object.ts @@ -0,0 +1,17 @@ +/** + * Pick a list of properties from an object + * into a new object + */ +export const pick = ( + obj: T, + keys: TKeys[] +): Pick => { + if (!obj) return {} as Pick; + return keys.reduce( + (acc, key) => { + if (Object.prototype.hasOwnProperty.call(obj, key)) acc[key] = obj[key]; + return acc; + }, + {} as Pick + ); +}; diff --git a/backend-pg/src/lib/knex/index.ts b/backend-pg/src/lib/knex/index.ts index 543da9255..ed41e069b 100644 --- a/backend-pg/src/lib/knex/index.ts +++ b/backend-pg/src/lib/knex/index.ts @@ -16,7 +16,7 @@ export const withTransaction = (db: Knex, dal: K) => ({ }); export type TFindFilter = Partial & { - $in?: Partial<{ [K in keyof R]: R[K][] }>; + $in?: Partial<{ [k in keyof R]: R[k][] }>; }; export const buildFindFilter = ({ $in, ...filter }: TFindFilter) => @@ -30,6 +30,13 @@ export const buildFindFilter = return bd; }; +export type TFindOpt = { + limit?: number; + offset?: number; + sort?: Array<[keyof R, "asc" | "desc"] | [keyof R, "asc" | "desc", "first" | "last"]>; + tx?: Knex; +}; + // What is ormify // It is to inject typical operations like find, findOne, update, delete, create // This will avoid writing most common ones each time @@ -60,9 +67,20 @@ export const ormify = ( throw new DatabaseError({ error, name: "Find one" }); } }, - find: async (filter: TFindFilter, tx?: Knex) => { + find: async ( + filter: TFindFilter, + { offset, limit, sort, tx }: TFindOpt = {} + ) => { try { - const res = await (tx || db)(tableName).where(buildFindFilter(filter)); + const query = (tx || db)(tableName).where(buildFindFilter(filter)); + if (limit) query.limit(limit); + if (offset) query.offset(offset); + if (sort) { + query.orderBy( + sort.map(([column, order, nulls]) => ({ column: column as string, order, nulls })) + ); + } + const res = await query; return res; } catch (error) { throw new DatabaseError({ error, name: "Find one" }); @@ -78,6 +96,7 @@ export const ormify = ( }, insertMany: async (data: readonly Tables[Tname]["insert"][], tx?: Knex) => { try { + if (!data.length) return []; const res = await (tx || db)(tableName) .insert(data as any) .returning("*"); diff --git a/backend-pg/src/lib/knex/join.ts b/backend-pg/src/lib/knex/join.ts index 884934f50..21b24880e 100644 --- a/backend-pg/src/lib/knex/join.ts +++ b/backend-pg/src/lib/knex/join.ts @@ -76,12 +76,14 @@ export const sqlNestRelationships = < parentLookup.add(pk); } childrenMapper.forEach(({ label, mapper, key: cKey }) => { - const ck = `${pk}-${label}-${el[cKey]}`; - if (!childLookUp.has(ck)) { - if (!recordsGroupedByPk[pk][label]) recordsGroupedByPk[pk][label as keyof Cm] = [] as any; - const val = mapper(el); - if (typeof val !== "undefined") recordsGroupedByPk[pk][label].push(val); - childLookUp.add(ck); + if (!recordsGroupedByPk[pk][label]) recordsGroupedByPk[pk][label as keyof Cm] = [] as any; + if (el[cKey] !== null && typeof el[cKey] !== "undefined") { + const ck = `${pk}-${label}-${el[cKey]}`; + if (!childLookUp.has(ck)) { + const val = mapper(el); + if (typeof val !== "undefined" && val !== null) recordsGroupedByPk[pk][label].push(val); + childLookUp.add(ck); + } } }); }); diff --git a/backend-pg/src/queue/queue-service.ts b/backend-pg/src/queue/queue-service.ts index 0833555b5..c1042a9a4 100644 --- a/backend-pg/src/queue/queue-service.ts +++ b/backend-pg/src/queue/queue-service.ts @@ -18,7 +18,7 @@ export type TQueueJobTypes = { export type TQueueServiceFactory = ReturnType; export const queueServiceFactory = (redisUrl: string) => { - const connection = new Redis(redisUrl); + const connection = new Redis(redisUrl, { maxRetriesPerRequest: null }); const queueContainer: Record< QueueName, Queue @@ -46,7 +46,7 @@ export const queueServiceFactory = (redisUrl: string) => { TQueueJobTypes[T]["payload"], void, TQueueJobTypes[T]["name"] - >(name, jobFn); + >(name, jobFn, { connection }); }; const listen = async < diff --git a/backend-pg/src/server/routes/index.ts b/backend-pg/src/server/routes/index.ts index 7999c3fe4..e153c6664 100644 --- a/backend-pg/src/server/routes/index.ts +++ b/backend-pg/src/server/routes/index.ts @@ -14,6 +14,10 @@ import { secretApprovalRequestServiceFactory } from "@app/ee/services/secret-app import { secretRotationDalFactory } from "@app/ee/services/secret-rotation/secret-rotation-dal"; import { secretRotationQueueFactory } from "@app/ee/services/secret-rotation/secret-rotation-queue"; import { secretRotationServiceFactory } from "@app/ee/services/secret-rotation/secret-rotation-service"; +import { secretSnapshotServiceFactory } from "@app/ee/services/secret-snapshot/secret-snapshot-service"; +import { snapshotDalFactory } from "@app/ee/services/secret-snapshot/snapshot-dal"; +import { snapshotFolderDalFactory } from "@app/ee/services/secret-snapshot/snapshot-folder-dal"; +import { snapshotSecretDalFactory } from "@app/ee/services/secret-snapshot/snapshot-secret-dal"; import { getConfig } from "@app/lib/config/env"; import { TQueueServiceFactory } from "@app/queue"; import { apiKeyDalFactory } from "@app/services/api-key/api-key-dal"; @@ -61,6 +65,7 @@ import { secretServiceFactory } from "@app/services/secret/secret-service"; import { secretVersionDalFactory } from "@app/services/secret/secret-version-dal"; import { secretFolderDalFactory } from "@app/services/secret-folder/secret-folder-dal"; import { secretFolderServiceFactory } from "@app/services/secret-folder/secret-folder-service"; +import { secretFolderVersionDalFactory } from "@app/services/secret-folder/secret-folder-version-dal"; import { secretImportDalFactory } from "@app/services/secret-import/secret-import-dal"; import { secretImportServiceFactory } from "@app/services/secret-import/secret-import-service"; import { secretTagDalFactory } from "@app/services/secret-tag/secret-tag-dal"; @@ -109,6 +114,7 @@ export const registerRoutes = async ( const secretDal = secretDalFactory(db); const secretTagDal = secretTagDalFactory(db); const folderDal = secretFolderDalFactory(db); + const folderVersionDal = secretFolderVersionDalFactory(db); const secretImportDal = secretImportDalFactory(db); const secretVersionDal = secretVersionDalFactory(db); const secretBlindIndexDal = secretBlindIndexDalFactory(db); @@ -135,6 +141,9 @@ export const registerRoutes = async ( const sarSecretDal = sarSecretDalFactory(db); const secretRotationDal = secretRotationDalFactory(db); + const snapshotDal = snapshotDalFactory(db); + const snapshotSecretDal = snapshotSecretDalFactory(db); + const snapshotFolderDal = snapshotFolderDalFactory(db); const permissionService = permissionServiceFactory({ permissionDal, orgRoleDal, projectRoleDal }); const sapService = secretApprovalPolicyServiceFactory({ @@ -215,19 +224,33 @@ export const registerRoutes = async ( }); const projectRoleService = projectRoleServiceFactory({ permissionService, projectRoleDal }); + const snapshotService = secretSnapshotServiceFactory({ + folderDal, + secretDal, + snapshotDal, + snapshotFolderDal, + snapshotSecretDal, + secretVersionDal, + folderVersionDal, + permissionService + }); + const secretService = secretServiceFactory({ folderDal, secretVersionDal, secretBlindIndexDal, permissionService, secretDal, - secretTagDal + secretTagDal, + snapshotService }); const secretTagService = secretTagServiceFactory({ secretTagDal, permissionService }); const folderService = secretFolderServiceFactory({ permissionService, folderDal, - projectEnvDal + folderVersionDal, + projectEnvDal, + snapshotService }); const secretImportService = secretImportServiceFactory({ projectEnvDal, @@ -329,7 +352,8 @@ export const registerRoutes = async ( identityUa: identityUaService, secretApprovalPolicy: sapService, secretApprovalRequest: sarService, - secretRotation: secretRotationService + secretRotation: secretRotationService, + snapshot: snapshotService }); server.decorate("store", { diff --git a/backend-pg/src/services/identity-ua/identity-ua-types.ts b/backend-pg/src/services/identity-ua/identity-ua-types.ts index 5f812cbcf..2cc4762a8 100644 --- a/backend-pg/src/services/identity-ua/identity-ua-types.ts +++ b/backend-pg/src/services/identity-ua/identity-ua-types.ts @@ -24,7 +24,7 @@ export type TGetUaDTO = { export type TCreateUaClientSecretDTO = { identityId: string; - description?: string; + description: string; numUsesLimit: number; ttl: number; } & Omit; diff --git a/backend-pg/src/services/secret-folder/secret-folder-dal.ts b/backend-pg/src/services/secret-folder/secret-folder-dal.ts index c425f2d63..289359f25 100644 --- a/backend-pg/src/services/secret-folder/secret-folder-dal.ts +++ b/backend-pg/src/services/secret-folder/secret-folder-dal.ts @@ -1,7 +1,7 @@ import { Knex } from "knex"; import { TDbClient } from "@app/db"; -import { TableName, TSecretFolders } from "@app/db/schemas"; +import { TableName, TSecretFolders, TSecretFoldersUpdate } from "@app/db/schemas"; import { BadRequestError, DatabaseError } from "@app/lib/errors"; import { ormify, selectAllTableCols } from "@app/lib/knex"; @@ -35,9 +35,9 @@ const sqlFindFolderByPathQuery = ( baseQb .select({ depth: 1, + // latestFolderVerId: db.raw("NULL::uuid"), path: db.raw("'/'") }) - .select(selectAllTableCols(TableName.SecretFolder)) .from(TableName.SecretFolder) .join( TableName.Environment, @@ -49,6 +49,7 @@ const sqlFindFolderByPathQuery = ( parentId: null }) .where(`${TableName.Environment}.slug`, environment) + .select(selectAllTableCols(TableName.SecretFolder)) .union((qb) => // for here on we keep going to next child node. // we also keep a measure of depth then we check the depth matches the array path segment and folder name @@ -100,5 +101,44 @@ export const secretFolderDalFactory = (db: TDbClient) => { } }; - return { ...secretFolderOrm, findBySecretPath }; + const update = async (filter: Partial, data: TSecretFoldersUpdate, tx?: Knex) => { + try { + const folder = await (tx || db)(TableName.SecretFolder) + .where(filter) + .update(data) + .increment("version", 1) + .returning("*"); + return folder; + } catch (error) { + throw new DatabaseError({ error, name: "SecretFolderUpdate" }); + } + }; + + const findById = async (id: string, tx?: Knex) => { + try { + const folder = await (tx || db)(TableName.SecretFolder) + .where({ [`${TableName.SecretFolder}.id` as "id"]: id }) + .join( + TableName.Environment, + `${TableName.SecretFolder}.envId`, + `${TableName.Environment}.id` + ) + .select(selectAllTableCols(TableName.SecretFolder)) + .select( + db.ref("id").withSchema(TableName.Environment).as("envId"), + db.ref("slug").withSchema(TableName.Environment).as("envSlug"), + db.ref("name").withSchema(TableName.Environment).as("envName"), + db.ref("projectId").withSchema(TableName.Environment) + ) + .first(); + if (folder) { + const { envId, envName, envSlug, ...el } = folder; + return { ...el, environment: { envId, envName, envSlug } }; + } + } catch (error) { + throw new DatabaseError({ error, name: "Find by id" }); + } + }; + + return { ...secretFolderOrm, update, findBySecretPath, findById }; }; diff --git a/backend-pg/src/services/secret-folder/secret-folder-service.ts b/backend-pg/src/services/secret-folder/secret-folder-service.ts index d666c7f9e..05acc1420 100644 --- a/backend-pg/src/services/secret-folder/secret-folder-service.ts +++ b/backend-pg/src/services/secret-folder/secret-folder-service.ts @@ -5,6 +5,7 @@ import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; +import { TSecretSnapshotServiceFactory } from "@app/ee/services/secret-snapshot/secret-snapshot-service"; import { BadRequestError } from "@app/lib/errors"; import { TProjectEnvDalFactory } from "../project-env/project-env-dal"; @@ -15,19 +16,24 @@ import { TGetFolderDTO, TUpdateFolderDTO } from "./secret-folder-types"; +import { TSecretFolderVersionDalFactory } from "./secret-folder-version-dal"; type TSecretFolderServiceFactoryDep = { permissionService: Pick; + snapshotService: Pick; folderDal: TSecretFolderDalFactory; projectEnvDal: Pick; + folderVersionDal: TSecretFolderVersionDalFactory; }; export type TSecretFolderServiceFactory = ReturnType; export const secretFolderServiceFactory = ({ folderDal, + snapshotService, permissionService, - projectEnvDal + projectEnvDal, + folderVersionDal }: TSecretFolderServiceFactoryDep) => { const createFolder = async ({ projectId, @@ -54,9 +60,19 @@ export const secretFolderServiceFactory = ({ { name, envId: env.id, version: 1, parentId: parentFolder.id }, tx ); + await folderVersionDal.create( + { + name: doc.name, + envId: doc.envId, + version: doc.version, + folderId: doc.id + }, + tx + ); return doc; }); + await snapshotService.performSnapshot(folder.parentId as string); return folder; }; @@ -85,13 +101,23 @@ export const secretFolderServiceFactory = ({ const [doc] = await folderDal.update( { envId: env.id, id, parentId: parentFolder.id }, - { name, version: 1 }, + { name }, + tx + ); + await folderVersionDal.create( + { + name: doc.name, + envId: doc.envId, + version: doc.version, + folderId: doc.id + }, tx ); if (!doc) throw new BadRequestError({ message: "Folder not found", name: "Update folder" }); return doc; }); + await snapshotService.performSnapshot(folder.parentId as string); return folder; }; @@ -122,6 +148,7 @@ export const secretFolderServiceFactory = ({ return doc; }); + await snapshotService.performSnapshot(folder.parentId as string); return folder; }; diff --git a/backend-pg/src/services/secret-folder/secret-folder-version-dal.ts b/backend-pg/src/services/secret-folder/secret-folder-version-dal.ts new file mode 100644 index 000000000..32ce76f3c --- /dev/null +++ b/backend-pg/src/services/secret-folder/secret-folder-version-dal.ts @@ -0,0 +1,72 @@ +import { Knex } from "knex"; + +import { TDbClient } from "@app/db"; +import { TableName,TSecretFolderVersions } from "@app/db/schemas"; +import { DatabaseError } from "@app/lib/errors"; +import { ormify, selectAllTableCols } from "@app/lib/knex"; + +export type TSecretFolderVersionDalFactory = ReturnType; + +export const secretFolderVersionDalFactory = (db: TDbClient) => { + const secretFolderVerOrm = ormify(db, TableName.SecretFolderVersion); + + // This will fetch all latest secret versions from a folder + const findLatestVersionByFolderId = async (folderId: string, tx?: Knex) => { + try { + const docs = await (tx || db)(TableName.SecretFolderVersion) + .join( + TableName.SecretFolder, + `${TableName.SecretFolderVersion}.folderId`, + `${TableName.SecretFolder}.id` + ) + .where({ parentId: folderId }) + .join( + (tx || db)(TableName.SecretFolderVersion) + .groupBy("envId", "folderId") + .max("version") + .select("folderId") + .as("latestVersion"), + (bd) => { + bd.on(`${TableName.SecretFolderVersion}.folderId`, "latestVersion.folderId").andOn( + `${TableName.SecretFolderVersion}.version`, + "latestVersion.max" + ); + } + ) + .select(selectAllTableCols(TableName.SecretFolderVersion)); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindLatestVersionByFolderId" }); + } + }; + + const findLatestFolderVersions = async (folderIds: string[], tx?: Knex) => { + try { + const docs: Array = await (tx || db)( + TableName.SecretFolderVersion + ) + .whereIn("folderId", folderIds) + .join( + (tx || db)(TableName.SecretFolderVersion) + .groupBy("folderId") + .max("version") + .select("folderId") + .as("latestVersion"), + (bd) => { + bd.on(`${TableName.SecretFolderVersion}.folderId`, "latestVersion.folderId").andOn( + `${TableName.SecretFolderVersion}.version`, + "latestVersion.max" + ); + } + ); + return docs.reduce>( + (prev, curr) => ({ ...prev, [curr.folderId || ""]: curr }), + {} + ); + } catch (error) { + throw new DatabaseError({ error, name: "FindLatestFolderVersions" }); + } + }; + + return { ...secretFolderVerOrm, findLatestFolderVersions, findLatestVersionByFolderId }; +}; diff --git a/backend-pg/src/services/secret/secret-dal.ts b/backend-pg/src/services/secret/secret-dal.ts index 36df01d88..039133845 100644 --- a/backend-pg/src/services/secret/secret-dal.ts +++ b/backend-pg/src/services/secret/secret-dal.ts @@ -2,7 +2,7 @@ import { Knex } from "knex"; import { TDbClient } from "@app/db"; import { SecretType, TableName, TSecrets, TSecretsInsert, TSecretsUpdate } from "@app/db/schemas"; -import { DatabaseError } from "@app/lib/errors"; +import { BadRequestError, DatabaseError } from "@app/lib/errors"; import { mergeOneToManyRelation, ormify, selectAllTableCols } from "@app/lib/knex"; export type TSecretDalFactory = ReturnType; @@ -29,10 +29,7 @@ export const secretDalFactory = (db: TDbClient) => { // the idea is to use postgres specific function // insert with id this will cause a conflict then merge the data - const bulkUpdate = async ( - data: Array, - tx?: Knex - ) => { + const bulkUpdate = async (data: Array, tx?: Knex) => { try { const secs = await (tx || db)(TableName.Secret) .insert(data as TSecretsInsert[]) @@ -59,7 +56,7 @@ export const secretDalFactory = (db: TDbClient) => { bd.orWhere({ secretBlindIndex: el.blindIndex, type: el.type, - userId: el.type === SecretType.Personal ? userId : null + ...(el.type === SecretType.Personal ? { userId } : {}) }); }); }) @@ -123,6 +120,9 @@ export const secretDalFactory = (db: TDbClient) => { .where({ folderId }) .where((bd) => { blindIndexes.forEach((el) => { + if (el.type === SecretType.Personal && !userId) { + throw new BadRequestError({ message: "Missing personal user id" }); + } bd.orWhere({ secretBlindIndex: el.blindIndex, type: el.type, diff --git a/backend-pg/src/services/secret/secret-service.ts b/backend-pg/src/services/secret/secret-service.ts index 4460b04bb..7fb76f620 100644 --- a/backend-pg/src/services/secret/secret-service.ts +++ b/backend-pg/src/services/secret/secret-service.ts @@ -5,18 +5,20 @@ import { SecretKeyEncoding, SecretType, TableName, - TSecretBlindIndexes, - TSecrets + TSecretBlindIndexes } from "@app/db/schemas"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; +import { TSecretSnapshotServiceFactory } from "@app/ee/services/secret-snapshot/secret-snapshot-service"; import { getConfig } from "@app/lib/config/env"; import { buildSecretBlindIndexFromName } from "@app/lib/crypto"; import { BadRequestError } from "@app/lib/errors"; +import { groupBy, pick } from "@app/lib/fn"; +import { ActorType } from "../auth/auth-type"; import { TSecretFolderDalFactory } from "../secret-folder/secret-folder-dal"; import { TSecretTagDalFactory } from "../secret-tag/secret-tag-dal"; import { TSecretBlindIndexDalFactory } from "./secret-blind-index-dal"; @@ -26,8 +28,13 @@ import { TCreateSecretDTO, TDeleteBulkSecretDTO, TDeleteSecretDTO, + TFnSecretBlindIndexCheck, + TFnSecretBulkDelete, + TFnSecretBulkInsert, + TFnSecretBulkUpdate, TGetASecretDTO, TGetSecretsDTO, + TListSecretVersionDTO, TUpdateBulkSecretDTO, TUpdateSecretDTO } from "./secret-types"; @@ -37,9 +44,10 @@ type TSecretServiceFactoryDep = { secretDal: TSecretDalFactory; secretTagDal: TSecretTagDalFactory; secretVersionDal: TSecretVersionDalFactory; - folderDal: TSecretFolderDalFactory; + folderDal: Pick; secretBlindIndexDal: TSecretBlindIndexDalFactory; permissionService: Pick; + snapshotService: Pick; }; export type TSecretServiceFactory = ReturnType; @@ -67,10 +75,11 @@ export const secretServiceFactory = ({ secretVersionDal, folderDal, secretBlindIndexDal, - permissionService + permissionService, + snapshotService }: TSecretServiceFactoryDep) => { // utility function to get secret blind index data - const generateSecretBlindIndexByName = async (projectId: string, secretName: string) => { + const interalGenSecBlindIndexByName = async (projectId: string, secretName: string) => { const appCfg = getConfig(); const secretBlindIndexDoc = await secretBlindIndexDal.findOne({ projectId }); @@ -90,6 +99,136 @@ export const secretServiceFactory = ({ return secretBlindIndex; }; + // these functions are special functions shared by a couple of resources + // used by secret approval, rotation or anywhere in which secret needs to modified + const fnSecretBulkInsert = async ({ folderId, inputSecrets, tx }: TFnSecretBulkInsert) => { + const newSecrets = await secretDal.insertMany( + inputSecrets.map(({ tags, ...el }) => ({ ...el, folderId })), + tx + ); + const newSecretGroupByBlindIndex = groupBy(newSecrets, (item) => item.secretBlindIndex); + const newSecretTags = inputSecrets.flatMap(({ tags: secretTags = [], secretBlindIndex }) => + secretTags.map((tag) => ({ + [`${TableName.SecretTag}Id`]: tag, + [`${TableName.Secret}Id`]: newSecretGroupByBlindIndex[secretBlindIndex][0].id + })) + ); + if (newSecretTags.length) { + await secretTagDal.saveTagsToSecret(newSecretTags, tx); + } + await secretVersionDal.insertMany( + inputSecrets.map(({ tags, ...el }) => ({ + ...el, + folderId, + secretId: newSecretGroupByBlindIndex[el.secretBlindIndex][0].id + })), + tx + ); + + return newSecrets; + }; + + const fnSecretBulkUpdate = async ({ + tx, + inputSecrets, + folderId, + projectId + }: TFnSecretBulkUpdate) => { + const newSecrets = await secretDal.bulkUpdate( + inputSecrets.map(({ tags, ...el }) => ({ ...el, folderId })), + tx + ); + const secsUpdatedTag = inputSecrets.filter(({ tags }) => Boolean(tags)); + if (secsUpdatedTag.length) { + await secretTagDal.deleteTagsManySecret( + projectId, + secsUpdatedTag.map(({ id }) => id), + tx + ); + const newSecretTags = secsUpdatedTag.flatMap(({ tags: secretTags = [], id }) => + secretTags.map((tag) => ({ + [`${TableName.SecretTag}Id`]: tag, + [`${TableName.Secret}Id`]: id + })) + ); + await secretTagDal.saveTagsToSecret(newSecretTags, tx); + } + await secretVersionDal.insertMany( + newSecrets.map(({ id, createdAt, updatedAt, ...el }) => ({ + ...el, + secretId: id + })), + tx + ); + + return newSecrets; + }; + + const fnSecretBulkDelete = async ({ + folderId, + inputSecrets, + tx, + actorId + }: TFnSecretBulkDelete) => { + const deletedSecrets = await secretDal.deleteMany( + inputSecrets.map(({ type, secretBlindIndex }) => ({ + blindIndex: secretBlindIndex, + type + })), + folderId, + actorId, + tx + ); + return deletedSecrets; + }; + + // this is a utility function for secret modification + // this will check given secret name blind index exist or not + // if its a created secret set isNew to true + // thus if these blindindex exist it will throw an error + // vice versa when u need to check for updated secret + // this will also return the blind index grouped by secretName + const fnSecretBlindIndexCheck = async ({ + inputSecrets, + folderId, + isNew, + userId, + blindIndexCfg + }: TFnSecretBlindIndexCheck) => { + const blindIndex2KeyName: Record = {}; // used at audit log point + const keyName2BlindIndex = await Promise.all( + inputSecrets.map(({ secretName }) => + generateSecretBlindIndexBySalt(secretName, blindIndexCfg) + ) + ).then((blindIndexes) => + blindIndexes.reduce>((prev, curr, i) => { + // eslint-disable-next-line + prev[inputSecrets[i].secretName] = curr; + blindIndex2KeyName[curr] = inputSecrets[i].secretName; + return prev; + }, {}) + ); + if (inputSecrets.some(({ type }) => type === SecretType.Personal) && !userId) { + throw new BadRequestError({ message: "Missing user id for personal secret" }); + } + + const secrets = await secretDal.findByBlindIndexes( + folderId, + inputSecrets.map(({ secretName, type }) => ({ + blindIndex: keyName2BlindIndex[secretName], + type: type || SecretType.Shared + })), + userId + ); + + if (isNew) { + if (secrets.length) throw new BadRequestError({ message: "Secret already exist" }); + } else if (secrets.length !== inputSecrets.length) + throw new BadRequestError({ message: "Secret not found" }); + + return { blindIndex2KeyName, keyName2BlindIndex, secrets }; + }; + const createSecret = async ({ path, actor, @@ -104,27 +243,29 @@ export const secretServiceFactory = ({ subject(ProjectPermissionSub.Secrets, { environment, secretPath: path }) ); - const secretBlindIndex = await generateSecretBlindIndexByName( - projectId, - inputSecret.secretName - ); const folder = await folderDal.findBySecretPath(projectId, environment, path); if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Create secret" }); const folderId = folder.id; - // check if secret exist by finding the secret blindIndex - const existingSecret = await secretDal.findOne({ - secretBlindIndex, + const blindIndexCfg = await secretBlindIndexDal.findOne({ projectId }); + if (!blindIndexCfg) + throw new BadRequestError({ message: "Blind index not found", name: "CreateSecret" }); + + if (ActorType.USER !== actor && inputSecret.type === SecretType.Personal) { + throw new BadRequestError({ message: "Must be user to create personal secret" }); + } + + const { keyName2BlindIndex } = await fnSecretBlindIndexCheck({ + inputSecrets: [{ secretName: inputSecret.secretName }], folderId, - type: inputSecret.type, - userId: inputSecret.type === SecretType.Personal ? actorId : null + isNew: true, + blindIndexCfg }); - if (existingSecret) throw new BadRequestError({ message: "Secret already exist" }); // if user creating personal check its shared also exist if (inputSecret.type === SecretType.Personal) { const sharedExist = await secretDal.findOne({ - secretBlindIndex, + secretBlindIndex: keyName2BlindIndex[inputSecret.secretName], folderId, type: SecretType.Shared }); @@ -142,46 +283,29 @@ export const secretServiceFactory = ({ if ((inputSecret.tags || []).length !== tags.length) throw new BadRequestError({ message: "Tag not found" }); - const secret = await secretDal.transaction(async (tx) => { - const { secretName, type, ...el } = inputSecret; - const doc = await secretDal.create( - { - version: 1, - folderId, - secretBlindIndex, - type, - ...el, - userId: inputSecret.type === SecretType.Personal ? actorId : null, - algorithm: SecretEncryptionAlgo.AES_256_GCM, - keyEncoding: SecretKeyEncoding.UTF8 - }, + const { secretName, type, ...el } = inputSecret; + const secret = await secretDal.transaction((tx) => + fnSecretBulkInsert({ + folderId, + inputSecrets: [ + { + version: 1, + secretBlindIndex: keyName2BlindIndex[secretName], + type, + ...el, + userId: inputSecret.type === SecretType.Personal ? actorId : null, + algorithm: SecretEncryptionAlgo.AES_256_GCM, + keyEncoding: SecretKeyEncoding.UTF8, + tags: inputSecret.tags + } + ], tx - ); - if (tags.length) { - await secretTagDal.saveTagsToSecret( - tags.map(({ id }) => ({ secretsId: doc.id, secret_tagsId: id })), - tx - ); - } - await secretVersionDal.create( - { - secretBlindIndex, - folderId, - version: 1, - type, - ...el, - userId: inputSecret.type === SecretType.Personal ? actorId : null, - algorithm: SecretEncryptionAlgo.AES_256_GCM, - keyEncoding: SecretKeyEncoding.UTF8, - secretId: doc.id - }, - tx - ); - return doc; - }); + }) + ); + await snapshotService.performSnapshot(folderId); // TODO(akhilmhdh-pg): licence check, posthog service and snapshot - return { ...secret, tags }; + return { ...secret[0], tags }; }; const updateSecret = async ({ @@ -198,33 +322,38 @@ export const secretServiceFactory = ({ subject(ProjectPermissionSub.Secrets, { environment, secretPath: path }) ); - const blindIndexDoc = await secretBlindIndexDal.findOne({ projectId }); - if (!blindIndexDoc) - throw new BadRequestError({ message: "Blind index not found", name: "Update secret" }); - const oldBlindIndex = await generateSecretBlindIndexBySalt( - inputSecret.secretName, - blindIndexDoc - ); - if (!oldBlindIndex) throw new BadRequestError({ message: "Secret not found" }); - const folder = await folderDal.findBySecretPath(projectId, environment, path); if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Create secret" }); const folderId = folder.id; - let newSecretNameBlindIndex: string; - if (inputSecret?.newSecretName && inputSecret.type === SecretType.Shared) { - newSecretNameBlindIndex = await generateSecretBlindIndexBySalt( - inputSecret.newSecretName, - blindIndexDoc - ); - const doesSecretExist = await secretDal.findOne({ - secretBlindIndex: newSecretNameBlindIndex, + const blindIndexCfg = await secretBlindIndexDal.findOne({ projectId }); + if (!blindIndexCfg) + throw new BadRequestError({ message: "Blind index not found", name: "CreateSecret" }); + + if (ActorType.USER !== actor && inputSecret.type === SecretType.Personal) { + throw new BadRequestError({ message: "Must be user to create personal secret" }); + } + + const { secrets, keyName2BlindIndex } = await fnSecretBlindIndexCheck({ + inputSecrets: [{ secretName: inputSecret.secretName, type: inputSecret.type as SecretType }], + folderId, + isNew: false, + blindIndexCfg, + userId: actorId + }); + if (inputSecret.newSecretName && inputSecret.type === SecretType.Personal) { + throw new BadRequestError({ message: "Personal secret cannot change the key name" }); + } + + let newSecretNameBlindIndex: string | undefined; + if (inputSecret?.newSecretName) { + const { keyName2BlindIndex: kN2NewBlindIndex } = await fnSecretBlindIndexCheck({ + inputSecrets: [{ secretName: inputSecret.newSecretName }], folderId, - type: inputSecret.type + isNew: true, + blindIndexCfg }); - if (doesSecretExist) { - throw new BadRequestError({ message: "Secret with the provided name already exist" }); - } + newSecretNameBlindIndex = kN2NewBlindIndex[inputSecret.newSecretName]; } const tags = inputSecret.tags @@ -233,41 +362,43 @@ export const secretServiceFactory = ({ if ((inputSecret.tags || []).length !== tags.length) throw new BadRequestError({ message: "Tag not found" }); - const updatedSecret = await secretDal.transaction(async (tx) => { - const { secretName, ...el } = inputSecret; - const [doc] = await secretDal.update( - { - secretBlindIndex: oldBlindIndex, - folderId, - type: inputSecret.type, - userId: inputSecret.type === SecretType.Personal ? actorId : null - }, - { - secretBlindIndex: newSecretNameBlindIndex, - ...el - }, + const { secretName, ...el } = inputSecret; + const updatedSecret = await secretDal.transaction(async (tx) => + fnSecretBulkUpdate({ + folderId, + projectId, + inputSecrets: [ + { + id: secrets[0].id, + version: (secrets[0].version || 0) + 1, + ...pick(el, [ + "type", + "secretCommentCiphertext", + "secretCommentTag", + "secretCommentIV", + "secretValueIV", + "secretValueTag", + "secretValueCiphertext", + "secretKeyCiphertext", + "secretKeyTag", + "secretKeyIV", + "metadata", + "skipMultilineEncoding", + "secretReminderNote", + "secretReminderRepeatDays", + "tags" + ]), + secretBlindIndex: newSecretNameBlindIndex || keyName2BlindIndex[secretName] + } + ], tx - ); - // replace tags - await secretTagDal.deleteTagsToSecret({ secretsId: doc.id }, tx); - await secretTagDal.saveTagsToSecret( - tags.map(({ id }) => ({ secretsId: doc.id, secret_tagsId: id })), - tx - ); - const { id, createdAt, updatedAt, ...newVersion } = doc; - await secretVersionDal.create( - { - userId: inputSecret.type === SecretType.Personal ? actorId : null, - secretId: doc.id, - ...newVersion - }, - tx - ); - return doc; - }); + }) + ); + + await snapshotService.performSnapshot(folderId); // TODO(akhilmhdh-pg): licence check, posthog service and snapshot - return updatedSecret; + return updatedSecret[0]; }; const deleteSecret = async ({ @@ -284,30 +415,44 @@ export const secretServiceFactory = ({ subject(ProjectPermissionSub.Secrets, { environment, secretPath: path }) ); - const secretBlindIndex = await generateSecretBlindIndexByName( - projectId, - inputSecret.secretName - ); - const folder = await folderDal.findBySecretPath(projectId, environment, path); if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Create secret" }); const folderId = folder.id; - const deletedSecret = await secretDal.transaction(async (tx) => { - const [doc] = await secretDal.delete( - { - secretBlindIndex, - folderId, - type: inputSecret.type, - userId: inputSecret.type === SecretType.Personal ? actorId : null - }, - tx - ); - return doc; + const blindIndexCfg = await secretBlindIndexDal.findOne({ projectId }); + if (!blindIndexCfg) + throw new BadRequestError({ message: "Blind index not found", name: "CreateSecret" }); + + if (ActorType.USER !== actor && inputSecret.type === SecretType.Personal) { + throw new BadRequestError({ message: "Must be user to create personal secret" }); + } + + const { keyName2BlindIndex } = await fnSecretBlindIndexCheck({ + inputSecrets: [{ secretName: inputSecret.secretName }], + folderId, + isNew: false, + blindIndexCfg }); + const deletedSecret = await secretDal.transaction(async (tx) => + fnSecretBulkDelete({ + projectId, + folderId, + actorId, + inputSecrets: [ + { + type: inputSecret.type as SecretType, + secretBlindIndex: keyName2BlindIndex[inputSecret.secretName] + } + ], + tx + }) + ); + + await snapshotService.performSnapshot(folderId); + // TODO(akhilmhdh-pg): licence check, posthog service and snapshot - return deletedSecret; + return deletedSecret[0]; }; const getSecrets = async ({ actorId, path, environment, projectId, actor }: TGetSecretsDTO) => { @@ -343,7 +488,7 @@ export const secretServiceFactory = ({ if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Create secret" }); const folderId = folder.id; - const secretBlindIndex = await generateSecretBlindIndexByName(projectId, secretName); + const secretBlindIndex = await interalGenSecBlindIndexByName(projectId, secretName); const secret = await secretDal.findOne({ folderId, @@ -374,76 +519,38 @@ export const secretServiceFactory = ({ if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Create secret" }); const folderId = folder.id; - const blindIndexDoc = await secretBlindIndexDal.findOne({ projectId }); - if (!blindIndexDoc) + const blindIndexCfg = await secretBlindIndexDal.findOne({ projectId }); + if (!blindIndexCfg) throw new BadRequestError({ message: "Blind index not found", name: "Update secret" }); - const secretBlindIndexToKey: Record = {}; // used at audit log point - const secretBlindIndexes = await Promise.all( - inputSecrets.map(({ secretName }) => - generateSecretBlindIndexBySalt(secretName, blindIndexDoc) - ) - ).then((blindIndexes) => - blindIndexes.reduce>((prev, curr, i) => { - // eslint-disable-next-line - prev[inputSecrets[i].secretName] = curr; - secretBlindIndexToKey[curr] = inputSecrets[i].secretName; - return prev; - }, {}) - ); - - const exists = await secretDal.findByBlindIndexes( + const { keyName2BlindIndex } = await fnSecretBlindIndexCheck({ + inputSecrets, folderId, - inputSecrets.map(({ type, secretName }) => ({ - blindIndex: secretBlindIndexes[secretName], - type - })) - ); - if (exists.length) throw new BadRequestError({ message: "Secret already exist" }); + isNew: true, + blindIndexCfg + }); // get all tags const tagIds = inputSecrets.flatMap(({ tags = [] }) => tags); const tags = tagIds.length ? await secretTagDal.findManyTagsById(projectId, tagIds) : []; + if (tags.length !== tagIds.length) throw new BadRequestError({ message: "Tag not found" }); - const secrets = await secretDal.transaction(async (tx) => { - const newSecrets = await secretDal.insertMany( - inputSecrets.map(({ secretName, type, ...el }) => ({ - version: 1, - folderId, - type, - secretBlindIndex: secretBlindIndexes[secretName], + const newSecrets = await secretDal.transaction(async (tx) => + fnSecretBulkInsert({ + inputSecrets: inputSecrets.map(({ secretName, ...el }) => ({ ...el, - userId: type === SecretType.Personal ? actorId : null, + secretBlindIndex: keyName2BlindIndex[secretName], + type: SecretType.Shared, algorithm: SecretEncryptionAlgo.AES_256_GCM, keyEncoding: SecretKeyEncoding.UTF8 })), + folderId, tx - ); - if (tags.length) { - await secretTagDal.saveTagsToSecret( - inputSecrets.flatMap(({ tags: secretTags = [], secretName }) => { - const secret = newSecrets.find( - ({ secretBlindIndex }) => secretBlindIndexes[secretName] === secretBlindIndex - ); - return secretTags.map((tag) => ({ - [`${TableName.SecretTag}Id`]: tag, - [`${TableName.Secret}Id`]: secret?.id || "" - })); - }), - tx - ); - } - await secretVersionDal.insertMany( - newSecrets.map(({ id, updatedAt, createdAt, ...el }) => ({ - ...el, - secretId: id - })), - tx - ); + }) + ); - return newSecrets; - }); - return secrets; + await snapshotService.performSnapshot(folderId); + return newSecrets; }; const updateManySecret = async ({ @@ -464,121 +571,58 @@ export const secretServiceFactory = ({ if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Create secret" }); const folderId = folder.id; - const blindIndexDoc = await secretBlindIndexDal.findOne({ projectId }); - if (!blindIndexDoc) + const blindIndexCfg = await secretBlindIndexDal.findOne({ projectId }); + if (!blindIndexCfg) throw new BadRequestError({ message: "Blind index not found", name: "Update secret" }); - // get all blind index - // Find all those secrets - // if not throw not found - const secretBlindIndexToKey: Record = {}; // used at audit log point - const secretBlindIndexes = await Promise.all( - inputSecrets.map(({ secretName }) => - generateSecretBlindIndexBySalt(secretName, blindIndexDoc) - ) - ).then((blindIndexes) => - blindIndexes.reduce>((prev, curr, i) => { - // eslint-disable-next-line - prev[inputSecrets[i].secretName] = curr; - secretBlindIndexToKey[curr] = inputSecrets[i].secretName; - return prev; - }, {}) - ); - - const secretsToBeUpdated = await secretDal.findByBlindIndexes( + const { keyName2BlindIndex, secrets: secretsToBeUpdated } = await fnSecretBlindIndexCheck({ + inputSecrets, folderId, - inputSecrets.map(({ type, secretName }) => ({ - blindIndex: secretBlindIndexes[secretName], - type - })) - ); - if (secretsToBeUpdated.length !== inputSecrets.length) - throw new BadRequestError({ message: "Secret not found" }); + isNew: false, + blindIndexCfg + }); // now find any secret that needs to update its name // same process as above const nameUpdatedSecrets = inputSecrets.filter(({ newSecretName }) => Boolean(newSecretName)); - const newSecretBlindIndexes = await Promise.all( - nameUpdatedSecrets.map(({ newSecretName }) => - generateSecretBlindIndexBySalt(newSecretName as string, blindIndexDoc) - ) - ).then((blindIndexes) => - blindIndexes.reduce>((prev, curr, i) => { - // eslint-disable-next-line - prev[nameUpdatedSecrets[i].secretName] = curr; - return prev; - }, {}) - ); - const secretsWithNewName = await secretDal.findByBlindIndexes( + const { keyName2BlindIndex: newKeyName2BlindIndex } = await fnSecretBlindIndexCheck({ + inputSecrets: nameUpdatedSecrets, folderId, - nameUpdatedSecrets.map(({ type, newSecretName }) => ({ - blindIndex: newSecretBlindIndexes[newSecretName as string], - type - })) - ); - if (secretsWithNewName.length) - throw new BadRequestError({ message: "Secret with new name already exist" }); - - const secretsGroupedByBlindIndex = secretsToBeUpdated.reduce>( - (prev, curr) => { - // eslint-disable-next-line - if (curr.secretBlindIndex) prev[curr.secretBlindIndex] = curr; - return prev; - }, - {} - ); + isNew: true, + blindIndexCfg + }); + const secsGroupedByBlindIndex = groupBy(secretsToBeUpdated, (el) => el.secretBlindIndex); // get all tags const tagIds = inputSecrets.flatMap(({ tags = [] }) => tags); const tags = tagIds.length ? await secretTagDal.findManyTagsById(projectId, tagIds) : []; if (tagIds.length !== tags.length) throw new BadRequestError({ message: "Tag not found" }); - - const secrets = await secretDal.transaction(async (tx) => { - const newSecrets = await secretDal.bulkUpdate( - inputSecrets.map(({ secretName, type, ...el }) => { + const secrets = await secretDal.transaction(async (tx) => + fnSecretBulkUpdate({ + folderId, + projectId, + tx, + inputSecrets: inputSecrets.map(({ secretName, newSecretName, ...el }) => { const { version, updatedAt, ...info } = - secretsGroupedByBlindIndex[secretBlindIndexes[secretName]]; + secsGroupedByBlindIndex[keyName2BlindIndex[secretName]][0]; return { + ...el, version: (version || 0) + 1, ...info, folderId, - type, + type: SecretType.Shared, secretBlindIndex: - el?.newSecretName && newSecretBlindIndexes[el.newSecretName] - ? newSecretBlindIndexes[el.newSecretName] - : secretBlindIndexes[secretName], - ...el, - userId: type === SecretType.Personal ? actorId : null, + newSecretName && newKeyName2BlindIndex[newSecretName] + ? newKeyName2BlindIndex[newSecretName] + : keyName2BlindIndex[secretName], algorithm: SecretEncryptionAlgo.AES_256_GCM, keyEncoding: SecretKeyEncoding.UTF8 }; - }), - tx - ); - await secretTagDal.deleteTagsManySecret( - projectId, - newSecrets.map(({ id }) => id), - tx - ); - await secretTagDal.saveTagsToSecret( - inputSecrets.flatMap(({ secretName, tags: secretTags = [] }) => - secretTags.map((secretTag) => ({ - [`${TableName.Secret}Id`]: secretsGroupedByBlindIndex[secretName].id, - [`${TableName.SecretTag}Id`]: secretTag - })) - ), - tx - ); - await secretVersionDal.insertMany( - newSecrets.map(({ id, updatedAt, createdAt, ...el }) => ({ - ...el, - secretId: id - })), - tx - ); + }) + }) + ); - return newSecrets; - }); + await snapshotService.performSnapshot(folderId); return secrets; }; @@ -600,51 +644,63 @@ export const secretServiceFactory = ({ if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Create secret" }); const folderId = folder.id; - const blindIndexDoc = await secretBlindIndexDal.findOne({ projectId }); - if (!blindIndexDoc) + const blindIndexCfg = await secretBlindIndexDal.findOne({ projectId }); + if (!blindIndexCfg) throw new BadRequestError({ message: "Blind index not found", name: "Update secret" }); - // get all blind index - // Find all those secrets - // if not throw not found - const secretBlindIndexToKey: Record = {}; // used at audit log point - const secretBlindIndexes = await Promise.all( - inputSecrets.map(({ secretName }) => - generateSecretBlindIndexBySalt(secretName, blindIndexDoc) - ) - ).then((blindIndexes) => - blindIndexes.reduce>((prev, curr, i) => { - // eslint-disable-next-line - prev[inputSecrets[i].secretName] = curr; - secretBlindIndexToKey[curr] = inputSecrets[i].secretName; - return prev; - }, {}) - ); - // not find those secrets. if any of them not found throw an not found error - const secretsToBeDeleted = await secretDal.findByBlindIndexes( + const { keyName2BlindIndex } = await fnSecretBlindIndexCheck({ + inputSecrets, folderId, - inputSecrets.map(({ type, secretName }) => ({ - blindIndex: secretBlindIndexes[secretName], - type - })) - ); - if (secretsToBeDeleted.length !== inputSecrets.length) - throw new BadRequestError({ message: "Secret not found" }); + isNew: false, + blindIndexCfg + }); + const secretsDeleted = await secretDal.transaction(async (tx) => - secretDal.deleteMany( - inputSecrets.map(({ type, secretName }) => ({ - blindIndex: secretBlindIndexes[secretName], + fnSecretBulkDelete({ + inputSecrets: inputSecrets.map(({ type, secretName }) => ({ + secretBlindIndex: keyName2BlindIndex[secretName], type })), + projectId, folderId, actorId, tx - ) + }) ); + await snapshotService.performSnapshot(folderId); return secretsDeleted; }; + const listSecretVersionsBySecretId = async ({ + actorId, + actor, + limit, + offset, + secretId + }: TListSecretVersionDTO) => { + const secret = await secretDal.findById(secretId); + if (!secret) throw new BadRequestError({ message: "Failed to find secret" }); + + const folder = await folderDal.findById(secret.folderId); + if (!folder) throw new BadRequestError({ message: "Folder not found" }); + const { permission } = await permissionService.getProjectPermission( + actor, + actorId, + folder.projectId + ); + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionActions.Read, + ProjectPermissionSub.SecretRollback + ); + + const secretVersions = await secretVersionDal.find( + { secretId }, + { limit, offset, sort: [["createdAt", "desc"]] } + ); + return secretVersions; + }; + return { createSecret, deleteSecret, @@ -653,6 +709,12 @@ export const secretServiceFactory = ({ updateManySecret, deleteManySecret, getASecret, - getSecrets + getSecrets, + listSecretVersionsBySecretId, + // external services function + fnSecretBulkDelete, + fnSecretBulkUpdate, + fnSecretBlindIndexCheck, + fnSecretBulkInsert }; }; diff --git a/backend-pg/src/services/secret/secret-types.ts b/backend-pg/src/services/secret/secret-types.ts index 48ceb094e..ac9229725 100644 --- a/backend-pg/src/services/secret/secret-types.ts +++ b/backend-pg/src/services/secret/secret-types.ts @@ -1,4 +1,6 @@ -import { SecretType } from "@app/db/schemas"; +import { Knex } from "knex"; + +import { SecretType, TSecretBlindIndexes, TSecretsInsert, TSecretsUpdate } from "@app/db/schemas"; import { TProjectPermission } from "@app/lib/types"; export type TCreateSecretDTO = { @@ -73,7 +75,6 @@ export type TCreateBulkSecretDTO = { environment: string; secrets: Array<{ secretName: string; - type: SecretType; secretKeyCiphertext: string; secretKeyIV: string; secretKeyTag: string; @@ -117,3 +118,38 @@ export type TDeleteBulkSecretDTO = { secretName: string; }>; } & TProjectPermission; + +export type TListSecretVersionDTO = { + secretId: string; + offset?: number; + limit?: number; +} & Omit; + +export type TFnSecretBulkInsert = { + folderId: string; + tx?: Knex; + inputSecrets: Array & { tags?: string[] }>; +}; + +export type TFnSecretBulkUpdate = { + folderId: string; + projectId: string; + inputSecrets: Array; + tx?: Knex; +}; + +export type TFnSecretBulkDelete = { + folderId: string; + projectId: string; + inputSecrets: Array<{ type: SecretType; secretBlindIndex: string }>; + actorId: string; + tx?: Knex; +}; + +export type TFnSecretBlindIndexCheck = { + folderId: string; + userId?: string; + blindIndexCfg: TSecretBlindIndexes; + inputSecrets: Array<{ secretName: string; type?: SecretType }>; + isNew: boolean; +}; diff --git a/backend-pg/src/services/secret/secret-version-dal.ts b/backend-pg/src/services/secret/secret-version-dal.ts index dcf3ee4a4..58a968da6 100644 --- a/backend-pg/src/services/secret/secret-version-dal.ts +++ b/backend-pg/src/services/secret/secret-version-dal.ts @@ -1,15 +1,41 @@ import { Knex } from "knex"; import { TDbClient } from "@app/db"; -import { TableName,TSecretVersions } from "@app/db/schemas"; +import { TableName, TSecretVersions } from "@app/db/schemas"; import { DatabaseError } from "@app/lib/errors"; -import { ormify } from "@app/lib/knex"; +import { ormify, selectAllTableCols } from "@app/lib/knex"; export type TSecretVersionDalFactory = ReturnType; export const secretVersionDalFactory = (db: TDbClient) => { const secretVersionOrm = ormify(db, TableName.SecretVersion); + // This will fetch all latest secret versions from a folder + const findLatestVersionByFolderId = async (folderId: string, tx?: Knex) => { + try { + const docs = await (tx || db)(TableName.SecretVersion) + .where(`${TableName.SecretVersion}.folderId`, folderId) + .join(TableName.Secret, `${TableName.Secret}.id`, `${TableName.SecretVersion}.secretId`) + .join( + (tx || db)(TableName.SecretVersion) + .groupBy("folderId", "secretId") + .max("version") + .select("secretId") + .as("latestVersion"), + (bd) => { + bd.on(`${TableName.SecretVersion}.secretId`, "latestVersion.secretId").andOn( + `${TableName.SecretVersion}.version`, + "latestVersion.max" + ); + } + ) + .select(selectAllTableCols(TableName.SecretVersion)); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindLatestVersionByFolderId" }); + } + }; + const findLatestVersionMany = async (folderId: string, secretIds: string[], tx?: Knex) => { try { const docs: Array = await (tx || db)( @@ -31,7 +57,7 @@ export const secretVersionDalFactory = (db: TDbClient) => { } ); return docs.reduce>( - (prev, curr) => ({ ...prev, [curr.secretId]: curr }), + (prev, curr) => ({ ...prev, [curr.secretId || ""]: curr }), {} ); } catch (error) { @@ -39,5 +65,5 @@ export const secretVersionDalFactory = (db: TDbClient) => { } }; - return { ...secretVersionOrm, findLatestVersionMany }; + return { ...secretVersionOrm, findLatestVersionMany, findLatestVersionByFolderId }; }; diff --git a/docker-compose.pg.yml b/docker-compose.pg.yml index 3ff77bc50..7f4d029f4 100644 --- a/docker-compose.pg.yml +++ b/docker-compose.pg.yml @@ -24,6 +24,16 @@ services: POSTGRES_USER: infisical POSTGRES_DB: infisical + redis: + image: redis + container_name: infisical-dev-redis + environment: + - ALLOW_EMPTY_PASSWORD=yes + ports: + - 6379:6379 + volumes: + - redis_data:/data + db-test: profiles: ["test"] image: postgres:14-alpine @@ -67,3 +77,5 @@ services: volumes: postgres-data: driver: local + redis_data: + driver: local diff --git a/frontend/src/hooks/api/secretFolders/queries.tsx b/frontend/src/hooks/api/secretFolders/queries.tsx index faea30531..6abf6cc30 100644 --- a/frontend/src/hooks/api/secretFolders/queries.tsx +++ b/frontend/src/hooks/api/secretFolders/queries.tsx @@ -19,7 +19,7 @@ import { TUpdateFolderDTO } from "./types"; -const queryKeys = { +export const folderQueryKeys = { getSecretFolders: ({ projectId, environment, path }: TGetProjectFoldersDTO) => ["secret-folders", { projectId, environment, path }] as const }; @@ -46,14 +46,14 @@ export const useGetProjectFolders = ({ TSecretFolder[], unknown, TSecretFolder[], - ReturnType + ReturnType >, "queryKey" | "queryFn" >; }) => useQuery({ ...options, - queryKey: queryKeys.getSecretFolders({ projectId, environment, path }), + queryKey: folderQueryKeys.getSecretFolders({ projectId, environment, path }), enabled: Boolean(projectId) && Boolean(environment) && (options?.enabled ?? true), queryFn: async () => fetchProjectFolders(projectId, environment, path) }); @@ -65,7 +65,7 @@ export const useGetFoldersByEnv = ({ }: TGetFoldersByEnvDTO) => { const folders = useQueries({ queries: environments.map((environment) => ({ - queryKey: queryKeys.getSecretFolders({ projectId, environment, path }), + queryKey: folderQueryKeys.getSecretFolders({ projectId, environment, path }), queryFn: async () => fetchProjectFolders(projectId, environment, path), enabled: Boolean(projectId) && Boolean(environment) })) @@ -106,7 +106,9 @@ export const useCreateFolder = () => { return data; }, onSuccess: (_, { projectId, environment, path }) => { - queryClient.invalidateQueries(queryKeys.getSecretFolders({ projectId, environment, path })); + queryClient.invalidateQueries( + folderQueryKeys.getSecretFolders({ projectId, environment, path }) + ); queryClient.invalidateQueries( secretSnapshotKeys.list({ workspaceId: projectId, environment, directory: path }) ); @@ -131,7 +133,9 @@ export const useUpdateFolder = () => { return data; }, onSuccess: (_, { projectId, environment, path }) => { - queryClient.invalidateQueries(queryKeys.getSecretFolders({ projectId, environment, path })); + queryClient.invalidateQueries( + folderQueryKeys.getSecretFolders({ projectId, environment, path }) + ); queryClient.invalidateQueries( secretSnapshotKeys.list({ workspaceId: projectId, environment, directory: path }) ); @@ -157,7 +161,9 @@ export const useDeleteFolder = () => { return data; }, onSuccess: (_, { path = "/", projectId, environment }) => { - queryClient.invalidateQueries(queryKeys.getSecretFolders({ projectId, environment, path })); + queryClient.invalidateQueries( + folderQueryKeys.getSecretFolders({ projectId, environment, path }) + ); queryClient.invalidateQueries( secretSnapshotKeys.list({ workspaceId: projectId, environment, directory: path }) ); diff --git a/frontend/src/hooks/api/secretSnapshots/queries.tsx b/frontend/src/hooks/api/secretSnapshots/queries.tsx index 842f9d44c..cbbe7e743 100644 --- a/frontend/src/hooks/api/secretSnapshots/queries.tsx +++ b/frontend/src/hooks/api/secretSnapshots/queries.tsx @@ -35,13 +35,13 @@ const fetchWorkspaceSnaphots = async ({ offset = 0 }: TGetSecretSnapshotsDTO & { offset: number }) => { const res = await apiRequest.get<{ secretSnapshots: TSecretSnapshot[] }>( - `/api/v1/workspace/${workspaceId}/secret-snapshots`, + `/api/ee/v1/workspace/${workspaceId}/secret-snapshots`, { params: { limit, offset, environment, - directory + path: directory } } ); @@ -60,12 +60,12 @@ export const useGetWorkspaceSnapshotList = (dto: TGetSecretSnapshotsDTO & { isPa const fetchSnapshotEncSecrets = async (snapshotId: string) => { const res = await apiRequest.get<{ secretSnapshot: TSnapshotData }>( - `/api/v1/secret-snapshot/${snapshotId}` + `/api/ee/v1/secret-snapshot/${snapshotId}` ); return res.data.secretSnapshot; }; -export const useGetSnapshotSecrets = ({ decryptFileKey, env, snapshotId }: TSnapshotDataProps) => +export const useGetSnapshotSecrets = ({ decryptFileKey, snapshotId }: TSnapshotDataProps) => useQuery({ queryKey: secretSnapshotKeys.snapshotData(snapshotId), enabled: Boolean(snapshotId && decryptFileKey), @@ -82,45 +82,43 @@ export const useGetSnapshotSecrets = ({ decryptFileKey, env, snapshotId }: TSnap const sharedSecrets: DecryptedSecret[] = []; const personalSecrets: Record = {}; - data.secretVersions - .filter(({ environment }) => environment === env) - .forEach((encSecret) => { - const secretKey = decryptSymmetric({ - ciphertext: encSecret.secretKeyCiphertext, - iv: encSecret.secretKeyIV, - tag: encSecret.secretKeyTag, - key - }); - - const secretValue = decryptSymmetric({ - ciphertext: encSecret.secretValueCiphertext, - iv: encSecret.secretValueIV, - tag: encSecret.secretValueTag, - key - }); - - const secretComment = ""; - - const decryptedSecret = { - id: encSecret.secret, - env: encSecret.environment, - key: secretKey, - value: secretValue, - tags: encSecret.tags, - comment: secretComment, - createdAt: encSecret.createdAt, - updatedAt: encSecret.updatedAt, - type: "modified", - version: encSecret.version - }; - - if (encSecret.type === "personal") { - personalSecrets[decryptedSecret.key] = { id: encSecret.secret, value: secretValue }; - } else { - sharedSecrets.push(decryptedSecret); - } + data.secretVersions.forEach((encSecret) => { + const secretKey = decryptSymmetric({ + ciphertext: encSecret.secretKeyCiphertext, + iv: encSecret.secretKeyIV, + tag: encSecret.secretKeyTag, + key }); + const secretValue = decryptSymmetric({ + ciphertext: encSecret.secretValueCiphertext, + iv: encSecret.secretValueIV, + tag: encSecret.secretValueTag, + key + }); + + const secretComment = ""; + + const decryptedSecret = { + id: encSecret.secretId, + env: data.environment.slug, + key: secretKey, + value: secretValue, + tags: encSecret.tags, + comment: secretComment, + createdAt: encSecret.createdAt, + updatedAt: encSecret.updatedAt, + type: "modified", + version: encSecret.version + }; + + if (encSecret.type === "personal") { + personalSecrets[decryptedSecret.key] = { id: encSecret.secretId, value: secretValue }; + } else { + sharedSecrets.push(decryptedSecret); + } + }); + sharedSecrets.forEach((val) => { if (personalSecrets?.[val.key]) { val.idOverride = personalSecrets[val.key].id; @@ -130,7 +128,7 @@ export const useGetSnapshotSecrets = ({ decryptFileKey, env, snapshotId }: TSnap }); return { - version: data.version, + id: data.id, secrets: sharedSecrets, createdAt: data.createdAt, folders: data.folderVersion @@ -144,11 +142,11 @@ const fetchWorkspaceSecretSnaphotCount = async ( directory = "/" ) => { const res = await apiRequest.get<{ count: number }>( - `/api/v1/workspace/${workspaceId}/secret-snapshots/count`, + `/api/ee/v1/workspace/${workspaceId}/secret-snapshots/count`, { params: { environment, - directory + path: directory } } ); @@ -171,11 +169,8 @@ export const usePerformSecretRollback = () => { const queryClient = useQueryClient(); return useMutation<{}, {}, TSecretRollbackDTO>({ - mutationFn: async ({ workspaceId, ...dto }) => { - const { data } = await apiRequest.post( - `/api/v1/workspace/${workspaceId}/secret-snapshots/rollback`, - dto - ); + mutationFn: async ({ snapshotId }) => { + const { data } = await apiRequest.post(`/api/ee/v1/secret-snapshot/${snapshotId}/rollback`); return data; }, onSuccess: (_, { workspaceId, environment, directory }) => { @@ -183,6 +178,10 @@ export const usePerformSecretRollback = () => { { workspaceId, environment, secretPath: directory }, "secrets" ]); + queryClient.invalidateQueries([ + "secret-folders", + { projectId: workspaceId, environment, path: directory } + ]); queryClient.invalidateQueries( secretSnapshotKeys.list({ workspaceId, environment, directory }) ); diff --git a/frontend/src/hooks/api/secretSnapshots/types.ts b/frontend/src/hooks/api/secretSnapshots/types.ts index 9f1fb4da6..2eb956a8a 100644 --- a/frontend/src/hooks/api/secretSnapshots/types.ts +++ b/frontend/src/hooks/api/secretSnapshots/types.ts @@ -1,19 +1,20 @@ import { UserWsKeyPair } from "../keys/types"; import { EncryptedSecretVersion } from "../secrets/types"; +import { WorkspaceEnv } from "../types"; export type TSecretSnapshot = { id: string; workspace: string; - version: number; secretVersions: string[]; createdAt: string; updatedAt: string; - __v: number; }; export type TSnapshotData = Omit & { + id: string; secretVersions: EncryptedSecretVersion[]; folderVersion: Array<{ name: string; id: string }>; + environment: WorkspaceEnv; }; export type TSnapshotDataProps = { @@ -30,8 +31,8 @@ export type TGetSecretSnapshotsDTO = { }; export type TSecretRollbackDTO = { + snapshotId: string; workspaceId: string; - version: number; environment: string; directory?: string; }; diff --git a/frontend/src/hooks/api/secrets/types.ts b/frontend/src/hooks/api/secrets/types.ts index 66ad3f1fc..f36872e43 100644 --- a/frontend/src/hooks/api/secrets/types.ts +++ b/frontend/src/hooks/api/secrets/types.ts @@ -46,12 +46,12 @@ export type DecryptedSecret = { export type EncryptedSecretVersion = { id: string; - secret: string; + secretId: string; version: number; workspace: string; type: string; - environment: string; isDeleted: boolean; + envId: string; secretKeyCiphertext: string; secretKeyIV: string; secretKeyTag: string; diff --git a/frontend/src/views/SecretMainPage/components/SnapshotView/SnapshotView.tsx b/frontend/src/views/SecretMainPage/components/SnapshotView/SnapshotView.tsx index 4235cd2a3..e541921e2 100644 --- a/frontend/src/views/SecretMainPage/components/SnapshotView/SnapshotView.tsx +++ b/frontend/src/views/SecretMainPage/components/SnapshotView/SnapshotView.tsx @@ -97,6 +97,7 @@ export const SnapshotView = ({ {} ); const diffView: Array> = []; + console.log({ rollingSecrets, secrets }); rollingSecrets.forEach((rollSecret) => { const { id } = rollSecret; const doesExist = Boolean(secretGroupById?.[id]); @@ -118,9 +119,9 @@ export const SnapshotView = ({ }); return diffView; }, [secrets, rollingSecrets]); - + console.log(secretDiffView); const handleClickRollback = async () => { - if (!snapshotData?.version) { + if (!snapshotData?.id) { createNotification({ text: "Failed to find secret version", type: "success" @@ -130,7 +131,7 @@ export const SnapshotView = ({ try { await performRollback({ workspaceId, - version: snapshotData.version, + snapshotId: snapshotData.id, environment, directory: secretPath });