From e9358cd1d8fab675d000923ea9ae64d05b05a7a2 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Thu, 19 Dec 2024 15:05:16 +0800 Subject: [PATCH] misc: backend setup + create secret metadata --- backend/src/@types/knex.d.ts | 8 ++++++ .../20241218165837_resource-metadata.ts | 26 +++++++++++++++++++ backend/src/db/schemas/index.ts | 1 + backend/src/db/schemas/models.ts | 1 + backend/src/db/schemas/resource-metadata.ts | 24 +++++++++++++++++ .../secret-approval-request-service.ts | 8 +++++- .../secret-replication-service.ts | 11 +++++++- backend/src/server/routes/index.ts | 15 ++++++++--- backend/src/server/routes/v3/secret-router.ts | 3 +++ .../external-migration-fns.ts | 6 +++++ .../resource-metadata-dal.ts | 11 ++++++++ .../resource-metadata-schema.ts | 10 +++++++ .../secret-v2-bridge/secret-v2-bridge-fns.ts | 21 +++++++++++++++ .../secret-v2-bridge-service.ts | 23 ++++++++++++++-- .../secret-v2-bridge-types.ts | 13 +++++++++- backend/src/services/secret/secret-fns.ts | 7 +++-- backend/src/services/secret/secret-queue.ts | 13 ++++++++-- backend/src/services/secret/secret-service.ts | 16 ++++++++++-- backend/src/services/secret/secret-types.ts | 13 +++++++++- 19 files changed, 214 insertions(+), 16 deletions(-) create mode 100644 backend/src/db/migrations/20241218165837_resource-metadata.ts create mode 100644 backend/src/db/schemas/resource-metadata.ts create mode 100644 backend/src/services/resource-metadata/resource-metadata-dal.ts create mode 100644 backend/src/services/resource-metadata/resource-metadata-schema.ts diff --git a/backend/src/@types/knex.d.ts b/backend/src/@types/knex.d.ts index 5c7787920..31258dc30 100644 --- a/backend/src/@types/knex.d.ts +++ b/backend/src/@types/knex.d.ts @@ -218,6 +218,9 @@ import { TRateLimit, TRateLimitInsert, TRateLimitUpdate, + TResourceMetadata, + TResourceMetadataInsert, + TResourceMetadataUpdate, TSamlConfigs, TSamlConfigsInsert, TSamlConfigsUpdate, @@ -846,5 +849,10 @@ declare module "knex/types/tables" { TProjectSplitBackfillIdsInsert, TProjectSplitBackfillIdsUpdate >; + [TableName.ResourceMetadata]: KnexOriginal.CompositeTableType< + TResourceMetadata, + TResourceMetadataInsert, + TResourceMetadataUpdate + >; } } diff --git a/backend/src/db/migrations/20241218165837_resource-metadata.ts b/backend/src/db/migrations/20241218165837_resource-metadata.ts new file mode 100644 index 000000000..90313b344 --- /dev/null +++ b/backend/src/db/migrations/20241218165837_resource-metadata.ts @@ -0,0 +1,26 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + if (!(await knex.schema.hasTable(TableName.ResourceMetadata))) { + await knex.schema.createTable(TableName.ResourceMetadata, (tb) => { + tb.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid()); + tb.string("key").notNullable(); + tb.string("value", 1020).notNullable(); + tb.uuid("orgId").notNullable(); + tb.foreign("orgId").references("id").inTable(TableName.Organization).onDelete("CASCADE"); + tb.uuid("userId"); + tb.foreign("userId").references("id").inTable(TableName.Users).onDelete("CASCADE"); + tb.uuid("identityId"); + tb.foreign("identityId").references("id").inTable(TableName.Identity).onDelete("CASCADE"); + tb.uuid("secretId"); + tb.foreign("secretId").references("id").inTable(TableName.SecretV2).onDelete("CASCADE"); + tb.timestamps(true, true, true); + }); + } +} + +export async function down(knex: Knex): Promise { + await knex.schema.dropTableIfExists(TableName.ResourceMetadata); +} diff --git a/backend/src/db/schemas/index.ts b/backend/src/db/schemas/index.ts index 02d6c404a..e6d845b7f 100644 --- a/backend/src/db/schemas/index.ts +++ b/backend/src/db/schemas/index.ts @@ -71,6 +71,7 @@ export * from "./project-user-additional-privilege"; export * from "./project-user-membership-roles"; export * from "./projects"; export * from "./rate-limit"; +export * from "./resource-metadata"; export * from "./saml-configs"; export * from "./scim-tokens"; export * from "./secret-approval-policies"; diff --git a/backend/src/db/schemas/models.ts b/backend/src/db/schemas/models.ts index f670ad6e9..dc9420c9e 100644 --- a/backend/src/db/schemas/models.ts +++ b/backend/src/db/schemas/models.ts @@ -75,6 +75,7 @@ export enum TableName { IdentityProjectAdditionalPrivilege = "identity_project_additional_privilege", // used by both identity and users IdentityMetadata = "identity_metadata", + ResourceMetadata = "resource_metadata", ScimToken = "scim_tokens", AccessApprovalPolicy = "access_approval_policies", AccessApprovalPolicyApprover = "access_approval_policies_approvers", diff --git a/backend/src/db/schemas/resource-metadata.ts b/backend/src/db/schemas/resource-metadata.ts new file mode 100644 index 000000000..f496b29db --- /dev/null +++ b/backend/src/db/schemas/resource-metadata.ts @@ -0,0 +1,24 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const ResourceMetadataSchema = z.object({ + id: z.string().uuid(), + key: z.string(), + value: z.string(), + orgId: z.string().uuid(), + userId: z.string().uuid().nullable().optional(), + identityId: z.string().uuid().nullable().optional(), + secretId: z.string().uuid().nullable().optional(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TResourceMetadata = z.infer; +export type TResourceMetadataInsert = Omit, TImmutableDBKeys>; +export type TResourceMetadataUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts index 96ab03b4e..7be71b951 100644 --- a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts +++ b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts @@ -22,6 +22,7 @@ import { KmsDataKey } from "@app/services/kms/kms-types"; import { TProjectDALFactory } from "@app/services/project/project-dal"; import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service"; import { TProjectEnvDALFactory } from "@app/services/project-env/project-env-dal"; +import { TResourceMetadataDALFactory } from "@app/services/resource-metadata/resource-metadata-dal"; import { TSecretDALFactory } from "@app/services/secret/secret-dal"; import { decryptSecretWithBot, @@ -91,6 +92,7 @@ type TSecretApprovalRequestServiceFactoryDep = { secretBlindIndexDAL: Pick; snapshotService: Pick; secretVersionDAL: Pick; + resourceMetadataDAL: Pick; secretVersionTagDAL: Pick; smtpService: Pick; userDAL: Pick; @@ -138,7 +140,8 @@ export const secretApprovalRequestServiceFactory = ({ secretVersionV2BridgeDAL, secretVersionTagV2BridgeDAL, licenseService, - projectSlackConfigDAL + projectSlackConfigDAL, + resourceMetadataDAL }: TSecretApprovalRequestServiceFactoryDep) => { const requestCount = async ({ projectId, actor, actorId, actorOrgId, actorAuthMethod }: TApprovalRequestCountDTO) => { if (actor === ActorType.SERVICE) throw new BadRequestError({ message: "Cannot use service token" }); @@ -543,6 +546,7 @@ export const secretApprovalRequestServiceFactory = ({ ? await fnSecretV2BridgeBulkInsert({ tx, folderId, + orgId: actorOrgId, inputSecrets: secretCreationCommits.map((el) => ({ tagIds: el?.tags.map(({ id }) => id), version: 1, @@ -559,6 +563,7 @@ export const secretApprovalRequestServiceFactory = ({ : [], type: SecretType.Shared })), + resourceMetadataDAL, secretDAL: secretV2BridgeDAL, secretVersionDAL: secretVersionV2BridgeDAL, secretTagDAL, @@ -824,6 +829,7 @@ export const secretApprovalRequestServiceFactory = ({ } await secretQueueService.syncSecrets({ projectId, + orgId: actorOrgId, secretPath: folder.path, environmentSlug: folder.environmentSlug, actorId, diff --git a/backend/src/ee/services/secret-replication/secret-replication-service.ts b/backend/src/ee/services/secret-replication/secret-replication-service.ts index 81d467bab..53ef8b657 100644 --- a/backend/src/ee/services/secret-replication/secret-replication-service.ts +++ b/backend/src/ee/services/secret-replication/secret-replication-service.ts @@ -13,6 +13,7 @@ import { ActorType } from "@app/services/auth/auth-type"; import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { KmsDataKey } from "@app/services/kms/kms-types"; import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service"; +import { TResourceMetadataDALFactory } from "@app/services/resource-metadata/resource-metadata-dal"; import { TSecretDALFactory } from "@app/services/secret/secret-dal"; import { fnSecretBulkInsert, fnSecretBulkUpdate } from "@app/services/secret/secret-fns"; import { TSecretQueueFactory, uniqueSecretQueueKey } from "@app/services/secret/secret-queue"; @@ -56,6 +57,7 @@ type TSecretReplicationServiceFactoryDep = { >; secretVersionTagDAL: Pick; secretVersionV2TagBridgeDAL: Pick; + resourceMetadataDAL: Pick; secretQueueService: Pick; queueService: Pick; secretApprovalPolicyService: Pick; @@ -121,7 +123,8 @@ export const secretReplicationServiceFactory = ({ secretVersionV2TagBridgeDAL, secretVersionV2BridgeDAL, secretV2BridgeDAL, - kmsService + kmsService, + resourceMetadataDAL }: TSecretReplicationServiceFactoryDep) => { const $getReplicatedSecrets = ( botKey: string, @@ -178,6 +181,7 @@ export const secretReplicationServiceFactory = ({ secretPath, environmentSlug, projectId, + orgId, actorId, actor, pickOnlyImportIds, @@ -222,6 +226,7 @@ export const secretReplicationServiceFactory = ({ .map(({ folderId }) => secretQueueService.replicateSecrets({ projectId, + orgId, secretPath: foldersGroupedById[folderId][0]?.path as string, environmentSlug: foldersGroupedById[folderId][0]?.environmentSlug as string, actorId, @@ -406,10 +411,12 @@ export const secretReplicationServiceFactory = ({ if (locallyCreatedSecrets.length) { await fnSecretV2BridgeBulkInsert({ folderId: destinationReplicationFolderId, + orgId, secretVersionDAL: secretVersionV2BridgeDAL, secretDAL: secretV2BridgeDAL, tx, secretTagDAL, + resourceMetadataDAL, secretVersionTagDAL: secretVersionV2TagBridgeDAL, inputSecrets: locallyCreatedSecrets.map((doc) => { return { @@ -466,6 +473,7 @@ export const secretReplicationServiceFactory = ({ await secretQueueService.syncSecrets({ projectId, + orgId, secretPath: destinationFolder.path, environmentSlug: destinationFolder.environmentSlug, actorId, @@ -751,6 +759,7 @@ export const secretReplicationServiceFactory = ({ await secretQueueService.syncSecrets({ projectId, + orgId, secretPath: destinationFolder.path, environmentSlug: destinationFolder.environmentSlug, actorId, diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 65a878bbf..a7e257f3a 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -172,6 +172,7 @@ import { projectUserMembershipRoleDALFactory } from "@app/services/project-membe import { projectRoleDALFactory } from "@app/services/project-role/project-role-dal"; import { projectRoleServiceFactory } from "@app/services/project-role/project-role-service"; import { dailyResourceCleanUpQueueServiceFactory } from "@app/services/resource-cleanup/resource-cleanup-queue"; +import { resourceMetadataDALFactory } from "@app/services/resource-metadata/resource-metadata-dal"; import { secretDALFactory } from "@app/services/secret/secret-dal"; import { secretQueueFactory } from "@app/services/secret/secret-queue"; import { secretServiceFactory } from "@app/services/secret/secret-service"; @@ -358,6 +359,7 @@ export const registerRoutes = async ( const externalGroupOrgRoleMappingDAL = externalGroupOrgRoleMappingDALFactory(db); const projectTemplateDAL = projectTemplateDALFactory(db); + const resourceMetadataDAL = resourceMetadataDALFactory(db); const permissionService = permissionServiceFactory({ permissionDAL, @@ -822,7 +824,8 @@ export const registerRoutes = async ( secretApprovalRequestDAL, projectKeyDAL, projectUserMembershipRoleDAL, - orgService + orgService, + resourceMetadataDAL }); const projectService = projectServiceFactory({ @@ -945,7 +948,8 @@ export const registerRoutes = async ( secretApprovalPolicyService, secretApprovalRequestSecretDAL, kmsService, - snapshotService + snapshotService, + resourceMetadataDAL }); const secretApprovalRequestService = secretApprovalRequestServiceFactory({ @@ -972,7 +976,8 @@ export const registerRoutes = async ( projectEnvDAL, userDAL, licenseService, - projectSlackConfigDAL + projectSlackConfigDAL, + resourceMetadataDAL }); const secretService = secretServiceFactory({ @@ -1051,8 +1056,10 @@ export const registerRoutes = async ( kmsService, secretV2BridgeDAL, secretVersionV2TagBridgeDAL: secretVersionTagV2BridgeDAL, - secretVersionV2BridgeDAL + secretVersionV2BridgeDAL, + resourceMetadataDAL }); + const secretRotationQueue = secretRotationQueueFactory({ telemetryService, secretRotationDAL, diff --git a/backend/src/server/routes/v3/secret-router.ts b/backend/src/server/routes/v3/secret-router.ts index 61981bef5..2b03ca589 100644 --- a/backend/src/server/routes/v3/secret-router.ts +++ b/backend/src/server/routes/v3/secret-router.ts @@ -18,6 +18,7 @@ import { getUserAgentType } from "@app/server/plugins/audit-log"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; import { ActorType, AuthMode } from "@app/services/auth/auth-type"; import { ProjectFilterType } from "@app/services/project/project-types"; +import { ResourceMetadataSchema } from "@app/services/resource-metadata/resource-metadata-schema"; import { SecretOperations, SecretProtectionType } from "@app/services/secret/secret-types"; import { PostHogEventTypes } from "@app/services/telemetry/telemetry-types"; @@ -450,6 +451,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { .transform((val) => (val.at(-1) === "\n" ? `${val.trim()}\n` : val.trim())) .describe(RAW_SECRETS.CREATE.secretValue), secretComment: z.string().trim().optional().default("").describe(RAW_SECRETS.CREATE.secretComment), + secretMetadata: ResourceMetadataSchema.optional(), tagIds: z.string().array().optional().describe(RAW_SECRETS.CREATE.tagIds), skipMultilineEncoding: z.boolean().optional().describe(RAW_SECRETS.CREATE.skipMultilineEncoding), type: z.nativeEnum(SecretType).default(SecretType.Shared).describe(RAW_SECRETS.CREATE.type), @@ -484,6 +486,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { secretValue: req.body.secretValue, skipMultilineEncoding: req.body.skipMultilineEncoding, secretComment: req.body.secretComment, + secretMetadata: req.body.secretMetadata, tagIds: req.body.tagIds, secretReminderNote: req.body.secretReminderNote, secretReminderRepeatDays: req.body.secretReminderRepeatDays diff --git a/backend/src/services/external-migration/external-migration-fns.ts b/backend/src/services/external-migration/external-migration-fns.ts index 7ae0d0aad..744678792 100644 --- a/backend/src/services/external-migration/external-migration-fns.ts +++ b/backend/src/services/external-migration/external-migration-fns.ts @@ -16,6 +16,7 @@ import { TProjectDALFactory } from "../project/project-dal"; import { TProjectServiceFactory } from "../project/project-service"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TProjectEnvServiceFactory } from "../project-env/project-env-service"; +import { TResourceMetadataDALFactory } from "../resource-metadata/resource-metadata-dal"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal"; import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal"; @@ -35,6 +36,8 @@ export type TImportDataIntoInfisicalDTO = { secretTagDAL: Pick; secretVersionTagDAL: Pick; + resourceMetadataDAL: Pick; + folderDAL: Pick; projectService: Pick; projectEnvService: Pick; @@ -503,6 +506,7 @@ export const importDataIntoInfisicalFn = async ({ secretTagDAL, secretVersionTagDAL, folderDAL, + resourceMetadataDAL, input: { data, actor, actorId, actorOrgId, actorAuthMethod } }: TImportDataIntoInfisicalDTO) => { // Import data to infisical @@ -762,6 +766,8 @@ export const importDataIntoInfisicalFn = async ({ }; }), folderId: selectedFolder.id, + orgId: actorOrgId, + resourceMetadataDAL, secretDAL, secretVersionDAL, secretTagDAL, diff --git a/backend/src/services/resource-metadata/resource-metadata-dal.ts b/backend/src/services/resource-metadata/resource-metadata-dal.ts new file mode 100644 index 000000000..b8b7a1541 --- /dev/null +++ b/backend/src/services/resource-metadata/resource-metadata-dal.ts @@ -0,0 +1,11 @@ +import { TDbClient } from "@app/db"; +import { TableName } from "@app/db/schemas"; +import { ormify } from "@app/lib/knex"; + +export type TResourceMetadataDALFactory = ReturnType; + +export const resourceMetadataDALFactory = (db: TDbClient) => { + const orm = ormify(db, TableName.ResourceMetadata); + + return orm; +}; diff --git a/backend/src/services/resource-metadata/resource-metadata-schema.ts b/backend/src/services/resource-metadata/resource-metadata-schema.ts new file mode 100644 index 000000000..f8a8b08fd --- /dev/null +++ b/backend/src/services/resource-metadata/resource-metadata-schema.ts @@ -0,0 +1,10 @@ +import z from "zod"; + +export const ResourceMetadataSchema = z + .object({ + key: z.string().trim().min(1), + value: z.string().trim().min(1) + }) + .array(); + +export type ResourceMetadataDTO = z.infer; diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts index 79aa22799..e88487319 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts @@ -54,9 +54,11 @@ export const getAllSecretReferences = (maybeSecretReference: string) => { export const fnSecretBulkInsert = async ({ // TODO: Pick types here folderId, + orgId, inputSecrets, secretDAL, secretVersionDAL, + resourceMetadataDAL, secretTagDAL, secretVersionTagDAL, tx @@ -91,6 +93,7 @@ export const fnSecretBulkInsert = async ({ sanitizedInputSecrets.map((el) => ({ ...el, folderId })), tx ); + const newSecretGroupedByKeyName = groupBy(newSecrets, (item) => item.key); const newSecretTags = inputSecrets.flatMap(({ tagIds: secretTags = [], key }) => secretTags.map((tag) => ({ @@ -106,6 +109,7 @@ export const fnSecretBulkInsert = async ({ })), tx ); + await secretDAL.upsertSecretReferences( inputSecrets.map(({ references = [], key }) => ({ secretId: newSecretGroupedByKeyName[key][0].id, @@ -113,6 +117,22 @@ export const fnSecretBulkInsert = async ({ })), tx ); + + await resourceMetadataDAL.insertMany( + inputSecrets.flatMap(({ key: secretKey, secretMetadata }) => { + if (secretMetadata) { + return secretMetadata.map(({ key, value }) => ({ + key, + value, + secretId: newSecretGroupedByKeyName[secretKey][0].id, + orgId + })); + } + return []; + }), + tx + ); + if (newSecretTags.length) { const secTags = await secretTagDAL.saveTagsToSecretV2(newSecretTags, tx); const secVersionsGroupBySecId = groupBy(secretVersions, (i) => i.secretId); @@ -120,6 +140,7 @@ export const fnSecretBulkInsert = async ({ [`${TableName.SecretVersionV2}Id` as const]: secVersionsGroupBySecId[secrets_v2Id][0].id, [`${TableName.SecretTag}Id` as const]: secret_tagsId })); + await secretVersionTagDAL.insertMany(newSecretVersionTags, tx); } diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts index e13a77432..809a8b8c1 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts @@ -18,6 +18,7 @@ import { ActorType } from "../auth/auth-type"; import { TKmsServiceFactory } from "../kms/kms-service"; import { KmsDataKey } from "../kms/kms-types"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; +import { TResourceMetadataDALFactory } from "../resource-metadata/resource-metadata-dal"; import { TSecretQueueFactory } from "../secret/secret-queue"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; import { TSecretImportDALFactory } from "../secret-import/secret-import-dal"; @@ -74,6 +75,7 @@ type TSecretV2BridgeServiceFactoryDep = { "insertV2Bridge" | "insertApprovalSecretV2Tags" >; snapshotService: Pick; + resourceMetadataDAL: Pick; }; export type TSecretV2BridgeServiceFactory = ReturnType; @@ -95,7 +97,8 @@ export const secretV2BridgeServiceFactory = ({ secretApprovalPolicyService, secretApprovalRequestDAL, secretApprovalRequestSecretDAL, - kmsService + kmsService, + resourceMetadataDAL }: TSecretV2BridgeServiceFactoryDep) => { const $validateSecretReferences = async ( projectId: string, @@ -186,6 +189,7 @@ export const secretV2BridgeServiceFactory = ({ actorAuthMethod, projectId, secretPath, + secretMetadata, ...inputSecret }: TCreateSecretDTO) => { const { permission, ForbidOnInvalidProjectType } = await permissionService.getProjectPermission( @@ -255,6 +259,7 @@ export const secretV2BridgeServiceFactory = ({ const secret = await secretDAL.transaction((tx) => fnSecretBulkInsert({ folderId, + orgId: actorOrgId, inputSecrets: [ { version: 1, @@ -272,9 +277,11 @@ export const secretV2BridgeServiceFactory = ({ key: secretName, userId: inputSecret.type === SecretType.Personal ? actorId : null, tagIds: inputSecret.tagIds, - references: nestedReferences + references: nestedReferences, + secretMetadata } ], + resourceMetadataDAL, secretDAL, secretVersionDAL, secretTagDAL, @@ -287,6 +294,7 @@ export const secretV2BridgeServiceFactory = ({ await snapshotService.performSnapshot(folderId); await secretQueueService.syncSecrets({ secretPath, + orgId: actorOrgId, actorId, actor, projectId, @@ -475,6 +483,7 @@ export const secretV2BridgeServiceFactory = ({ actorId, actor, projectId, + orgId: actorOrgId, environmentSlug: folder.environment.slug }); } @@ -562,6 +571,7 @@ export const secretV2BridgeServiceFactory = ({ actorId, actor, projectId, + orgId: actorOrgId, environmentSlug: folder.environment.slug }); } @@ -1189,7 +1199,9 @@ export const secretV2BridgeServiceFactory = ({ }; }), folderId, + orgId: actorOrgId, secretDAL, + resourceMetadataDAL, secretVersionDAL, secretTagDAL, secretVersionTagDAL, @@ -1203,6 +1215,7 @@ export const secretV2BridgeServiceFactory = ({ actorId, secretPath, projectId, + orgId: actorOrgId, environmentSlug: folder.environment.slug }); @@ -1410,6 +1423,7 @@ export const secretV2BridgeServiceFactory = ({ actorId, secretPath, projectId, + orgId: actorOrgId, environmentSlug: folder.environment.slug }); @@ -1512,6 +1526,7 @@ export const secretV2BridgeServiceFactory = ({ actorId, secretPath, projectId, + orgId: actorOrgId, environmentSlug: folder.environment.slug }); @@ -1815,10 +1830,12 @@ export const secretV2BridgeServiceFactory = ({ if (locallyCreatedSecrets.length) { await fnSecretBulkInsert({ folderId: destinationFolder.id, + orgId: actorOrgId, secretVersionDAL, secretDAL, tx, secretTagDAL, + resourceMetadataDAL, secretVersionTagDAL, inputSecrets: locallyCreatedSecrets.map((doc) => { return { @@ -1938,6 +1955,7 @@ export const secretV2BridgeServiceFactory = ({ await snapshotService.performSnapshot(destinationFolder.id); await secretQueueService.syncSecrets({ projectId, + orgId: actorOrgId, secretPath: destinationFolder.path, environmentSlug: destinationFolder.environment.slug, actorId, @@ -1949,6 +1967,7 @@ export const secretV2BridgeServiceFactory = ({ await snapshotService.performSnapshot(sourceFolder.id); await secretQueueService.syncSecrets({ projectId, + orgId: actorOrgId, secretPath: sourceFolder.path, environmentSlug: sourceFolder.environment.slug, actorId, diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts index 7216989ff..6eab637ff 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts @@ -7,6 +7,8 @@ import { SecretsOrderBy } from "@app/services/secret/secret-types"; import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal"; import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; +import { TResourceMetadataDALFactory } from "../resource-metadata/resource-metadata-dal"; +import { ResourceMetadataDTO } from "../resource-metadata/resource-metadata-schema"; import { TSecretV2BridgeDALFactory } from "./secret-v2-bridge-dal"; import { TSecretVersionV2DALFactory } from "./secret-version-dal"; import { TSecretVersionV2TagDALFactory } from "./secret-version-tag-dal"; @@ -58,6 +60,7 @@ export type TCreateSecretDTO = TProjectPermission & { skipMultilineEncoding?: boolean; secretReminderRepeatDays?: number | null; secretReminderNote?: string | null; + secretMetadata?: ResourceMetadataDTO; }; export type TUpdateSecretDTO = TProjectPermission & { @@ -136,8 +139,16 @@ export type TSecretReference = { environment: string; secretPath: string; secret export type TFnSecretBulkInsert = { folderId: string; + orgId: string; tx?: Knex; - inputSecrets: Array & { tagIds?: string[]; references: TSecretReference[] }>; + inputSecrets: Array< + Omit & { + tagIds?: string[]; + references: TSecretReference[]; + secretMetadata?: ResourceMetadataDTO; + } + >; + resourceMetadataDAL: Pick; secretDAL: Pick; secretVersionDAL: Pick; secretTagDAL: Pick; diff --git a/backend/src/services/secret/secret-fns.ts b/backend/src/services/secret/secret-fns.ts index 6336c479d..d4f3fe345 100644 --- a/backend/src/services/secret/secret-fns.ts +++ b/backend/src/services/secret/secret-fns.ts @@ -749,7 +749,8 @@ export const createManySecretsRawFnFactory = ({ secretVersionV2BridgeDAL, secretV2BridgeDAL, secretVersionTagV2BridgeDAL, - kmsService + kmsService, + resourceMetadataDAL }: TCreateManySecretsRawFnFactory) => { const getBotKeyFn = getBotKeyFnFactory(projectBotDAL, projectDAL); const createManySecretsRawFn = async ({ @@ -760,7 +761,7 @@ export const createManySecretsRawFnFactory = ({ userId }: TCreateManySecretsRawFn) => { const { botKey, shouldUseSecretV2Bridge } = await getBotKeyFn(projectId); - + const project = await projectDAL.findById(projectId); const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); if (!folder) throw new NotFoundError({ @@ -814,7 +815,9 @@ export const createManySecretsRawFnFactory = ({ tagIds: el.tags })), folderId, + orgId: project.orgId, secretDAL: secretV2BridgeDAL, + resourceMetadataDAL, secretVersionDAL: secretVersionV2BridgeDAL, secretTagDAL, secretVersionTagDAL: secretVersionTagV2BridgeDAL, diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index 57d32ab11..b8ad734f8 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -46,6 +46,7 @@ import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TProjectKeyDALFactory } from "../project-key/project-key-dal"; import { TProjectMembershipDALFactory } from "../project-membership/project-membership-dal"; import { TProjectUserMembershipRoleDALFactory } from "../project-membership/project-user-membership-role-dal"; +import { TResourceMetadataDALFactory } from "../resource-metadata/resource-metadata-dal"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; import { TSecretImportDALFactory } from "../secret-import/secret-import-dal"; import { fnSecretsV2FromImports } from "../secret-import/secret-import-fns"; @@ -103,6 +104,7 @@ type TSecretQueueFactoryDep = { auditLogService: Pick; orgService: Pick; projectUserMembershipRoleDAL: Pick; + resourceMetadataDAL: Pick; }; export type TGetSecrets = { @@ -156,7 +158,8 @@ export const secretQueueFactory = ({ auditLogService, orgService, projectUserMembershipRoleDAL, - projectKeyDAL + projectKeyDAL, + resourceMetadataDAL }: TSecretQueueFactoryDep) => { const removeSecretReminder = async (dto: TRemoveSecretReminderDTO) => { const appCfg = getConfig(); @@ -299,7 +302,8 @@ export const secretQueueFactory = ({ kmsService, secretVersionV2BridgeDAL, secretV2BridgeDAL, - secretVersionTagV2BridgeDAL + secretVersionTagV2BridgeDAL, + resourceMetadataDAL }); const updateManySecretsRawFn = updateManySecretsRawFnFactory({ @@ -590,6 +594,7 @@ export const secretQueueFactory = ({ _depth: depth, secretPath, projectId, + orgId, environmentSlug: environment, excludeReplication, actorId, @@ -618,6 +623,7 @@ export const secretQueueFactory = ({ _deDupeReplicationQueue: deDupeReplicationQueue, _depth: depth, projectId, + orgId, secretPath, actorId, actor, @@ -674,6 +680,7 @@ export const secretQueueFactory = ({ if (!folder) { throw new Error("Secret path not found"); } + const project = await projectDAL.findById(projectId); // find all imports made with the given environment and secret path const linkSourceDto = { @@ -708,6 +715,7 @@ export const secretQueueFactory = ({ .map(({ folderId }) => syncSecrets({ projectId, + orgId: project.orgId, secretPath: foldersGroupedById[folderId][0]?.path as string, environmentSlug: foldersGroupedById[folderId][0]?.environmentSlug as string, _deDupeQueue: deDupeQueue, @@ -760,6 +768,7 @@ export const secretQueueFactory = ({ .map((folderId) => syncSecrets({ projectId, + orgId: project.orgId, secretPath: referencedFoldersGroupedById[folderId][0]?.path as string, environmentSlug: referencedFoldersGroupedById[folderId][0]?.environmentSlug as string, _deDupeQueue: deDupeQueue, diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index fbf90a7f8..0b8f7def6 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -288,6 +288,7 @@ export const secretServiceFactory = ({ actorId, actor, projectId, + orgId: actorOrgId, environmentSlug: folder.environment.slug }); } @@ -429,6 +430,7 @@ export const secretServiceFactory = ({ await snapshotService.performSnapshot(folderId); await secretQueueService.syncSecrets({ secretPath: path, + orgId: actorOrgId, actorId, actor, projectId, @@ -526,6 +528,7 @@ export const secretServiceFactory = ({ actorId, actor, projectId, + orgId: actorOrgId, environmentSlug: folder.environment.slug }); } @@ -820,6 +823,7 @@ export const secretServiceFactory = ({ actorId, secretPath: path, projectId, + orgId: actorOrgId, environmentSlug: folder.environment.slug }); @@ -928,6 +932,7 @@ export const secretServiceFactory = ({ actorId, secretPath: path, projectId, + orgId: actorOrgId, environmentSlug: folder.environment.slug }); @@ -1014,6 +1019,7 @@ export const secretServiceFactory = ({ actorId, secretPath: path, projectId, + orgId: actorOrgId, environmentSlug: folder.environment.slug }); @@ -1385,7 +1391,8 @@ export const secretServiceFactory = ({ skipMultilineEncoding, tagIds, secretReminderNote, - secretReminderRepeatDays + secretReminderRepeatDays, + secretMetadata }: TCreateSecretRawDTO) => { const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId); const policy = @@ -1435,7 +1442,8 @@ export const secretServiceFactory = ({ tagIds, secretReminderNote, skipMultilineEncoding, - secretReminderRepeatDays + secretReminderRepeatDays, + secretMetadata }); return { secret, type: SecretProtectionType.Direct as const }; } @@ -2262,6 +2270,7 @@ export const secretServiceFactory = ({ await secretQueueService.syncSecrets({ secretPath, projectId: project.id, + orgId: project.orgId, environmentSlug: environment, excludeReplication: true }); @@ -2370,6 +2379,7 @@ export const secretServiceFactory = ({ await secretQueueService.syncSecrets({ secretPath, projectId: project.id, + orgId: project.orgId, environmentSlug: environment, excludeReplication: true }); @@ -2828,6 +2838,7 @@ export const secretServiceFactory = ({ await snapshotService.performSnapshot(destinationFolder.id); await secretQueueService.syncSecrets({ projectId: project.id, + orgId: project.orgId, secretPath: destinationFolder.path, environmentSlug: destinationFolder.environment.slug, actorId, @@ -2839,6 +2850,7 @@ export const secretServiceFactory = ({ await snapshotService.performSnapshot(sourceFolder.id); await secretQueueService.syncSecrets({ projectId: project.id, + orgId: project.orgId, secretPath: sourceFolder.path, environmentSlug: sourceFolder.environment.slug, actorId, diff --git a/backend/src/services/secret/secret-types.ts b/backend/src/services/secret/secret-types.ts index ca5c5a74b..0d6fb4e29 100644 --- a/backend/src/services/secret/secret-types.ts +++ b/backend/src/services/secret/secret-types.ts @@ -14,6 +14,8 @@ import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; import { ActorType } from "../auth/auth-type"; import { TKmsServiceFactory } from "../kms/kms-service"; +import { TResourceMetadataDALFactory } from "../resource-metadata/resource-metadata-dal"; +import { ResourceMetadataDTO } from "../resource-metadata/resource-metadata-schema"; import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal"; import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-dal"; import { TSecretVersionV2TagDALFactory } from "../secret-v2-bridge/secret-version-tag-dal"; @@ -211,6 +213,7 @@ export type TCreateSecretRawDTO = TProjectPermission & { skipMultilineEncoding?: boolean; secretReminderRepeatDays?: number | null; secretReminderNote?: string | null; + secretMetadata?: ResourceMetadataDTO; }; export type TUpdateSecretRawDTO = TProjectPermission & { @@ -293,7 +296,13 @@ export type TSecretReference = { environment: string; secretPath: string }; export type TFnSecretBulkInsert = { folderId: string; tx?: Knex; - inputSecrets: Array & { tags?: string[]; references?: TSecretReference[] }>; + inputSecrets: Array< + Omit & { + tags?: string[]; + references?: TSecretReference[]; + secretMetadata?: ResourceMetadataDTO; + } + >; secretDAL: Pick; secretVersionDAL: Pick; secretTagDAL: Pick; @@ -389,6 +398,7 @@ export type TCreateManySecretsRawFnFactory = { >; secretVersionV2BridgeDAL: Pick; secretVersionTagV2BridgeDAL: Pick; + resourceMetadataDAL: Pick; }; export type TCreateManySecretsRawFn = { @@ -460,6 +470,7 @@ export type TSyncSecretsDTO = { _depth?: number; secretPath: string; projectId: string; + orgId: string; environmentSlug: string; // cases for just doing sync integration and webhook excludeReplication?: T;