From 718cabe49b83bf04942dea8a64513ccae4ed7169 Mon Sep 17 00:00:00 2001 From: Akhil Mohan Date: Fri, 19 Apr 2024 20:53:54 +0530 Subject: [PATCH] feat(server): added batch raw bulk secret ops api --- backend/e2e-test/routes/v3/secrets.spec.ts | 107 ++++++++ backend/src/lib/api-docs/constants.ts | 1 + backend/src/server/routes/v3/secret-router.ts | 259 ++++++++++++++++++ backend/src/services/secret/secret-service.ts | 131 +++++++++ backend/src/services/secret/secret-types.ts | 30 ++ 5 files changed, 528 insertions(+) diff --git a/backend/e2e-test/routes/v3/secrets.spec.ts b/backend/e2e-test/routes/v3/secrets.spec.ts index 03e1c2f50..ab73a7f1f 100644 --- a/backend/e2e-test/routes/v3/secrets.spec.ts +++ b/backend/e2e-test/routes/v3/secrets.spec.ts @@ -942,6 +942,113 @@ describe.each([{ auth: AuthMode.JWT }, { auth: AuthMode.IDENTITY_ACCESS_TOKEN }] const secrets = await getSecrets(seedData1.environment.slug, path); expect(secrets).toEqual([]); }); + + test.each(testRawSecrets)("Bulk create secret raw in path $path", async ({ path, secret }) => { + const createSecretReqBody = { + workspaceId: seedData1.project.id, + environment: seedData1.environment.slug, + secretPath: path, + secrets: [ + { + secretKey: secret.key, + secretValue: secret.value, + secretComment: secret.comment + } + ] + }; + const createSecRes = await testServer.inject({ + method: "POST", + url: `/api/v3/secrets/batch/raw`, + headers: { + authorization: `Bearer ${authToken}` + }, + body: createSecretReqBody + }); + expect(createSecRes.statusCode).toBe(200); + const createdSecretPayload = JSON.parse(createSecRes.payload); + expect(createdSecretPayload).toHaveProperty("secrets"); + + // fetch secrets + const secrets = await getSecrets(seedData1.environment.slug, path); + expect(secrets).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + key: secret.key, + value: secret.value, + type: SecretType.Shared + }) + ]) + ); + + await deleteRawSecret({ path, key: secret.key }); + }); + + test.each(testRawSecrets)("Bulk update secret raw in path $path", async ({ secret, path }) => { + await createRawSecret({ path, ...secret }); + const updateSecretReqBody = { + workspaceId: seedData1.project.id, + environment: seedData1.environment.slug, + secretPath: path, + secrets: [ + { + secretValue: "new-value", + secretKey: secret.key + } + ] + }; + const updateSecRes = await testServer.inject({ + method: "PATCH", + url: `/api/v3/secrets/batch/raw`, + headers: { + authorization: `Bearer ${authToken}` + }, + body: updateSecretReqBody + }); + expect(updateSecRes.statusCode).toBe(200); + const updatedSecretPayload = JSON.parse(updateSecRes.payload); + expect(updatedSecretPayload).toHaveProperty("secrets"); + + // fetch secrets + const secrets = await getSecrets(seedData1.environment.slug, path); + expect(secrets).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + key: secret.key, + value: "new-value", + version: 2, + type: SecretType.Shared + }) + ]) + ); + + await deleteRawSecret({ path, key: secret.key }); + }); + + test.each(testRawSecrets)("Bulk delete secret raw in path $path", async ({ path, secret }) => { + await createRawSecret({ path, ...secret }); + + const deletedSecretReqBody = { + workspaceId: seedData1.project.id, + environment: seedData1.environment.slug, + secretPath: path, + secrets: [{ secretKey: secret.key }] + }; + const deletedSecRes = await testServer.inject({ + method: "DELETE", + url: `/api/v3/secrets/batch/raw`, + headers: { + authorization: `Bearer ${authToken}` + }, + body: deletedSecretReqBody + }); + expect(deletedSecRes.statusCode).toBe(200); + const deletedSecretPayload = JSON.parse(deletedSecRes.payload); + expect(deletedSecretPayload).toHaveProperty("secrets"); + + // fetch secrets + const secrets = await getSecrets(seedData1.environment.slug, path); + expect(secrets).toEqual([]); + }); } ); diff --git a/backend/src/lib/api-docs/constants.ts b/backend/src/lib/api-docs/constants.ts index c83234c1f..4a98f73c8 100644 --- a/backend/src/lib/api-docs/constants.ts +++ b/backend/src/lib/api-docs/constants.ts @@ -301,6 +301,7 @@ export const RAW_SECRETS = { }, UPDATE: { secretName: "The name of the secret to update.", + secretComment: "Update comment to the secret.", environment: "The slug of the environment where the secret is located.", secretPath: "The path of the secret to update", secretValue: "The new value of the secret.", diff --git a/backend/src/server/routes/v3/secret-router.ts b/backend/src/server/routes/v3/secret-router.ts index b1d852a88..ceecc0805 100644 --- a/backend/src/server/routes/v3/secret-router.ts +++ b/backend/src/server/routes/v3/secret-router.ts @@ -1656,4 +1656,263 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { return { secrets }; } }); + + server.route({ + method: "POST", + url: "/batch/raw", + config: { + rateLimit: secretsLimit + }, + schema: { + description: "Create many secrets", + security: [ + { + bearerAuth: [] + } + ], + body: z.object({ + workspaceId: z.string().trim().describe(RAW_SECRETS.CREATE.workspaceId), + environment: z.string().trim().describe(RAW_SECRETS.CREATE.environment), + secretPath: z + .string() + .trim() + .default("/") + .transform(removeTrailingSlash) + .describe(RAW_SECRETS.CREATE.secretPath), + secrets: z + .object({ + secretKey: z.string().trim().describe(RAW_SECRETS.CREATE.secretName), + secretValue: z + .string() + .transform((val) => (val.at(-1) === "\n" ? `${val.trim()}\n` : val.trim())) + .describe(RAW_SECRETS.CREATE.secretValue), + secretComment: z.string().trim().optional().default("").describe(RAW_SECRETS.CREATE.secretComment), + skipMultilineEncoding: z.boolean().optional().describe(RAW_SECRETS.CREATE.skipMultilineEncoding) + }) + .array() + .min(1) + }), + response: { + 200: z.object({ + secrets: secretRawSchema.array() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY, AuthMode.SERVICE_TOKEN, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const { environment, workspaceId: projectId, secretPath, secrets: inputSecrets } = req.body; + + const secrets = await server.services.secret.createManySecretsRaw({ + actorId: req.permission.id, + actor: req.permission.type, + actorAuthMethod: req.permission.authMethod, + actorOrgId: req.permission.orgId, + secretPath, + environment, + projectId, + secrets: inputSecrets + }); + + await server.services.auditLog.createAuditLog({ + projectId: req.body.workspaceId, + ...req.auditLogInfo, + event: { + type: EventType.CREATE_SECRETS, + metadata: { + environment: req.body.environment, + secretPath: req.body.secretPath, + secrets: secrets.map((secret, i) => ({ + secretId: secret.id, + secretKey: inputSecrets[i].secretKey, + secretVersion: secret.version + })) + } + } + }); + + await server.services.telemetry.sendPostHogEvents({ + event: PostHogEventTypes.SecretCreated, + distinctId: getTelemetryDistinctId(req), + properties: { + numberOfSecrets: secrets.length, + workspaceId: req.body.workspaceId, + environment: req.body.environment, + secretPath: req.body.secretPath, + channel: getUserAgentType(req.headers["user-agent"]), + ...req.auditLogInfo + } + }); + return { secrets }; + } + }); + + server.route({ + method: "PATCH", + url: "/batch/raw", + config: { + rateLimit: secretsLimit + }, + schema: { + description: "Update many secrets", + security: [ + { + bearerAuth: [] + } + ], + body: z.object({ + workspaceId: z.string().trim().describe(RAW_SECRETS.UPDATE.workspaceId), + environment: z.string().trim().describe(RAW_SECRETS.UPDATE.environment), + secretPath: z + .string() + .trim() + .default("/") + .transform(removeTrailingSlash) + .describe(RAW_SECRETS.UPDATE.secretPath), + secrets: z + .object({ + secretKey: z.string().trim().describe(RAW_SECRETS.UPDATE.secretName), + secretValue: z + .string() + .transform((val) => (val.at(-1) === "\n" ? `${val.trim()}\n` : val.trim())) + .describe(RAW_SECRETS.UPDATE.secretValue), + secretComment: z.string().trim().optional().describe(RAW_SECRETS.UPDATE.secretComment), + skipMultilineEncoding: z.boolean().optional().describe(RAW_SECRETS.UPDATE.skipMultilineEncoding) + }) + .array() + .min(1) + }), + response: { + 200: z.object({ + secrets: secretRawSchema.array() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY, AuthMode.SERVICE_TOKEN, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const { environment, workspaceId: projectId, secretPath, secrets: inputSecrets } = req.body; + const secrets = await server.services.secret.updateManySecretsRaw({ + actorId: req.permission.id, + actor: req.permission.type, + actorAuthMethod: req.permission.authMethod, + actorOrgId: req.permission.orgId, + secretPath, + environment, + projectId, + secrets: inputSecrets + }); + + await server.services.auditLog.createAuditLog({ + projectId: req.body.workspaceId, + ...req.auditLogInfo, + event: { + type: EventType.UPDATE_SECRETS, + metadata: { + environment: req.body.environment, + secretPath: req.body.secretPath, + secrets: secrets.map((secret, i) => ({ + secretId: secret.id, + secretKey: inputSecrets[i].secretKey, + secretVersion: secret.version + })) + } + } + }); + + await server.services.telemetry.sendPostHogEvents({ + event: PostHogEventTypes.SecretUpdated, + distinctId: getTelemetryDistinctId(req), + properties: { + numberOfSecrets: secrets.length, + workspaceId: req.body.workspaceId, + environment: req.body.environment, + secretPath: req.body.secretPath, + channel: getUserAgentType(req.headers["user-agent"]), + ...req.auditLogInfo + } + }); + return { secrets }; + } + }); + + server.route({ + method: "DELETE", + url: "/batch/raw", + config: { + rateLimit: secretsLimit + }, + schema: { + description: "Delete many secrets", + security: [ + { + bearerAuth: [] + } + ], + body: z.object({ + workspaceId: z.string().trim().describe(RAW_SECRETS.DELETE.workspaceId), + environment: z.string().trim().describe(RAW_SECRETS.DELETE.environment), + secretPath: z + .string() + .trim() + .default("/") + .transform(removeTrailingSlash) + .describe(RAW_SECRETS.DELETE.secretPath), + secrets: z + .object({ + secretKey: z.string().trim().describe(RAW_SECRETS.DELETE.secretName) + }) + .array() + .min(1) + }), + response: { + 200: z.object({ + secrets: secretRawSchema.array() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY, AuthMode.SERVICE_TOKEN, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const { environment, workspaceId: projectId, secretPath, secrets: inputSecrets } = req.body; + const secrets = await server.services.secret.deleteManySecretsRaw({ + actorId: req.permission.id, + actor: req.permission.type, + actorAuthMethod: req.permission.authMethod, + actorOrgId: req.permission.orgId, + environment, + projectId, + secretPath, + secrets: inputSecrets + }); + + await server.services.auditLog.createAuditLog({ + projectId: req.body.workspaceId, + ...req.auditLogInfo, + event: { + type: EventType.DELETE_SECRETS, + metadata: { + environment: req.body.environment, + secretPath: req.body.secretPath, + secrets: secrets.map((secret, i) => ({ + secretId: secret.id, + secretKey: inputSecrets[i].secretKey, + secretVersion: secret.version + })) + } + } + }); + + await server.services.telemetry.sendPostHogEvents({ + event: PostHogEventTypes.SecretDeleted, + distinctId: getTelemetryDistinctId(req), + properties: { + numberOfSecrets: secrets.length, + workspaceId: req.body.workspaceId, + environment: req.body.environment, + secretPath: req.body.secretPath, + channel: getUserAgentType(req.headers["user-agent"]), + ...req.auditLogInfo + } + }); + return { secrets }; + } + }); }; diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index 3b504fbf4..9c7c0cbd6 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -33,9 +33,11 @@ import { TSecretQueueFactory } from "./secret-queue"; import { TAttachSecretTagsDTO, TCreateBulkSecretDTO, + TCreateManySecretRawDTO, TCreateSecretDTO, TCreateSecretRawDTO, TDeleteBulkSecretDTO, + TDeleteManySecretRawDTO, TDeleteSecretDTO, TDeleteSecretRawDTO, TFnSecretBlindIndexCheckV2, @@ -46,6 +48,7 @@ import { TGetSecretsRawDTO, TGetSecretVersionsDTO, TUpdateBulkSecretDTO, + TUpdateManySecretRawDTO, TUpdateSecretDTO, TUpdateSecretRawDTO } from "./secret-types"; @@ -1036,6 +1039,131 @@ export const secretServiceFactory = ({ return decryptSecretRaw(secret, botKey); }; + const createManySecretsRaw = async ({ + actorId, + projectId, + environment, + actor, + actorOrgId, + actorAuthMethod, + secretPath, + secrets: inputSecrets = [] + }: TCreateManySecretRawDTO) => { + const botKey = await projectBotService.getBotKey(projectId); + if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" }); + + const secrets = await createManySecret({ + projectId, + environment, + path: secretPath, + actor, + actorId, + actorOrgId, + actorAuthMethod, + secrets: inputSecrets.map(({ secretComment, secretKey, secretValue, skipMultilineEncoding }) => { + const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secretKey, botKey); + const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secretValue || "", botKey); + const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secretComment || "", botKey); + return { + secretName: secretKey, + skipMultilineEncoding, + secretKeyCiphertext: secretKeyEncrypted.ciphertext, + secretKeyIV: secretKeyEncrypted.iv, + secretKeyTag: secretKeyEncrypted.tag, + secretValueCiphertext: secretValueEncrypted.ciphertext, + secretValueIV: secretValueEncrypted.iv, + secretValueTag: secretValueEncrypted.tag, + secretCommentCiphertext: secretCommentEncrypted.ciphertext, + secretCommentIV: secretCommentEncrypted.iv, + secretCommentTag: secretCommentEncrypted.tag + }; + }) + }); + + await snapshotService.performSnapshot(secrets[0].folderId); + await secretQueueService.syncSecrets({ secretPath, projectId, environment }); + + return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment }, botKey)); + }; + + const updateManySecretsRaw = async ({ + actorId, + projectId, + environment, + actor, + actorOrgId, + actorAuthMethod, + secretPath, + secrets: inputSecrets = [] + }: TUpdateManySecretRawDTO) => { + const botKey = await projectBotService.getBotKey(projectId); + if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" }); + + const secrets = await updateManySecret({ + projectId, + environment, + path: secretPath, + actor, + actorId, + actorOrgId, + actorAuthMethod, + secrets: inputSecrets.map(({ secretComment, secretKey, secretValue, skipMultilineEncoding }) => { + const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secretKey, botKey); + const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secretValue || "", botKey); + const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secretComment || "", botKey); + return { + secretName: secretKey, + type: SecretType.Shared, + skipMultilineEncoding, + secretKeyCiphertext: secretKeyEncrypted.ciphertext, + secretKeyIV: secretKeyEncrypted.iv, + secretKeyTag: secretKeyEncrypted.tag, + secretValueCiphertext: secretValueEncrypted.ciphertext, + secretValueIV: secretValueEncrypted.iv, + secretValueTag: secretValueEncrypted.tag, + secretCommentCiphertext: secretCommentEncrypted.ciphertext, + secretCommentIV: secretCommentEncrypted.iv, + secretCommentTag: secretCommentEncrypted.tag + }; + }) + }); + + await snapshotService.performSnapshot(secrets[0].folderId); + await secretQueueService.syncSecrets({ secretPath, projectId, environment }); + + return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment }, botKey)); + }; + + const deleteManySecretsRaw = async ({ + actorId, + projectId, + environment, + actor, + actorOrgId, + actorAuthMethod, + secretPath, + secrets: inputSecrets = [] + }: TDeleteManySecretRawDTO) => { + const botKey = await projectBotService.getBotKey(projectId); + if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" }); + + const secrets = await deleteManySecret({ + projectId, + environment, + path: secretPath, + actor, + actorId, + actorOrgId, + actorAuthMethod, + secrets: inputSecrets.map(({ secretKey }) => ({ secretName: secretKey, type: SecretType.Shared })) + }); + + await snapshotService.performSnapshot(secrets[0].folderId); + await secretQueueService.syncSecrets({ secretPath, projectId, environment }); + + return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment }, botKey)); + }; + const getSecretVersions = async ({ actorId, actor, @@ -1280,6 +1408,9 @@ export const secretServiceFactory = ({ createSecretRaw, updateSecretRaw, deleteSecretRaw, + createManySecretsRaw, + updateManySecretsRaw, + deleteManySecretsRaw, getSecretVersions, // external services function fnSecretBulkDelete, diff --git a/backend/src/services/secret/secret-types.ts b/backend/src/services/secret/secret-types.ts index 22347de4e..34b8bc822 100644 --- a/backend/src/services/secret/secret-types.ts +++ b/backend/src/services/secret/secret-types.ts @@ -181,6 +181,36 @@ export type TDeleteSecretRawDTO = TProjectPermission & { type: SecretType; }; +export type TCreateManySecretRawDTO = TProjectPermission & { + secretPath: string; + environment: string; + secrets: { + secretKey: string; + secretValue: string; + secretComment?: string; + skipMultilineEncoding?: boolean; + }[]; +}; + +export type TUpdateManySecretRawDTO = TProjectPermission & { + secretPath: string; + environment: string; + secrets: { + secretKey: string; + secretValue: string; + secretComment?: string; + skipMultilineEncoding?: boolean; + }[]; +}; + +export type TDeleteManySecretRawDTO = TProjectPermission & { + secretPath: string; + environment: string; + secrets: { + secretKey: string; + }[]; +}; + export type TGetSecretVersionsDTO = Omit & { limit?: number; offset?: number;