diff --git a/backend/src/db/migrations/20240307232900_integration-last-used.ts b/backend/src/db/migrations/20240307232900_integration-last-used.ts new file mode 100644 index 000000000..c64c31881 --- /dev/null +++ b/backend/src/db/migrations/20240307232900_integration-last-used.ts @@ -0,0 +1,15 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + await knex.schema.alterTable(TableName.Integration, (t) => { + t.datetime("lastUsed"); + }); +} + +export async function down(knex: Knex): Promise { + await knex.schema.alterTable(TableName.Integration, (t) => { + t.dropColumn("lastUsed"); + }); +} diff --git a/backend/src/db/schemas/integrations.ts b/backend/src/db/schemas/integrations.ts index cf8c88154..203498c85 100644 --- a/backend/src/db/schemas/integrations.ts +++ b/backend/src/db/schemas/integrations.ts @@ -27,7 +27,8 @@ export const IntegrationsSchema = z.object({ envId: z.string().uuid(), secretPath: z.string().default("/"), createdAt: z.date(), - updatedAt: z.date() + updatedAt: z.date(), + lastUsed: z.date().nullable().optional() }); export type TIntegrations = z.infer; diff --git a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts index c53136f13..d2203fea3 100644 --- a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts +++ b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts @@ -12,14 +12,14 @@ import { groupBy, pick, unique } from "@app/lib/fn"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { ActorType } from "@app/services/auth/auth-type"; import { TProjectDALFactory } from "@app/services/project/project-dal"; +import { TSecretDALFactory } from "@app/services/secret/secret-dal"; import { TSecretQueueFactory } from "@app/services/secret/secret-queue"; import { TSecretServiceFactory } from "@app/services/secret/secret-service"; import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal"; +import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal"; import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal"; import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal"; import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; -import { TSecretDALFactory } from "@app/services/secret/secret-dal"; -import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal"; import { TPermissionServiceFactory } from "../permission/permission-service"; import { ProjectPermissionActions, ProjectPermissionSub } from "../permission/project-permission"; @@ -47,7 +47,7 @@ type TSecretApprovalRequestServiceFactoryDep = { secretApprovalRequestReviewerDAL: TSecretApprovalRequestReviewerDALFactory; folderDAL: Pick; secretDAL: TSecretDALFactory; - secretTagDAL: Pick; + secretTagDAL: Pick; secretBlindIndexDAL: Pick; snapshotService: Pick; secretVersionDAL: Pick; @@ -377,7 +377,11 @@ export const secretApprovalRequestServiceFactory = ({ "secretBlindIndex" ]) } - })) + })), + secretDAL, + secretVersionDAL, + secretTagDAL, + secretVersionTagDAL }) : []; const deletedSecret = secretDeletionCommits.length diff --git a/backend/src/services/integration-auth/integration-sync-secret.ts b/backend/src/services/integration-auth/integration-sync-secret.ts index 7e613ceab..0dd298fa7 100644 --- a/backend/src/services/integration-auth/integration-sync-secret.ts +++ b/backend/src/services/integration-auth/integration-sync-secret.ts @@ -20,20 +20,20 @@ import sodium from "libsodium-wrappers"; import isEqual from "lodash.isequal"; import { z } from "zod"; -import { TIntegrationAuths, TIntegrations, SecretType } from "@app/db/schemas"; +import { SecretType, TIntegrationAuths, TIntegrations } from "@app/db/schemas"; import { request } from "@app/lib/config/request"; import { BadRequestError } from "@app/lib/errors"; - -import { Integrations, IntegrationUrls, IntegrationSyncBehavior } from "./integration-list"; import { TProjectDALFactory } from "@app/services/project/project-dal"; import { TSecretDALFactory } from "@app/services/secret/secret-dal"; -import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal"; -import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; -import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal"; import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal"; import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal"; +import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal"; +import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal"; +import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; -import { createManySecretsRawHelper } from "../secret/secret-fns"; +import { TIntegrationDALFactory } from "../integration/integration-dal"; +import { createManySecretsRawHelper, updateManySecretsRawHelper } from "../secret/secret-fns"; +import { Integrations, IntegrationSyncBehavior, IntegrationUrls } from "./integration-list"; const getSecretKeyValuePair = (secrets: Record) => Object.keys(secrets).reduce>((prev, key) => { @@ -589,10 +589,11 @@ const syncSecretsAWSSecretManager = async ({ /** * Sync/push [secrets] to Heroku app named [integration.app] - * server.services... + * server.services... */ const syncSecretsHeroku = async ({ projectDAL, + integrationDAL, secretDAL, secretVersionDAL, secretBlindIndexDAL, @@ -608,6 +609,7 @@ const syncSecretsHeroku = async ({ accessToken }: { projectDAL: TProjectDALFactory; + integrationDAL: TIntegrationDALFactory; secretDAL: TSecretDALFactory; secretVersionDAL: TSecretVersionDALFactory; secretBlindIndexDAL: TSecretBlindIndexDALFactory; @@ -622,7 +624,6 @@ const syncSecretsHeroku = async ({ secrets: Record; accessToken: string; }) => { - const herokuSecrets = ( await request.get(`${IntegrationUrls.HEROKU_API_URL}/apps/${integration.app}/config-vars`, { headers: { @@ -632,62 +633,90 @@ const syncSecretsHeroku = async ({ } }) ).data; - - let secretsToAdd: { [key: string]: string } = {}; - let secretsToUpdate: { [key: string]: string } = {}; + + const secretsToAdd: { [key: string]: string } = {}; + const secretsToUpdate: { [key: string]: string } = {}; const metadata = z.record(z.any()).parse(integration.metadata); Object.keys(herokuSecrets).forEach((key) => { - switch (metadata.syncBehavior) { - case IntegrationSyncBehavior.OVERWRITE_TARGET: { - if (!(key in secrets)) secrets[key] = null; - break; - }; - case IntegrationSyncBehavior.PREFER_TARGET: { - secrets[key] = herokuSecrets[key]; - if (!(key in secrets)) { - secretsToAdd[key] = herokuSecrets[key]; - } else { - if (secrets[key] !== herokuSecrets[key]) { - secretsToUpdate[key] = secrets[key]?.value as string; + if (!integration.lastUsed) { + // first time using integration + // -> apply initial sync behavior rule + switch (metadata.syncBehavior) { + case IntegrationSyncBehavior.OVERWRITE_TARGET: { + if (!(key in secrets)) secrets[key] = null; + break; + } + case IntegrationSyncBehavior.PREFER_TARGET: { + if (!(key in secrets)) { + secretsToAdd[key] = herokuSecrets[key]; + } else if (secrets[key]?.value !== herokuSecrets[key]) { + secretsToUpdate[key] = herokuSecrets[key]; } + secrets[key] = { + value: herokuSecrets[key] + }; + break; } - break; - }; - case IntegrationSyncBehavior.PREFER_SOURCE: { - if(!(key in secrets)) { - secrets[key] = herokuSecrets[key]; - secretsToAdd[key] = herokuSecrets[key]; + case IntegrationSyncBehavior.PREFER_SOURCE: { + if (!(key in secrets)) { + secrets[key] = herokuSecrets[key]; + secretsToAdd[key] = herokuSecrets[key]; + } + break; } - break; - }; - default: { - if (!(key in secrets)) secrets[key] = null; - break; - }; - } + default: { + if (!(key in secrets)) secrets[key] = null; + break; + } + } + } else if (!(key in secrets)) secrets[key] = null; }); - await createManySecretsRawHelper({ - botKey, - projectDAL, - secretDAL, - secretVersionDAL, - secretBlindIndexDAL, - secretTagDAL, - secretVersionTagDAL, - folderDAL, - projectId, - environment, - path: secretPath, - secrets: Object.keys(secretsToAdd).map((key) => ({ - secretName: key, - secretValue: secretsToAdd[key], - type: SecretType.Shared, - secretComment: "" - })) - }); + if (Object.keys(secretsToAdd).length) { + await createManySecretsRawHelper({ + botKey, + projectDAL, + secretDAL, + secretVersionDAL, + secretBlindIndexDAL, + secretTagDAL, + secretVersionTagDAL, + folderDAL, + projectId, + environment, + path: secretPath, + secrets: Object.keys(secretsToAdd).map((key) => ({ + secretName: key, + secretValue: secretsToAdd[key], + type: SecretType.Shared, + secretComment: "" + })) + }); + } + + if (Object.keys(secretsToUpdate).length) { + await updateManySecretsRawHelper({ + projectId, + environment, + path: secretPath, + secrets: Object.keys(secretsToUpdate).map((key) => ({ + secretName: key, + secretValue: secretsToUpdate[key], + type: SecretType.Shared, + secretComment: "" + })), + botKey, // TODO: consider getting botKey inside this fn + projectDAL, + secretDAL, + secretVersionDAL, + secretBlindIndexDAL, + secretTagDAL, + secretVersionTagDAL, + folderDAL + }); + } await request.patch( `${IntegrationUrls.HEROKU_API_URL}/apps/${integration.app}/config-vars`, @@ -700,6 +729,10 @@ const syncSecretsHeroku = async ({ } } ); + + await integrationDAL.updateById(integration.id, { + lastUsed: new Date() + }); }; /** @@ -3013,12 +3046,13 @@ const syncSecretsHasuraCloud = async ({ /** * Sync/push [secrets] to [app] in integration named [integration] - * + * * Do this in terms of DAL - * + * */ export const syncIntegrationSecrets = async ({ projectDAL, + integrationDAL, secretDAL, secretVersionDAL, secretBlindIndexDAL, @@ -3037,6 +3071,7 @@ export const syncIntegrationSecrets = async ({ appendices }: { projectDAL: TProjectDALFactory; + integrationDAL: TIntegrationDALFactory; secretDAL: TSecretDALFactory; secretVersionDAL: TSecretVersionDALFactory; secretBlindIndexDAL: TSecretBlindIndexDALFactory; @@ -3088,6 +3123,7 @@ export const syncIntegrationSecrets = async ({ case Integrations.HEROKU: await syncSecretsHeroku({ projectDAL, + integrationDAL, secretDAL, secretVersionDAL, secretBlindIndexDAL, @@ -3100,7 +3136,7 @@ export const syncIntegrationSecrets = async ({ secretPath, integration, secrets, - accessToken, + accessToken }); break; case Integrations.VERCEL: diff --git a/backend/src/services/secret/secret-fns.ts b/backend/src/services/secret/secret-fns.ts index 382e87310..cc557f2b4 100644 --- a/backend/src/services/secret/secret-fns.ts +++ b/backend/src/services/secret/secret-fns.ts @@ -1,32 +1,33 @@ /* eslint-disable no-await-in-loop */ import path from "path"; -import { - SecretKeyEncoding, - TSecretBlindIndexes, - TSecrets, +import { SecretEncryptionAlgo, + SecretKeyEncoding, SecretType, - TableName + TableName, + TSecretBlindIndexes, + TSecrets } from "@app/db/schemas"; import { getConfig } from "@app/lib/config/env"; -import { - buildSecretBlindIndexFromName, - encryptSymmetric128BitHexKeyUTF8, - decryptSymmetric128BitHexKeyUTF8 +import { + buildSecretBlindIndexFromName, + decryptSymmetric128BitHexKeyUTF8, + encryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto"; +import { BadRequestError } from "@app/lib/errors"; +import { groupBy, unique } from "@app/lib/fn"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; import { TSecretDALFactory } from "./secret-dal"; -import { +import { TCreateManySecretsRawHelper, + TFnSecretBlindIndexCheck, TFnSecretBulkInsert, - TFnSecretBlindIndexCheck + TFnSecretBulkUpdate, + TUpdateManySecretsRawHelper } from "./secret-types"; -import { groupBy, unique } from "@app/lib/fn"; -import { BadRequestError } from "@app/lib/errors"; - export const generateSecretBlindIndexBySalt = async (secretName: string, secretBlindIndexDoc: TSecretBlindIndexes) => { const appCfg = getConfig(); const secretBlindIndex = await buildSecretBlindIndexFromName({ @@ -308,14 +309,15 @@ export const fnSecretBlindIndexCheck = async ({ // these functions are special functions shared by a couple of resources // used by secret approval, rotation or anywhere in which secret needs to modified -export const fnSecretBulkInsert = async ({ // TODO: Pick types here - folderId, - inputSecrets, +export const fnSecretBulkInsert = async ({ + // TODO: Pick types here + folderId, + inputSecrets, secretDAL, secretVersionDAL, secretTagDAL, secretVersionTagDAL, - tx + tx }: TFnSecretBulkInsert) => { const newSecrets = await secretDAL.insertMany( inputSecrets.map(({ tags, ...el }) => ({ ...el, folderId })), @@ -349,10 +351,63 @@ export const fnSecretBulkInsert = async ({ // TODO: Pick types here return newSecrets.map((secret) => ({ ...secret, _id: secret.id })); }; -export const createManySecretsRawHelper = async ({ // TODO: place on top +export const fnSecretBulkUpdate = async ({ + tx, + inputSecrets, + folderId, + projectId, + secretDAL, + secretVersionDAL, + secretTagDAL, + secretVersionTagDAL +}: TFnSecretBulkUpdate) => { + const newSecrets = await secretDAL.bulkUpdate( + inputSecrets.map(({ filter, data: { tags, ...data } }) => ({ + filter: { ...filter, folderId }, + data + })), + tx + ); + const secretVersions = await secretVersionDAL.insertMany( + newSecrets.map(({ id, createdAt, updatedAt, ...el }) => ({ + ...el, + secretId: id + })), + tx + ); + const secsUpdatedTag = inputSecrets.flatMap(({ data: { tags } }, i) => + tags !== undefined ? { tags, secretId: newSecrets[i].id } : [] + ); + if (secsUpdatedTag.length) { + await secretTagDAL.deleteTagsManySecret( + projectId, + secsUpdatedTag.map(({ secretId }) => secretId), + tx + ); + const newSecretTags = secsUpdatedTag.flatMap(({ tags: secretTags = [], secretId }) => + secretTags.map((tag) => ({ + [`${TableName.SecretTag}Id` as const]: tag, + [`${TableName.Secret}Id` as const]: secretId + })) + ); + if (newSecretTags.length) { + const secTags = await secretTagDAL.saveTagsToSecret(newSecretTags, tx); + const secVersionsGroupBySecId = groupBy(secretVersions, (i) => i.secretId); + const newSecretVersionTags = secTags.flatMap(({ secretsId, secret_tagsId }) => ({ + [`${TableName.SecretVersion}Id` as const]: secVersionsGroupBySecId[secretsId][0].id, + [`${TableName.SecretTag}Id` as const]: secret_tagsId + })); + await secretVersionTagDAL.insertMany(newSecretVersionTags, tx); + } + } + + return newSecrets.map((secret) => ({ ...secret, _id: secret.id })); +}; + +export const createManySecretsRawHelper = async ({ projectId, environment, - path, + path: secretPath, secrets, userId, botKey, // TODO: consider getting botKey inside this fn @@ -364,67 +419,66 @@ export const createManySecretsRawHelper = async ({ // TODO: place on top secretVersionTagDAL, folderDAL }: TCreateManySecretsRawHelper) => { -await projectDAL.checkProjectUpgradeStatus(projectId); + await projectDAL.checkProjectUpgradeStatus(projectId); -const folder = await folderDAL.findBySecretPath(projectId, environment, path); -if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Create secret" }); -const folderId = folder.id; + const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); + if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Create secret" }); + const folderId = folder.id; -const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId }); -if (!blindIndexCfg) throw new BadRequestError({ message: "Blind index not found", name: "Update secret" }); - -const inputSecrets = await Promise.all( - secrets.map(async (secret) => { - const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey); - const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey); - const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey); - - if (secret.type === SecretType.Personal) { - const sharedExist = await secretDAL.findOne({ - secretBlindIndex: keyName2BlindIndex[secret.secretName], - folderId, - type: SecretType.Shared - }); - - if (!sharedExist) - throw new BadRequestError({ - message: "Failed to create personal secret override for no corresponding shared secret" - }); - - if (!userId) throw new BadRequestError({ message: "Missing user id for personal secret" }); - } - - const tags = secret.tags ? await secretTagDAL.findManyTagsById(projectId, secret.tags) : []; - if ((secret.tags || []).length !== tags.length) throw new BadRequestError({ message: "Tag not found" }); - - return ({ - type: secret.type, - userId: secret.type === SecretType.Personal ? userId : null, - secretName: secret.secretName, - secretKeyCiphertext: secretKeyEncrypted.ciphertext, - secretKeyIV: secretKeyEncrypted.iv, - secretKeyTag: secretKeyEncrypted.tag, - secretValueCiphertext: secretValueEncrypted.ciphertext, - secretValueIV: secretValueEncrypted.iv, - secretValueTag: secretValueEncrypted.tag, - secretCommentCiphertext: secretCommentEncrypted.ciphertext, - secretCommentIV: secretCommentEncrypted.iv, - secretCommentTag: secretCommentEncrypted.tag, - skipMultilineEncoding: secret.skipMultilineEncoding, - tags: secret.tags - }); - }) -); + const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId }); + if (!blindIndexCfg) throw new BadRequestError({ message: "Blind index not found", name: "Create secret" }); // insert operation const { keyName2BlindIndex } = await fnSecretBlindIndexCheck({ - inputSecrets, + inputSecrets: secrets, folderId, isNew: true, blindIndexCfg, secretDAL }); + const inputSecrets = await Promise.all( + secrets.map(async (secret) => { + const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey); + const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey); + const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey); + + if (secret.type === SecretType.Personal) { + if (!userId) throw new BadRequestError({ message: "Missing user id for personal secret" }); + const sharedExist = await secretDAL.findOne({ + secretBlindIndex: keyName2BlindIndex[secret.secretName], + folderId, + type: SecretType.Shared + }); + + if (!sharedExist) + throw new BadRequestError({ + message: "Failed to create personal secret override for no corresponding shared secret" + }); + } + + const tags = secret.tags ? await secretTagDAL.findManyTagsById(projectId, secret.tags) : []; + if ((secret.tags || []).length !== tags.length) throw new BadRequestError({ message: "Tag not found" }); + + return { + type: secret.type, + userId: secret.type === SecretType.Personal ? userId : null, + secretName: secret.secretName, + secretKeyCiphertext: secretKeyEncrypted.ciphertext, + secretKeyIV: secretKeyEncrypted.iv, + secretKeyTag: secretKeyEncrypted.tag, + secretValueCiphertext: secretValueEncrypted.ciphertext, + secretValueIV: secretValueEncrypted.iv, + secretValueTag: secretValueEncrypted.tag, + secretCommentCiphertext: secretCommentEncrypted.ciphertext, + secretCommentIV: secretCommentEncrypted.iv, + secretCommentTag: secretCommentEncrypted.tag, + skipMultilineEncoding: secret.skipMultilineEncoding, + tags: secret.tags + }; + }) + ); + const newSecrets = await secretDAL.transaction(async (tx) => fnSecretBulkInsert({ inputSecrets: inputSecrets.map(({ secretName, ...el }) => ({ @@ -444,4 +498,130 @@ const inputSecrets = await Promise.all( ); return newSecrets; -} \ No newline at end of file +}; + +export const updateManySecretsRawHelper = async ({ + projectId, + environment, + path: secretPath, + secrets, + userId, + botKey, // TODO: consider getting botKey inside this fn + projectDAL, + secretDAL, + secretVersionDAL, + secretBlindIndexDAL, + secretTagDAL, + secretVersionTagDAL, + folderDAL +}: TUpdateManySecretsRawHelper) => { + await projectDAL.checkProjectUpgradeStatus(projectId); + + const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); + if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Update secret" }); + const folderId = folder.id; + + const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId }); + if (!blindIndexCfg) throw new BadRequestError({ message: "Blind index not found", name: "Update secret" }); + + const { keyName2BlindIndex } = await fnSecretBlindIndexCheck({ + inputSecrets: secrets, + folderId, + isNew: false, + blindIndexCfg, + secretDAL, + userId + }); + + const inputSecrets = await Promise.all( + secrets.map(async (secret) => { + if (secret.newSecretName === "") { + throw new BadRequestError({ message: "New secret name cannot be empty" }); + } + + const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey); + const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey); + const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey); + + if (secret.type === SecretType.Personal) { + if (!userId) throw new BadRequestError({ message: "Missing user id for personal secret" }); + + const sharedExist = await secretDAL.findOne({ + secretBlindIndex: keyName2BlindIndex[secret.secretName], + folderId, + type: SecretType.Shared + }); + + if (!sharedExist) + throw new BadRequestError({ + message: "Failed to update personal secret override for no corresponding shared secret" + }); + + if (secret.newSecretName) throw new BadRequestError({ message: "Personal secret cannot change the key name" }); + } + + const tags = secret.tags ? await secretTagDAL.findManyTagsById(projectId, secret.tags) : []; + if ((secret.tags || []).length !== tags.length) throw new BadRequestError({ message: "Tag not found" }); + + return { + type: secret.type, + userId: secret.type === SecretType.Personal ? userId : null, + secretName: secret.secretName, + newSecretName: secret.newSecretName, + secretKeyCiphertext: secretKeyEncrypted.ciphertext, + secretKeyIV: secretKeyEncrypted.iv, + secretKeyTag: secretKeyEncrypted.tag, + secretValueCiphertext: secretValueEncrypted.ciphertext, + secretValueIV: secretValueEncrypted.iv, + secretValueTag: secretValueEncrypted.tag, + secretCommentCiphertext: secretCommentEncrypted.ciphertext, + secretCommentIV: secretCommentEncrypted.iv, + secretCommentTag: secretCommentEncrypted.tag, + skipMultilineEncoding: secret.skipMultilineEncoding, + tags: secret.tags + }; + }) + ); + + const tagIds = inputSecrets.flatMap(({ tags = [] }) => tags); + const tags = tagIds.length ? await secretTagDAL.findManyTagsById(projectId, tagIds) : []; + if (tagIds.length !== tags.length) throw new BadRequestError({ message: "Tag not found" }); + + // now find any secret that needs to update its name + // same process as above + const nameUpdatedSecrets = inputSecrets.filter(({ newSecretName }) => Boolean(newSecretName)); + const { keyName2BlindIndex: newKeyName2BlindIndex } = await fnSecretBlindIndexCheck({ + inputSecrets: nameUpdatedSecrets, + folderId, + isNew: true, + blindIndexCfg, + secretDAL + }); + + const updatedSecrets = await secretDAL.transaction(async (tx) => + fnSecretBulkUpdate({ + folderId, + projectId, + tx, + inputSecrets: inputSecrets.map(({ secretName, newSecretName, ...el }) => ({ + filter: { secretBlindIndex: keyName2BlindIndex[secretName], type: SecretType.Shared }, + data: { + ...el, + folderId, + secretBlindIndex: + newSecretName && newKeyName2BlindIndex[newSecretName] + ? newKeyName2BlindIndex[newSecretName] + : keyName2BlindIndex[secretName], + algorithm: SecretEncryptionAlgo.AES_256_GCM, + keyEncoding: SecretKeyEncoding.UTF8 + } + })), + secretDAL, + secretVersionDAL, + secretTagDAL, + secretVersionTagDAL + }) + ); + + return updatedSecrets; +}; diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index 2b5a69298..02aa28c7e 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -6,6 +6,10 @@ import { BadRequestError } from "@app/lib/errors"; import { isSamePath } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; +import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal"; +import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal"; +import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal"; +import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; import { TIntegrationDALFactory } from "../integration/integration-dal"; import { TIntegrationAuthServiceFactory } from "../integration-auth/integration-auth-service"; @@ -25,16 +29,11 @@ import { TSecretDALFactory } from "./secret-dal"; import { interpolateSecrets } from "./secret-fns"; import { TCreateSecretReminderDTO, THandleReminderDTO, TRemoveSecretReminderDTO } from "./secret-types"; -import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal"; -import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; -import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal"; -import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal"; - export type TSecretQueueFactory = ReturnType; type TSecretQueueFactoryDep = { queueService: TQueueServiceFactory; - integrationDAL: Pick; + integrationDAL: Pick; projectBotService: Pick; integrationAuthService: Pick; folderDAL: TSecretFolderDALFactory; @@ -75,7 +74,7 @@ export const secretQueueFactory = ({ secretVersionDAL, secretBlindIndexDAL, secretTagDAL, - secretVersionTagDAL, + secretVersionTagDAL }: TSecretQueueFactoryDep) => { const syncIntegrations = async (dto: TGetSecrets) => { await queueService.queue(QueueName.IntegrationSync, QueueJobs.IntegrationSync, dto, { @@ -321,6 +320,7 @@ export const secretQueueFactory = ({ await syncIntegrationSecrets({ projectDAL, + integrationDAL, secretDAL, secretVersionDAL, secretBlindIndexDAL, diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index 6b4aaf2d5..51136e40f 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -1,6 +1,6 @@ import { ForbiddenError, subject } from "@casl/ability"; -import { SecretEncryptionAlgo, SecretKeyEncoding, SecretsSchema, SecretType, TableName } from "@app/db/schemas"; +import { SecretEncryptionAlgo, SecretKeyEncoding, SecretsSchema, SecretType } from "@app/db/schemas"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; import { TSecretSnapshotServiceFactory } from "@app/ee/services/secret-snapshot/secret-snapshot-service"; @@ -19,14 +19,9 @@ import { TSecretImportDALFactory } from "../secret-import/secret-import-dal"; import { fnSecretsFromImports } from "../secret-import/secret-import-fns"; import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal"; import { TSecretDALFactory } from "./secret-dal"; -import { - decryptSecretRaw, - fnSecretBlindIndexCheck, - fnSecretBulkInsert -} from "./secret-fns"; +import { decryptSecretRaw, fnSecretBlindIndexCheck, fnSecretBulkInsert, fnSecretBulkUpdate } from "./secret-fns"; import { TSecretQueueFactory } from "./secret-queue"; import { - TCreateManySecretsRawHelper, TCreateBulkSecretDTO, TCreateSecretDTO, TCreateSecretRawDTO, @@ -35,7 +30,6 @@ import { TDeleteSecretRawDTO, TFnSecretBlindIndexCheckV2, TFnSecretBulkDelete, - TFnSecretBulkUpdate, TGetASecretDTO, TGetASecretRawDTO, TGetSecretsDTO, @@ -98,50 +92,6 @@ export const secretServiceFactory = ({ return secretBlindIndex; }; - const fnSecretBulkUpdate = async ({ tx, inputSecrets, folderId, projectId }: TFnSecretBulkUpdate) => { - const newSecrets = await secretDAL.bulkUpdate( - inputSecrets.map(({ filter, data: { tags, ...data } }) => ({ - filter: { ...filter, folderId }, - data - })), - tx - ); - const secretVersions = await secretVersionDAL.insertMany( - newSecrets.map(({ id, createdAt, updatedAt, ...el }) => ({ - ...el, - secretId: id - })), - tx - ); - const secsUpdatedTag = inputSecrets.flatMap(({ data: { tags } }, i) => - tags !== undefined ? { tags, secretId: newSecrets[i].id } : [] - ); - if (secsUpdatedTag.length) { - await secretTagDAL.deleteTagsManySecret( - projectId, - secsUpdatedTag.map(({ secretId }) => secretId), - tx - ); - const newSecretTags = secsUpdatedTag.flatMap(({ tags: secretTags = [], secretId }) => - secretTags.map((tag) => ({ - [`${TableName.SecretTag}Id` as const]: tag, - [`${TableName.Secret}Id` as const]: secretId - })) - ); - if (newSecretTags.length) { - const secTags = await secretTagDAL.saveTagsToSecret(newSecretTags, tx); - const secVersionsGroupBySecId = groupBy(secretVersions, (i) => i.secretId); - const newSecretVersionTags = secTags.flatMap(({ secretsId, secret_tagsId }) => ({ - [`${TableName.SecretVersion}Id` as const]: secVersionsGroupBySecId[secretsId][0].id, - [`${TableName.SecretTag}Id` as const]: secret_tagsId - })); - await secretVersionTagDAL.insertMany(newSecretVersionTags, tx); - } - } - - return newSecrets.map((secret) => ({ ...secret, _id: secret.id })); - }; - const fnSecretBulkDelete = async ({ folderId, inputSecrets, tx, actorId }: TFnSecretBulkDelete) => { const deletedSecrets = await secretDAL.deleteMany( inputSecrets.map(({ type, secretBlindIndex }) => ({ @@ -372,6 +322,10 @@ export const secretServiceFactory = ({ } } ], + secretDAL, + secretVersionDAL, + secretTagDAL, + secretVersionTagDAL, tx }) ); @@ -598,7 +552,7 @@ export const secretServiceFactory = ({ const folderId = folder.id; const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId }); - if (!blindIndexCfg) throw new BadRequestError({ message: "Blind index not found", name: "Update secret" }); + if (!blindIndexCfg) throw new BadRequestError({ message: "Blind index not found", name: "Create secret" }); const { keyName2BlindIndex } = await fnSecretBlindIndexCheck({ inputSecrets, @@ -656,7 +610,7 @@ export const secretServiceFactory = ({ await projectDAL.checkProjectUpgradeStatus(projectId); const folder = await folderDAL.findBySecretPath(projectId, environment, path); - if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Create secret" }); + if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Update secret" }); const folderId = folder.id; const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId }); @@ -703,7 +657,11 @@ export const secretServiceFactory = ({ algorithm: SecretEncryptionAlgo.AES_256_GCM, keyEncoding: SecretKeyEncoding.UTF8 } - })) + })), + secretDAL, + secretVersionDAL, + secretTagDAL, + secretVersionTagDAL }) ); @@ -826,98 +784,6 @@ export const secretServiceFactory = ({ }); return decryptSecretRaw(secret, botKey); }; - - const createManySecretsRawHelper = async ({ // TODO: place on top - projectId, - environment, - path, - secrets, - userId - }: TCreateManySecretsRawHelper) => { - const botKey = await projectBotService.getBotKey(projectId); - if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" }); - - await projectDAL.checkProjectUpgradeStatus(projectId); - - const folder = await folderDAL.findBySecretPath(projectId, environment, path); - if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Create secret" }); - const folderId = folder.id; - - const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId }); - if (!blindIndexCfg) throw new BadRequestError({ message: "Blind index not found", name: "Update secret" }); - - const inputSecrets = await Promise.all( - secrets.map(async (secret) => { - const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey); - const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey); - const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey); - - if (secret.type === SecretType.Personal) { - const sharedExist = await secretDAL.findOne({ - secretBlindIndex: keyName2BlindIndex[secret.secretName], - folderId, - type: SecretType.Shared - }); - - if (!sharedExist) - throw new BadRequestError({ - message: "Failed to create personal secret override for no corresponding shared secret" - }); - - if (!userId) throw new BadRequestError({ message: "Missing user id for personal secret" }); - } - - const tags = secret.tags ? await secretTagDAL.findManyTagsById(projectId, secret.tags) : []; - if ((secret.tags || []).length !== tags.length) throw new BadRequestError({ message: "Tag not found" }); - - return ({ - type: secret.type, - userId: secret.type === SecretType.Personal ? userId : null, - secretName: secret.secretName, - secretKeyCiphertext: secretKeyEncrypted.ciphertext, - secretKeyIV: secretKeyEncrypted.iv, - secretKeyTag: secretKeyEncrypted.tag, - secretValueCiphertext: secretValueEncrypted.ciphertext, - secretValueIV: secretValueEncrypted.iv, - secretValueTag: secretValueEncrypted.tag, - secretCommentCiphertext: secretCommentEncrypted.ciphertext, - secretCommentIV: secretCommentEncrypted.iv, - secretCommentTag: secretCommentEncrypted.tag, - skipMultilineEncoding: secret.skipMultilineEncoding, - tags: secret.tags - }); - }) - ); - - // insert operation - const { keyName2BlindIndex } = await fnSecretBlindIndexCheck({ - inputSecrets, - folderId, - isNew: true, - blindIndexCfg, - secretDAL - }); - - const newSecrets = await secretDAL.transaction(async (tx) => - fnSecretBulkInsert({ - inputSecrets: inputSecrets.map(({ secretName, ...el }) => ({ - ...el, - version: 0, - secretBlindIndex: keyName2BlindIndex[secretName], - algorithm: SecretEncryptionAlgo.AES_256_GCM, - keyEncoding: SecretKeyEncoding.UTF8 - })), - folderId, - secretDAL, - secretVersionDAL, - secretTagDAL, - secretVersionTagDAL, - tx - }) - ); - - return newSecrets; - } const createSecretRaw = async ({ secretName, diff --git a/backend/src/services/secret/secret-types.ts b/backend/src/services/secret/secret-types.ts index 6b0b205ed..acdd32c62 100644 --- a/backend/src/services/secret/secret-types.ts +++ b/backend/src/services/secret/secret-types.ts @@ -4,11 +4,11 @@ import { SecretType, TSecretBlindIndexes, TSecrets, TSecretsInsert, TSecretsUpda import { TProjectPermission } from "@app/lib/types"; import { TProjectDALFactory } from "@app/services/project/project-dal"; import { TSecretDALFactory } from "@app/services/secret/secret-dal"; -import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal"; -import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; -import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal"; import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal"; import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal"; +import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal"; +import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal"; +import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; type TPartialSecret = Pick; @@ -198,6 +198,10 @@ export type TFnSecretBulkUpdate = { folderId: string; projectId: string; inputSecrets: { filter: Partial; data: TSecretsUpdate & { tags?: string[] } }[]; + secretDAL: Pick; + secretVersionDAL: Pick; + secretTagDAL: Pick; + secretVersionTagDAL: Pick; tx?: Knex; }; @@ -257,7 +261,7 @@ export type TCreateManySecretsRawHelper = { tags?: string[]; metadata?: { source?: string; - } + }; }[]; userId?: string; // only relevant for personal secret(s) botKey: string; @@ -268,4 +272,33 @@ export type TCreateManySecretsRawHelper = { secretTagDAL: TSecretTagDALFactory; secretVersionTagDAL: TSecretVersionTagDALFactory; folderDAL: TSecretFolderDALFactory; -} \ No newline at end of file +}; + +export type TUpdateManySecretsRawHelper = { + projectId: string; + environment: string; + path: string; + secrets: { + secretName: string; + newSecretName?: string; + secretValue: string; + type: SecretType; + secretComment?: string; + skipMultilineEncoding?: boolean; + secretReminderRepeatDays?: number | null; + secretReminderNote?: string | null; + tags?: string[]; + metadata?: { + source?: string; + }; + }[]; + userId?: string; // only relevant for personal secret(s) + botKey: string; + projectDAL: TProjectDALFactory; + secretDAL: TSecretDALFactory; + secretVersionDAL: TSecretVersionDALFactory; + secretBlindIndexDAL: TSecretBlindIndexDALFactory; + secretTagDAL: TSecretTagDALFactory; + secretVersionTagDAL: TSecretVersionTagDALFactory; + folderDAL: TSecretFolderDALFactory; +};