awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-09-22 13:39:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

checkpoint

Browse Source
This commit is contained in:
=
2024-07-21 22:09:24 +05:30
parent ef2816b2ee
commit 643d13b0ec
7 changed files with 432 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts
View File

@@ -730,7 +730,7 @@ export const secretApprovalRequestServiceFactory = ({
});
}
await snapshotService.performSnapshot(folderId);
await snapshotService.performSnapshot(folderId, shouldUseSecretV2Bridge);
const [folder] = await folderDAL.findSecretPathByFolderIds(projectId, [folderId]);
if (!folder) throw new BadRequestError({ message: "Folder not found" });
await secretQueueService.syncSecrets({

163
backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts
View File

@@ -39,6 +39,10 @@ import {
secretRotationPreSetFn
} from "./secret-rotation-queue-fn";
import { TSecretRotationData, TSecretRotationDbFn, TSecretRotationEncData } from "./secret-rotation-queue-types";
import { TSecretV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-dal";
import { TSecretVersionV2DALFactory } from "@app/services/secret-v2-bridge/secret-version-dal";
import { TKmsServiceFactory } from "@app/services/kms/kms-service";
import { KmsDataKey } from "@app/services/kms/kms-types";
export type TSecretRotationQueueFactory = ReturnType<typeof secretRotationQueueFactory>;
@@ -47,8 +51,11 @@ type TSecretRotationQueueFactoryDep = {
secretRotationDAL: TSecretRotationDALFactory;
projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
secretDAL: Pick<TSecretDALFactory, "bulkUpdate" | "find">;
secretV2BridgeDAL: Pick<TSecretV2BridgeDALFactory, "bulkUpdate" | "find">;
secretVersionDAL: Pick<TSecretVersionDALFactory, "insertMany" | "findLatestVersionMany">;
secretVersionV2BridgeDAL: Pick<TSecretVersionV2DALFactory, "insertMany" | "findLatestVersionMany">;
telemetryService: Pick<TTelemetryServiceFactory, "sendPostHogEvents">;
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
};
// These error should stop the repeatable job and ask user to reconfigure rotation
@@ -70,7 +77,10 @@ export const secretRotationQueueFactory = ({
projectBotService,
secretDAL,
secretVersionDAL,
telemetryService
telemetryService,
secretV2BridgeDAL,
secretVersionV2BridgeDAL,
kmsService
}: TSecretRotationQueueFactoryDep) => {
const addToQueue = async (rotationId: string, interval: number) => {
const appCfg = getConfig();
@@ -267,62 +277,113 @@ export const secretRotationQueueFactory = ({
internal: newCredential.internal
});
const encVarData = infisicalSymmetricEncypt(JSON.stringify(variables));
const key = await projectBotService.getBotKey(secretRotation.projectId);
const encryptedSecrets = rotationOutputs.map(({ key: outputKey, secretId }) => ({
secretId,
value: encryptSymmetric128BitHexKeyUTF8(
typeof newCredential.outputs[outputKey] === "object"
? JSON.stringify(newCredential.outputs[outputKey])
: String(newCredential.outputs[outputKey]),
key
)
}));
// map the final values to output keys in the board
await secretRotationDAL.transaction(async (tx) => {
await secretRotationDAL.updateById(
rotationId,
{
encryptedData: encVarData.ciphertext,
encryptedDataIV: encVarData.iv,
encryptedDataTag: encVarData.tag,
keyEncoding: encVarData.encoding,
algorithm: encVarData.algorithm,
lastRotatedAt: new Date(),
statusMessage: "Rotated successfull",
status: "success"
},
tx
);
const updatedSecrets = await secretDAL.bulkUpdate(
encryptedSecrets.map(({ secretId, value }) => ({
// this secret id is validated when user is inserted
filter: { id: secretId, type: SecretType.Shared },
data: {
secretValueCiphertext: value.ciphertext,
secretValueIV: value.iv,
secretValueTag: value.tag
}
})),
tx
);
await secretVersionDAL.insertMany(
updatedSecrets.map(({ id, updatedAt, createdAt, ...el }) => {
if (!el.secretBlindIndex) throw new BadRequestError({ message: "Missing blind index" });
return {
...el,
secretId: id,
secretBlindIndex: el.secretBlindIndex
};
}),
tx
);
const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(secretRotation.projectId);
const { encryptor: secretManagerEncryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.SecretManager,
projectId: secretRotation.projectId
});
const numberOfSecretsRotated = rotationOutputs.length;
if (shouldUseSecretV2Bridge) {
const encryptedSecrets = rotationOutputs.map(({ key: outputKey, secretId }) => ({
secretId,
value:
typeof newCredential.outputs[outputKey] === "object"
? JSON.stringify(newCredential.outputs[outputKey])
: String(newCredential.outputs[outputKey])
}));
// map the final values to output keys in the board
await secretRotationDAL.transaction(async (tx) => {
await secretRotationDAL.updateById(
rotationId,
{
encryptedData: encVarData.ciphertext,
encryptedDataIV: encVarData.iv,
encryptedDataTag: encVarData.tag,
keyEncoding: encVarData.encoding,
algorithm: encVarData.algorithm,
lastRotatedAt: new Date(),
statusMessage: "Rotated successfull",
status: "success"
},
tx
);
const updatedSecrets = await secretV2BridgeDAL.bulkUpdate(
encryptedSecrets.map(({ secretId, value }) => ({
// this secret id is validated when user is inserted
filter: { id: secretId, type: SecretType.Shared },
data: {
encryptedValue: secretManagerEncryptor({ plainText: Buffer.from(value) }).cipherTextBlob
}
})),
tx
);
await secretVersionV2BridgeDAL.insertMany(
updatedSecrets.map(({ id, updatedAt, createdAt, ...el }) => ({
...el,
secretId: id
})),
tx
);
});
} else {
if (!botKey) throw new BadRequestError({ message: "Bot not found" });
const encryptedSecrets = rotationOutputs.map(({ key: outputKey, secretId }) => ({
secretId,
value: encryptSymmetric128BitHexKeyUTF8(
typeof newCredential.outputs[outputKey] === "object"
? JSON.stringify(newCredential.outputs[outputKey])
: String(newCredential.outputs[outputKey]),
botKey
)
}));
// map the final values to output keys in the board
await secretRotationDAL.transaction(async (tx) => {
await secretRotationDAL.updateById(
rotationId,
{
encryptedData: encVarData.ciphertext,
encryptedDataIV: encVarData.iv,
encryptedDataTag: encVarData.tag,
keyEncoding: encVarData.encoding,
algorithm: encVarData.algorithm,
lastRotatedAt: new Date(),
statusMessage: "Rotated successfull",
status: "success"
},
tx
);
const updatedSecrets = await secretDAL.bulkUpdate(
encryptedSecrets.map(({ secretId, value }) => ({
// this secret id is validated when user is inserted
filter: { id: secretId, type: SecretType.Shared },
data: {
secretValueCiphertext: value.ciphertext,
secretValueIV: value.iv,
secretValueTag: value.tag
}
})),
tx
);
await secretVersionDAL.insertMany(
updatedSecrets.map(({ id, updatedAt, createdAt, ...el }) => {
if (!el.secretBlindIndex) throw new BadRequestError({ message: "Missing blind index" });
return {
...el,
secretId: id,
secretBlindIndex: el.secretBlindIndex
};
}),
tx
);
});
}
await telemetryService.sendPostHogEvents({
event: PostHogEventTypes.SecretRotated,
distinctId: "",
properties: {
numberOfSecrets: encryptedSecrets.length,
numberOfSecrets: numberOfSecretsRotated,
environment: secretRotation.environment.slug,
secretPath: secretRotation.secretPath,
workspaceId: secretRotation.projectId

18
backend/src/services/integration-auth/integration-sync-secret.ts
View File

@@ -26,7 +26,7 @@ import sodium from "libsodium-wrappers";
import isEqual from "lodash.isequal";
import { z } from "zod";
import { SecretType, TIntegrationAuths, TIntegrations, TSecrets } from "@app/db/schemas";
import { SecretType, TIntegrationAuths, TIntegrations } from "@app/db/schemas";
import { getConfig } from "@app/lib/config/env";
import { request } from "@app/lib/config/request";
import { BadRequestError } from "@app/lib/errors";
@@ -275,8 +275,8 @@ const syncSecretsAzureKeyVault = async ({
};
secrets: Record<string, { value: string; comment?: string }>;
accessToken: string;
createManySecretsRawFn: (params: TCreateManySecretsRawFn) => Promise<Array<TSecrets & { _id: string }>>;
updateManySecretsRawFn: (params: TUpdateManySecretsRawFn) => Promise<Array<TSecrets & { _id: string }>>;
createManySecretsRawFn: (params: TCreateManySecretsRawFn) => Promise<Array<{ id: string }>>;
updateManySecretsRawFn: (params: TUpdateManySecretsRawFn) => Promise<Array<{ id: string }>>;
}) => {
interface GetAzureKeyVaultSecret {
id: string; // secret URI
@@ -903,8 +903,8 @@ const syncSecretsHeroku = async ({
secrets,
accessToken
}: {
createManySecretsRawFn: (params: TCreateManySecretsRawFn) => Promise<Array<TSecrets & { _id: string }>>;
updateManySecretsRawFn: (params: TUpdateManySecretsRawFn) => Promise<Array<TSecrets & { _id: string }>>;
createManySecretsRawFn: (params: TCreateManySecretsRawFn) => Promise<Array<{ id: string }>>;
updateManySecretsRawFn: (params: TUpdateManySecretsRawFn) => Promise<Array<{ id: string }>>;
integration: TIntegrations & {
projectId: string;
environment: {
@@ -2464,8 +2464,8 @@ const syncSecretsTerraformCloud = async ({
accessToken,
integrationDAL
}: {
createManySecretsRawFn: (params: TCreateManySecretsRawFn) => Promise<Array<TSecrets & { _id: string }>>;
updateManySecretsRawFn: (params: TUpdateManySecretsRawFn) => Promise<Array<TSecrets & { _id: string }>>;
createManySecretsRawFn: (params: TCreateManySecretsRawFn) => Promise<Array<{ id: string }>>;
updateManySecretsRawFn: (params: TUpdateManySecretsRawFn) => Promise<Array<{ id: string }>>;
integration: TIntegrations & {
projectId: string;
environment: {
@@ -3612,8 +3612,8 @@ export const syncIntegrationSecrets = async ({
appendices,
projectId
}: {
createManySecretsRawFn: (params: TCreateManySecretsRawFn) => Promise<Array<TSecrets & { _id: string }>>;
updateManySecretsRawFn: (params: TUpdateManySecretsRawFn) => Promise<Array<TSecrets & { _id: string }>>;
createManySecretsRawFn: (params: TCreateManySecretsRawFn) => Promise<Array<{ id: string }>>;
updateManySecretsRawFn: (params: TUpdateManySecretsRawFn) => Promise<Array<{ id: string }>>;
integrationDAL: Pick<TIntegrationDALFactory, "updateById">;
integration: TIntegrations & {
projectId: string;

22
backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts
View File

@@ -179,7 +179,7 @@ export const secretV2BridgeServiceFactory = ({
})
);
await snapshotService.performSnapshot(folderId);
await snapshotService.performSnapshot(folderId, true);
await secretQueueService.syncSecrets({
secretPath,
actorId,
@@ -319,7 +319,7 @@ export const secretV2BridgeServiceFactory = ({
projectId
});
await snapshotService.performSnapshot(folderId);
await snapshotService.performSnapshot(folderId, true);
await secretQueueService.syncSecrets({
actor,
actorId,
@@ -385,7 +385,7 @@ export const secretV2BridgeServiceFactory = ({
})
);
await snapshotService.performSnapshot(folderId);
await snapshotService.performSnapshot(folderId, true);
await secretQueueService.syncSecrets({
actor,
actorId,
@@ -717,7 +717,7 @@ export const secretV2BridgeServiceFactory = ({
})
);
await snapshotService.performSnapshot(folderId);
await snapshotService.performSnapshot(folderId, true);
await secretQueueService.syncSecrets({
actor,
actorId,
@@ -775,6 +775,7 @@ export const secretV2BridgeServiceFactory = ({
);
if (secretsToUpdate.length !== inputSecrets.length)
throw new BadRequestError({ message: `Secret not exist: ${secretsToUpdate.map((el) => el.key).join(",")}` });
const secretsToUpdateInDBGroupedByKey = groupBy(secretsToUpdate, (i) => i.key);
// now find any secret that needs to update its name
// same process as above
@@ -788,7 +789,9 @@ export const secretV2BridgeServiceFactory = ({
}))
);
if (secrets.length)
throw new BadRequestError({ message: `Secret not exist: ${secretsToUpdate.map((el) => el.key).join(",")}` });
throw new BadRequestError({
message: `Secret with new name exist: ${secretsWithNewName.map((el) => el.newSecretName).join(",")}`
});
}
// get all tags
@@ -804,6 +807,7 @@ export const secretV2BridgeServiceFactory = ({
folderId,
tx,
inputSecrets: inputSecrets.map((el) => {
const originalSecret = secretsToUpdateInDBGroupedByKey[el.secretKey][0];
const encryptedValue =
typeof el.secretValue !== "undefined"
? {
@@ -812,7 +816,7 @@ export const secretV2BridgeServiceFactory = ({
}
: {};
return {
filter: { key: el.secretKey, type: SecretType.Shared },
filter: { id: originalSecret.id, type: SecretType.Shared },
data: {
reminderRepeatDays: el.secretReminderRepeatDays,
encryptedComment: secretEncryptionHelper.encryptValue(secretManagerEncryptor, el.secretComment),
@@ -830,7 +834,7 @@ export const secretV2BridgeServiceFactory = ({
secretVersionTagDAL
})
);
await snapshotService.performSnapshot(folderId);
await snapshotService.performSnapshot(folderId, true);
await secretQueueService.syncSecrets({
actor,
actorId,
@@ -1309,7 +1313,7 @@ export const secretV2BridgeServiceFactory = ({
});
if (isDestinationUpdated) {
await snapshotService.performSnapshot(destinationFolder.id);
await snapshotService.performSnapshot(destinationFolder.id, true);
await secretQueueService.syncSecrets({
projectId,
secretPath: destinationFolder.path,
@@ -1320,7 +1324,7 @@ export const secretV2BridgeServiceFactory = ({
}
if (isSourceUpdated) {
await snapshotService.performSnapshot(sourceFolder.id);
await snapshotService.performSnapshot(sourceFolder.id, true);
await secretQueueService.syncSecrets({
projectId,
secretPath: sourceFolder.path,

162
backend/src/services/secret/secret-fns.ts
View File

@@ -39,6 +39,12 @@ import {
TUpdateManySecretsRawFn,
TUpdateManySecretsRawFnFactory
} from "./secret-types";
import { KmsDataKey } from "../kms/kms-types";
import {
fnSecretBulkInsert as fnSecretV2BridgeBulkInsert,
fnSecretBulkUpdate as fnSecretV2BridgeBulkUpdate,
getAllNestedSecretReferences as getAllNestedSecretReferencesV2Bridge
} from "@app/services/secret-v2-bridge/secret-v2-bridge-fns";
export const generateSecretBlindIndexBySalt = async (secretName: string, secretBlindIndexDoc: TSecretBlindIndexes) => {
const appCfg = getConfig();
@@ -679,7 +685,11 @@ export const createManySecretsRawFnFactory = ({
secretBlindIndexDAL,
secretTagDAL,
secretVersionTagDAL,
folderDAL
folderDAL,
secretVersionV2BridgeDAL,
secretV2BridgeDAL,
secretVersionTagV2BridgeDAL,
kmsService
}: TCreateManySecretsRawFnFactory) => {
const getBotKeyFn = getBotKeyFnFactory(projectBotDAL, projectDAL);
const createManySecretsRawFn = async ({
@@ -689,10 +699,7 @@ export const createManySecretsRawFnFactory = ({
secrets,
userId
}: TCreateManySecretsRawFn) => {
const { botKey } = await getBotKeyFn(projectId);
if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" });
await projectDAL.checkProjectUpgradeStatus(projectId);
const { botKey, shouldUseSecretV2Bridge } = await getBotKeyFn(projectId);
const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
if (!folder)
@@ -701,6 +708,62 @@ export const createManySecretsRawFnFactory = ({
name: "Create secret"
});
const folderId = folder.id;
if (shouldUseSecretV2Bridge) {
const { encryptor: secretManagerEncryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.SecretManager,
projectId
});
const secretsStoredInDB = await secretV2BridgeDAL.findBySecretKeys(
folderId,
secrets.map((el) => ({
key: el.secretName,
type: SecretType.Shared
}))
);
if (secretsStoredInDB.length)
throw new BadRequestError({
message: `Secret already exist: ${secretsStoredInDB.map((el) => el.key).join(",")}`
});
const inputSecrets = secrets.map((secret) => {
return {
type: secret.type,
userId: secret.type === SecretType.Personal ? userId : null,
key: secret.secretName,
encryptedValue: secretManagerEncryptor({ plainText: Buffer.from(secret.secretValue) }).cipherTextBlob,
encryptedComent: secret.secretComment
? secretManagerEncryptor({ plainText: Buffer.from(secret.secretComment) }).cipherTextBlob
: null,
skipMultilineEncoding: secret.skipMultilineEncoding,
tags: secret.tags,
references: getAllNestedSecretReferencesV2Bridge(secret.secretValue)
};
});
// get all tags
const tagIds = inputSecrets.flatMap(({ tags = [] }) => tags);
const tags = tagIds.length ? await secretTagDAL.findManyTagsById(projectId, tagIds) : [];
if (tags.length !== tagIds.length) throw new BadRequestError({ message: "Tag not found" });
const newSecrets = await secretDAL.transaction(async (tx) =>
fnSecretV2BridgeBulkInsert({
inputSecrets: inputSecrets.map((el) => ({
...el,
version: 1,
tagIds: el.tags
})),
folderId,
secretDAL: secretV2BridgeDAL,
secretVersionDAL: secretVersionV2BridgeDAL,
secretTagDAL,
secretVersionTagDAL: secretVersionTagV2BridgeDAL,
tx
})
);
return newSecrets;
}
const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId });
if (!blindIndexCfg) throw new BadRequestError({ message: "Blind index not found", name: "Create secret" });
@@ -715,6 +778,7 @@ export const createManySecretsRawFnFactory = ({
secretDAL
});
if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" });
const inputSecrets = secrets.map((secret) => {
const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey);
const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey);
@@ -777,7 +841,11 @@ export const updateManySecretsRawFnFactory = ({
secretBlindIndexDAL,
secretTagDAL,
secretVersionTagDAL,
folderDAL
folderDAL,
secretVersionTagV2BridgeDAL,
secretVersionV2BridgeDAL,
secretV2BridgeDAL,
kmsService
}: TUpdateManySecretsRawFnFactory) => {
const getBotKeyFn = getBotKeyFnFactory(projectBotDAL, projectDAL);
const updateManySecretsRawFn = async ({
@@ -786,11 +854,8 @@ export const updateManySecretsRawFnFactory = ({
path: secretPath,
secrets, // consider accepting instead ciphertext secrets
userId
}: TUpdateManySecretsRawFn): Promise<Array<TSecrets & { _id: string }>> => {
const { botKey } = await getBotKeyFn(projectId);
if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" });
await projectDAL.checkProjectUpgradeStatus(projectId);
}: TUpdateManySecretsRawFn): Promise<Array<{ id: string }>> => {
const { botKey, shouldUseSecretV2Bridge } = await getBotKeyFn(projectId);
const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
if (!folder)
@@ -799,7 +864,82 @@ export const updateManySecretsRawFnFactory = ({
name: "Update secret"
});
const folderId = folder.id;
if (shouldUseSecretV2Bridge) {
const { encryptor: secretManagerEncryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.SecretManager,
projectId
});
const secretsToUpdate = await secretV2BridgeDAL.findBySecretKeys(
folderId,
secrets.map((el) => ({
key: el.secretName,
type: SecretType.Shared
}))
);
if (secretsToUpdate.length !== secrets.length)
throw new BadRequestError({ message: `Secret not exist: ${secretsToUpdate.map((el) => el.key).join(",")}` });
// now find any secret that needs to update its name
// same process as above
const secretsWithNewName = secrets.filter(({ newSecretName }) => Boolean(newSecretName));
if (secretsWithNewName.length) {
const secretsWithNewNameInDB = await secretV2BridgeDAL.findBySecretKeys(
folderId,
secrets.map((el) => ({
key: el.secretName,
type: SecretType.Shared
}))
);
if (secretsWithNewNameInDB.length)
throw new BadRequestError({
message: `Secret not exist: ${secretsWithNewName.map((el) => el.newSecretName).join(",")}`
});
}
const secretsToUpdateInDBGroupedByKey = groupBy(secretsToUpdate, (i) => i.key);
const inputSecrets = secrets.map((secret) => {
if (secret.newSecretName === "") {
throw new BadRequestError({ message: "New secret name cannot be empty" });
}
return {
type: secret.type,
userId: secret.type === SecretType.Personal ? userId : null,
key: secret.newSecretName || secret.secretName,
encryptedValue: secretManagerEncryptor({ plainText: Buffer.from(secret.secretValue) }).cipherTextBlob,
encryptedComent: secret.secretComment
? secretManagerEncryptor({ plainText: Buffer.from(secret.secretComment) }).cipherTextBlob
: null,
skipMultilineEncoding: secret.skipMultilineEncoding,
tags: secret.tags,
references: getAllNestedSecretReferencesV2Bridge(secret.secretValue)
};
});
const tagIds = inputSecrets.flatMap(({ tags = [] }) => tags);
const tags = tagIds.length ? await secretTagDAL.findManyTagsById(projectId, tagIds) : [];
if (tagIds.length !== tags.length) throw new BadRequestError({ message: "Tag not found" });
const updatedSecrets = await secretDAL.transaction(async (tx) =>
fnSecretV2BridgeBulkUpdate({
folderId,
tx,
inputSecrets: inputSecrets.map((el) => ({
filter: { id: secretsToUpdateInDBGroupedByKey[el.key][0].id, type: SecretType.Shared },
data: el
})),
secretDAL: secretV2BridgeDAL,
secretVersionDAL: secretVersionV2BridgeDAL,
secretTagDAL,
secretVersionTagDAL: secretVersionTagV2BridgeDAL
})
);
return updatedSecrets;
}
if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" });
const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId });
if (!blindIndexCfg) throw new BadRequestError({ message: "Blind index not found", name: "Update secret" });

150
backend/src/services/secret/secret-queue.ts
View File

@@ -36,6 +36,10 @@ import {
TRemoveSecretReminderDTO,
TSyncSecretsDTO
} from "./secret-types";
import { TKmsServiceFactory } from "../kms/kms-service";
import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal";
import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-dal";
import { TSecretVersionV2TagDALFactory } from "../secret-v2-bridge/secret-version-tag-dal";
export type TSecretQueueFactory = ReturnType<typeof secretQueueFactory>;
type TSecretQueueFactoryDep = {
@@ -57,6 +61,10 @@ type TSecretQueueFactoryDep = {
secretBlindIndexDAL: TSecretBlindIndexDALFactory;
secretTagDAL: TSecretTagDALFactory;
secretVersionTagDAL: TSecretVersionTagDALFactory;
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
secretV2BridgeDAL: TSecretV2BridgeDALFactory;
secretVersionV2BridgeDAL: Pick<TSecretVersionV2DALFactory, "insertMany" | "findLatestVersionMany">;
secretVersionTagV2BridgeDAL: Pick<TSecretVersionV2TagDALFactory, "insertMany">;
};
export type TGetSecrets = {
@@ -91,7 +99,11 @@ export const secretQueueFactory = ({
secretVersionDAL,
secretBlindIndexDAL,
secretTagDAL,
secretVersionTagDAL
secretVersionTagDAL,
secretV2BridgeDAL,
secretVersionV2BridgeDAL,
kmsService,
secretVersionTagV2BridgeDAL
}: TSecretQueueFactoryDep) => {
const removeSecretReminder = async (dto: TRemoveSecretReminderDTO) => {
const appCfg = getConfig();
@@ -195,7 +207,11 @@ export const secretQueueFactory = ({
secretBlindIndexDAL,
secretTagDAL,
secretVersionTagDAL,
folderDAL
folderDAL,
kmsService,
secretVersionV2BridgeDAL,
secretV2BridgeDAL,
secretVersionTagV2BridgeDAL
});
const updateManySecretsRawFn = updateManySecretsRawFnFactory({
@@ -206,9 +222,82 @@ export const secretQueueFactory = ({
secretBlindIndexDAL,
secretTagDAL,
secretVersionTagDAL,
folderDAL
folderDAL,
kmsService,
secretVersionV2BridgeDAL,
secretV2BridgeDAL,
secretVersionTagV2BridgeDAL
});
/**
* Return the secrets in a given [folderId] including secrets from
* nested imported folders recursively.
*/
const getIntegrationSecretsV2 = async (dto: {
projectId: string;
environment: string;
folderId: string;
depth: number;
decryptor: (value: Buffer | null | undefined) => string;
}) => {
let content: TIntegrationSecret = {};
if (dto.depth > MAX_SYNC_SECRET_DEPTH) {
logger.info(
`getIntegrationSecrets: secret depth exceeded for [projectId=${dto.projectId}] [folderId=${dto.folderId}] [depth=${dto.depth}]`
);
return content;
}
// process secrets in current folder
const secrets = await secretV2BridgeDAL.findByFolderId(dto.folderId);
secrets.forEach((secret) => {
const secretKey = secret.key;
const secretValue = dto.decryptor(secret.encryptedValue);
content[secretKey] = { value: secretValue };
if (secret.encryptedComment) {
const commentValue = dto.decryptor(secret.encryptedComment);
content[secretKey].comment = commentValue;
}
content[secretKey].skipMultilineEncoding = Boolean(secret.skipMultilineEncoding);
});
// TODO(akhilmhdh-sev2): change this to v2 expand secrets
// check if current folder has any imports from other folders
const secretImport = await secretImportDAL.find({ folderId: dto.folderId, isReplication: false });
// if no imports then return secrets in the current folder
if (!secretImport) return content;
const importedFolders = await folderDAL.findByManySecretPath(
secretImport.map(({ importEnv, importPath }) => ({
envId: importEnv.id,
secretPath: importPath
}))
);
for await (const folder of importedFolders) {
if (folder) {
// get secrets contained in each imported folder by recursively calling
// this function against the imported folder
const importedSecrets = await getIntegrationSecretsV2({
environment: dto.environment,
projectId: dto.projectId,
folderId: folder.id,
depth: dto.depth + 1,
decryptor: dto.decryptor
});
// add the imported secrets to the current folder secrets
content = { ...importedSecrets, ...content };
}
}
return content;
};
/**
* Return the secrets in a given [folderId] including secrets from
* nested imported folders recursively.
@@ -467,24 +556,35 @@ export const secretQueueFactory = ({
);
}
const secretReferences = await secretDAL.findReferencedSecretReferences(
projectId,
folder.environment.slug,
secretPath
);
if (secretReferences.length) {
const referencedFolderIds = unique(secretReferences, (i) => i.folderId).map(({ folderId }) => folderId);
const { shouldUseSecretV2Bridge, botKey } = await projectBotService.getBotKey(projectId);
let referencedFolderIds;
if (shouldUseSecretV2Bridge) {
const secretReferences = await secretV2BridgeDAL.findReferencedSecretReferences(
projectId,
folder.environment.slug,
secretPath
);
referencedFolderIds = unique(secretReferences, (i) => i.folderId).map(({ folderId }) => folderId);
} else {
const secretReferences = await secretDAL.findReferencedSecretReferences(
projectId,
folder.environment.slug,
secretPath
);
referencedFolderIds = unique(secretReferences, (i) => i.folderId).map(({ folderId }) => folderId);
}
if (referencedFolderIds.length) {
const referencedFolders = await folderDAL.findSecretPathByFolderIds(projectId, referencedFolderIds);
const referencedFoldersGroupedById = groupBy(referencedFolders.filter(Boolean), (i) => i?.id as string);
logger.info(
`getIntegrationSecrets: Syncing secret due to reference change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}] [secretPath=${job.data.secretPath}] [depth=${depth}]`
);
await Promise.all(
secretReferences
.filter(({ folderId }) => Boolean(referencedFoldersGroupedById[folderId][0]?.path))
referencedFolderIds
.filter((folderId) => Boolean(referencedFoldersGroupedById[folderId][0]?.path))
// filter out already synced ones
.filter(
({ folderId }) =>
(folderId) =>
!deDupeQueue[
uniqueSecretQueueKey(
referencedFoldersGroupedById[folderId][0]?.environmentSlug as string,
@@ -492,7 +592,7 @@ export const secretQueueFactory = ({
)
]
)
.map(({ folderId }) =>
.map((folderId) =>
syncSecrets({
projectId,
secretPath: referencedFoldersGroupedById[folderId][0]?.path as string,
@@ -523,7 +623,6 @@ export const secretQueueFactory = ({
projectId: integration.projectId
};
const botKey = await projectBotService.getBotKey(projectId);
const { accessToken, accessId } = await integrationAuthService.getIntegrationAccessToken(integrationAuth, botKey);
const awsAssumeRoleArn =
integrationAuth.awsAssumeIamRoleArnTag &&
@@ -537,13 +636,20 @@ export const secretQueueFactory = ({
})
: null;
const secrets = await getIntegrationSecrets({
environment,
projectId,
folderId: folder.id,
key: botKey,
depth: 1
});
const secrets = shouldUseSecretV2Bridge
? await getIntegrationSecretsV2({
environment,
projectId,
folderId: folder.id,
depth: 1
})
: await getIntegrationSecrets({
environment,
projectId,
folderId: folder.id,
key: botKey,
depth: 1
});
const suffixedSecrets: typeof secrets = {};
const metadata = integration.metadata as Record<string, string>;
if (metadata) {

18
backend/src/services/secret/secret-types.ts
View File

@@ -12,6 +12,10 @@ import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-fold
import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
import { ActorType } from "../auth/auth-type";
import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal";
import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-dal";
import { TSecretVersionV2TagDALFactory } from "../secret-v2-bridge/secret-version-tag-dal";
import { TKmsServiceFactory } from "../kms/kms-service";
type TPartialSecret = Pick<TSecrets, "id" | "secretReminderRepeatDays" | "secretReminderNote">;
@@ -341,6 +345,13 @@ export type TCreateManySecretsRawFnFactory = {
secretTagDAL: TSecretTagDALFactory;
secretVersionTagDAL: TSecretVersionTagDALFactory;
folderDAL: TSecretFolderDALFactory;
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
secretV2BridgeDAL: Pick<
TSecretV2BridgeDALFactory,
"insertMany" | "upsertSecretReferences" | "findBySecretKeys" | "bulkUpdate" | "deleteMany"
>;
secretVersionV2BridgeDAL: Pick<TSecretVersionV2DALFactory, "insertMany" | "findLatestVersionMany">;
secretVersionTagV2BridgeDAL: Pick<TSecretVersionV2TagDALFactory, "insertMany">;
};
export type TCreateManySecretsRawFn = {
@@ -370,6 +381,13 @@ export type TUpdateManySecretsRawFnFactory = {
secretTagDAL: TSecretTagDALFactory;
secretVersionTagDAL: TSecretVersionTagDALFactory;
folderDAL: TSecretFolderDALFactory;
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
secretV2BridgeDAL: Pick<
TSecretV2BridgeDALFactory,
"insertMany" | "upsertSecretReferences" | "findBySecretKeys" | "bulkUpdate" | "deleteMany"
>;
secretVersionV2BridgeDAL: Pick<TSecretVersionV2DALFactory, "insertMany" | "findLatestVersionMany">;
secretVersionTagV2BridgeDAL: Pick<TSecretVersionV2TagDALFactory, "insertMany">;
};
export type TUpdateManySecretsRawFn = {