feat: added bridge logic in secret replication, snapshot and approval for raw endpoint

This commit is contained in:
=
2024-07-20 21:25:16 +05:30
parent 9e314d7a09
commit ef2816b2ee
14 changed files with 2205 additions and 283 deletions

View File

@@ -356,5 +356,144 @@ export const secretApprovalRequestDALFactory = (db: TDbClient) => {
}
};
return { ...secretApprovalRequestOrm, findById, findProjectRequestCount, findByProjectId };
const findByProjectIdBridgeSecretV2 = async (
{ status, limit = 20, offset = 0, projectId, committer, environment, userId }: TFindQueryFilter,
tx?: Knex
) => {
try {
// akhilmhdh: If ever u wanted a 1 to so many relationship connected with pagination
// this is the place u wanna look at.
const query = (tx || db.replicaNode())(TableName.SecretApprovalRequest)
.join(TableName.SecretFolder, `${TableName.SecretApprovalRequest}.folderId`, `${TableName.SecretFolder}.id`)
.join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`)
.join(
TableName.SecretApprovalPolicy,
`${TableName.SecretApprovalRequest}.policyId`,
`${TableName.SecretApprovalPolicy}.id`
)
.join(
TableName.SecretApprovalPolicyApprover,
`${TableName.SecretApprovalPolicy}.id`,
`${TableName.SecretApprovalPolicyApprover}.policyId`
)
.join<TUsers>(
db(TableName.Users).as("committerUser"),
`${TableName.SecretApprovalRequest}.committerUserId`,
`committerUser.id`
)
.leftJoin(
TableName.SecretApprovalRequestReviewer,
`${TableName.SecretApprovalRequest}.id`,
`${TableName.SecretApprovalRequestReviewer}.requestId`
)
.leftJoin<TSecretApprovalRequestsSecrets>(
TableName.SecretApprovalRequestSecretV2,
`${TableName.SecretApprovalRequestSecretV2}.requestId`,
`${TableName.SecretApprovalRequest}.id`
)
.where(
stripUndefinedInWhere({
projectId,
[`${TableName.Environment}.slug` as "slug"]: environment,
[`${TableName.SecretApprovalRequest}.status`]: status,
committerUserId: committer
})
)
.andWhere(
(bd) =>
void bd
.where(`${TableName.SecretApprovalPolicyApprover}.approverUserId`, userId)
.orWhere(`${TableName.SecretApprovalRequest}.committerUserId`, userId)
)
.select(selectAllTableCols(TableName.SecretApprovalRequest))
.select(
db.ref("projectId").withSchema(TableName.Environment),
db.ref("slug").withSchema(TableName.Environment).as("environment"),
db.ref("id").withSchema(TableName.SecretApprovalRequestReviewer).as("reviewerId"),
db.ref("reviewerUserId").withSchema(TableName.SecretApprovalRequestReviewer),
db.ref("status").withSchema(TableName.SecretApprovalRequestReviewer).as("reviewerStatus"),
db.ref("id").withSchema(TableName.SecretApprovalPolicy).as("policyId"),
db.ref("name").withSchema(TableName.SecretApprovalPolicy).as("policyName"),
db.ref("op").withSchema(TableName.SecretApprovalRequestSecretV2).as("commitOp"),
db.ref("secretId").withSchema(TableName.SecretApprovalRequestSecretV2).as("commitSecretId"),
db.ref("id").withSchema(TableName.SecretApprovalRequestSecretV2).as("commitId"),
db.raw(
`DENSE_RANK() OVER (partition by ${TableName.Environment}."projectId" ORDER BY ${TableName.SecretApprovalRequest}."id" DESC) as rank`
),
db.ref("secretPath").withSchema(TableName.SecretApprovalPolicy).as("policySecretPath"),
db.ref("approvals").withSchema(TableName.SecretApprovalPolicy).as("policyApprovals"),
db.ref("approverUserId").withSchema(TableName.SecretApprovalPolicyApprover),
db.ref("email").withSchema("committerUser").as("committerUserEmail"),
db.ref("username").withSchema("committerUser").as("committerUserUsername"),
db.ref("firstName").withSchema("committerUser").as("committerUserFirstName"),
db.ref("lastName").withSchema("committerUser").as("committerUserLastName")
)
.orderBy("createdAt", "desc");
const docs = await (tx || db)
.with("w", query)
.select("*")
.from<Awaited<typeof query>[number]>("w")
.where("w.rank", ">=", offset)
.andWhere("w.rank", "<", offset + limit);
const formatedDoc = sqlNestRelationships({
data: docs,
key: "id",
parentMapper: (el) => ({
...SecretApprovalRequestsSchema.parse(el),
environment: el.environment,
projectId: el.projectId,
policy: {
id: el.policyId,
name: el.policyName,
approvals: el.policyApprovals,
secretPath: el.policySecretPath
},
committerUser: {
userId: el.committerUserId,
email: el.committerUserEmail,
firstName: el.committerUserFirstName,
lastName: el.committerUserLastName,
username: el.committerUserUsername
}
}),
childrenMapper: [
{
key: "reviewerId",
label: "reviewers" as const,
mapper: ({ reviewerUserId, reviewerStatus: s }) =>
reviewerUserId ? { userId: reviewerUserId, status: s } : undefined
},
{
key: "approverUserId",
label: "approvers" as const,
mapper: ({ approverUserId }) => approverUserId
},
{
key: "commitId",
label: "commits" as const,
mapper: ({ commitSecretId: secretId, commitId: id, commitOp: op }) => ({
op,
id,
secretId
})
}
]
});
return formatedDoc.map((el) => ({
...el,
policy: { ...el.policy, approvers: el.approvers }
}));
} catch (error) {
throw new DatabaseError({ error, name: "FindSAR" });
}
};
return {
...secretApprovalRequestOrm,
findById,
findProjectRequestCount,
findByProjectId,
findByProjectIdBridgeSecretV2
};
};

View File

@@ -3,6 +3,7 @@ import { Knex } from "knex";
import { TDbClient } from "@app/db";
import {
SecretApprovalRequestsSecretsSchema,
SecretApprovalRequestsSecretsV2Schema,
TableName,
TSecretApprovalRequestsSecrets,
TSecretTags
@@ -15,6 +16,7 @@ export type TSecretApprovalRequestSecretDALFactory = ReturnType<typeof secretApp
export const secretApprovalRequestSecretDALFactory = (db: TDbClient) => {
const secretApprovalRequestSecretOrm = ormify(db, TableName.SecretApprovalRequestSecret);
const secretApprovalRequestSecretTagOrm = ormify(db, TableName.SecretApprovalRequestSecretTag);
const secretApprovalRequestSecretV2Orm = ormify(db, TableName.SecretApprovalRequestSecretV2);
const bulkUpdateNoVersionIncrement = async (data: TSecretApprovalRequestsSecrets[], tx?: Knex) => {
try {
@@ -221,9 +223,141 @@ export const secretApprovalRequestSecretDALFactory = (db: TDbClient) => {
throw new DatabaseError({ error, name: "FindByRequestId" });
}
};
const findByRequestIdBridgeSecretV2 = async (requestId: string, tx?: Knex) => {
try {
const doc = await (tx || db.replicaNode())({
secVerTag: TableName.SecretTag
})
.from(TableName.SecretApprovalRequestSecretV2)
.where({ requestId })
.leftJoin(
TableName.SecretApprovalRequestSecretTagV2,
`${TableName.SecretApprovalRequestSecretV2}.id`,
`${TableName.SecretApprovalRequestSecretTagV2}.secretId`
)
.leftJoin(
TableName.SecretTag,
`${TableName.SecretApprovalRequestSecretTagV2}.tagId`,
`${TableName.SecretTag}.id`
)
.leftJoin(TableName.SecretV2, `${TableName.SecretApprovalRequestSecretV2}.secretId`, `${TableName.Secret}.id`)
.leftJoin(
TableName.SecretVersionV2,
`${TableName.SecretVersionV2}.id`,
`${TableName.SecretApprovalRequestSecretV2}.secretVersion`
)
.leftJoin(
TableName.SecretVersionV2Tag,
`${TableName.SecretVersionV2Tag}.${TableName.SecretVersionV2}Id`,
`${TableName.SecretVersionV2}.id`
)
.leftJoin<TSecretTags>(
db.ref(TableName.SecretTag).as("secVerTag"),
`${TableName.SecretVersionV2Tag}.${TableName.SecretTag}Id`,
db.ref("id").withSchema("secVerTag")
)
.select(selectAllTableCols(TableName.SecretApprovalRequestSecretV2))
.select({
secVerTagId: "secVerTag.id",
secVerTagColor: "secVerTag.color",
secVerTagSlug: "secVerTag.slug",
secVerTagName: "secVerTag.name"
})
.select(
db.ref("id").withSchema(TableName.SecretTag).as("tagId"),
db.ref("id").withSchema(TableName.SecretApprovalRequestSecretTagV2).as("tagJnId"),
db.ref("color").withSchema(TableName.SecretTag).as("tagColor"),
db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug"),
db.ref("name").withSchema(TableName.SecretTag).as("tagName")
)
.select(
db.ref("version").withSchema(TableName.SecretV2).as("orgSecVersion"),
db.ref("key").withSchema(TableName.SecretV2).as("orgSecKey"),
db.ref("encryptedValue").withSchema(TableName.SecretV2).as("orgSecValue"),
db.ref("encryptedComment").withSchema(TableName.SecretV2).as("orgSecComment")
)
.select(
db.ref("version").withSchema(TableName.SecretVersionV2).as("secVerVersion"),
db.ref("key").withSchema(TableName.SecretVersionV2).as("secVerKey"),
db.ref("encryptedValue").withSchema(TableName.SecretVersionV2).as("secVerValue"),
db.ref("encryptedComment").withSchema(TableName.SecretVersionV2).as("secVerComment")
);
const formatedDoc = sqlNestRelationships({
data: doc,
key: "id",
parentMapper: (data) => SecretApprovalRequestsSecretsV2Schema.omit({ secretVersion: true }).parse(data),
childrenMapper: [
{
key: "tagJnId",
label: "tags" as const,
mapper: ({ tagId: id, tagName: name, tagSlug: slug, tagColor: color }) => ({
id,
name,
slug,
color
})
},
{
key: "secretId",
label: "secret" as const,
mapper: ({ orgSecVersion, orgSecKey, orgSecValue, orgSecComment, secretId }) =>
secretId
? {
id: secretId,
version: orgSecVersion,
key: orgSecKey,
encryptedValue: orgSecValue,
encryptedComment: orgSecComment
}
: undefined
},
{
key: "secretVersion",
label: "secretVersion" as const,
mapper: ({ secretVersion, secVerVersion, secVerKey, secVerValue, secVerComment }) =>
secretVersion
? {
version: secVerVersion,
id: secretVersion,
key: secVerKey,
encryptedValue: secVerValue,
encryptedComment: secVerComment
}
: undefined,
childrenMapper: [
{
key: "secVerTagId",
label: "tags" as const,
mapper: ({ secVerTagId: id, secVerTagName: name, secVerTagSlug: slug, secVerTagColor: color }) => ({
// eslint-disable-next-line
id,
// eslint-disable-next-line
name,
// eslint-disable-next-line
slug,
// eslint-disable-next-line
color
})
}
]
}
]
});
return formatedDoc?.map(({ secret, secretVersion, ...el }) => ({
...el,
secret: secret?.[0],
secretVersion: secretVersion?.[0]
}));
} catch (error) {
throw new DatabaseError({ error, name: "FindByRequestId" });
}
};
return {
...secretApprovalRequestSecretOrm,
insertV2Bridge: secretApprovalRequestSecretV2Orm.insertMany,
findByRequestId,
findByRequestIdBridgeSecretV2,
bulkUpdateNoVersionIncrement,
insertApprovalSecretTags: secretApprovalRequestSecretTagOrm.insertMany
};

View File

@@ -5,7 +5,8 @@ import {
SecretEncryptionAlgo,
SecretKeyEncoding,
SecretType,
TSecretApprovalRequestsSecretsInsert
TSecretApprovalRequestsSecretsInsert,
TSecretApprovalRequestsSecretsV2Insert
} from "@app/db/schemas";
import { getConfig } from "@app/lib/config/env";
import { decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
@@ -19,6 +20,7 @@ import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot
import { TProjectEnvDALFactory } from "@app/services/project-env/project-env-dal";
import { TSecretDALFactory } from "@app/services/secret/secret-dal";
import {
decryptSecretWithBot,
fnSecretBlindIndexCheck,
fnSecretBlindIndexCheckV2,
fnSecretBulkDelete,
@@ -26,6 +28,13 @@ import {
fnSecretBulkUpdate,
getAllNestedSecretReferences
} from "@app/services/secret/secret-fns";
import {
fnSecretBulkInsert as fnSecretV2BridgeBulkInsert,
fnSecretBulkUpdate as fnSecretV2BridgeBulkUpdate,
fnSecretBulkDelete as fnSecretV2BridgeBulkDelete,
getAllNestedSecretReferences as getAllNestedSecretReferencesV2Bridge,
secretEncryptionHelper
} from "@app/services/secret-v2-bridge/secret-v2-bridge-fns";
import { TSecretQueueFactory } from "@app/services/secret/secret-queue";
import { SecretOperations } from "@app/services/secret/secret-types";
import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
@@ -47,12 +56,17 @@ import {
RequestState,
TApprovalRequestCountDTO,
TGenerateSecretApprovalRequestDTO,
TGenerateSecretApprovalRequestV2BridgeDTO,
TListApprovalsDTO,
TMergeSecretApprovalRequestDTO,
TReviewRequestDTO,
TSecretApprovalDetailsDTO,
TStatusChangeDTO
} from "./secret-approval-request-types";
import { TKmsServiceFactory } from "@app/services/kms/kms-service";
import { TSecretV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-dal";
import { TSecretVersionV2DALFactory } from "@app/services/secret-v2-bridge/secret-version-dal";
import { TSecretVersionV2TagDALFactory } from "@app/services/secret-v2-bridge/secret-version-tag-dal";
type TSecretApprovalRequestServiceFactoryDep = {
permissionService: Pick<TPermissionServiceFactory, "getProjectPermission">;
@@ -62,16 +76,29 @@ type TSecretApprovalRequestServiceFactoryDep = {
secretApprovalRequestReviewerDAL: TSecretApprovalRequestReviewerDALFactory;
folderDAL: Pick<TSecretFolderDALFactory, "findBySecretPath" | "findSecretPathByFolderIds">;
secretDAL: TSecretDALFactory;
secretTagDAL: Pick<TSecretTagDALFactory, "findManyTagsById" | "saveTagsToSecret" | "deleteTagsManySecret">;
secretTagDAL: Pick<
TSecretTagDALFactory,
"findManyTagsById" | "saveTagsToSecret" | "deleteTagsManySecret" | "saveTagsToSecretV2" | "deleteTagsToSecretV2"
>;
secretBlindIndexDAL: Pick<TSecretBlindIndexDALFactory, "findOne">;
snapshotService: Pick<TSecretSnapshotServiceFactory, "performSnapshot">;
secretVersionDAL: Pick<TSecretVersionDALFactory, "findLatestVersionMany" | "insertMany">;
secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
projectDAL: Pick<TProjectDALFactory, "checkProjectUpgradeStatus" | "findProjectById">;
secretQueueService: Pick<TSecretQueueFactory, "syncSecrets" | "removeSecretReminder">;
smtpService: Pick<TSmtpService, "sendMail">;
userDAL: Pick<TUserDALFactory, "find" | "findOne">;
projectEnvDAL: Pick<TProjectEnvDALFactory, "findOne">;
projectDAL: Pick<TProjectDALFactory, "checkProjectUpgradeStatus" | "findById" | "findProjectById">;
secretQueueService: Pick<TSecretQueueFactory, "syncSecrets" | "removeSecretReminder">;
kmsService: Pick<
TKmsServiceFactory,
"getProjectSecretManagerKmsDataKey" | "encryptWithInputKey" | "decryptWithInputKey"
>;
secretV2BridgeDAL: Pick<
TSecretV2BridgeDALFactory,
"insertMany" | "upsertSecretReferences" | "findBySecretKeys" | "bulkUpdate" | "deleteMany"
>;
secretVersionV2BridgeDAL: Pick<TSecretVersionV2DALFactory, "insertMany" | "findLatestVersionMany">;
secretVersionTagV2BridgeDAL: Pick<TSecretVersionV2TagDALFactory, "insertMany">;
};
export type TSecretApprovalRequestServiceFactory = ReturnType<typeof secretApprovalRequestServiceFactory>;
@@ -93,7 +120,11 @@ export const secretApprovalRequestServiceFactory = ({
projectBotService,
smtpService,
userDAL,
projectEnvDAL
projectEnvDAL,
kmsService,
secretV2BridgeDAL,
secretVersionV2BridgeDAL,
secretVersionTagV2BridgeDAL
}: TSecretApprovalRequestServiceFactoryDep) => {
const requestCount = async ({ projectId, actor, actorId, actorOrgId, actorAuthMethod }: TApprovalRequestCountDTO) => {
if (actor === ActorType.SERVICE) throw new BadRequestError({ message: "Cannot use service token" });
@@ -125,6 +156,19 @@ export const secretApprovalRequestServiceFactory = ({
if (actor === ActorType.SERVICE) throw new BadRequestError({ message: "Cannot use service token" });
await permissionService.getProjectPermission(actor, actorId, projectId, actorAuthMethod, actorOrgId);
const { shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId);
if (shouldUseSecretV2Bridge) {
return secretApprovalRequestDAL.findByProjectIdBridgeSecretV2({
projectId,
committer,
environment,
status,
userId: actorId,
limit,
offset
});
}
const approvals = await secretApprovalRequestDAL.findByProjectId({
projectId,
committer,
@@ -149,11 +193,14 @@ export const secretApprovalRequestServiceFactory = ({
const secretApprovalRequest = await secretApprovalRequestDAL.findById(id);
if (!secretApprovalRequest) throw new BadRequestError({ message: "Secret approval request not found" });
const { projectId } = secretApprovalRequest;
const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId);
const { policy } = secretApprovalRequest;
const { hasRole } = await permissionService.getProjectPermission(
actor,
actorId,
secretApprovalRequest.projectId,
projectId,
actorAuthMethod,
actorOrgId
);
@@ -165,7 +212,55 @@ export const secretApprovalRequestServiceFactory = ({
throw new UnauthorizedError({ message: "User has no access" });
}
const secrets = await secretApprovalRequestSecretDAL.findByRequestId(secretApprovalRequest.id);
let secrets;
if (shouldUseSecretV2Bridge) {
const secretManagerDataKey = await kmsService.getProjectSecretManagerKmsDataKey(projectId);
const secretManagerDecryptor = await kmsService.decryptWithInputKey({ key: secretManagerDataKey });
const encrypedSecrets = await secretApprovalRequestSecretDAL.findByRequestIdBridgeSecretV2(
secretApprovalRequest.id
);
secrets = encrypedSecrets.map((el) => ({
...el,
secret: {
secretKey: el.secret.key,
id: el.secret.id,
version: el.secret.version,
secretValue: el.secret.encryptedValue
? secretManagerDecryptor({ cipherTextBlob: el.secret.encryptedValue }).toString()
: undefined,
secretComment: el.secret.encryptedComment
? secretManagerDecryptor({ cipherTextBlob: el.secret.encryptedComment }).toString()
: undefined
},
secretVersion: {
secretKey: el.secretVersion.key,
id: el.secretVersion.id,
version: el.secretVersion.version,
secretValue: el.secretVersion.encryptedValue
? secretManagerDecryptor({ cipherTextBlob: el.secretVersion.encryptedValue }).toString()
: undefined,
secretComment: el.secretVersion.encryptedComment
? secretManagerDecryptor({ cipherTextBlob: el.secretVersion.encryptedComment }).toString()
: undefined
}
}));
} else {
if (!botKey) throw new BadRequestError({ message: "Bot key not found" });
const encrypedSecrets = await secretApprovalRequestSecretDAL.findByRequestId(secretApprovalRequest.id);
secrets = encrypedSecrets.map((el) => ({
...el,
secret: {
id: el.secret.id,
version: el.secret.version,
...decryptSecretWithBot(el.secret, botKey)
},
secretVersion: {
id: el.secretVersion.id,
version: el.secretVersion.version,
...decryptSecretWithBot(el.secretVersion, botKey)
}
}));
}
const secretPath = await folderDAL.findSecretPathByFolderIds(secretApprovalRequest.projectId, [
secretApprovalRequest.folderId
]);
@@ -300,48 +395,165 @@ export const secretApprovalRequestServiceFactory = ({
secretApprovalRequest.policy.approvers.filter(
({ userId: approverId }) => reviewers[approverId.toString()] === ApprovalStatus.APPROVED
).length;
const isSoftEnforcement = secretApprovalRequest.policy.enforcementLevel === EnforcementLevel.Soft;
const isSoftEnforcement = secretApprovalRequest.policy.enforcementLevel === EnforcementLevel.Soft;
if (!hasMinApproval && !isSoftEnforcement)
throw new BadRequestError({ message: "Doesn't have minimum approvals needed" });
if (!hasMinApproval && !isSoftEnforcement)
throw new BadRequestError({ message: "Doesn't have minimum approvals needed" });
const secretApprovalSecrets = await secretApprovalRequestSecretDAL.findByRequestId(secretApprovalRequest.id);
if (!secretApprovalSecrets) throw new BadRequestError({ message: "No secrets found" });
const conflicts: Array<{ secretId: string; op: SecretOperations }> = [];
let secretCreationCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Create);
if (secretCreationCommits.length) {
const { secsGroupedByBlindIndex: conflictGroupByBlindIndex } = await fnSecretBlindIndexCheckV2({
folderId,
secretDAL,
inputSecrets: secretCreationCommits.map(({ secretBlindIndex }) => {
if (!secretBlindIndex) {
throw new BadRequestError({
message: "Missing secret blind index"
});
}
return { secretBlindIndex };
})
});
secretCreationCommits
.filter(({ secretBlindIndex }) => conflictGroupByBlindIndex[secretBlindIndex || ""])
.forEach((el) => {
conflicts.push({ op: SecretOperations.Create, secretId: el.id });
});
secretCreationCommits = secretCreationCommits.filter(
({ secretBlindIndex }) => !conflictGroupByBlindIndex[secretBlindIndex || ""]
const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId);
let mergeStatus;
if (shouldUseSecretV2Bridge) {
// this cycle if for bridged secrets
const secretApprovalSecrets = await secretApprovalRequestSecretDAL.findByRequestIdBridgeSecretV2(
secretApprovalRequest.id
);
}
if (!secretApprovalSecrets) throw new BadRequestError({ message: "No secrets found" });
let secretUpdationCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Update);
if (secretUpdationCommits.length) {
const { secsGroupedByBlindIndex: conflictGroupByBlindIndex } = await fnSecretBlindIndexCheckV2({
folderId,
secretDAL,
userId: "",
inputSecrets: secretUpdationCommits
.filter(({ secretBlindIndex, secret }) => secret && secret.secretBlindIndex !== secretBlindIndex)
.map(({ secretBlindIndex }) => {
const secretManagerDataKey = await kmsService.getProjectSecretManagerKmsDataKey(projectId);
const secretManagerDecryptor = await kmsService.decryptWithInputKey({ key: secretManagerDataKey });
const conflicts: Array<{ secretId: string; op: SecretOperations }> = [];
let secretCreationCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Create);
if (secretCreationCommits.length) {
const secrets = await secretV2BridgeDAL.findBySecretKeys(
folderId,
secretCreationCommits.map((el) => ({
key: el.key,
type: SecretType.Shared
}))
);
const creationConflictSecretsGroupByKey = groupBy(secrets, (i) => i.key);
secretCreationCommits
.filter(({ key }) => creationConflictSecretsGroupByKey[key])
.forEach((el) => {
conflicts.push({ op: SecretOperations.Create, secretId: el.id });
});
secretCreationCommits = secretCreationCommits.filter(({ key }) => !creationConflictSecretsGroupByKey[key]);
}
let secretUpdationCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Update);
if (secretUpdationCommits.length) {
const secrets = await secretV2BridgeDAL.findBySecretKeys(
folderId,
secretCreationCommits.map((el) => ({
key: el.key,
type: SecretType.Shared
}))
);
const updationConflictSecretsGroupByKey = groupBy(secrets, (i) => i.key);
secretUpdationCommits
.filter(({ key, secretId }) => updationConflictSecretsGroupByKey[key] || !secretId)
.forEach((el) => {
conflicts.push({ op: SecretOperations.Update, secretId: el.id });
});
secretUpdationCommits = secretUpdationCommits.filter(
({ key, secretId }) => Boolean(secretId) && !updationConflictSecretsGroupByKey[key]
);
}
const secretDeletionCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Delete);
mergeStatus = await secretApprovalRequestDAL.transaction(async (tx) => {
const newSecrets = secretCreationCommits.length
? await fnSecretV2BridgeBulkInsert({
tx,
folderId,
inputSecrets: secretCreationCommits.map((el) => ({
tagIds: el?.tags.map(({ id }) => id),
version: 1,
encryptedComment: el.encryptedComment,
encryptedValue: el.encryptedValue,
skipMultilineEncoding: el.skipMultilineEncoding,
key: el.key,
references: el.encryptedValue
? getAllNestedSecretReferencesV2Bridge(
secretManagerDecryptor({
cipherTextBlob: el.encryptedValue
}).toString()
)
: [],
type: SecretType.Shared
})),
secretDAL: secretV2BridgeDAL,
secretVersionDAL: secretVersionV2BridgeDAL,
secretTagDAL,
secretVersionTagDAL: secretVersionTagV2BridgeDAL
})
: [];
const updatedSecrets = secretUpdationCommits.length
? await fnSecretV2BridgeBulkUpdate({
folderId,
tx,
inputSecrets: secretUpdationCommits.map((el) => {
const encryptedValue =
typeof el.encryptedValue !== "undefined"
? {
encryptedValue: el.encryptedValue as Buffer,
references: el.encryptedValue
? getAllNestedSecretReferencesV2Bridge(
secretManagerDecryptor({
cipherTextBlob: el.encryptedValue
}).toString()
)
: []
}
: {};
return {
filter: { id: el.secretId as string, type: SecretType.Shared },
data: {
reminderRepeatDays: el.reminderRepeatDays,
encryptedComment: el.encryptedComment,
reminderNote: el.reminderNote,
skipMultilineEncoding: el.skipMultilineEncoding,
key: el.key,
tagIds: el?.tags.map(({ id }) => id),
...encryptedValue
}
};
}),
secretDAL: secretV2BridgeDAL,
secretVersionDAL: secretVersionV2BridgeDAL,
secretTagDAL,
secretVersionTagDAL: secretVersionTagV2BridgeDAL
})
: [];
const deletedSecret = secretDeletionCommits.length
? await fnSecretV2BridgeBulkDelete({
projectId,
folderId,
tx,
actorId: "",
secretDAL: secretV2BridgeDAL,
secretQueueService,
inputSecrets: secretDeletionCommits.map(({ key }) => ({ secretKey: key, type: SecretType.Shared }))
})
: [];
const updatedSecretApproval = await secretApprovalRequestDAL.updateById(
secretApprovalRequest.id,
{
conflicts: JSON.stringify(conflicts),
hasMerged: true,
status: RequestState.Closed,
statusChangedByUserId: actorId
},
tx
);
return {
secrets: { created: newSecrets, updated: updatedSecrets, deleted: deletedSecret },
approval: updatedSecretApproval
};
});
} else {
const secretApprovalSecrets = await secretApprovalRequestSecretDAL.findByRequestId(secretApprovalRequest.id);
if (!secretApprovalSecrets) throw new BadRequestError({ message: "No secrets found" });
const conflicts: Array<{ secretId: string; op: SecretOperations }> = [];
let secretCreationCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Create);
if (secretCreationCommits.length) {
const { secsGroupedByBlindIndex: conflictGroupByBlindIndex } = await fnSecretBlindIndexCheckV2({
folderId,
secretDAL,
inputSecrets: secretCreationCommits.map(({ secretBlindIndex }) => {
if (!secretBlindIndex) {
throw new BadRequestError({
message: "Missing secret blind index"
@@ -349,80 +561,56 @@ export const secretApprovalRequestServiceFactory = ({
}
return { secretBlindIndex };
})
});
secretUpdationCommits
.filter(
({ secretBlindIndex, secretId }) =>
(secretBlindIndex && conflictGroupByBlindIndex[secretBlindIndex]) || !secretId
)
.forEach((el) => {
conflicts.push({ op: SecretOperations.Update, secretId: el.id });
});
secretCreationCommits
.filter(({ secretBlindIndex }) => conflictGroupByBlindIndex[secretBlindIndex || ""])
.forEach((el) => {
conflicts.push({ op: SecretOperations.Create, secretId: el.id });
});
secretCreationCommits = secretCreationCommits.filter(
({ secretBlindIndex }) => !conflictGroupByBlindIndex[secretBlindIndex || ""]
);
}
secretUpdationCommits = secretUpdationCommits.filter(
({ secretBlindIndex, secretId }) =>
Boolean(secretId) && (secretBlindIndex ? !conflictGroupByBlindIndex[secretBlindIndex] : true)
);
}
let secretUpdationCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Update);
if (secretUpdationCommits.length) {
const { secsGroupedByBlindIndex: conflictGroupByBlindIndex } = await fnSecretBlindIndexCheckV2({
folderId,
secretDAL,
userId: "",
inputSecrets: secretUpdationCommits
.filter(({ secretBlindIndex, secret }) => secret && secret.secretBlindIndex !== secretBlindIndex)
.map(({ secretBlindIndex }) => {
if (!secretBlindIndex) {
throw new BadRequestError({
message: "Missing secret blind index"
});
}
return { secretBlindIndex };
})
});
secretUpdationCommits
.filter(
({ secretBlindIndex, secretId }) =>
(secretBlindIndex && conflictGroupByBlindIndex[secretBlindIndex]) || !secretId
)
.forEach((el) => {
conflicts.push({ op: SecretOperations.Update, secretId: el.id });
});
const secretDeletionCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Delete);
const botKey = await projectBotService.getBotKey(projectId).catch(() => null);
const mergeStatus = await secretApprovalRequestDAL.transaction(async (tx) => {
const newSecrets = secretCreationCommits.length
? await fnSecretBulkInsert({
tx,
folderId,
inputSecrets: secretCreationCommits.map((el) => ({
...pick(el, [
"secretCommentCiphertext",
"secretCommentTag",
"secretCommentIV",
"secretValueIV",
"secretValueTag",
"secretValueCiphertext",
"secretKeyCiphertext",
"secretKeyTag",
"secretKeyIV",
"metadata",
"skipMultilineEncoding",
"secretReminderNote",
"secretReminderRepeatDays",
"algorithm",
"keyEncoding",
"secretBlindIndex"
]),
tags: el?.tags.map(({ id }) => id),
version: 1,
type: SecretType.Shared,
references: botKey
? getAllNestedSecretReferences(
decryptSymmetric128BitHexKeyUTF8({
ciphertext: el.secretValueCiphertext,
iv: el.secretValueIV,
tag: el.secretValueTag,
key: botKey
})
)
: undefined
})),
secretDAL,
secretVersionDAL,
secretTagDAL,
secretVersionTagDAL
})
: [];
const updatedSecrets = secretUpdationCommits.length
? await fnSecretBulkUpdate({
folderId,
projectId,
tx,
inputSecrets: secretUpdationCommits.map((el) => ({
filter: {
id: el.secretId as string, // this null check is already checked at top on conflict strategy
type: SecretType.Shared
},
data: {
tags: el?.tags.map(({ id }) => id),
secretUpdationCommits = secretUpdationCommits.filter(
({ secretBlindIndex, secretId }) =>
Boolean(secretId) && (secretBlindIndex ? !conflictGroupByBlindIndex[secretBlindIndex] : true)
);
}
const secretDeletionCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Delete);
mergeStatus = await secretApprovalRequestDAL.transaction(async (tx) => {
const newSecrets = secretCreationCommits.length
? await fnSecretBulkInsert({
tx,
folderId,
inputSecrets: secretCreationCommits.map((el) => ({
...pick(el, [
"secretCommentCiphertext",
"secretCommentTag",
@@ -437,8 +625,13 @@ export const secretApprovalRequestServiceFactory = ({
"skipMultilineEncoding",
"secretReminderNote",
"secretReminderRepeatDays",
"algorithm",
"keyEncoding",
"secretBlindIndex"
]),
tags: el?.tags.map(({ id }) => id),
version: 1,
type: SecretType.Shared,
references: botKey
? getAllNestedSecretReferences(
decryptSymmetric128BitHexKeyUTF8({
@@ -449,48 +642,94 @@ export const secretApprovalRequestServiceFactory = ({
})
)
: undefined
}
})),
secretDAL,
secretVersionDAL,
secretTagDAL,
secretVersionTagDAL
})
: [];
const deletedSecret = secretDeletionCommits.length
? await fnSecretBulkDelete({
projectId,
folderId,
tx,
actorId: "",
secretDAL,
secretQueueService,
inputSecrets: secretDeletionCommits.map(({ secretBlindIndex }) => {
if (!secretBlindIndex) {
throw new BadRequestError({
message: "Missing secret blind index"
});
}
return { secretBlindIndex, type: SecretType.Shared };
})),
secretDAL,
secretVersionDAL,
secretTagDAL,
secretVersionTagDAL
})
})
: [];
const updatedSecretApproval = await secretApprovalRequestDAL.updateById(
secretApprovalRequest.id,
{
conflicts: JSON.stringify(conflicts),
hasMerged: true,
status: RequestState.Closed,
statusChangedByUserId: actorId,
bypassReason
},
tx
);
return {
secrets: { created: newSecrets, updated: updatedSecrets, deleted: deletedSecret },
approval: updatedSecretApproval
};
});
: [];
const updatedSecrets = secretUpdationCommits.length
? await fnSecretBulkUpdate({
folderId,
projectId,
tx,
inputSecrets: secretUpdationCommits.map((el) => ({
filter: {
id: el.secretId as string, // this null check is already checked at top on conflict strategy
type: SecretType.Shared
},
data: {
tags: el?.tags.map(({ id }) => id),
...pick(el, [
"secretCommentCiphertext",
"secretCommentTag",
"secretCommentIV",
"secretValueIV",
"secretValueTag",
"secretValueCiphertext",
"secretKeyCiphertext",
"secretKeyTag",
"secretKeyIV",
"metadata",
"skipMultilineEncoding",
"secretReminderNote",
"secretReminderRepeatDays",
"secretBlindIndex"
]),
references: botKey
? getAllNestedSecretReferences(
decryptSymmetric128BitHexKeyUTF8({
ciphertext: el.secretValueCiphertext,
iv: el.secretValueIV,
tag: el.secretValueTag,
key: botKey
})
)
: undefined
}
})),
secretDAL,
secretVersionDAL,
secretTagDAL,
secretVersionTagDAL
})
: [];
const deletedSecret = secretDeletionCommits.length
? await fnSecretBulkDelete({
projectId,
folderId,
tx,
actorId: "",
secretDAL,
secretQueueService,
inputSecrets: secretDeletionCommits.map(({ secretBlindIndex }) => {
if (!secretBlindIndex) {
throw new BadRequestError({
message: "Missing secret blind index"
});
}
return { secretBlindIndex, type: SecretType.Shared };
})
})
: [];
const updatedSecretApproval = await secretApprovalRequestDAL.updateById(
secretApprovalRequest.id,
{
conflicts: JSON.stringify(conflicts),
hasMerged: true,
status: RequestState.Closed,
statusChangedByUserId: actorId
},
tx
);
return {
secrets: { created: newSecrets, updated: updatedSecrets, deleted: deletedSecret },
approval: updatedSecretApproval
};
});
}
await snapshotService.performSnapshot(folderId);
const [folder] = await folderDAL.findSecretPathByFolderIds(projectId, [folderId]);
if (!folder) throw new BadRequestError({ message: "Folder not found" });
@@ -779,8 +1018,248 @@ export const secretApprovalRequestServiceFactory = ({
});
return secretApprovalRequest;
};
const generateSecretApprovalRequestV2Bridge = async ({
data,
actorId,
actor,
actorOrgId,
actorAuthMethod,
policy,
projectId,
secretPath,
environment
}: TGenerateSecretApprovalRequestV2BridgeDTO) => {
if (actor === ActorType.SERVICE || actor === ActorType.Machine)
throw new BadRequestError({ message: "Cannot use service token or machine token over protected branches" });
const { permission } = await permissionService.getProjectPermission(
actor,
actorId,
projectId,
actorAuthMethod,
actorOrgId
);
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionActions.Read,
subject(ProjectPermissionSub.Secrets, { environment, secretPath })
);
const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
if (!folder)
throw new BadRequestError({
message: "Folder not found for the given environment slug & secret path",
name: "GenSecretApproval"
});
const folderId = folder.id;
const commits: Omit<TSecretApprovalRequestsSecretsV2Insert, "requestId">[] = [];
const commitTagIds: Record<string, string[]> = {};
const secretManagerDataKey = await kmsService.getProjectSecretManagerKmsDataKey(projectId);
const secretManagerEncryptor = await kmsService.encryptWithInputKey({ key: secretManagerDataKey });
// for created secret approval change
const createdSecrets = data[SecretOperations.Create];
if (createdSecrets && createdSecrets?.length) {
const secrets = await secretV2BridgeDAL.findBySecretKeys(
folderId,
createdSecrets.map((el) => ({
key: el.secretKey,
type: SecretType.Shared
}))
);
if (secrets.length)
throw new BadRequestError({ message: `Secret already exist: ${secrets.map((el) => el.key).join(",")}` });
commits.push(
...createdSecrets.map((createdSecret) => ({
op: SecretOperations.Create,
version: 1,
encryptedComment: secretEncryptionHelper.encryptValue(secretManagerEncryptor, createdSecret.secretComment),
encryptedValue: secretEncryptionHelper.encryptValue(secretManagerEncryptor, createdSecret.secretValue),
skipMultilineEncoding: createdSecret.skipMultilineEncoding,
key: createdSecret.secretKey,
type: SecretType.Shared
}))
);
createdSecrets.forEach(({ tagIds, secretKey }) => {
if (tagIds?.length) commitTagIds[secretKey] = tagIds;
});
}
// not secret approval for update operations
const secretsToUpdate = data[SecretOperations.Update];
if (secretsToUpdate && secretsToUpdate?.length) {
const secretsToUpdateStoredInDB = await secretV2BridgeDAL.findBySecretKeys(
folderId,
secretsToUpdate.map((el) => ({
key: el.secretKey,
type: SecretType.Shared
}))
);
if (secretsToUpdateStoredInDB.length !== secretsToUpdate.length)
throw new BadRequestError({
message: `Secret not exist: ${secretsToUpdateStoredInDB.map((el) => el.key).join(",")}`
});
// now find any secret that needs to update its name
// same process as above
const secretsWithNewName = secretsToUpdate.filter(({ newSecretName }) => Boolean(newSecretName));
if (secretsWithNewName.length) {
const secrets = await secretV2BridgeDAL.findBySecretKeys(
folderId,
secretsWithNewName.map((el) => ({
key: el.secretKey,
type: SecretType.Shared
}))
);
if (secrets.length)
throw new BadRequestError({
message: `Secret not exist: ${secretsToUpdateStoredInDB.map((el) => el.key).join(",")}`
});
}
const updatingSecretsGroupByKey = groupBy(secretsToUpdateStoredInDB, (el) => el.key);
const latestSecretVersions = await secretVersionV2BridgeDAL.findLatestVersionMany(
folderId,
secretsToUpdateStoredInDB.map(({ id }) => id)
);
commits.push(
...secretsToUpdate.map(
({
newSecretName,
secretKey,
tagIds,
secretValue,
reminderRepeatDays,
reminderNote,
secretComment,
metadata,
skipMultilineEncoding
}) => {
const secretId = updatingSecretsGroupByKey[secretKey][0].id;
if (tagIds?.length) commitTagIds[secretKey] = tagIds;
return {
...latestSecretVersions[secretId],
key: newSecretName || secretKey,
encryptedValue: secretEncryptionHelper.encryptValue(secretManagerEncryptor, secretValue) as Buffer,
encryptedComment: secretEncryptionHelper.encryptValue(secretManagerEncryptor, secretComment) as Buffer,
reminderRepeatDays,
reminderNote,
metadata,
skipMultilineEncoding,
op: SecretOperations.Update as const,
secret: secretId,
secretVersion: latestSecretVersions[secretId].id,
version: updatingSecretsGroupByKey[secretKey][0].version || 1
};
}
)
);
}
// deleted secrets
const deletedSecrets = data[SecretOperations.Delete];
if (deletedSecrets && deletedSecrets.length) {
const secretsToDeleteInDB = await secretV2BridgeDAL.findBySecretKeys(
folderId,
deletedSecrets.map((el) => ({
key: el.secretKey,
type: SecretType.Shared
}))
);
if (secretsToDeleteInDB.length !== deletedSecrets.length)
throw new BadRequestError({
message: `Secret not exist: ${secretsToDeleteInDB.map((el) => el.key).join(",")}`
});
const secretsGroupedByKey = groupBy(secretsToDeleteInDB, (i) => i.key);
const deletedSecretIds = deletedSecrets.map((el) => secretsGroupedByKey[el.secretKey][0].id);
const latestSecretVersions = await secretVersionV2BridgeDAL.findLatestVersionMany(folderId, deletedSecretIds);
commits.push(
...deletedSecrets.map(({ secretKey }) => {
const secretId = secretsGroupedByKey[secretKey][0].id;
return {
op: SecretOperations.Delete as const,
...latestSecretVersions[secretId],
key: secretKey,
secret: secretId,
secretVersion: latestSecretVersions[secretId].id
};
})
);
}
if (!commits.length) throw new BadRequestError({ message: "Empty commits" });
const tagIds = unique(Object.values(commitTagIds).flat());
const tags = tagIds.length ? await secretTagDAL.findManyTagsById(projectId, tagIds) : [];
if (tagIds.length !== tags.length) throw new BadRequestError({ message: "Tag not found" });
const secretApprovalRequest = await secretApprovalRequestDAL.transaction(async (tx) => {
const doc = await secretApprovalRequestDAL.create(
{
folderId,
slug: alphaNumericNanoId(),
policyId: policy.id,
status: "open",
hasMerged: false,
committerUserId: actorId
},
tx
);
const approvalCommits = await secretApprovalRequestSecretDAL.insertV2Bridge(
commits.map(
({
version,
op,
key,
encryptedComment,
skipMultilineEncoding,
metadata,
reminderNote,
reminderRepeatDays,
encryptedValue,
secretId,
secretVersion
}) => ({
version,
requestId: doc.id,
op,
secretId,
metadata,
secretVersion,
skipMultilineEncoding,
encryptedValue,
reminderRepeatDays,
reminderNote,
encryptedComment,
key
})
),
tx
);
const commitsGroupByKey = groupBy(approvalCommits, (i) => i.key);
if (tagIds.length) {
await secretApprovalRequestSecretDAL.insertApprovalSecretTags(
Object.keys(commitTagIds).flatMap((blindIndex) =>
commitTagIds[blindIndex]
? commitTagIds[blindIndex].map((tagId) => ({
secretId: commitsGroupByKey[blindIndex][0].id,
tagId
}))
: []
),
tx
);
}
return { ...doc, commits: approvalCommits };
});
return secretApprovalRequest;
};
return {
generateSecretApprovalRequest,
generateSecretApprovalRequestV2Bridge,
mergeSecretApprovalRequest,
reviewApproval,
updateApprovalStatus,

View File

@@ -26,6 +26,23 @@ export type TApprovalUpdateSecret = Partial<TApprovalCreateSecret> & {
tagIds?: string[];
};
export type TApprovalCreateSecretV2Bridge = {
secretKey: string;
secretValue?: string;
secretComment?: string;
reminderNote?: string | null;
reminderRepeatDays?: number | null;
skipMultilineEncoding?: boolean;
metadata?: Record<string, string>;
tagIds?: string[];
};
export type TApprovalUpdateSecretV2Bridge = Partial<TApprovalCreateSecretV2Bridge> & {
secretKey: string;
newSecretName?: string;
tagIds?: string[];
};
export type TGenerateSecretApprovalRequestDTO = {
environment: string;
secretPath: string;
@@ -37,6 +54,17 @@ export type TGenerateSecretApprovalRequestDTO = {
};
} & TProjectPermission;
export type TGenerateSecretApprovalRequestV2BridgeDTO = {
environment: string;
secretPath: string;
policy: TSecretApprovalPolicies;
data: {
[SecretOperations.Create]?: TApprovalCreateSecretV2Bridge[];
[SecretOperations.Update]?: TApprovalUpdateSecretV2Bridge[];
[SecretOperations.Delete]?: { secretKey: string }[];
};
} & TProjectPermission;
export type TMergeSecretApprovalRequestDTO = {
approvalId: string;
bypassReason?: string;

View File

@@ -1,4 +1,4 @@
import { SecretType, TSecrets } from "@app/db/schemas";
import { SecretType, TSecrets, TSecretsV2 } from "@app/db/schemas";
import { TSecretApprovalPolicyServiceFactory } from "@app/ee/services/secret-approval-policy/secret-approval-policy-service";
import { TSecretApprovalRequestDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-dal";
import { TSecretApprovalRequestSecretDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-secret-dal";
@@ -13,18 +13,28 @@ import { ActorType } from "@app/services/auth/auth-type";
import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service";
import { TSecretDALFactory } from "@app/services/secret/secret-dal";
import { fnSecretBulkInsert, fnSecretBulkUpdate } from "@app/services/secret/secret-fns";
import {
fnSecretBulkInsert as fnSecretV2BridgeBulkInsert,
fnSecretBulkUpdate as fnSecretV2BridgeBulkUpdate,
getAllNestedSecretReferences,
getAllNestedSecretReferences as getAllNestedSecretReferencesV2Bridge
} from "@app/services/secret-v2-bridge/secret-v2-bridge-fns";
import { TSecretQueueFactory, uniqueSecretQueueKey } from "@app/services/secret/secret-queue";
import { SecretOperations } from "@app/services/secret/secret-types";
import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal";
import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal";
import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";
import { ReservedFolders } from "@app/services/secret-folder/secret-folder-types";
import { TSecretImportDALFactory } from "@app/services/secret-import/secret-import-dal";
import { fnSecretsFromImports } from "@app/services/secret-import/secret-import-fns";
import { fnSecretsFromImports, fnSecretsV2FromImports } from "@app/services/secret-import/secret-import-fns";
import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
import { MAX_REPLICATION_DEPTH } from "./secret-replication-constants";
import { TSecretV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-dal";
import { TSecretVersionV2DALFactory } from "@app/services/secret-v2-bridge/secret-version-dal";
import { TSecretVersionV2TagDALFactory } from "@app/services/secret-v2-bridge/secret-version-tag-dal";
import { TKmsServiceFactory } from "@app/services/kms/kms-service";
import { KmsDataKey } from "@app/services/kms/kms-types";
type TSecretReplicationServiceFactoryDep = {
secretDAL: Pick<
@@ -32,24 +42,42 @@ type TSecretReplicationServiceFactoryDep = {
"find" | "findByBlindIndexes" | "insertMany" | "bulkUpdate" | "delete" | "upsertSecretReferences" | "transaction"
>;
secretVersionDAL: Pick<TSecretVersionDALFactory, "find" | "insertMany" | "update" | "findLatestVersionMany">;
secretV2BridgeDAL: Pick<
TSecretV2BridgeDALFactory,
"find" | "findBySecretKeys" | "insertMany" | "bulkUpdate" | "delete" | "upsertSecretReferences" | "transaction"
>;
secretVersionV2BridgeDAL: Pick<
TSecretVersionV2DALFactory,
"find" | "insertMany" | "update" | "findLatestVersionMany"
>;
secretImportDAL: Pick<TSecretImportDALFactory, "find" | "updateById" | "findByFolderIds">;
folderDAL: Pick<
TSecretFolderDALFactory,
"findSecretPathByFolderIds" | "findBySecretPath" | "create" | "findOne" | "findByManySecretPath"
>;
secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "find" | "insertMany">;
secretVersionV2TagBridgeDAL: Pick<TSecretVersionV2TagDALFactory, "find" | "insertMany">;
secretQueueService: Pick<TSecretQueueFactory, "syncSecrets" | "replicateSecrets">;
queueService: Pick<TQueueServiceFactory, "start" | "listen" | "queue" | "stopJobById">;
secretApprovalPolicyService: Pick<TSecretApprovalPolicyServiceFactory, "getSecretApprovalPolicy">;
keyStore: Pick<TKeyStoreFactory, "acquireLock" | "setItemWithExpiry" | "getItem">;
secretBlindIndexDAL: Pick<TSecretBlindIndexDALFactory, "findOne">;
secretTagDAL: Pick<TSecretTagDALFactory, "findManyTagsById" | "saveTagsToSecret" | "deleteTagsManySecret" | "find">;
secretTagDAL: Pick<
TSecretTagDALFactory,
| "findManyTagsById"
| "saveTagsToSecret"
| "deleteTagsManySecret"
| "find"
| "saveTagsToSecretV2"
| "deleteTagsToSecretV2"
>;
secretApprovalRequestDAL: Pick<TSecretApprovalRequestDALFactory, "create" | "transaction">;
secretApprovalRequestSecretDAL: Pick<
TSecretApprovalRequestSecretDALFactory,
"insertMany" | "insertApprovalSecretTags"
"insertMany" | "insertApprovalSecretTags" | "insertV2Bridge"
>;
projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
};
export type TSecretReplicationServiceFactory = ReturnType<typeof secretReplicationServiceFactory>;
@@ -90,9 +118,13 @@ export const secretReplicationServiceFactory = ({
secretApprovalRequestSecretDAL,
secretApprovalRequestDAL,
secretQueueService,
projectBotService
projectBotService,
secretVersionV2TagBridgeDAL,
secretVersionV2BridgeDAL,
secretV2BridgeDAL,
kmsService
}: TSecretReplicationServiceFactoryDep) => {
const getReplicatedSecrets = (
const $getReplicatedSecrets = (
botKey: string,
localSecrets: TSecrets[],
importedSecrets: { secrets: TSecrets[] }[]
@@ -119,6 +151,25 @@ export const secretReplicationServiceFactory = ({
return secrets;
};
const $getReplicatedSecretsV2 = (
localSecrets: (TSecretsV2 & { secretKey: string; secretValue?: string })[],
importedSecrets: { secrets: (TSecretsV2 & { secretKey: string; secretValue?: string })[] }[]
) => {
const deDupe = new Set<string>();
const secrets = [...localSecrets];
for (let i = importedSecrets.length - 1; i >= 0; i = -1) {
importedSecrets[i].secrets.forEach((el) => {
if (deDupe.has(el.key)) {
return;
}
deDupe.add(el.key);
secrets.push(el);
});
}
return secrets;
};
// IMPORTANT NOTE BEFORE READING THE FUNCTION
// SOURCE - Where secrets are copied from
// DESTINATION - Where the replicated imports that points to SOURCE from Destination
@@ -139,6 +190,7 @@ export const secretReplicationServiceFactory = ({
const folder = await folderDAL.findBySecretPath(projectId, environmentSlug, secretPath);
if (!folder) return;
const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId);
// the the replicated imports made to the source. These are the destinations
const destinationSecretImports = await secretImportDAL.find({
@@ -191,8 +243,270 @@ export const secretReplicationServiceFactory = ({
: destinationReplicatedSecretImports;
if (!destinationReplicatedSecretImports.length) return;
const botKey = await projectBotService.getBotKey(projectId);
if (shouldUseSecretV2Bridge) {
const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.SecretManager,
projectId
});
// these are the secrets to be added in replicated folders
const sourceLocalSecrets = await secretV2BridgeDAL.find({ folderId: folder.id, type: SecretType.Shared });
const sourceSecretImports = await secretImportDAL.find({ folderId: folder.id });
const sourceImportedSecrets = await fnSecretsV2FromImports({
allowedImports: sourceSecretImports,
secretDAL: secretV2BridgeDAL,
folderDAL,
secretImportDAL,
decryptor: (value) => (value ? secretManagerDecryptor({ cipherTextBlob: value }).toString() : undefined)
});
// secrets that gets replicated across imports
const sourceDecryptedLocalSecrets = sourceLocalSecrets.map((el) => ({
...el,
secretKey: el.key,
secretValue: el.encryptedValue
? secretManagerDecryptor({ cipherTextBlob: el.encryptedValue }).toString()
: undefined
}));
const sourceSecrets = $getReplicatedSecretsV2(sourceDecryptedLocalSecrets, sourceImportedSecrets);
const sourceSecretsGroupByKey = groupBy(sourceSecrets, (i) => i.key);
const lock = await keyStore.acquireLock(
[getReplicationKeyLockPrefix(projectId, environmentSlug, secretPath)],
5000
);
try {
/* eslint-disable no-await-in-loop */
for (const destinationSecretImport of destinationReplicatedSecretImports) {
try {
const hasJobCompleted = await keyStore.getItem(
keystoreReplicationSuccessKey(job.id as string, destinationSecretImport.id),
KeyStorePrefixes.SecretReplication
);
if (hasJobCompleted) {
logger.info(
{ jobId: job.id, importId: destinationSecretImport.id },
"Skipping this job as this has been successfully replicated."
);
// eslint-disable-next-line
continue;
}
const [destinationFolder] = await folderDAL.findSecretPathByFolderIds(projectId, [
destinationSecretImport.folderId
]);
if (!destinationFolder) throw new BadRequestError({ message: "Imported folder not found" });
let destinationReplicationFolder = await folderDAL.findOne({
parentId: destinationFolder.id,
name: getReplicationFolderName(destinationSecretImport.id),
isReserved: true
});
if (!destinationReplicationFolder) {
destinationReplicationFolder = await folderDAL.create({
parentId: destinationFolder.id,
name: getReplicationFolderName(destinationSecretImport.id),
envId: destinationFolder.envId,
isReserved: true
});
}
const destinationReplicationFolderId = destinationReplicationFolder.id;
const destinationLocalSecretsFromDB = await secretV2BridgeDAL.find({
folderId: destinationReplicationFolderId
});
const destinationLocalSecrets = destinationLocalSecretsFromDB.map((el) => ({
...el,
secretKey: el.key,
secretValue: el.encryptedValue
? secretManagerDecryptor({ cipherTextBlob: el.encryptedValue }).toString()
: undefined
}));
const destinationLocalSecretsGroupedByKey = groupBy(destinationLocalSecrets, (i) => i.key);
const locallyCreatedSecrets = sourceSecrets
.filter(({ key }) => !destinationLocalSecretsGroupedByKey[key]?.[0])
.map((el) => ({ ...el, operation: SecretOperations.Create })); // rewrite update ops to create
const locallyUpdatedSecrets = sourceSecrets
.filter(
({ key, secretKey, secretValue }) =>
destinationLocalSecretsGroupedByKey[key]?.[0] &&
// if key or value changed
(destinationLocalSecretsGroupedByKey[key]?.[0]?.secretKey !== secretKey ||
destinationLocalSecretsGroupedByKey[key]?.[0]?.secretValue !== secretValue)
)
.map((el) => ({ ...el, operation: SecretOperations.Update })); // rewrite update ops to create
const locallyDeletedSecrets = destinationLocalSecrets
.filter(({ key }) => !sourceSecretsGroupByKey[key]?.[0])
.map((el) => ({ ...el, operation: SecretOperations.Delete }));
const isEmtpy =
locallyCreatedSecrets.length + locallyUpdatedSecrets.length + locallyDeletedSecrets.length === 0;
// eslint-disable-next-line
if (isEmtpy) continue;
const policy = await secretApprovalPolicyService.getSecretApprovalPolicy(
projectId,
destinationFolder.environmentSlug,
destinationFolder.path
);
// this means it should be a approval request rather than direct replication
if (policy && actor === ActorType.USER) {
const localSecretsLatestVersions = destinationLocalSecrets.map(({ id }) => id);
const latestSecretVersions = await secretVersionV2BridgeDAL.findLatestVersionMany(
destinationReplicationFolderId,
localSecretsLatestVersions
);
await secretApprovalRequestDAL.transaction(async (tx) => {
const approvalRequestDoc = await secretApprovalRequestDAL.create(
{
folderId: destinationReplicationFolderId,
slug: alphaNumericNanoId(),
policyId: policy.id,
status: "open",
hasMerged: false,
committerUserId: actorId,
isReplicated: true
},
tx
);
const commits = locallyCreatedSecrets
.concat(locallyUpdatedSecrets)
.concat(locallyDeletedSecrets)
.map((doc) => {
const { operation } = doc;
const localSecret = destinationLocalSecretsGroupedByKey[doc.key]?.[0];
return {
op: operation,
requestId: approvalRequestDoc.id,
metadata: doc.metadata,
key: doc.key,
encryptedValue: doc.encryptedValue,
encryptedComment: doc.encryptedComment,
skipMultilineEncoding: doc.skipMultilineEncoding,
// except create operation other two needs the secret id and version id
...(operation !== SecretOperations.Create
? { secretId: localSecret.id, secretVersion: latestSecretVersions[localSecret.id].id }
: {})
};
});
const approvalCommits = await secretApprovalRequestSecretDAL.insertV2Bridge(commits, tx);
return { ...approvalRequestDoc, commits: approvalCommits };
});
} else {
await secretDAL.transaction(async (tx) => {
if (locallyCreatedSecrets.length) {
await fnSecretV2BridgeBulkInsert({
folderId: destinationReplicationFolderId,
secretVersionDAL: secretVersionV2BridgeDAL,
secretDAL: secretV2BridgeDAL,
tx,
secretTagDAL,
secretVersionTagDAL: secretVersionV2TagBridgeDAL,
inputSecrets: locallyCreatedSecrets.map((doc) => {
return {
type: doc.type,
metadata: doc.metadata,
key: doc.key,
encryptedValue: doc.encryptedValue,
encryptedComment: doc.encryptedComment,
skipMultilineEncoding: doc.skipMultilineEncoding,
references: doc.secretValue ? getAllNestedSecretReferencesV2Bridge(doc.secretValue) : []
};
})
});
}
if (locallyUpdatedSecrets.length) {
await fnSecretV2BridgeBulkUpdate({
folderId: destinationReplicationFolderId,
secretVersionDAL: secretVersionV2BridgeDAL,
secretDAL: secretV2BridgeDAL,
tx,
secretTagDAL,
secretVersionTagDAL: secretVersionV2TagBridgeDAL,
inputSecrets: locallyUpdatedSecrets.map((doc) => {
return {
filter: {
folderId: destinationReplicationFolderId,
id: destinationLocalSecretsGroupedByKey[doc.key][0].id
},
data: {
type: doc.type,
metadata: doc.metadata,
key: doc.key,
encryptedValue: doc.encryptedValue as Buffer,
encryptedComment: doc.encryptedComment,
skipMultilineEncoding: doc.skipMultilineEncoding,
references: doc.secretValue ? getAllNestedSecretReferencesV2Bridge(doc.secretValue) : []
}
};
})
});
}
if (locallyDeletedSecrets.length) {
await secretDAL.delete(
{
$in: {
id: locallyDeletedSecrets.map(({ id }) => id)
},
folderId: destinationReplicationFolderId
},
tx
);
}
});
await secretQueueService.syncSecrets({
projectId,
secretPath: destinationFolder.path,
environmentSlug: destinationFolder.environmentSlug,
actorId,
actor,
_depth: depth + 1,
_deDupeReplicationQueue: deDupeReplicationQueue,
_deDupeQueue: deDupeQueue
});
}
// this is used to avoid multiple times generating secret approval by failed one
await keyStore.setItemWithExpiry(
keystoreReplicationSuccessKey(job.id as string, destinationSecretImport.id),
SECRET_IMPORT_SUCCESS_LOCK,
1,
KeyStorePrefixes.SecretReplication
);
await secretImportDAL.updateById(destinationSecretImport.id, {
lastReplicated: new Date(),
replicationStatus: null,
isReplicationSuccess: true
});
} catch (err) {
logger.error(
err,
`Failed to replicate secret with import id=[${destinationSecretImport.id}] env=[${destinationSecretImport.importEnv.slug}] path=[${destinationSecretImport.importPath}]`
);
await secretImportDAL.updateById(destinationSecretImport.id, {
lastReplicated: new Date(),
replicationStatus: (err as Error)?.message.slice(0, 500),
isReplicationSuccess: false
});
}
}
/* eslint-enable no-await-in-loop */
} finally {
await lock.release();
logger.info(job.data, "Replication finished");
}
return;
}
if (!botKey) throw new BadRequestError({ message: "Bot not found" });
// these are the secrets to be added in replicated folders
const sourceLocalSecrets = await secretDAL.find({ folderId: folder.id, type: SecretType.Shared });
const sourceSecretImports = await secretImportDAL.find({ folderId: folder.id });
@@ -203,7 +517,7 @@ export const secretReplicationServiceFactory = ({
secretImportDAL
});
// secrets that gets replicated across imports
const sourceSecrets = getReplicatedSecrets(botKey, sourceLocalSecrets, sourceImportedSecrets);
const sourceSecrets = $getReplicatedSecrets(botKey, sourceLocalSecrets, sourceImportedSecrets);
const sourceSecretsGroupByBlindIndex = groupBy(sourceSecrets, (i) => i.secretBlindIndex as string);
const lock = await keyStore.acquireLock(
@@ -372,7 +686,8 @@ export const secretReplicationServiceFactory = ({
secretCommentIV: doc.secretCommentIV,
secretCommentTag: doc.secretCommentTag,
secretCommentCiphertext: doc.secretCommentCiphertext,
skipMultilineEncoding: doc.skipMultilineEncoding
skipMultilineEncoding: doc.skipMultilineEncoding,
references: getAllNestedSecretReferences(doc.secretValue)
};
})
});
@@ -407,7 +722,8 @@ export const secretReplicationServiceFactory = ({
secretCommentIV: doc.secretCommentIV,
secretCommentTag: doc.secretCommentTag,
secretCommentCiphertext: doc.secretCommentCiphertext,
skipMultilineEncoding: doc.skipMultilineEncoding
skipMultilineEncoding: doc.skipMultilineEncoding,
references: getAllNestedSecretReferences(doc.secretValue)
}
};
})

View File

@@ -1,6 +1,6 @@
import { ForbiddenError, subject } from "@casl/ability";
import { TableName, TSecretTagJunctionInsert } from "@app/db/schemas";
import { TableName, TSecretTagJunctionInsert, TSecretV2TagJunctionInsert } from "@app/db/schemas";
import { BadRequestError, InternalServerError } from "@app/lib/errors";
import { groupBy } from "@app/lib/fn";
import { logger } from "@app/lib/logger";
@@ -24,16 +24,24 @@ import { TSnapshotDALFactory } from "./snapshot-dal";
import { TSnapshotFolderDALFactory } from "./snapshot-folder-dal";
import { TSnapshotSecretDALFactory } from "./snapshot-secret-dal";
import { getFullFolderPath } from "./snapshot-service-fns";
import { TSecretVersionV2DALFactory } from "@app/services/secret-v2-bridge/secret-version-dal";
import { TSnapshotSecretV2DALFactory } from "./snapshot-secret-v2-dal";
import { TSecretV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-dal";
import { TSecretVersionV2TagDALFactory } from "@app/services/secret-v2-bridge/secret-version-tag-dal";
type TSecretSnapshotServiceFactoryDep = {
snapshotDAL: TSnapshotDALFactory;
snapshotSecretDAL: TSnapshotSecretDALFactory;
snapshotSecretV2BridgeDAL: TSnapshotSecretV2DALFactory;
snapshotFolderDAL: TSnapshotFolderDALFactory;
secretVersionDAL: Pick<TSecretVersionDALFactory, "insertMany" | "findLatestVersionByFolderId">;
secretVersionV2BridgeDAL: Pick<TSecretVersionV2DALFactory, "insertMany" | "findLatestVersionByFolderId">;
folderVersionDAL: Pick<TSecretFolderVersionDALFactory, "findLatestVersionByFolderId" | "insertMany">;
secretDAL: Pick<TSecretDALFactory, "delete" | "insertMany">;
secretTagDAL: Pick<TSecretTagDALFactory, "saveTagsToSecret">;
secretV2BridgeDAL: Pick<TSecretV2BridgeDALFactory, "delete" | "insertMany">;
secretTagDAL: Pick<TSecretTagDALFactory, "saveTagsToSecret" | "saveTagsToSecretV2">;
secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
secretVersionV2TagBridgeDAL: Pick<TSecretVersionV2TagDALFactory, "insertMany">;
folderDAL: Pick<TSecretFolderDALFactory, "findById" | "findBySecretPath" | "delete" | "insertMany" | "find">;
permissionService: Pick<TPermissionServiceFactory, "getProjectPermission">;
licenseService: Pick<TLicenseServiceFactory, "isValidLicense">;
@@ -52,7 +60,11 @@ export const secretSnapshotServiceFactory = ({
permissionService,
licenseService,
secretTagDAL,
secretVersionTagDAL
secretVersionTagDAL,
secretVersionV2BridgeDAL,
secretV2BridgeDAL,
snapshotSecretV2BridgeDAL,
secretVersionV2TagBridgeDAL
}: TSecretSnapshotServiceFactoryDep) => {
const projectSecretSnapshotCount = async ({
environment,
@@ -118,7 +130,7 @@ export const secretSnapshotServiceFactory = ({
};
const getSnapshotData = async ({ actorId, actor, actorOrgId, actorAuthMethod, id }: TGetSnapshotDataDTO) => {
const snapshot = await snapshotDAL.findSecretSnapshotDataById(id);
const snapshot = await snapshotDAL.findById(id);
if (!snapshot) throw new BadRequestError({ message: "Snapshot not found" });
const { permission } = await permissionService.getProjectPermission(
actor,
@@ -127,26 +139,73 @@ export const secretSnapshotServiceFactory = ({
actorAuthMethod,
actorOrgId
);
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretRollback);
const shouldUseBridge = snapshot.projectVersion === 3;
let snapshotDetails;
if (shouldUseBridge) {
snapshotDetails = await snapshotDAL.findSecretSnapshotV2DataById(id);
} else {
snapshotDetails = await snapshotDAL.findSecretSnapshotDataById(id);
}
const fullFolderPath = await getFullFolderPath({
folderDAL,
folderId: snapshot.folderId,
envId: snapshot.environment.id
folderId: snapshotDetails.folderId,
envId: snapshotDetails.environment.id
});
// We need to check if the user has access to the secrets in the folder. If we don't do this, a user could theoretically access snapshot secret values even if they don't have read access to the secrets in the folder.
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionActions.Read,
subject(ProjectPermissionSub.Secrets, { environment: snapshot.environment.slug, secretPath: fullFolderPath })
subject(ProjectPermissionSub.Secrets, {
environment: snapshotDetails.environment.slug,
secretPath: fullFolderPath
})
);
return snapshot;
return snapshotDetails;
};
const performSnapshot = async (folderId: string) => {
const performSnapshot = async (folderId: string, shouldUseSecretV2Bridge: boolean) => {
try {
if (!licenseService.isValidLicense) throw new InternalServerError({ message: "Invalid license" });
if (shouldUseSecretV2Bridge) {
const snapshot = await snapshotDAL.transaction(async (tx) => {
const folder = await folderDAL.findById(folderId, tx);
if (!folder) throw new BadRequestError({ message: "Folder not found" });
const secretVersions = await secretVersionV2BridgeDAL.findLatestVersionByFolderId(folderId, tx);
const folderVersions = await folderVersionDAL.findLatestVersionByFolderId(folderId, tx);
const newSnapshot = await snapshotDAL.create(
{
folderId,
envId: folder.environment.envId,
parentFolderId: folder.parentId
},
tx
);
const snapshotSecrets = await snapshotSecretV2BridgeDAL.insertMany(
secretVersions.map(({ id }) => ({
secretVersionId: id,
envId: folder.environment.envId,
snapshotId: newSnapshot.id
})),
tx
);
const snapshotFolders = await snapshotFolderDAL.insertMany(
folderVersions.map(({ id }) => ({
folderVersionId: id,
envId: folder.environment.envId,
snapshotId: newSnapshot.id
})),
tx
);
return { ...newSnapshot, secrets: snapshotSecrets, folder: snapshotFolders };
});
return snapshot;
}
const snapshot = await snapshotDAL.transaction(async (tx) => {
const folder = await folderDAL.findById(folderId, tx);
@@ -199,6 +258,7 @@ export const secretSnapshotServiceFactory = ({
}: TRollbackSnapshotDTO) => {
const snapshot = await snapshotDAL.findById(snapshotId);
if (!snapshot) throw new BadRequestError({ message: "Snapshot not found" });
const shouldUseBridge = snapshot.projectVersion === 3;
const { permission } = await permissionService.getProjectPermission(
actor,
@@ -212,6 +272,117 @@ export const secretSnapshotServiceFactory = ({
ProjectPermissionSub.SecretRollback
);
if (shouldUseBridge) {
const rollback = await snapshotDAL.transaction(async (tx) => {
const rollbackSnaps = await snapshotDAL.findRecursivelySnapshotsV2Bridge(snapshot.id, tx);
// this will remove all secrets in current folder
const deletedTopLevelSecs = await secretV2BridgeDAL.delete({ folderId: snapshot.folderId }, tx);
const deletedTopLevelSecsGroupById = groupBy(deletedTopLevelSecs, (item) => item.id);
// this will remove all secrets and folders on child
// due to sql foreign key and link list connection removing the folders removes everything below too
const deletedFolders = await folderDAL.delete({ parentId: snapshot.folderId, isReserved: false }, tx);
const deletedTopLevelFolders = groupBy(
deletedFolders.filter(({ parentId }) => parentId === snapshot.folderId),
(item) => item.id
);
const folders = await folderDAL.insertMany(
rollbackSnaps.flatMap(({ folderVersion, folderId }) =>
folderVersion.map(({ name, id, latestFolderVersion }) => ({
envId: snapshot.envId,
id,
// this means don't bump up the version if not root folder
// because below ones can be same version as nothing changed
version: deletedTopLevelFolders[folderId] ? latestFolderVersion + 1 : latestFolderVersion,
name,
parentId: folderId
}))
),
tx
);
const secrets = await secretV2BridgeDAL.insertMany(
rollbackSnaps.flatMap(({ secretVersions, folderId }) =>
secretVersions.map(
({ latestSecretVersion, version, updatedAt, createdAt, secretId, envId, id, tags, ...el }) => ({
...el,
id: secretId,
version: deletedTopLevelSecsGroupById[secretId] ? latestSecretVersion + 1 : latestSecretVersion,
folderId
})
)
),
tx
);
const secretTagsToBeInsert: TSecretV2TagJunctionInsert[] = [];
const secretVerTagToBeInsert: Record<string, string[]> = {};
rollbackSnaps.forEach(({ secretVersions }) => {
secretVersions.forEach((secVer) => {
secVer.tags.forEach((tag) => {
secretTagsToBeInsert.push({ secrets_v2Id: secVer.secretId, secret_tagsId: tag.id });
if (!secretVerTagToBeInsert?.[secVer.secretId]) secretVerTagToBeInsert[secVer.secretId] = [];
secretVerTagToBeInsert[secVer.secretId].push(tag.id);
});
});
});
await secretTagDAL.saveTagsToSecretV2(secretTagsToBeInsert, tx);
const folderVersions = await folderVersionDAL.insertMany(
folders.map(({ version, name, id, envId }) => ({
name,
version,
folderId: id,
envId
})),
tx
);
const secretVersions = await secretVersionV2BridgeDAL.insertMany(
secrets.map(({ id, updatedAt, createdAt, ...el }) => ({ ...el, secretId: id })),
tx
);
await secretVersionV2TagBridgeDAL.insertMany(
secretVersions.flatMap(({ secretId, id }) =>
secretVerTagToBeInsert?.[secretId]?.length
? secretVerTagToBeInsert[secretId].map((tagId) => ({
[`${TableName.SecretTag}Id` as const]: tagId,
[`${TableName.SecretVersionV2}Id` as const]: id
}))
: []
),
tx
);
const newSnapshot = await snapshotDAL.create(
{
folderId: snapshot.folderId,
envId: snapshot.envId,
parentFolderId: snapshot.parentFolderId
},
tx
);
const snapshotSecrets = await snapshotSecretV2BridgeDAL.insertMany(
secretVersions
.filter(({ secretId }) => Boolean(deletedTopLevelSecsGroupById?.[secretId]))
.map(({ id }) => ({
secretVersionId: id,
envId: newSnapshot.envId,
snapshotId: newSnapshot.id
})),
tx
);
const snapshotFolders = await snapshotFolderDAL.insertMany(
folderVersions
.filter(({ folderId }) => Boolean(deletedTopLevelFolders?.[folderId]))
.map(({ id }) => ({
folderVersionId: id,
envId: newSnapshot.envId,
snapshotId: newSnapshot.id
})),
tx
);
return { ...newSnapshot, snapshotSecrets, snapshotFolders };
});
return rollback;
}
const rollback = await snapshotDAL.transaction(async (tx) => {
const rollbackSnaps = await snapshotDAL.findRecursivelySnapshots(snapshot.id, tx);
// this will remove all secrets in current folder

View File

@@ -4,11 +4,13 @@ import { Knex } from "knex";
import { TDbClient } from "@app/db";
import {
SecretVersionsSchema,
SecretVersionsV2Schema,
TableName,
TSecretFolderVersions,
TSecretSnapshotFolders,
TSecretSnapshots,
TSecretVersions
TSecretVersions,
TSecretVersionsV2
} from "@app/db/schemas";
import { DatabaseError } from "@app/lib/errors";
import { ormify, selectAllTableCols, sqlNestRelationships } from "@app/lib/knex";
@@ -24,12 +26,14 @@ export const snapshotDALFactory = (db: TDbClient) => {
const data = await (tx || db.replicaNode())(TableName.Snapshot)
.where(`${TableName.Snapshot}.id`, id)
.join(TableName.Environment, `${TableName.Snapshot}.envId`, `${TableName.Environment}.id`)
.join(TableName.Project, `${TableName.Environment}.projectId`, `${TableName.Project}.id`)
.select(selectAllTableCols(TableName.Snapshot))
.select(
db.ref("id").withSchema(TableName.Environment).as("envId"),
db.ref("projectId").withSchema(TableName.Environment),
db.ref("name").withSchema(TableName.Environment).as("envName"),
db.ref("slug").withSchema(TableName.Environment).as("envSlug")
db.ref("slug").withSchema(TableName.Environment).as("envSlug"),
db.ref("version").withSchema(TableName.Project).as("projectVersion")
)
.first();
if (data) {
@@ -149,6 +153,101 @@ export const snapshotDALFactory = (db: TDbClient) => {
}
};
const findSecretSnapshotV2DataById = async (snapshotId: string, tx?: Knex) => {
try {
const data = await (tx || db.replicaNode())(TableName.Snapshot)
.where(`${TableName.Snapshot}.id`, snapshotId)
.join(TableName.Environment, `${TableName.Snapshot}.envId`, `${TableName.Environment}.id`)
.leftJoin(TableName.SnapshotSecretV2, `${TableName.Snapshot}.id`, `${TableName.SnapshotSecretV2}.snapshotId`)
.leftJoin(
TableName.SecretVersionV2,
`${TableName.SnapshotSecretV2}.secretVersionId`,
`${TableName.SecretVersionV2}.id`
)
.leftJoin(
TableName.SecretVersionV2Tag,
`${TableName.SecretVersionV2Tag}.${TableName.SecretVersionV2}Id`,
`${TableName.SecretVersionV2}.id`
)
.leftJoin(
TableName.SecretTag,
`${TableName.SecretVersionV2Tag}.${TableName.SecretTag}Id`,
`${TableName.SecretTag}.id`
)
.leftJoin(TableName.SnapshotFolder, `${TableName.SnapshotFolder}.snapshotId`, `${TableName.Snapshot}.id`)
.leftJoin<TSecretFolderVersions>(
TableName.SecretFolderVersion,
`${TableName.SnapshotFolder}.folderVersionId`,
`${TableName.SecretFolderVersion}.id`
)
.select(selectAllTableCols(TableName.SecretVersionV2))
.select(
db.ref("id").withSchema(TableName.Snapshot).as("snapshotId"),
db.ref("createdAt").withSchema(TableName.Snapshot).as("snapshotCreatedAt"),
db.ref("updatedAt").withSchema(TableName.Snapshot).as("snapshotUpdatedAt"),
db.ref("id").withSchema(TableName.Environment).as("envId"),
db.ref("name").withSchema(TableName.Environment).as("envName"),
db.ref("slug").withSchema(TableName.Environment).as("envSlug"),
db.ref("projectId").withSchema(TableName.Environment),
db.ref("name").withSchema(TableName.SecretFolderVersion).as("folderVerName"),
db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("folderVerId"),
db.ref("id").withSchema(TableName.SecretTag).as("tagId"),
db.ref("id").withSchema(TableName.SecretVersionV2Tag).as("tagVersionId"),
db.ref("color").withSchema(TableName.SecretTag).as("tagColor"),
db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug"),
db.ref("name").withSchema(TableName.SecretTag).as("tagName")
);
return sqlNestRelationships({
data,
key: "snapshotId",
parentMapper: ({
snapshotId: id,
folderId,
projectId,
envId,
envSlug,
envName,
snapshotCreatedAt: createdAt,
snapshotUpdatedAt: updatedAt
}) => ({
id,
folderId,
projectId,
createdAt,
updatedAt,
environment: { id: envId, slug: envSlug, name: envName }
}),
childrenMapper: [
{
key: "id",
label: "secretVersions" as const,
mapper: (el) => SecretVersionsV2Schema.parse(el),
childrenMapper: [
{
key: "tagVersionId",
label: "tags" as const,
mapper: ({ tagId: id, tagName: name, tagSlug: slug, tagColor: color, tagVersionId: vId }) => ({
id,
name,
slug,
color,
vId
})
}
]
},
{
key: "folderVerId",
label: "folderVersion" as const,
mapper: ({ folderVerId: id, folderVerName: name }) => ({ id, name })
}
]
})?.[0];
} catch (error) {
throw new DatabaseError({ error, name: "FindSecretSnapshotDataById" });
}
};
// this is used for rollback
// from a starting snapshot it will collect all the secrets and folder of that
// then it will start go through recursively the below folders latest snapshots then their child folder snapshot until leaf node
@@ -304,6 +403,161 @@ export const snapshotDALFactory = (db: TDbClient) => {
}
};
// this is used for rollback
// from a starting snapshot it will collect all the secrets and folder of that
// then it will start go through recursively the below folders latest snapshots then their child folder snapshot until leaf node
// the recursive part find all snapshot id
// then joins with respective secrets and folder
const findRecursivelySnapshotsV2Bridge = async (snapshotId: string, tx?: Knex) => {
try {
const data = await (tx || db)
.withRecursive("parent", (qb) => {
void qb
.from(TableName.Snapshot)
.leftJoin<TSecretSnapshotFolders>(
TableName.SnapshotFolder,
`${TableName.SnapshotFolder}.snapshotId`,
`${TableName.Snapshot}.id`
)
.leftJoin<TSecretFolderVersions>(
TableName.SecretFolderVersion,
`${TableName.SnapshotFolder}.folderVersionId`,
`${TableName.SecretFolderVersion}.id`
)
.select(selectAllTableCols(TableName.Snapshot))
.select({ depth: 1 })
.select(
db.ref("name").withSchema(TableName.SecretFolderVersion).as("folderVerName"),
db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("folderVerId")
)
.where(`${TableName.Snapshot}.id`, snapshotId)
.union(
(cb) =>
void cb
.select(selectAllTableCols(TableName.Snapshot))
.select({ depth: db.raw("parent.depth + 1") })
.select(
db.ref("name").withSchema(TableName.SecretFolderVersion).as("folderVerName"),
db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("folderVerId")
)
.from(TableName.Snapshot)
.join<TSecretSnapshots, TSecretSnapshots & { secretId: string; max: number }>(
db(TableName.Snapshot).groupBy("folderId").max("createdAt").select("folderId").as("latestVersion"),
`${TableName.Snapshot}.createdAt`,
"latestVersion.max"
)
.leftJoin<TSecretSnapshotFolders>(
TableName.SnapshotFolder,
`${TableName.SnapshotFolder}.snapshotId`,
`${TableName.Snapshot}.id`
)
.leftJoin<TSecretFolderVersions>(
TableName.SecretFolderVersion,
`${TableName.SnapshotFolder}.folderVersionId`,
`${TableName.SecretFolderVersion}.id`
)
.join("parent", "parent.folderVerId", `${TableName.Snapshot}.folderId`)
);
})
.orderBy("depth", "asc")
.from<TSecretSnapshots & { folderVerId: string; folderVerName: string }>("parent")
.leftJoin<TSecretSnapshots>(TableName.SnapshotSecretV2, `parent.id`, `${TableName.SnapshotSecretV2}.snapshotId`)
.leftJoin<TSecretVersionsV2>(
TableName.SecretVersionV2,
`${TableName.SnapshotSecretV2}.secretVersionId`,
`${TableName.SecretVersionV2}.id`
)
.leftJoin(
TableName.SecretVersionV2Tag,
`${TableName.SecretVersionV2Tag}.${TableName.SecretVersionV2}Id`,
`${TableName.SecretVersionV2}.id`
)
.leftJoin(
TableName.SecretTag,
`${TableName.SecretVersionV2Tag}.${TableName.SecretTag}Id`,
`${TableName.SecretTag}.id`
)
.leftJoin<{ latestSecretVersion: number }>(
(tx || db)(TableName.SecretVersionV2)
.groupBy("secretId")
.select("secretId")
.max("version")
.as("secGroupByMaxVersion"),
`${TableName.SecretVersionV2}.secretId`,
"secGroupByMaxVersion.secretId"
)
.leftJoin<{ latestFolderVersion: number }>(
(tx || db)(TableName.SecretFolderVersion)
.groupBy("folderId")
.select("folderId")
.max("version")
.as("folderGroupByMaxVersion"),
`parent.folderId`,
"folderGroupByMaxVersion.folderId"
)
.select(selectAllTableCols(TableName.SecretVersionV2))
.select(
db.ref("id").withSchema("parent").as("snapshotId"),
db.ref("folderId").withSchema("parent").as("snapshotFolderId"),
db.ref("parentFolderId").withSchema("parent").as("snapshotParentFolderId"),
db.ref("folderVerName").withSchema("parent"),
db.ref("folderVerId").withSchema("parent"),
db.ref("max").withSchema("secGroupByMaxVersion").as("latestSecretVersion"),
db.ref("max").withSchema("folderGroupByMaxVersion").as("latestFolderVersion"),
db.ref("id").withSchema(TableName.SecretTag).as("tagId"),
db.ref("id").withSchema(TableName.SecretVersionTag).as("tagVersionId"),
db.ref("color").withSchema(TableName.SecretTag).as("tagColor"),
db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug"),
db.ref("name").withSchema(TableName.SecretTag).as("tagName")
);
const formated = sqlNestRelationships({
data,
key: "snapshotId",
parentMapper: ({ snapshotId: id, snapshotFolderId: folderId, snapshotParentFolderId: parentFolderId }) => ({
id,
folderId,
parentFolderId
}),
childrenMapper: [
{
key: "id",
label: "secretVersions" as const,
mapper: (el) => ({
...SecretVersionsV2Schema.parse(el),
latestSecretVersion: el.latestSecretVersion as number
}),
childrenMapper: [
{
key: "tagVersionId",
label: "tags" as const,
mapper: ({ tagId: id, tagName: name, tagSlug: slug, tagColor: color, tagVersionId: vId }) => ({
id,
name,
slug,
color,
vId
})
}
]
},
{
key: "folderVerId",
label: "folderVersion" as const,
mapper: ({ folderVerId: id, folderVerName: name, latestFolderVersion }) => ({
id,
name,
latestFolderVersion: latestFolderVersion as number
})
}
]
});
return formated;
} catch (error) {
throw new DatabaseError({ error, name: "FindRecursivelySnapshots" });
}
};
// instead of copying all child folders
// we will take the latest snapshot of those folders
// when we need to rollback we will pull from these snapshots
@@ -470,8 +724,10 @@ export const snapshotDALFactory = (db: TDbClient) => {
findById,
findLatestSnapshotByFolderId,
findRecursivelySnapshots,
findRecursivelySnapshotsV2Bridge,
countOfSnapshotsByFolderId,
findSecretSnapshotDataById,
findSecretSnapshotV2DataById,
pruneExcessSnapshots
};
};

View File

@@ -0,0 +1,10 @@
import { TDbClient } from "@app/db";
import { TableName } from "@app/db/schemas";
import { ormify } from "@app/lib/knex";
export type TSnapshotSecretV2DALFactory = ReturnType<typeof snapshotSecretV2DALFactory>;
export const snapshotSecretV2DALFactory = (db: TDbClient) => {
const snapshotSecretOrm = ormify(db, TableName.SnapshotSecretV2);
return snapshotSecretOrm;
};

View File

@@ -22,9 +22,11 @@ import { TInternalKmsDALFactory } from "./internal-kms-dal";
import { TKmsKeyDALFactory } from "./kms-key-dal";
import { TKmsRootConfigDALFactory } from "./kms-root-config-dal";
import {
KmsDataKey,
TDecryptWithKeyDTO,
TDecryptWithKmsDTO,
TEncryptionWithKeyDTO,
TEncryptWithKmsDataKeyDTO,
TEncryptWithKmsDTO,
TGenerateKMSDTO
} from "./kms-types";
@@ -678,6 +680,37 @@ export const kmsServiceFactory = ({
return kms;
};
const $getDataKey = async (dto: TEncryptWithKmsDataKeyDTO) => {
switch (dto.type) {
case KmsDataKey.SecretManager: {
return getProjectSecretManagerKmsDataKey(dto.projectId);
}
default: {
return getProjectSecretManagerKmsDataKey(dto.orgId);
}
}
};
const createCipherPairWithDataKey = async (encryptionContext: TEncryptWithKmsDataKeyDTO) => {
const dataKey = await $getDataKey(encryptionContext);
const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256);
return {
encryptor: ({ plainText }: Pick<TEncryptWithKmsDTO, "plainText">) => {
const encryptedPlainTextBlob = cipher.encrypt(plainText, dataKey);
// Buffer#1 encrypted text + Buffer#2 version number
const versionBlob = Buffer.from(KMS_VERSION, "utf8"); // length is 3
const cipherTextBlob = Buffer.concat([encryptedPlainTextBlob, versionBlob]);
return { cipherTextBlob };
},
decryptor: ({ cipherTextBlob: versionedCipherTextBlob }: Pick<TDecryptWithKeyDTO, "cipherTextBlob">) => {
const cipherTextBlob = versionedCipherTextBlob.subarray(0, -KMS_VERSION_BLOB_LENGTH);
const decryptedBlob = cipher.decrypt(cipherTextBlob, dataKey);
return decryptedBlob;
}
};
};
const startService = async () => {
const appCfg = getConfig();
// This will switch to a seal process and HMS flow in future
@@ -738,6 +771,7 @@ export const kmsServiceFactory = ({
updateProjectSecretManagerKmsKey,
getProjectKeyBackup,
loadProjectKeyBackup,
getKmsById
getKmsById,
createCipherPairWithDataKey
};
};

View File

@@ -26,3 +26,18 @@ export type TDecryptWithKeyDTO = {
key: Buffer;
cipherTextBlob: Buffer;
};
export enum KmsDataKey {
Organization,
SecretManager
// CertificateManager
}
export type TEncryptWithKmsDataKeyDTO =
| { type: KmsDataKey.Organization; orgId: string }
| { type: KmsDataKey.SecretManager; projectId: string };
// akhilmhdh: not implemented yet
// | {
// type: KmsDataKey.CertificateManager;
// projectId: string;
// };

View File

@@ -8,6 +8,7 @@ import { TProjectEnvDALFactory } from "../project-env/project-env-dal";
import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
import { TSecretV2BridgeDALFactory } from "./secret-v2-bridge-dal";
import { TFnSecretBulkDelete, TFnSecretBulkInsert, TFnSecretBulkUpdate } from "./secret-v2-bridge-types";
import { TKmsServiceFactory } from "../kms/kms-service";
const INTERPOLATION_SYNTAX_REG = /\${([^}]+)}/g;
@@ -551,3 +552,14 @@ export const reshapeBridgeSecret = (
createdAt: secret.createdAt,
updatedAt: secret.updatedAt
});
export const secretEncryptionHelper = {
encryptValue: (encryptor: Awaited<ReturnType<TKmsServiceFactory["encryptWithKmsKey"]>>, value?: string) => {
if (typeof value === "undefined") return;
return encryptor({ plainText: Buffer.from(value) }).cipherTextBlob;
},
decryptValue: (decryptor: Awaited<ReturnType<TKmsServiceFactory["decryptWithInputKey"]>>, value?: Buffer | null) => {
if (!value) return;
return decryptor({ cipherTextBlob: value }).toString();
}
};

View File

@@ -22,6 +22,7 @@ import { fnSecretsV2FromImports } from "../secret-import/secret-import-fns";
import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal";
import { TSecretV2BridgeDALFactory } from "./secret-v2-bridge-dal";
import {
secretEncryptionHelper,
fnSecretBulkDelete,
fnSecretBulkInsert,
fnSecretBulkUpdate,
@@ -75,17 +76,6 @@ type TSecretV2BridgeServiceFactoryDep = {
export type TSecretV2BridgeServiceFactory = ReturnType<typeof secretV2BridgeServiceFactory>;
const encryptionHelper = {
encryptValue: (encryptor: Awaited<ReturnType<TKmsServiceFactory["encryptWithKmsKey"]>>, value?: string) => {
if (typeof value === "undefined") return;
return encryptor({ plainText: Buffer.from(value) }).cipherTextBlob;
},
decryptValue: (decryptor: Awaited<ReturnType<TKmsServiceFactory["decryptWithInputKey"]>>, value?: Buffer | null) => {
if (!value) return;
return decryptor({ cipherTextBlob: value }).toString();
}
};
/*
* This service is a bridge from our old architecture towards the new architecture
*/
@@ -171,8 +161,8 @@ export const secretV2BridgeServiceFactory = ({
version: 1,
type,
reminderRepeatDays: el.secretReminderRepeatDays,
encryptedComment: encryptionHelper.encryptValue(secretManagerEncryptor, el.secretComment),
encryptedValue: encryptionHelper.encryptValue(secretManagerEncryptor, el.secretValue),
encryptedComment: secretEncryptionHelper.encryptValue(secretManagerEncryptor, el.secretComment),
encryptedValue: secretEncryptionHelper.encryptValue(secretManagerEncryptor, el.secretValue),
reminderNote: el.secretReminderNote,
skipMultilineEncoding: el.skipMultilineEncoding,
key: secretName,
@@ -291,7 +281,7 @@ export const secretV2BridgeServiceFactory = ({
const encryptedValue =
typeof secretValue !== "undefined"
? {
encryptedValue: encryptionHelper.encryptValue(secretManagerEncryptor, secretValue) as Buffer,
encryptedValue: secretEncryptionHelper.encryptValue(secretManagerEncryptor, secretValue) as Buffer,
references: getAllNestedSecretReferences(secretValue)
}
: {};
@@ -304,7 +294,7 @@ export const secretV2BridgeServiceFactory = ({
filter: { id: secretId },
data: {
reminderRepeatDays: inputSecret.secretReminderRepeatDays,
encryptedComment: encryptionHelper.encryptValue(secretManagerEncryptor, secretComment),
encryptedComment: secretEncryptionHelper.encryptValue(secretManagerEncryptor, secretComment),
reminderNote: inputSecret.secretReminderNote,
skipMultilineEncoding: inputSecret.skipMultilineEncoding,
key: inputSecret.newSecretName || secretName,
@@ -408,8 +398,8 @@ export const secretV2BridgeServiceFactory = ({
const secretManagerDecryptor = await kmsService.decryptWithInputKey({ key: secretManagerDataKey });
return reshapeBridgeSecret(projectId, environment, secretPath, {
...deletedSecret[0],
value: encryptionHelper.decryptValue(secretManagerDecryptor, deletedSecret[0].encryptedValue),
comment: encryptionHelper.decryptValue(secretManagerDecryptor, deletedSecret[0].encryptedComment)
value: secretEncryptionHelper.decryptValue(secretManagerDecryptor, deletedSecret[0].encryptedValue),
comment: secretEncryptionHelper.decryptValue(secretManagerDecryptor, deletedSecret[0].encryptedComment)
});
};
@@ -496,15 +486,15 @@ export const secretV2BridgeServiceFactory = ({
secretDAL,
folderDAL,
secretImportDAL,
decryptor: (value) => encryptionHelper.decryptValue(secretManagerDecryptor, value)
decryptor: (value) => secretEncryptionHelper.decryptValue(secretManagerDecryptor, value)
});
return {
secrets: secrets.map((secret) =>
reshapeBridgeSecret(projectId, environment, groupedPaths[secret.folderId][0].path, {
...secret,
value: encryptionHelper.decryptValue(secretManagerDecryptor, secret.encryptedValue),
comment: encryptionHelper.decryptValue(secretManagerDecryptor, secret.encryptedComment)
value: secretEncryptionHelper.decryptValue(secretManagerDecryptor, secret.encryptedValue),
comment: secretEncryptionHelper.decryptValue(secretManagerDecryptor, secret.encryptedComment)
})
),
imports: importedSecrets
@@ -515,8 +505,8 @@ export const secretV2BridgeServiceFactory = ({
secrets: secrets.map((secret) =>
reshapeBridgeSecret(projectId, environment, groupedPaths[secret.folderId][0].path, {
...secret,
value: encryptionHelper.decryptValue(secretManagerDecryptor, secret.encryptedValue),
comment: encryptionHelper.decryptValue(secretManagerDecryptor, secret.encryptedComment)
value: secretEncryptionHelper.decryptValue(secretManagerDecryptor, secret.encryptedValue),
comment: secretEncryptionHelper.decryptValue(secretManagerDecryptor, secret.encryptedComment)
})
)
};
@@ -583,7 +573,7 @@ export const secretV2BridgeServiceFactory = ({
.then((el) => SecretsV2Schema.parse({ ...el, id: el.secretId })));
const interpolateInlineSecretReference = interpolateSecrets({
projectId,
decryptSecret: (encryptedValue) => encryptionHelper.decryptValue(secretManagerDecryptor, encryptedValue),
decryptSecret: (encryptedValue) => secretEncryptionHelper.decryptValue(secretManagerDecryptor, encryptedValue),
secretDAL,
folderDAL
});
@@ -610,14 +600,17 @@ export const secretV2BridgeServiceFactory = ({
secretDAL,
folderDAL,
secretImportDAL,
decryptor: (value) => encryptionHelper.decryptValue(secretManagerDecryptor, value)
decryptor: (value) => secretEncryptionHelper.decryptValue(secretManagerDecryptor, value)
});
for (let i = importedSecrets.length - 1; i >= 0; i -= 1) {
for (let j = 0; j < importedSecrets[i].secrets.length; j += 1) {
if (secretName === importedSecrets[i].secrets[j].key) {
const importedSecret = importedSecrets[i].secrets[j];
let secretValue = encryptionHelper.decryptValue(secretManagerDecryptor, importedSecret.encryptedValue);
let secretValue = secretEncryptionHelper.decryptValue(
secretManagerDecryptor,
importedSecret.encryptedValue
);
if (expandSecretReferences && secretValue) {
const secretReferenceExpandedString = {
[importedSecret.key]: { value: secretValue }
@@ -630,7 +623,7 @@ export const secretV2BridgeServiceFactory = ({
return reshapeBridgeSecret(projectId, importedSecrets[i].environment, importedSecrets[i].secretPath, {
...importedSecret,
value: secretValue,
comment: encryptionHelper.decryptValue(secretManagerDecryptor, importedSecret.encryptedComment)
comment: secretEncryptionHelper.decryptValue(secretManagerDecryptor, importedSecret.encryptedComment)
});
}
}
@@ -638,7 +631,7 @@ export const secretV2BridgeServiceFactory = ({
}
if (!secret) throw new BadRequestError({ message: "Secret not found" });
let secretValue = encryptionHelper.decryptValue(secretManagerDecryptor, secret.encryptedValue);
let secretValue = secretEncryptionHelper.decryptValue(secretManagerDecryptor, secret.encryptedValue);
if (expandSecretReferences && secretValue) {
const secretReferenceExpandedString = {
[secret.key]: { value: secretValue }
@@ -651,7 +644,7 @@ export const secretV2BridgeServiceFactory = ({
return reshapeBridgeSecret(projectId, environment, path, {
...secret,
value: secretValue,
comment: encryptionHelper.decryptValue(secretManagerDecryptor, secret.encryptedComment)
comment: secretEncryptionHelper.decryptValue(secretManagerDecryptor, secret.encryptedComment)
});
};
@@ -707,8 +700,8 @@ export const secretV2BridgeServiceFactory = ({
fnSecretBulkInsert({
inputSecrets: inputSecrets.map((el) => ({
version: 1,
encryptedComment: encryptionHelper.encryptValue(secretManagerEncryptor, el.secretComment),
encryptedValue: encryptionHelper.encryptValue(secretManagerEncryptor, el.secretValue),
encryptedComment: secretEncryptionHelper.encryptValue(secretManagerEncryptor, el.secretComment),
encryptedValue: secretEncryptionHelper.encryptValue(secretManagerEncryptor, el.secretValue),
skipMultilineEncoding: el.skipMultilineEncoding,
key: el.secretKey,
tagIds: el.tagIds,
@@ -737,8 +730,8 @@ export const secretV2BridgeServiceFactory = ({
return newSecrets.map((el) =>
reshapeBridgeSecret(projectId, environment, secretPath, {
...el,
value: encryptionHelper.decryptValue(secretManagerDecryptor, el.encryptedValue),
comment: encryptionHelper.decryptValue(secretManagerDecryptor, el.encryptedComment)
value: secretEncryptionHelper.decryptValue(secretManagerDecryptor, el.encryptedValue),
comment: secretEncryptionHelper.decryptValue(secretManagerDecryptor, el.encryptedComment)
})
);
};
@@ -814,7 +807,7 @@ export const secretV2BridgeServiceFactory = ({
const encryptedValue =
typeof el.secretValue !== "undefined"
? {
encryptedValue: encryptionHelper.encryptValue(secretManagerEncryptor, el.secretValue) as Buffer,
encryptedValue: secretEncryptionHelper.encryptValue(secretManagerEncryptor, el.secretValue) as Buffer,
references: getAllNestedSecretReferences(el.secretValue)
}
: {};
@@ -822,7 +815,7 @@ export const secretV2BridgeServiceFactory = ({
filter: { key: el.secretKey, type: SecretType.Shared },
data: {
reminderRepeatDays: el.secretReminderRepeatDays,
encryptedComment: encryptionHelper.encryptValue(secretManagerEncryptor, el.secretComment),
encryptedComment: secretEncryptionHelper.encryptValue(secretManagerEncryptor, el.secretComment),
reminderNote: el.secretReminderNote,
skipMultilineEncoding: el.skipMultilineEncoding,
key: el.newSecretName || el.secretKey,
@@ -850,8 +843,8 @@ export const secretV2BridgeServiceFactory = ({
return secrets.map((el) =>
reshapeBridgeSecret(projectId, environment, secretPath, {
...el,
value: encryptionHelper.decryptValue(secretManagerDecryptor, el.encryptedValue),
comment: encryptionHelper.decryptValue(secretManagerDecryptor, el.encryptedComment)
value: secretEncryptionHelper.decryptValue(secretManagerDecryptor, el.encryptedValue),
comment: secretEncryptionHelper.decryptValue(secretManagerDecryptor, el.encryptedComment)
})
);
};
@@ -925,8 +918,8 @@ export const secretV2BridgeServiceFactory = ({
return secretsDeleted.map((el) =>
reshapeBridgeSecret(projectId, environment, secretPath, {
...el,
value: encryptionHelper.decryptValue(secretManagerDecryptor, el.encryptedValue),
comment: encryptionHelper.decryptValue(secretManagerDecryptor, el.encryptedComment)
value: secretEncryptionHelper.decryptValue(secretManagerDecryptor, el.encryptedValue),
comment: secretEncryptionHelper.decryptValue(secretManagerDecryptor, el.encryptedComment)
})
);
};
@@ -991,7 +984,7 @@ export const secretV2BridgeServiceFactory = ({
secretId: id,
references: encryptedValue
? getAllNestedSecretReferences(
encryptionHelper.decryptValue(secretManagerDecryptor, encryptedValue) as string
secretEncryptionHelper.decryptValue(secretManagerDecryptor, encryptedValue) as string
)
: []
})),
@@ -1074,7 +1067,7 @@ export const secretV2BridgeServiceFactory = ({
const secretManagerDecryptor = await kmsService.decryptWithInputKey({ key: secretManagerDataKey });
const decryptedSourceSecrets = sourceSecrets.map((secret) => ({
...secret,
value: encryptionHelper.decryptValue(secretManagerDecryptor, secret.encryptedValue)
value: secretEncryptionHelper.decryptValue(secretManagerDecryptor, secret.encryptedValue)
}));
let isSourceUpdated = false;
@@ -1093,7 +1086,7 @@ export const secretV2BridgeServiceFactory = ({
const decryptedDestinationSecrets = destinationSecretsFromDB.map((secret) => {
return {
...secret,
value: encryptionHelper.decryptValue(secretManagerDecryptor, secret.encryptedValue)
value: secretEncryptionHelper.decryptValue(secretManagerDecryptor, secret.encryptedValue)
};
});

View File

@@ -887,3 +887,50 @@ export const updateManySecretsRawFnFactory = ({
return updateManySecretsRawFn;
};
export const decryptSecretWithBot = (
secret: Pick<
TSecrets,
| "secretKeyIV"
| "secretKeyTag"
| "secretKeyCiphertext"
| "secretValueIV"
| "secretValueTag"
| "secretValueCiphertext"
| "secretCommentIV"
| "secretCommentTag"
| "secretCommentCiphertext"
>,
key: string
) => {
const secretKey = decryptSymmetric128BitHexKeyUTF8({
ciphertext: secret.secretKeyCiphertext,
iv: secret.secretKeyIV,
tag: secret.secretKeyTag,
key
});
const secretValue = decryptSymmetric128BitHexKeyUTF8({
ciphertext: secret.secretValueCiphertext,
iv: secret.secretValueIV,
tag: secret.secretValueTag,
key
});
let secretComment = "";
if (secret.secretCommentCiphertext && secret.secretCommentIV && secret.secretCommentTag) {
secretComment = decryptSymmetric128BitHexKeyUTF8({
ciphertext: secret.secretCommentCiphertext,
iv: secret.secretCommentIV,
tag: secret.secretCommentTag,
key
});
}
return {
secretKey,
secretValue,
secretComment
};
};

View File

@@ -73,6 +73,7 @@ import {
} from "./secret-types";
import { TSecretVersionDALFactory } from "./secret-version-dal";
import { TSecretVersionTagDALFactory } from "./secret-version-tag-dal";
import { TSecretApprovalRequestServiceFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-service";
type TSecretServiceFactoryDep = {
secretDAL: TSecretDALFactory;
@@ -93,6 +94,10 @@ type TSecretServiceFactoryDep = {
secretImportDAL: Pick<TSecretImportDALFactory, "find" | "findByFolderIds">;
secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
secretApprovalPolicyService: Pick<TSecretApprovalPolicyServiceFactory, "getSecretApprovalPolicy">;
secretApprovalRequestService: Pick<
TSecretApprovalRequestServiceFactory,
"generateSecretApprovalRequest" | "generateSecretApprovalRequestV2Bridge"
>;
secretApprovalRequestDAL: Pick<TSecretApprovalRequestDALFactory, "create" | "transaction">;
secretApprovalRequestSecretDAL: Pick<
TSecretApprovalRequestSecretDALFactory,
@@ -118,7 +123,8 @@ export const secretServiceFactory = ({
secretApprovalPolicyService,
secretApprovalRequestDAL,
secretApprovalRequestSecretDAL,
secretV2BridgeService
secretV2BridgeService,
secretApprovalRequestService
}: TSecretServiceFactoryDep) => {
const getSecretReference = async (projectId: string) => {
// if bot key missing means e2e still exist
@@ -1216,7 +1222,37 @@ export const secretServiceFactory = ({
secretReminderRepeatDays
}: TCreateSecretRawDTO) => {
const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId);
const policy =
actor === ActorType.USER && type === SecretType.Shared
? await secretApprovalPolicyService.getSecretApprovalPolicy(projectId, environment, secretPath)
: undefined;
if (shouldUseSecretV2Bridge) {
if (policy) {
return secretApprovalRequestService.generateSecretApprovalRequestV2Bridge({
policy,
secretPath,
environment,
projectId,
actor,
actorId,
actorOrgId,
actorAuthMethod,
data: {
[SecretOperations.Create]: [
{
secretKey: secretName,
skipMultilineEncoding,
secretComment,
secretValue,
tagIds,
reminderNote: secretReminderNote,
reminderRepeatDays: secretReminderRepeatDays
}
]
}
});
}
const secret = await secretV2BridgeService.createSecret({
secretName,
type,
@@ -1241,6 +1277,36 @@ export const secretServiceFactory = ({
const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secretName, botKey);
const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secretValue || "", botKey);
const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secretComment || "", botKey);
if (policy) {
return secretApprovalRequestService.generateSecretApprovalRequest({
policy,
secretPath,
environment,
projectId,
actor,
actorId,
actorOrgId,
actorAuthMethod,
data: {
[SecretOperations.Create]: [
{
secretName,
secretKeyCiphertext: secretKeyEncrypted.ciphertext,
secretKeyIV: secretKeyEncrypted.iv,
secretKeyTag: secretKeyEncrypted.tag,
secretValueCiphertext: secretValueEncrypted.ciphertext,
secretValueIV: secretValueEncrypted.iv,
secretValueTag: secretValueEncrypted.tag,
secretCommentCiphertext: secretCommentEncrypted.ciphertext,
secretCommentIV: secretCommentEncrypted.iv,
secretCommentTag: secretCommentEncrypted.tag,
skipMultilineEncoding,
tagIds
}
]
}
});
}
const secret = await createSecret({
secretName,
@@ -1290,7 +1356,37 @@ export const secretServiceFactory = ({
newSecretName
}: TUpdateSecretRawDTO) => {
const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId);
const policy =
actor === ActorType.USER && type === SecretType.Shared
? await secretApprovalPolicyService.getSecretApprovalPolicy(projectId, environment, secretPath)
: undefined;
if (shouldUseSecretV2Bridge) {
if (policy) {
return secretApprovalRequestService.generateSecretApprovalRequestV2Bridge({
policy,
secretPath,
environment,
projectId,
actor,
actorId,
actorOrgId,
actorAuthMethod,
data: {
[SecretOperations.Update]: [
{
secretKey: secretName,
newSecretName,
skipMultilineEncoding,
secretComment,
secretValue,
tagIds,
reminderNote: secretReminderNote,
reminderRepeatDays: secretReminderRepeatDays
}
]
}
});
}
const secret = await secretV2BridgeService.updateSecret({
secretReminderRepeatDays,
skipMultilineEncoding,
@@ -1319,6 +1415,40 @@ export const secretServiceFactory = ({
const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secretComment || "", botKey);
const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(newSecretName || secretName, botKey);
if (policy) {
return secretApprovalRequestService.generateSecretApprovalRequest({
policy,
secretPath,
environment,
projectId,
actor,
actorId,
actorOrgId,
actorAuthMethod,
data: {
[SecretOperations.Update]: [
{
secretName,
newSecretName,
skipMultilineEncoding,
secretKeyCiphertext: secretKeyEncrypted.ciphertext,
secretKeyIV: secretKeyEncrypted.iv,
secretKeyTag: secretKeyEncrypted.tag,
secretValueCiphertext: secretValueEncrypted.ciphertext,
secretValueIV: secretValueEncrypted.iv,
secretValueTag: secretValueEncrypted.tag,
secretCommentCiphertext: secretCommentEncrypted.ciphertext,
secretCommentIV: secretCommentEncrypted.iv,
secretCommentTag: secretCommentEncrypted.tag,
tagIds,
secretReminderNote,
secretReminderRepeatDays
}
]
}
});
}
const secret = await updateSecret({
secretName,
projectId,
@@ -1362,7 +1492,30 @@ export const secretServiceFactory = ({
secretPath
}: TDeleteSecretRawDTO) => {
const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId);
const policy =
actor === ActorType.USER && type === SecretType.Shared
? await secretApprovalPolicyService.getSecretApprovalPolicy(projectId, environment, secretPath)
: undefined;
if (shouldUseSecretV2Bridge) {
if (policy) {
return secretApprovalRequestService.generateSecretApprovalRequestV2Bridge({
policy,
actorAuthMethod,
actorOrgId,
actorId,
actor,
projectId,
environment,
secretPath,
data: {
[SecretOperations.Delete]: [
{
secretKey: secretName
}
]
}
});
}
const secret = await secretV2BridgeService.deleteSecret({
secretName,
type,
@@ -1377,7 +1530,25 @@ export const secretServiceFactory = ({
return secret;
}
if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" });
if (policy) {
return secretApprovalRequestService.generateSecretApprovalRequest({
policy,
actorAuthMethod,
actorOrgId,
actorId,
actor,
projectId,
environment,
secretPath,
data: {
[SecretOperations.Delete]: [
{
secretName
}
]
}
});
}
const secret = await deleteSecret({
secretName,
projectId,
@@ -1416,7 +1587,33 @@ export const secretServiceFactory = ({
}
const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId);
const policy =
actor === ActorType.USER
? await secretApprovalPolicyService.getSecretApprovalPolicy(projectId, environment, secretPath)
: undefined;
if (shouldUseSecretV2Bridge) {
if (policy) {
return secretApprovalRequestService.generateSecretApprovalRequestV2Bridge({
policy,
secretPath,
environment,
projectId,
actor,
actorId,
actorOrgId,
actorAuthMethod,
data: {
[SecretOperations.Create]: inputSecrets.map((el) => ({
tagIds: el.tagIds,
secretValue: el.secretValue,
secretComment: el.secretComment,
metadata: el.metadata,
skipMultilineEncoding: el.skipMultilineEncoding,
secretKey: el.secretKey
}))
}
});
}
const secrets = await secretV2BridgeService.createManySecret({
secretPath,
environment,
@@ -1430,7 +1627,44 @@ export const secretServiceFactory = ({
return secrets;
}
if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" });
const sanitizedSecrets = inputSecrets.map(
({ secretComment, secretKey, metadata, tagIds, secretValue, skipMultilineEncoding }) => {
const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secretKey, botKey);
const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secretValue || "", botKey);
const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secretComment || "", botKey);
return {
secretName: secretKey,
skipMultilineEncoding,
secretKeyCiphertext: secretKeyEncrypted.ciphertext,
secretKeyIV: secretKeyEncrypted.iv,
secretKeyTag: secretKeyEncrypted.tag,
secretValueCiphertext: secretValueEncrypted.ciphertext,
secretValueIV: secretValueEncrypted.iv,
secretValueTag: secretValueEncrypted.tag,
secretCommentCiphertext: secretCommentEncrypted.ciphertext,
secretCommentIV: secretCommentEncrypted.iv,
secretCommentTag: secretCommentEncrypted.tag,
tags: tagIds,
tagIds,
metadata
};
}
);
if (policy) {
return secretApprovalRequestService.generateSecretApprovalRequest({
policy,
secretPath,
environment,
projectId,
actor,
actorId,
actorOrgId,
actorAuthMethod,
data: {
[SecretOperations.Create]: sanitizedSecrets
}
});
}
const secrets = await createManySecret({
projectId,
environment,
@@ -1439,28 +1673,7 @@ export const secretServiceFactory = ({
actorId,
actorOrgId,
actorAuthMethod,
secrets: inputSecrets.map(
({ secretComment, secretKey, metadata, tagIds, secretValue, skipMultilineEncoding }) => {
const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secretKey, botKey);
const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secretValue || "", botKey);
const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secretComment || "", botKey);
return {
secretName: secretKey,
skipMultilineEncoding,
secretKeyCiphertext: secretKeyEncrypted.ciphertext,
secretKeyIV: secretKeyEncrypted.iv,
secretKeyTag: secretKeyEncrypted.tag,
secretValueCiphertext: secretValueEncrypted.ciphertext,
secretValueIV: secretValueEncrypted.iv,
secretValueTag: secretValueEncrypted.tag,
secretCommentCiphertext: secretCommentEncrypted.ciphertext,
secretCommentIV: secretCommentEncrypted.iv,
secretCommentTag: secretCommentEncrypted.tag,
tags: tagIds,
metadata
};
}
)
secrets: sanitizedSecrets
});
return secrets.map((secret) =>
@@ -1490,7 +1703,32 @@ export const secretServiceFactory = ({
}
const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId);
const policy =
actor === ActorType.USER
? await secretApprovalPolicyService.getSecretApprovalPolicy(projectId, environment, secretPath)
: undefined;
if (shouldUseSecretV2Bridge) {
if (policy) {
return secretApprovalRequestService.generateSecretApprovalRequestV2Bridge({
policy,
secretPath,
environment,
projectId,
actor,
actorId,
actorOrgId,
actorAuthMethod,
data: {
[SecretOperations.Update]: inputSecrets.map((el) => ({
tagIds: el.tagIds,
secretValue: el.secretValue,
secretComment: el.secretComment,
skipMultilineEncoding: el.skipMultilineEncoding,
secretKey: el.secretKey
}))
}
});
}
const secrets = await secretV2BridgeService.updateManySecret({
secretPath,
environment,
@@ -1505,7 +1743,56 @@ export const secretServiceFactory = ({
}
if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" });
const sanitizedSecrets = inputSecrets.map(
({
secretComment,
secretKey,
secretValue,
skipMultilineEncoding,
tagIds: tags,
newSecretName,
secretReminderNote,
secretReminderRepeatDays
}) => {
const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(newSecretName || secretKey, botKey);
const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secretValue || "", botKey);
const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secretComment || "", botKey);
return {
secretName: secretKey,
newSecretName,
tags,
tagIds: tags,
secretReminderRepeatDays,
secretReminderNote,
type: SecretType.Shared,
skipMultilineEncoding,
secretKeyCiphertext: secretKeyEncrypted.ciphertext,
secretKeyIV: secretKeyEncrypted.iv,
secretKeyTag: secretKeyEncrypted.tag,
secretValueCiphertext: secretValueEncrypted.ciphertext,
secretValueIV: secretValueEncrypted.iv,
secretValueTag: secretValueEncrypted.tag,
secretCommentCiphertext: secretCommentEncrypted.ciphertext,
secretCommentIV: secretCommentEncrypted.iv,
secretCommentTag: secretCommentEncrypted.tag
};
}
);
if (policy) {
return secretApprovalRequestService.generateSecretApprovalRequest({
policy,
secretPath,
environment,
projectId,
actor,
actorId,
actorOrgId,
actorAuthMethod,
data: {
[SecretOperations.Update]: sanitizedSecrets
}
});
}
const secrets = await updateManySecret({
projectId,
environment,
@@ -1514,40 +1801,7 @@ export const secretServiceFactory = ({
actorId,
actorOrgId,
actorAuthMethod,
secrets: inputSecrets.map(
({
secretComment,
secretKey,
secretValue,
skipMultilineEncoding,
tagIds: tags,
newSecretName,
secretReminderNote,
secretReminderRepeatDays
}) => {
const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(newSecretName || secretKey, botKey);
const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secretValue || "", botKey);
const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secretComment || "", botKey);
return {
secretName: secretKey,
newSecretName,
tags,
secretReminderRepeatDays,
secretReminderNote,
type: SecretType.Shared,
skipMultilineEncoding,
secretKeyCiphertext: secretKeyEncrypted.ciphertext,
secretKeyIV: secretKeyEncrypted.iv,
secretKeyTag: secretKeyEncrypted.tag,
secretValueCiphertext: secretValueEncrypted.ciphertext,
secretValueIV: secretValueEncrypted.iv,
secretValueTag: secretValueEncrypted.tag,
secretCommentCiphertext: secretCommentEncrypted.ciphertext,
secretCommentIV: secretCommentEncrypted.iv,
secretCommentTag: secretCommentEncrypted.tag
};
}
)
secrets: sanitizedSecrets
});
return secrets.map((secret) =>
@@ -1577,7 +1831,26 @@ export const secretServiceFactory = ({
}
const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId);
const policy =
actor === ActorType.USER
? await secretApprovalPolicyService.getSecretApprovalPolicy(projectId, environment, secretPath)
: undefined;
if (shouldUseSecretV2Bridge) {
if (policy) {
return secretApprovalRequestService.generateSecretApprovalRequestV2Bridge({
policy,
actorAuthMethod,
actorOrgId,
actorId,
actor,
projectId,
environment,
secretPath,
data: {
[SecretOperations.Delete]: inputSecrets
}
});
}
const secrets = await secretV2BridgeService.deleteManySecret({
secretPath,
environment,
@@ -1593,6 +1866,21 @@ export const secretServiceFactory = ({
if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" });
if (policy) {
return secretApprovalRequestService.generateSecretApprovalRequest({
policy,
actorAuthMethod,
actorOrgId,
actorId,
actor,
projectId,
environment,
secretPath,
data: {
[SecretOperations.Delete]: inputSecrets.map((el) => ({ secretName: el.secretKey }))
}
});
}
const secrets = await deleteManySecret({
projectId,
environment,