awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-10-14 17:40:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: implemented inline secret reference integration sync

Browse Source
This commit is contained in:
=
2024-05-15 09:57:23 +05:30
parent c438479246
commit 393964c4ae
13 changed files with 471 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/services/secret/secret-dal.ts
+72 -1
View File
@@ -243,6 +243,74 @@ export const secretDALFactory = (db: TDbClient) => {
}
};
const upsertSecretReferences = async (
data: {
secretId: string;
references: Array<{ environment: string; secretPath: string }>;
}[] = [],
tx?: Knex
) => {
try {
if (!data.length) return;
await (tx || db)(TableName.SecretReference)
.whereIn(
"secretId",
data.map(({ secretId }) => secretId)
)
.delete();
const newSecretReferences = data
.filter(({ references }) => references.length)
.flatMap(({ secretId, references }) =>
references.map(({ environment, secretPath }) => ({
secretPath,
secretId,
environment
}))
);
if (!newSecretReferences.length) return;
const secretReferences = await (tx || db)(TableName.SecretReference).insert(newSecretReferences);
return secretReferences;
} catch (error) {
throw new DatabaseError({ error, name: "UpsertSecretReference" });
}
};
const findReferencedSecretReferences = async (projectId: string, envSlug: string, secretPath: string, tx?: Knex) => {
try {
const docs = await (tx || db)(TableName.SecretReference)
.where({
secretPath,
environment: envSlug
})
.join(TableName.Secret, `${TableName.Secret}.id`, `${TableName.SecretReference}.secretId`)
.join(TableName.SecretFolder, `${TableName.Secret}.folderId`, `${TableName.SecretFolder}.id`)
.join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`)
.where("projectId", projectId)
.select(selectAllTableCols(TableName.SecretReference))
.select("folderId");
return docs;
} catch (error) {
throw new DatabaseError({ error, name: "FindReferencedSecretReferences" });
}
};
// special query to backfill secret value
const findAllProjectSecretValues = async (projectId: string, tx?: Knex) => {
try {
const docs = await (tx || db)(TableName.Secret)
.join(TableName.SecretFolder, `${TableName.Secret}.folderId`, `${TableName.SecretFolder}.id`)
.join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`)
.where("projectId", projectId)
// not empty
.whereNotNull("secretValueCiphertext")
.select("secretValueTag", "secretValueCiphertext", "secretValueIV", `${TableName.Secret}.id` as "id");
return docs;
} catch (error) {
throw new DatabaseError({ error, name: "FindAllProjectSecretValues" });
}
};
return {
...secretOrm,
update,
@@ -252,6 +320,9 @@ export const secretDALFactory = (db: TDbClient) => {
getSecretTags,
findByFolderId,
findByFolderIds,
findByBlindIndexes
findByBlindIndexes,
upsertSecretReferences,
findReferencedSecretReferences,
findAllProjectSecretValues
};
};
backend/src/services/secret/secret-fns.ts
+83 -91
View File
@@ -194,6 +194,7 @@ type TInterpolateSecretArg = {
folderDAL: Pick<TSecretFolderDALFactory, "findBySecretPath">;
};
const INTERPOLATION_SYNTAX_REG = /\${([^}]+)}/g;
export const interpolateSecrets = ({ projectId, secretEncKey, secretDAL, folderDAL }: TInterpolateSecretArg) => {
const fetchSecretsCrossEnv = () => {
const fetchCache: Record<string, Record<string, string>> = {};
@@ -235,7 +236,6 @@ export const interpolateSecrets = ({ projectId, secretEncKey, secretDAL, folderD
};
};
const INTERPOLATION_SYNTAX_REG = /\${([^}]+)}/g;
const recursivelyExpandSecret = async (
expandedSec: Record<string, string>,
interpolatedSec: Record<string, string>,
@@ -396,6 +396,16 @@ export const decryptSecretRaw = (
};
};
export const getAllNestedSecretReferences = (value: string) => {
const references = Array.from(value.matchAll(INTERPOLATION_SYNTAX_REG), (m) => m[1]);
return references
.filter((el) => el.includes("."))
.map((el) => {
const [environment, ...secretPath] = el.split(".");
return { environment, secretPath: path.join("/", ...secretPath.slice(0, -1)) };
});
};
/**
* Checks and handles secrets using a blind index method.
* The function generates mappings between secret names and their blind indexes, validates user IDs for personal secrets, and retrieves secrets from the database based on their blind indexes.
@@ -467,7 +477,7 @@ export const fnSecretBulkInsert = async ({
tx
}: TFnSecretBulkInsert) => {
const newSecrets = await secretDAL.insertMany(
inputSecrets.map(({ tags, ...el }) => ({ ...el, folderId })),
inputSecrets.map(({ tags, references, ...el }) => ({ ...el, folderId })),
tx
);
const newSecretGroupByBlindIndex = groupBy(newSecrets, (item) => item.secretBlindIndex as string);
@@ -478,13 +488,19 @@ export const fnSecretBulkInsert = async ({
}))
);
const secretVersions = await secretVersionDAL.insertMany(
inputSecrets.map(({ tags, ...el }) => ({
inputSecrets.map(({ tags, references, ...el }) => ({
...el,
folderId,
secretId: newSecretGroupByBlindIndex[el.secretBlindIndex as string][0].id
})),
tx
);
await secretDAL.upsertSecretReferences(
inputSecrets.map(({ references = [], secretBlindIndex }) => ({
secretId: newSecretGroupByBlindIndex[secretBlindIndex as string][0].id,
references
}))
);
if (newSecretTags.length) {
const secTags = await secretTagDAL.saveTagsToSecret(newSecretTags, tx);
const secVersionsGroupBySecId = groupBy(secretVersions, (i) => i.secretId);
@@ -509,7 +525,7 @@ export const fnSecretBulkUpdate = async ({
secretVersionTagDAL
}: TFnSecretBulkUpdate) => {
const newSecrets = await secretDAL.bulkUpdate(
inputSecrets.map(({ filter, data: { tags, ...data } }) => ({
inputSecrets.map(({ filter, data: { tags, references, ...data } }) => ({
filter: { ...filter, folderId },
data
})),
@@ -522,6 +538,14 @@ export const fnSecretBulkUpdate = async ({
})),
tx
);
await secretDAL.upsertSecretReferences(
inputSecrets
.filter(({ data: { references } }) => Boolean(references))
.map(({ data: { references = [] } }, i) => ({
secretId: newSecrets[i].id,
references
}))
);
const secsUpdatedTag = inputSecrets.flatMap(({ data: { tags } }, i) =>
tags !== undefined ? { tags, secretId: newSecrets[i].id } : []
);
@@ -591,50 +615,39 @@ export const createManySecretsRawFnFactory = ({
folderId,
isNew: true,
blindIndexCfg,
userId,
secretDAL
});
const inputSecrets = await Promise.all(
secrets.map(async (secret) => {
const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey);
const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey);
const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey);
const inputSecrets = secrets.map((secret) => {
const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey);
const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey);
const secretReferences = getAllNestedSecretReferences(secret.secretValue || "");
const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey);
if (secret.type === SecretType.Personal) {
if (!userId) throw new BadRequestError({ message: "Missing user id for personal secret" });
const sharedExist = await secretDAL.findOne({
secretBlindIndex: keyName2BlindIndex[secret.secretName],
folderId,
type: SecretType.Shared
});
return {
type: secret.type,
userId: secret.type === SecretType.Personal ? userId : null,
secretName: secret.secretName,
secretKeyCiphertext: secretKeyEncrypted.ciphertext,
secretKeyIV: secretKeyEncrypted.iv,
secretKeyTag: secretKeyEncrypted.tag,
secretValueCiphertext: secretValueEncrypted.ciphertext,
secretValueIV: secretValueEncrypted.iv,
secretValueTag: secretValueEncrypted.tag,
secretCommentCiphertext: secretCommentEncrypted.ciphertext,
secretCommentIV: secretCommentEncrypted.iv,
secretCommentTag: secretCommentEncrypted.tag,
skipMultilineEncoding: secret.skipMultilineEncoding,
tags: secret.tags,
references: secretReferences
};
});
if (!sharedExist)
throw new BadRequestError({
message: "Failed to create personal secret override for no corresponding shared secret"
});
}
const tags = secret.tags ? await secretTagDAL.findManyTagsById(projectId, secret.tags) : [];
if ((secret.tags || []).length !== tags.length) throw new BadRequestError({ message: "Tag not found" });
return {
type: secret.type,
userId: secret.type === SecretType.Personal ? userId : null,
secretName: secret.secretName,
secretKeyCiphertext: secretKeyEncrypted.ciphertext,
secretKeyIV: secretKeyEncrypted.iv,
secretKeyTag: secretKeyEncrypted.tag,
secretValueCiphertext: secretValueEncrypted.ciphertext,
secretValueIV: secretValueEncrypted.iv,
secretValueTag: secretValueEncrypted.tag,
secretCommentCiphertext: secretCommentEncrypted.ciphertext,
secretCommentIV: secretCommentEncrypted.iv,
secretCommentTag: secretCommentEncrypted.tag,
skipMultilineEncoding: secret.skipMultilineEncoding,
tags: secret.tags
};
})
);
// get all tags
const tagIds = inputSecrets.flatMap(({ tags = [] }) => tags);
const tags = tagIds.length ? await secretTagDAL.findManyTagsById(projectId, tagIds) : [];
if (tags.length !== tagIds.length) throw new BadRequestError({ message: "Tag not found" });
const newSecrets = await secretDAL.transaction(async (tx) =>
fnSecretBulkInsert({
@@ -703,56 +716,35 @@ export const updateManySecretsRawFnFactory = ({
userId
});
const inputSecrets = await Promise.all(
secrets.map(async (secret) => {
if (secret.newSecretName === "") {
throw new BadRequestError({ message: "New secret name cannot be empty" });
}
const inputSecrets = secrets.map((secret) => {
if (secret.newSecretName === "") {
throw new BadRequestError({ message: "New secret name cannot be empty" });
}
const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey);
const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey);
const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey);
const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey);
const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey);
const secretReferences = getAllNestedSecretReferences(secret.secretValue || "");
const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey);
if (secret.type === SecretType.Personal) {
if (!userId) throw new BadRequestError({ message: "Missing user id for personal secret" });
const sharedExist = await secretDAL.findOne({
secretBlindIndex: keyName2BlindIndex[secret.secretName],
folderId,
type: SecretType.Shared
});
if (!sharedExist)
throw new BadRequestError({
message: "Failed to update personal secret override for no corresponding shared secret"
});
if (secret.newSecretName)
throw new BadRequestError({ message: "Personal secret cannot change the key name" });
}
const tags = secret.tags ? await secretTagDAL.findManyTagsById(projectId, secret.tags) : [];
if ((secret.tags || []).length !== tags.length) throw new BadRequestError({ message: "Tag not found" });
return {
type: secret.type,
userId: secret.type === SecretType.Personal ? userId : null,
secretName: secret.secretName,
newSecretName: secret.newSecretName,
secretKeyCiphertext: secretKeyEncrypted.ciphertext,
secretKeyIV: secretKeyEncrypted.iv,
secretKeyTag: secretKeyEncrypted.tag,
secretValueCiphertext: secretValueEncrypted.ciphertext,
secretValueIV: secretValueEncrypted.iv,
secretValueTag: secretValueEncrypted.tag,
secretCommentCiphertext: secretCommentEncrypted.ciphertext,
secretCommentIV: secretCommentEncrypted.iv,
secretCommentTag: secretCommentEncrypted.tag,
skipMultilineEncoding: secret.skipMultilineEncoding,
tags: secret.tags
};
})
);
return {
type: secret.type,
userId: secret.type === SecretType.Personal ? userId : null,
secretName: secret.secretName,
newSecretName: secret.newSecretName,
secretKeyCiphertext: secretKeyEncrypted.ciphertext,
secretKeyIV: secretKeyEncrypted.iv,
secretKeyTag: secretKeyEncrypted.tag,
secretValueCiphertext: secretValueEncrypted.ciphertext,
secretValueIV: secretValueEncrypted.iv,
secretValueTag: secretValueEncrypted.tag,
secretCommentCiphertext: secretCommentEncrypted.ciphertext,
secretCommentIV: secretCommentEncrypted.iv,
secretCommentTag: secretCommentEncrypted.tag,
skipMultilineEncoding: secret.skipMultilineEncoding,
tags: secret.tags,
references: secretReferences
};
});
const tagIds = inputSecrets.flatMap(({ tags = [] }) => tags);
const tags = tagIds.length ? await secretTagDAL.findManyTagsById(projectId, tagIds) : [];
backend/src/services/secret/secret-queue.ts
+73 -18
View File
@@ -59,6 +59,7 @@ export type TGetSecrets = {
};
const MAX_SYNC_SECRET_DEPTH = 5;
const uniqueIntegrationKey = (environment: string, secretPath: string) => `integration-${environment}-${secretPath}`;
export const secretQueueFactory = ({
queueService,
@@ -102,28 +103,35 @@ export const secretQueueFactory = ({
folderDAL
});
const syncIntegrations = async (dto: TGetSecrets) => {
const syncIntegrations = async (dto: TGetSecrets & { deDupeQueue?: Record<string, boolean> }) => {
await queueService.queue(QueueName.IntegrationSync, QueueJobs.IntegrationSync, dto, {
attempts: 5,
attempts: 3,
delay: 1000,
backoff: {
type: "exponential",
delay: 3000
},
removeOnComplete: true,
removeOnFail: {
count: 5 // keep the most recent jobs
}
removeOnFail: true
});
};
const syncSecrets = async (dto: TGetSecrets & { depth?: number }) => {
const syncSecrets = async ({
deDupeQueue = {},
...dto
}: TGetSecrets & { depth?: number; deDupeQueue?: Record<string, boolean> }) => {
const deDuplicationKey = uniqueIntegrationKey(dto.environment, dto.secretPath);
if (deDupeQueue?.[deDuplicationKey]) {
return;
}
// eslint-disable-next-line
deDupeQueue[deDuplicationKey] = true;
logger.info(
`syncSecrets: syncing project secrets where [projectId=${dto.projectId}] [environment=${dto.environment}] [path=${dto.secretPath}]`
);
await queueService.queue(QueueName.SecretWebhook, QueueJobs.SecWebhook, dto, {
jobId: `secret-webhook-${dto.environment}-${dto.projectId}-${dto.secretPath}`,
removeOnFail: { count: 5 },
removeOnFail: true,
removeOnComplete: true,
delay: 1000,
attempts: 5,
@@ -132,7 +140,7 @@ export const secretQueueFactory = ({
delay: 3000
}
});
await syncIntegrations(dto);
await syncIntegrations({ ...dto, deDupeQueue });
};
const removeSecretReminder = async (dto: TRemoveSecretReminderDTO) => {
@@ -326,7 +334,7 @@ export const secretQueueFactory = ({
};
queueService.start(QueueName.IntegrationSync, async (job) => {
const { environment, projectId, secretPath, depth = 1 } = job.data;
const { environment, projectId, secretPath, depth = 1, deDupeQueue = {} } = job.data;
const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
if (!folder) {
@@ -349,21 +357,68 @@ export const secretQueueFactory = ({
const importedFolderIds = unique(imports, (i) => i.folderId).map(({ folderId }) => folderId);
const importedFolders = await folderDAL.findSecretPathByFolderIds(projectId, importedFolderIds);
const foldersGroupedById = groupBy(importedFolders, (i) => i.child || i.id);
logger.info(
`getIntegrationSecrets: Syncing secret due to link change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}] [secretPath=${job.data.secretPath}] [depth=${depth}]`
);
await Promise.all(
imports
.filter(({ folderId }) => Boolean(foldersGroupedById[folderId][0].path))
.map(({ folderId }) => {
const syncDto = {
// filter out already synced ones
.filter(
({ folderId }) =>
!deDupeQueue[
uniqueIntegrationKey(
foldersGroupedById[folderId][0].environmentSlug,
foldersGroupedById[folderId][0].path
)
]
)
.map(({ folderId }) =>
syncSecrets({
depth: depth + 1,
projectId,
secretPath: foldersGroupedById[folderId][0].path,
environment: foldersGroupedById[folderId][0].environmentSlug
};
logger.info(
`getIntegrationSecrets: Syncing secret due to link change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}] [secretPath=${job.data.secretPath}] [depth=${depth}]`
);
return syncSecrets(syncDto);
})
environment: foldersGroupedById[folderId][0].environmentSlug,
deDupeQueue
})
)
);
}
const secretReferences = await secretDAL.findReferencedSecretReferences(
projectId,
folder.environment.slug,
secretPath
);
if (secretReferences.length) {
const referencedFolderIds = unique(secretReferences, (i) => i.folderId).map(({ folderId }) => folderId);
const referencedFolders = await folderDAL.findSecretPathByFolderIds(projectId, referencedFolderIds);
const referencedFoldersGroupedById = groupBy(referencedFolders, (i) => i.child || i.id);
logger.info(
`getIntegrationSecrets: Syncing secret due to reference change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}] [secretPath=${job.data.secretPath}] [depth=${depth}]`
);
await Promise.all(
secretReferences
.filter(({ folderId }) => Boolean(referencedFoldersGroupedById[folderId][0].path))
// filter out already synced ones
.filter(
({ folderId }) =>
!deDupeQueue[
uniqueIntegrationKey(
referencedFoldersGroupedById[folderId][0].environmentSlug,
referencedFoldersGroupedById[folderId][0].path
)
]
)
.map(({ folderId }) =>
syncSecrets({
depth: depth + 1,
projectId,
secretPath: referencedFoldersGroupedById[folderId][0].path,
environment: referencedFoldersGroupedById[folderId][0].environmentSlug,
deDupeQueue
})
)
);
}
} else {
backend/src/services/secret/secret-service.ts
+106 -6
View File
@@ -2,12 +2,22 @@
/* eslint-disable no-await-in-loop */
import { ForbiddenError, subject } from "@casl/ability";
import { SecretEncryptionAlgo, SecretKeyEncoding, SecretsSchema, SecretType } from "@app/db/schemas";
import {
ProjectMembershipRole,
SecretEncryptionAlgo,
SecretKeyEncoding,
SecretsSchema,
SecretType
} from "@app/db/schemas";
import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission";
import { TSecretSnapshotServiceFactory } from "@app/ee/services/secret-snapshot/secret-snapshot-service";
import { getConfig } from "@app/lib/config/env";
import { buildSecretBlindIndexFromName, encryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
import {
buildSecretBlindIndexFromName,
decryptSymmetric128BitHexKeyUTF8,
encryptSymmetric128BitHexKeyUTF8
} from "@app/lib/crypto";
import { BadRequestError } from "@app/lib/errors";
import { groupBy, pick } from "@app/lib/fn";
import { logger } from "@app/lib/logger";
@@ -27,12 +37,14 @@ import {
fnSecretBlindIndexCheck,
fnSecretBulkInsert,
fnSecretBulkUpdate,
getAllNestedSecretReferences,
interpolateSecrets,
recursivelyGetSecretPaths
} from "./secret-fns";
import { TSecretQueueFactory } from "./secret-queue";
import {
TAttachSecretTagsDTO,
TBackFillSecretReferencesDTO,
TCreateBulkSecretDTO,
TCreateManySecretRawDTO,
TCreateSecretDTO,
@@ -91,6 +103,22 @@ export const secretServiceFactory = ({
secretImportDAL,
secretVersionTagDAL
}: TSecretServiceFactoryDep) => {
const getSecretReference = async (projectId: string) => {
// if bot key missing means e2e still exist
const botKey = await projectBotService.getBotKey(projectId).catch(() => null);
return (el: { ciphertext?: string; iv: string; tag: string }) =>
botKey
? getAllNestedSecretReferences(
decryptSymmetric128BitHexKeyUTF8({
ciphertext: el.ciphertext || "",
iv: el.iv,
tag: el.tag,
key: botKey
})
)
: undefined;
};
// utility function to get secret blind index data
const interalGenSecBlindIndexByName = async (projectId: string, secretName: string) => {
const appCfg = getConfig();
@@ -225,6 +253,7 @@ export const secretServiceFactory = ({
if ((inputSecret.tags || []).length !== tags.length) throw new BadRequestError({ message: "Tag not found" });
const { secretName, type, ...el } = inputSecret;
const references = await getSecretReference(projectId);
const secret = await secretDAL.transaction((tx) =>
fnSecretBulkInsert({
folderId,
@@ -237,7 +266,12 @@ export const secretServiceFactory = ({
userId: inputSecret.type === SecretType.Personal ? actorId : null,
algorithm: SecretEncryptionAlgo.AES_256_GCM,
keyEncoding: SecretKeyEncoding.UTF8,
tags: inputSecret.tags
tags: inputSecret.tags,
references: references({
ciphertext: inputSecret.secretValueCiphertext,
iv: inputSecret.secretValueIV,
tag: inputSecret.secretValueTag
})
}
],
secretDAL,
@@ -335,6 +369,7 @@ export const secretServiceFactory = ({
const { secretName, ...el } = inputSecret;
const references = await getSecretReference(projectId);
const updatedSecret = await secretDAL.transaction(async (tx) =>
fnSecretBulkUpdate({
folderId,
@@ -360,7 +395,12 @@ export const secretServiceFactory = ({
"secretReminderRepeatDays",
"tags"
]),
secretBlindIndex: newSecretNameBlindIndex || keyName2BlindIndex[secretName]
secretBlindIndex: newSecretNameBlindIndex || keyName2BlindIndex[secretName],
references: references({
ciphertext: inputSecret.secretValueCiphertext,
iv: inputSecret.secretValueIV,
tag: inputSecret.secretValueTag
})
}
}
],
@@ -700,6 +740,7 @@ export const secretServiceFactory = ({
const tags = tagIds.length ? await secretTagDAL.findManyTagsById(projectId, tagIds) : [];
if (tags.length !== tagIds.length) throw new BadRequestError({ message: "Tag not found" });
const references = await getSecretReference(projectId);
const newSecrets = await secretDAL.transaction(async (tx) =>
fnSecretBulkInsert({
inputSecrets: inputSecrets.map(({ secretName, ...el }) => ({
@@ -708,7 +749,12 @@ export const secretServiceFactory = ({
secretBlindIndex: keyName2BlindIndex[secretName],
type: SecretType.Shared,
algorithm: SecretEncryptionAlgo.AES_256_GCM,
keyEncoding: SecretKeyEncoding.UTF8
keyEncoding: SecretKeyEncoding.UTF8,
references: references({
ciphertext: el.secretValueCiphertext,
iv: el.secretValueIV,
tag: el.secretValueTag
})
})),
folderId,
secretDAL,
@@ -783,6 +829,8 @@ export const secretServiceFactory = ({
const tagIds = inputSecrets.flatMap(({ tags = [] }) => tags);
const tags = tagIds.length ? await secretTagDAL.findManyTagsById(projectId, tagIds) : [];
if (tagIds.length !== tags.length) throw new BadRequestError({ message: "Tag not found" });
const references = await getSecretReference(projectId);
const secrets = await secretDAL.transaction(async (tx) =>
fnSecretBulkUpdate({
folderId,
@@ -799,7 +847,15 @@ export const secretServiceFactory = ({
? newKeyName2BlindIndex[newSecretName]
: keyName2BlindIndex[secretName],
algorithm: SecretEncryptionAlgo.AES_256_GCM,
keyEncoding: SecretKeyEncoding.UTF8
keyEncoding: SecretKeyEncoding.UTF8,
references:
el.secretValueIV && el.secretValueTag
? references({
ciphertext: el.secretValueCiphertext,
iv: el.secretValueIV,
tag: el.secretValueTag
})
: undefined
}
})),
secretDAL,
@@ -1488,6 +1544,49 @@ export const secretServiceFactory = ({
};
};
// this is a backfilling API for secret references
// what it does is it will go through all the secret values and parse all references
// populate the secret reference to do sync integrations
const backfillSecretReferences = async ({
projectId,
actor,
actorId,
actorOrgId,
actorAuthMethod
}: TBackFillSecretReferencesDTO) => {
const { hasRole } = await permissionService.getProjectPermission(
actor,
actorId,
projectId,
actorAuthMethod,
actorOrgId
);
if (!hasRole(ProjectMembershipRole.Admin)) throw new BadRequestError({ message: "Only admins are allowed" });
const botKey = await projectBotService.getBotKey(projectId);
if (!botKey) throw new BadRequestError({ message: "Kindly upgrade your project", name: "bot_not_found_error" });
await secretDAL.transaction(async (tx) => {
const secrets = await secretDAL.findAllProjectSecretValues(projectId, tx);
await secretDAL.upsertSecretReferences(
secrets.map(({ id, secretValueCiphertext, secretValueIV, secretValueTag }) => ({
secretId: id,
references: getAllNestedSecretReferences(
decryptSymmetric128BitHexKeyUTF8({
ciphertext: secretValueCiphertext,
iv: secretValueIV,
tag: secretValueTag,
key: botKey
})
)
}))
);
});
return { message: "Successfully backfilled secret references." };
};
return {
attachTags,
detachTags,
@@ -1508,6 +1607,7 @@ export const secretServiceFactory = ({
updateManySecretsRaw,
deleteManySecretsRaw,
getSecretVersions,
backfillSecretReferences,
// external services function
fnSecretBulkDelete,
fnSecretBulkUpdate,
backend/src/services/secret/secret-types.ts
+11 -4
View File
@@ -223,11 +223,13 @@ export type TGetSecretVersionsDTO = Omit<TProjectPermission, "projectId"> & {
secretId: string;
};
export type TSecretReference = { environment: string; secretPath: string };
export type TFnSecretBulkInsert = {
folderId: string;
tx?: Knex;
inputSecrets: Array<Omit<TSecretsInsert, "folderId"> & { tags?: string[] }>;
secretDAL: Pick<TSecretDALFactory, "insertMany">;
inputSecrets: Array<Omit<TSecretsInsert, "folderId"> & { tags?: string[]; references?: TSecretReference[] }>;
secretDAL: Pick<TSecretDALFactory, "insertMany" | "upsertSecretReferences">;
secretVersionDAL: Pick<TSecretVersionDALFactory, "insertMany">;
secretTagDAL: Pick<TSecretTagDALFactory, "saveTagsToSecret">;
secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
@@ -236,8 +238,11 @@ export type TFnSecretBulkInsert = {
export type TFnSecretBulkUpdate = {
folderId: string;
projectId: string;
inputSecrets: { filter: Partial<TSecrets>; data: TSecretsUpdate & { tags?: string[] } }[];
secretDAL: Pick<TSecretDALFactory, "bulkUpdate">;
inputSecrets: {
filter: Partial<TSecrets>;
data: TSecretsUpdate & { tags?: string[]; references?: TSecretReference[] };
}[];
secretDAL: Pick<TSecretDALFactory, "bulkUpdate" | "upsertSecretReferences">;
secretVersionDAL: Pick<TSecretVersionDALFactory, "insertMany">;
secretTagDAL: Pick<TSecretTagDALFactory, "saveTagsToSecret" | "deleteTagsManySecret">;
secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
@@ -294,6 +299,8 @@ export type TRemoveSecretReminderDTO = {
repeatDays: number;
};
export type TBackFillSecretReferencesDTO = TProjectPermission;
// ---
export type TCreateManySecretsRawFnFactory = {