awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-10-15 10:35:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into feat/native-slack-integration

Browse Source
This commit is contained in:
Sheen Capadngan
2024-09-04 23:10:15 +08:00
parent 956719f797 04cbbccd25
commit 27852607d1
93 changed files with 6556 additions and 1663 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/services/identity-access-token/identity-access-token-dal.ts
+5 -1
View File
@@ -4,6 +4,8 @@ import { TDbClient } from "@app/db";
import { IdentityAuthMethod, TableName, TIdentityAccessTokens } from "@app/db/schemas";
import { DatabaseError } from "@app/lib/errors";
import { ormify, selectAllTableCols } from "@app/lib/knex";
import { logger } from "@app/lib/logger";
import { QueueName } from "@app/queue";
export type TIdentityAccessTokenDALFactory = ReturnType<typeof identityAccessTokenDALFactory>;
@@ -95,6 +97,7 @@ export const identityAccessTokenDALFactory = (db: TDbClient) => {
};
const removeExpiredTokens = async (tx?: Knex) => {
logger.info(`${QueueName.DailyResourceCleanUp}: remove expired access token started`);
try {
const docs = (tx || db)(TableName.IdentityAccessToken)
.where({
@@ -131,7 +134,8 @@ export const identityAccessTokenDALFactory = (db: TDbClient) => {
});
})
.delete();
return await docs;
await docs;
logger.info(`${QueueName.DailyResourceCleanUp}: remove expired access token completed`);
} catch (error) {
throw new DatabaseError({ error, name: "IdentityAccessTokenPrune" });
}
backend/src/services/identity-ua/identity-ua-client-secret-dal.ts
+6 -1
View File
@@ -5,6 +5,7 @@ import { TableName } from "@app/db/schemas";
import { DatabaseError } from "@app/lib/errors";
import { ormify } from "@app/lib/knex";
import { logger } from "@app/lib/logger";
import { QueueName } from "@app/queue";
export type TIdentityUaClientSecretDALFactory = ReturnType<typeof identityUaClientSecretDALFactory>;
@@ -30,7 +31,9 @@ export const identityUaClientSecretDALFactory = (db: TDbClient) => {
let deletedClientSecret: { id: string }[] = [];
let numberOfRetryOnFailure = 0;
let isRetrying = false;
logger.info(`${QueueName.DailyResourceCleanUp}: remove expired univesal auth client secret started`);
do {
try {
const findExpiredClientSecretQuery = (tx || db)(TableName.IdentityUaClientSecret)
@@ -71,7 +74,9 @@ export const identityUaClientSecretDALFactory = (db: TDbClient) => {
setTimeout(resolve, 10); // time to breathe for db
});
}
} while (deletedClientSecret.length > 0 || numberOfRetryOnFailure < MAX_RETRY_ON_FAILURE);
isRetrying = numberOfRetryOnFailure > 0;
} while (deletedClientSecret.length > 0 || (isRetrying && numberOfRetryOnFailure < MAX_RETRY_ON_FAILURE));
logger.info(`${QueueName.DailyResourceCleanUp}: remove expired univesal auth client secret completed`);
};
return { ...uaClientSecretOrm, incrementUsage, removeExpiredClientSecrets };
backend/src/services/integration-auth/integration-delete-secret.ts
+2
View File
@@ -30,6 +30,7 @@ const getIntegrationSecretsV2 = async (
environment: string;
folderId: string;
depth: number;
secretPath: string;
decryptor: (value: Buffer | null | undefined) => string;
},
secretV2BridgeDAL: Pick<TSecretV2BridgeDALFactory, "find" | "findByFolderId">,
@@ -306,6 +307,7 @@ export const deleteIntegrationSecrets = async ({
? await getIntegrationSecretsV2(
{
environment: integration.environment.id,
secretPath: integration.secretPath,
projectId: integration.projectId,
folderId: folder.id,
depth: 1,
backend/src/services/integration-auth/integration-sync-secret.ts
+5 -1
View File
@@ -1625,7 +1625,11 @@ const syncSecretsGitHub = async ({
await octokit.request("PUT /orgs/{org}/actions/secrets/{secret_name}", {
org: integration.owner as string,
secret_name: key,
visibility: "all",
visibility: metadata.githubVisibility ?? "all",
...(metadata.githubVisibility === "selected" && {
// we need to map the githubVisibilityRepoIds to numbers
selected_repository_ids: metadata.githubVisibilityRepoIds?.map(Number) ?? []
}),
encrypted_value: encryptedSecret,
key_id: repoPublicKey.key_id
});
backend/src/services/integration/integration-schema.ts
+14 -1
View File
@@ -5,14 +5,18 @@ import { INTEGRATION } from "@app/lib/api-docs";
import { IntegrationMappingBehavior } from "../integration-auth/integration-list";
export const IntegrationMetadataSchema = z.object({
initialSyncBehavior: z.string().optional().describe(INTEGRATION.CREATE.metadata.initialSyncBehavoir),
secretPrefix: z.string().optional().describe(INTEGRATION.CREATE.metadata.secretPrefix),
secretSuffix: z.string().optional().describe(INTEGRATION.CREATE.metadata.secretSuffix),
initialSyncBehavior: z.string().optional().describe(INTEGRATION.CREATE.metadata.initialSyncBehavoir),
mappingBehavior: z
.nativeEnum(IntegrationMappingBehavior)
.optional()
.describe(INTEGRATION.CREATE.metadata.mappingBehavior),
shouldAutoRedeploy: z.boolean().optional().describe(INTEGRATION.CREATE.metadata.shouldAutoRedeploy),
secretGCPLabel: z
.object({
labelName: z.string(),
@@ -20,6 +24,7 @@ export const IntegrationMetadataSchema = z.object({
})
.optional()
.describe(INTEGRATION.CREATE.metadata.secretGCPLabel),
secretAWSTag: z
.array(
z.object({
@@ -29,7 +34,15 @@ export const IntegrationMetadataSchema = z.object({
)
.optional()
.describe(INTEGRATION.CREATE.metadata.secretAWSTag),
githubVisibility: z
.union([z.literal("selected"), z.literal("private"), z.literal("all")])
.optional()
.describe(INTEGRATION.CREATE.metadata.githubVisibility),
githubVisibilityRepoIds: z.array(z.string()).optional().describe(INTEGRATION.CREATE.metadata.githubVisibilityRepoIds),
kmsKeyId: z.string().optional().describe(INTEGRATION.CREATE.metadata.kmsKeyId),
shouldDisableDelete: z.boolean().optional().describe(INTEGRATION.CREATE.metadata.shouldDisableDelete),
shouldEnableDelete: z.boolean().optional().describe(INTEGRATION.CREATE.metadata.shouldEnableDelete),
shouldMaskSecrets: z.boolean().optional().describe(INTEGRATION.CREATE.metadata.shouldMaskSecrets),
backend/src/services/integration/integration-types.ts
+4
View File
@@ -27,6 +27,10 @@ export type TCreateIntegrationDTO = {
key: string;
value: string;
}[];
githubVisibility?: string;
githubVisibilityRepoIds?: string[];
kmsKeyId?: string;
shouldDisableDelete?: boolean;
shouldMaskSecrets?: boolean;
backend/src/services/org/org-dal.ts
+63
View File
@@ -12,6 +12,7 @@ import {
} from "@app/db/schemas";
import { DatabaseError } from "@app/lib/errors";
import { buildFindFilter, ormify, selectAllTableCols, TFindFilter, TFindOpt, withTransaction } from "@app/lib/knex";
import { generateKnexQueryFromScim } from "@app/lib/knex/scim";
export type TOrgDALFactory = ReturnType<typeof orgDALFactory>;
@@ -280,6 +281,67 @@ export const orgDALFactory = (db: TDbClient) => {
.select(
selectAllTableCols(TableName.OrgMembership),
db.ref("email").withSchema(TableName.Users),
db.ref("isEmailVerified").withSchema(TableName.Users),
db.ref("username").withSchema(TableName.Users),
db.ref("firstName").withSchema(TableName.Users),
db.ref("lastName").withSchema(TableName.Users),
db.ref("scimEnabled").withSchema(TableName.Organization),
db.ref("externalId").withSchema(TableName.UserAliases)
)
.where({ isGhost: false });
if (limit) void query.limit(limit);
if (offset) void query.offset(offset);
if (sort) {
void query.orderBy(sort.map(([column, order, nulls]) => ({ column: column as string, order, nulls })));
}
const res = await query;
return res;
} catch (error) {
throw new DatabaseError({ error, name: "Find one" });
}
};
const findMembershipWithScimFilter = async (
orgId: string,
scimFilter: string | undefined,
{ offset, limit, sort, tx }: TFindOpt<TOrgMemberships> = {}
) => {
try {
const query = (tx || db.replicaNode())(TableName.OrgMembership)
// eslint-disable-next-line
.where(`${TableName.OrgMembership}.orgId`, orgId)
.where((qb) => {
if (scimFilter) {
void generateKnexQueryFromScim(qb, scimFilter, (attrPath) => {
switch (attrPath) {
case "active":
return `${TableName.OrgMembership}.isActive`;
case "userName":
return `${TableName.UserAliases}.externalId`;
case "name.givenName":
return `${TableName.Users}.firstName`;
case "name.familyName":
return `${TableName.Users}.lastName`;
case "email.value":
return `${TableName.Users}.email`;
default:
return null;
}
});
}
})
.join(TableName.Users, `${TableName.Users}.id`, `${TableName.OrgMembership}.userId`)
.join(TableName.Organization, `${TableName.Organization}.id`, `${TableName.OrgMembership}.orgId`)
.leftJoin(TableName.UserAliases, function joinUserAlias() {
this.on(`${TableName.UserAliases}.userId`, "=", `${TableName.OrgMembership}.userId`)
.andOn(`${TableName.UserAliases}.orgId`, "=", `${TableName.OrgMembership}.orgId`)
.andOn(`${TableName.UserAliases}.aliasType`, "=", (tx || db).raw("?", ["saml"]));
})
.select(
selectAllTableCols(TableName.OrgMembership),
db.ref("email").withSchema(TableName.Users),
db.ref("isEmailVerified").withSchema(TableName.Users),
db.ref("username").withSchema(TableName.Users),
db.ref("firstName").withSchema(TableName.Users),
db.ref("lastName").withSchema(TableName.Users),
@@ -314,6 +376,7 @@ export const orgDALFactory = (db: TDbClient) => {
updateById,
deleteById,
findMembership,
findMembershipWithScimFilter,
createMembership,
updateMembershipById,
deleteMembershipById,
backend/src/services/secret-folder/secret-folder-version-dal.ts
+4
View File
@@ -4,6 +4,8 @@ import { TDbClient } from "@app/db";
import { TableName, TSecretFolderVersions } from "@app/db/schemas";
import { DatabaseError } from "@app/lib/errors";
import { ormify, selectAllTableCols } from "@app/lib/knex";
import { logger } from "@app/lib/logger";
import { QueueName } from "@app/queue";
export type TSecretFolderVersionDALFactory = ReturnType<typeof secretFolderVersionDALFactory>;
@@ -65,6 +67,7 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => {
};
const pruneExcessVersions = async () => {
logger.info(`${QueueName.DailyResourceCleanUp}: pruning secret folder versions started`);
try {
await db(TableName.SecretFolderVersion)
.with("folder_cte", (qb) => {
@@ -89,6 +92,7 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => {
name: "Secret Folder Version Prune"
});
}
logger.info(`${QueueName.DailyResourceCleanUp}: pruning secret folder versions completed`);
};
return { ...secretFolderVerOrm, findLatestFolderVersions, findLatestVersionByFolderId, pruneExcessVersions };
backend/src/services/secret-import/secret-import-fns.ts
+21 -23
View File
@@ -158,9 +158,12 @@ export const fnSecretsV2FromImports = async ({
depth?: number;
cyclicDetector?: Set<string>;
decryptor: (value?: Buffer | null) => string;
expandSecretReferences?: (
secrets: Record<string, { value?: string; comment?: string; skipMultilineEncoding?: boolean | null }>
) => Promise<Record<string, { value?: string; comment?: string; skipMultilineEncoding?: boolean | null }>>;
expandSecretReferences?: (inputSecret: {
value?: string;
skipMultilineEncoding?: boolean | null;
secretPath: string;
environment: string;
}) => Promise<string | undefined>;
}) => {
// avoid going more than a depth
if (depth >= LEVEL_BREAK) return [];
@@ -244,26 +247,21 @@ export const fnSecretsV2FromImports = async ({
});
if (expandSecretReferences) {
await Promise.all(
processedImports.map(async (processedImport) => {
const secretsGroupByKey = processedImport.secrets.reduce(
(acc, item) => {
acc[item.secretKey] = {
value: item.secretValue,
comment: item.secretComment,
skipMultilineEncoding: item.skipMultilineEncoding
};
return acc;
},
{} as Record<string, { value: string; comment?: string; skipMultilineEncoding?: boolean | null }>
);
// eslint-disable-next-line
await expandSecretReferences(secretsGroupByKey);
processedImport.secrets.forEach((decryptedSecret) => {
// eslint-disable-next-line no-param-reassign
decryptedSecret.secretValue = secretsGroupByKey[decryptedSecret.secretKey].value;
});
})
await Promise.allSettled(
processedImports.map((processedImport) =>
Promise.allSettled(
processedImport.secrets.map(async (decryptedSecret, index) => {
const expandedSecretValue = await expandSecretReferences({
value: decryptedSecret.secretValue,
secretPath: processedImport.secretPath,
environment: processedImport.environment,
skipMultilineEncoding: decryptedSecret.skipMultilineEncoding
});
// eslint-disable-next-line no-param-reassign
processedImport.secrets[index].secretValue = expandedSecretValue || "";
})
)
)
);
}
backend/src/services/secret-sharing/secret-sharing-dal.ts
+4
View File
@@ -4,6 +4,8 @@ import { TDbClient } from "@app/db";
import { TableName, TSecretSharing } from "@app/db/schemas";
import { DatabaseError } from "@app/lib/errors";
import { ormify, selectAllTableCols } from "@app/lib/knex";
import { logger } from "@app/lib/logger";
import { QueueName } from "@app/queue";
export type TSecretSharingDALFactory = ReturnType<typeof secretSharingDALFactory>;
@@ -30,6 +32,7 @@ export const secretSharingDALFactory = (db: TDbClient) => {
};
const pruneExpiredSharedSecrets = async (tx?: Knex) => {
logger.info(`${QueueName.DailyResourceCleanUp}: pruning expired shared secret started`);
try {
const today = new Date();
const docs = await (tx || db)(TableName.SecretSharing)
@@ -40,6 +43,7 @@ export const secretSharingDALFactory = (db: TDbClient) => {
tag: "",
iv: ""
});
logger.info(`${QueueName.DailyResourceCleanUp}: pruning expired shared secret completed`);
return docs;
} catch (error) {
throw new DatabaseError({ error, name: "pruneExpiredSharedSecrets" });
backend/src/services/secret-sharing/secret-sharing-service.ts
+47 -13
View File
@@ -1,3 +1,6 @@
import bcrypt from "bcrypt";
import { TSecretSharing } from "@app/db/schemas";
import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
import { BadRequestError, ForbiddenRequestError, NotFoundError, UnauthorizedError } from "@app/lib/errors";
import { SecretSharingAccessType } from "@app/lib/types";
@@ -36,6 +39,7 @@ export const secretSharingServiceFactory = ({
iv,
tag,
name,
password,
accessType,
expiresAt,
expiresAfterViews
@@ -60,8 +64,10 @@ export const secretSharingServiceFactory = ({
throw new BadRequestError({ message: "Shared secret value too long" });
}
const hashedPassword = password ? await bcrypt.hash(password, 10) : null;
const newSharedSecret = await secretSharingDAL.create({
name,
password: hashedPassword,
encryptedValue,
hashedHex,
iv,
@@ -77,6 +83,7 @@ export const secretSharingServiceFactory = ({
};
const createPublicSharedSecret = async ({
password,
encryptedValue,
hashedHex,
iv,
@@ -102,7 +109,9 @@ export const secretSharingServiceFactory = ({
throw new BadRequestError({ message: "Shared secret value too long" });
}
const hashedPassword = password ? await bcrypt.hash(password, 10) : null;
const newSharedSecret = await secretSharingDAL.create({
password: hashedPassword,
encryptedValue,
hashedHex,
iv,
@@ -111,6 +120,7 @@ export const secretSharingServiceFactory = ({
expiresAfterViews,
accessType
});
return { id: newSharedSecret.id };
};
@@ -152,7 +162,21 @@ export const secretSharingServiceFactory = ({
};
};
const getActiveSharedSecretById = async ({ sharedSecretId, hashedHex, orgId }: TGetActiveSharedSecretByIdDTO) => {
const $decrementSecretViewCount = async (sharedSecret: TSecretSharing, sharedSecretId: string) => {
const { expiresAfterViews } = sharedSecret;
if (expiresAfterViews) {
// decrement view count if view count expiry set
await secretSharingDAL.updateById(sharedSecretId, { $decr: { expiresAfterViews: 1 } });
}
await secretSharingDAL.updateById(sharedSecretId, {
lastViewedAt: new Date()
});
};
/** Get's passwordless secret. validates all secret's requested (must be fresh). */
const getSharedSecretById = async ({ sharedSecretId, hashedHex, orgId, password }: TGetActiveSharedSecretByIdDTO) => {
const sharedSecret = await secretSharingDAL.findOne({
id: sharedSecretId,
hashedHex
@@ -169,6 +193,8 @@ export const secretSharingServiceFactory = ({
if (accessType === SecretSharingAccessType.Organization && orgId !== sharedSecret.orgId)
throw new UnauthorizedError();
// all secrets pass through here, meaning we check if its expired first and then check if it needs verification
// or can be safely sent to the client.
if (expiresAt !== null && expiresAt < new Date()) {
// check lifetime expiry
await secretSharingDAL.softDeleteById(sharedSecretId);
@@ -185,21 +211,29 @@ export const secretSharingServiceFactory = ({
});
}
if (expiresAfterViews) {
// decrement view count if view count expiry set
await secretSharingDAL.updateById(sharedSecretId, { $decr: { expiresAfterViews: 1 } });
const isPasswordProtected = Boolean(sharedSecret.password);
const hasProvidedPassword = Boolean(password);
if (isPasswordProtected) {
if (hasProvidedPassword) {
const isMatch = await bcrypt.compare(password as string, sharedSecret.password as string);
if (!isMatch) throw new UnauthorizedError({ message: "Invalid credentials" });
} else {
return { isPasswordProtected };
}
}
await secretSharingDAL.updateById(sharedSecretId, {
lastViewedAt: new Date()
});
// decrement when we are sure the user will view secret.
await $decrementSecretViewCount(sharedSecret, sharedSecretId);
return {
...sharedSecret,
orgName:
sharedSecret.accessType === SecretSharingAccessType.Organization && orgId === sharedSecret.orgId
? orgName
: undefined
isPasswordProtected,
secret: {
...sharedSecret,
orgName:
sharedSecret.accessType === SecretSharingAccessType.Organization && orgId === sharedSecret.orgId
? orgName
: undefined
}
};
};
@@ -216,6 +250,6 @@ export const secretSharingServiceFactory = ({
createPublicSharedSecret,
getSharedSecrets,
deleteSharedSecretById,
getActiveSharedSecretById
getSharedSecretById
};
};
backend/src/services/secret-sharing/secret-sharing-types.ts
+7
View File
@@ -15,6 +15,7 @@ export type TSharedSecretPermission = {
orgId: string;
accessType?: SecretSharingAccessType;
name?: string;
password?: string;
};
export type TCreatePublicSharedSecretDTO = {
@@ -24,6 +25,7 @@ export type TCreatePublicSharedSecretDTO = {
tag: string;
expiresAt: string;
expiresAfterViews?: number;
password?: string;
accessType: SecretSharingAccessType;
};
@@ -31,6 +33,11 @@ export type TGetActiveSharedSecretByIdDTO = {
sharedSecretId: string;
hashedHex: string;
orgId?: string;
password?: string;
};
export type TValidateActiveSharedSecretDTO = TGetActiveSharedSecretByIdDTO & {
password: string;
};
export type TCreateSharedSecretDTO = TSharedSecretPermission & TCreatePublicSharedSecretDTO;
backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts
+81 -115
View File
@@ -377,150 +377,116 @@ type TInterpolateSecretArg = {
folderDAL: Pick<TSecretFolderDALFactory, "findBySecretPath">;
};
const MAX_SECRET_REFERENCE_DEPTH = 10;
export const expandSecretReferencesFactory = ({
projectId,
decryptSecretValue: decryptSecret,
secretDAL,
folderDAL
}: TInterpolateSecretArg) => {
const fetchSecretFactory = () => {
const secretCache: Record<string, Record<string, string>> = {};
const secretCache: Record<string, Record<string, string>> = {};
const getCacheUniqueKey = (environment: string, secretPath: string) => `${environment}-${secretPath}`;
return async (secRefEnv: string, secRefPath: string[], secRefKey: string) => {
const referredSecretPathURL = path.join("/", ...secRefPath);
const uniqueKey = `${secRefEnv}-${referredSecretPathURL}`;
const fetchSecret = async (environment: string, secretPath: string, secretKey: string) => {
const cacheKey = getCacheUniqueKey(environment, secretPath);
if (secretCache?.[uniqueKey]) {
return secretCache[uniqueKey][secRefKey];
}
if (secretCache?.[cacheKey]) {
return secretCache[cacheKey][secretKey] || "";
}
const folder = await folderDAL.findBySecretPath(projectId, secRefEnv, referredSecretPathURL);
if (!folder) return "";
const secrets = await secretDAL.findByFolderId(folder.id);
const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
if (!folder) return "";
const secrets = await secretDAL.findByFolderId(folder.id);
const decryptedSecret = secrets.reduce<Record<string, string>>((prev, secret) => {
// eslint-disable-next-line
prev[secret.key] = decryptSecret(secret.encryptedValue) || "";
return prev;
}, {});
const decryptedSecret = secrets.reduce<Record<string, string>>((prev, secret) => {
// eslint-disable-next-line no-param-reassign
prev[secret.key] = decryptSecret(secret.encryptedValue) || "";
return prev;
}, {});
secretCache[uniqueKey] = decryptedSecret;
secretCache[cacheKey] = decryptedSecret;
return secretCache[uniqueKey][secRefKey];
};
return secretCache[cacheKey][secretKey] || "";
};
const recursivelyExpandSecret = async (
expandedSec: Record<string, string | undefined>,
interpolatedSec: Record<string, string | undefined>,
fetchSecret: (env: string, secPath: string[], secKey: string) => Promise<string>,
recursionChainBreaker: Record<string, boolean>,
key: string
): Promise<string | undefined> => {
if (expandedSec?.[key] !== undefined) {
return expandedSec[key];
}
if (recursionChainBreaker?.[key]) {
return "";
}
// eslint-disable-next-line
recursionChainBreaker[key] = true;
const recursivelyExpandSecret = async (dto: { value?: string; secretPath: string; environment: string }) => {
if (!dto.value) return "";
let interpolatedValue = interpolatedSec[key];
if (!interpolatedValue) {
// eslint-disable-next-line no-console
console.error(`Couldn't find referenced value - ${key}`);
return "";
}
const stack = [{ ...dto, depth: 0 }];
let expandedValue = dto.value;
const refs = interpolatedValue.match(INTERPOLATION_SYNTAX_REG);
if (refs) {
for (const interpolationSyntax of refs) {
const interpolationKey = interpolationSyntax.slice(2, interpolationSyntax.length - 1);
const entities = interpolationKey.trim().split(".");
while (stack.length) {
const { value, secretPath, environment, depth } = stack.pop()!;
// eslint-disable-next-line no-continue
if (depth > MAX_SECRET_REFERENCE_DEPTH) continue;
const refs = value?.match(INTERPOLATION_SYNTAX_REG);
if (entities.length === 1) {
// eslint-disable-next-line
const val = await recursivelyExpandSecret(
expandedSec,
interpolatedSec,
fetchSecret,
recursionChainBreaker,
interpolationKey
);
if (val) {
interpolatedValue = interpolatedValue.replaceAll(interpolationSyntax, val);
}
// eslint-disable-next-line
continue;
}
if (refs) {
for (const interpolationSyntax of refs) {
const interpolationKey = interpolationSyntax.slice(2, interpolationSyntax.length - 1);
const entities = interpolationKey.trim().split(".");
if (entities.length > 1) {
const secRefEnv = entities[0];
const secRefPath = entities.slice(1, entities.length - 1);
const secRefKey = entities[entities.length - 1];
// eslint-disable-next-line no-continue
if (!entities.length) continue;
// eslint-disable-next-line
const val = await fetchSecret(secRefEnv, secRefPath, secRefKey);
if (val) {
interpolatedValue = interpolatedValue.replaceAll(interpolationSyntax, val);
if (entities.length === 1) {
const [secretKey] = entities;
// eslint-disable-next-line no-continue,no-await-in-loop
const referedValue = await fetchSecret(environment, secretPath, secretKey);
const cacheKey = getCacheUniqueKey(environment, secretPath);
secretCache[cacheKey][secretKey] = referedValue;
if (INTERPOLATION_SYNTAX_REG.test(referedValue)) {
stack.push({
value: referedValue,
secretPath,
environment,
depth: depth + 1
});
}
expandedValue = expandedValue.replaceAll(interpolationSyntax, referedValue);
} else {
const secretReferenceEnvironment = entities[0];
const secretReferencePath = path.join("/", ...entities.slice(1, entities.length - 1));
const secretReferenceKey = entities[entities.length - 1];
// eslint-disable-next-line no-await-in-loop
const referedValue = await fetchSecret(secretReferenceEnvironment, secretReferencePath, secretReferenceKey);
const cacheKey = getCacheUniqueKey(secretReferenceEnvironment, secretReferencePath);
secretCache[cacheKey][secretReferenceKey] = referedValue;
if (INTERPOLATION_SYNTAX_REG.test(referedValue)) {
stack.push({
value: referedValue,
secretPath: secretReferencePath,
environment: secretReferenceEnvironment,
depth: depth + 1
});
}
expandedValue = expandedValue.replaceAll(interpolationSyntax, referedValue);
}
}
}
}
// eslint-disable-next-line
expandedSec[key] = interpolatedValue;
return interpolatedValue;
return expandedValue;
};
const fetchSecret = fetchSecretFactory();
const expandSecrets = async (
inputSecrets: Record<string, { value?: string; comment?: string; skipMultilineEncoding?: boolean | null }>
) => {
const expandedSecrets: Record<string, string | undefined> = {};
const toBeExpandedSecrets: Record<string, string | undefined> = {};
const expandSecret = async (inputSecret: {
value?: string;
skipMultilineEncoding?: boolean | null;
secretPath: string;
environment: string;
}) => {
if (!inputSecret.value) return inputSecret.value;
Object.keys(inputSecrets).forEach((key) => {
if (inputSecrets[key].value?.match(INTERPOLATION_SYNTAX_REG)) {
toBeExpandedSecrets[key] = inputSecrets[key].value;
} else {
expandedSecrets[key] = inputSecrets[key].value;
}
});
const shouldExpand = Boolean(inputSecret.value?.match(INTERPOLATION_SYNTAX_REG));
if (!shouldExpand) return inputSecret.value;
for (const key of Object.keys(inputSecrets)) {
if (expandedSecrets?.[key]) {
// should not do multi line encoding if user has set it to skip
// eslint-disable-next-line
inputSecrets[key].value = inputSecrets[key].skipMultilineEncoding
? formatMultiValueEnv(expandedSecrets[key])
: expandedSecrets[key];
// eslint-disable-next-line
continue;
}
// this is to avoid recursion loop. So the graph should be direct graph rather than cyclic
// so for any recursion building if there is an entity two times same key meaning it will be looped
const recursionChainBreaker: Record<string, boolean> = {};
// eslint-disable-next-line
const expandedVal = await recursivelyExpandSecret(
expandedSecrets,
toBeExpandedSecrets,
fetchSecret,
recursionChainBreaker,
key
);
// eslint-disable-next-line
inputSecrets[key].value = inputSecrets[key].skipMultilineEncoding
? formatMultiValueEnv(expandedVal)
: expandedVal;
}
return inputSecrets;
const expandedSecretValue = await recursivelyExpandSecret(inputSecret);
return inputSecret.skipMultilineEncoding ? formatMultiValueEnv(expandedSecretValue) : expandedSecretValue;
};
return expandSecrets;
return expandSecret;
};
export const reshapeBridgeSecret = (
backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts
+23 -26
View File
@@ -521,27 +521,22 @@ export const secretV2BridgeServiceFactory = ({
if (shouldExpandSecretReferences) {
const secretsGroupByPath = groupBy(filteredSecrets, (i) => i.secretPath);
for (const secretPathKey in secretsGroupByPath) {
if (Object.hasOwn(secretsGroupByPath, secretPathKey)) {
const secretsGroupByKey = secretsGroupByPath[secretPathKey].reduce(
(acc, item) => {
acc[item.secretKey] = {
value: item.secretValue,
comment: item.secretComment,
skipMultilineEncoding: item.skipMultilineEncoding
};
return acc;
},
{} as Record<string, { value?: string; comment?: string; skipMultilineEncoding?: boolean | null }>
);
// eslint-disable-next-line
await expandSecretReferences(secretsGroupByKey);
secretsGroupByPath[secretPathKey].forEach((decryptedSecret) => {
// eslint-disable-next-line no-param-reassign
decryptedSecret.secretValue = secretsGroupByKey[decryptedSecret.secretKey].value || "";
});
}
}
await Promise.allSettled(
Object.keys(secretsGroupByPath).map((groupedPath) =>
Promise.allSettled(
secretsGroupByPath[groupedPath].map(async (decryptedSecret, index) => {
const expandedSecretValue = await expandSecretReferences({
value: decryptedSecret.secretValue,
secretPath: groupedPath,
environment,
skipMultilineEncoding: decryptedSecret.skipMultilineEncoding
});
// eslint-disable-next-line no-param-reassign
secretsGroupByPath[groupedPath][index].secretValue = expandedSecretValue || "";
})
)
)
);
}
if (!includeImports) {
@@ -693,12 +688,14 @@ export const secretV2BridgeServiceFactory = ({
? secretManagerDecryptor({ cipherTextBlob: secret.encryptedValue }).toString()
: "";
if (shouldExpandSecretReferences && secretValue) {
const secretReferenceExpandedRecord = {
[secret.key]: { value: secretValue }
};
// eslint-disable-next-line
await expandSecretReferences(secretReferenceExpandedRecord);
secretValue = secretReferenceExpandedRecord[secret.key].value;
const expandedSecretValue = await expandSecretReferences({
environment,
secretPath: path,
value: secretValue,
skipMultilineEncoding: secret.skipMultilineEncoding
});
secretValue = expandedSecretValue || "";
}
return reshapeBridgeSecret(projectId, environment, path, {
backend/src/services/secret-v2-bridge/secret-version-dal.ts
+4
View File
@@ -4,6 +4,8 @@ import { TDbClient } from "@app/db";
import { TableName, TSecretVersionsV2, TSecretVersionsV2Update } from "@app/db/schemas";
import { BadRequestError, DatabaseError } from "@app/lib/errors";
import { ormify, selectAllTableCols } from "@app/lib/knex";
import { logger } from "@app/lib/logger";
import { QueueName } from "@app/queue";
export type TSecretVersionV2DALFactory = ReturnType<typeof secretVersionV2BridgeDALFactory>;
@@ -87,6 +89,7 @@ export const secretVersionV2BridgeDALFactory = (db: TDbClient) => {
};
const pruneExcessVersions = async () => {
logger.info(`${QueueName.DailyResourceCleanUp}: pruning secret version v2 started`);
try {
await db(TableName.SecretVersionV2)
.with("version_cte", (qb) => {
@@ -112,6 +115,7 @@ export const secretVersionV2BridgeDALFactory = (db: TDbClient) => {
name: "Secret Version Prune"
});
}
logger.info(`${QueueName.DailyResourceCleanUp}: pruning secret version v2 completed`);
};
return {
backend/src/services/secret/secret-fns.ts
+98 -125
View File
@@ -196,6 +196,13 @@ export const recursivelyGetSecretPaths = ({
return getPaths;
};
// used to convert multi line ones to quotes ones with \n
const formatMultiValueEnv = (val?: string) => {
if (!val) return "";
if (!val.match("\n")) return val;
return `"${val.replace(/\n/g, "\\n")}"`;
};
type TInterpolateSecretArg = {
projectId: string;
secretEncKey: string;
@@ -203,162 +210,128 @@ type TInterpolateSecretArg = {
folderDAL: Pick<TSecretFolderDALFactory, "findBySecretPath">;
};
const MAX_SECRET_REFERENCE_DEPTH = 5;
const INTERPOLATION_SYNTAX_REG = /\${([^}]+)}/g;
export const interpolateSecrets = ({ projectId, secretEncKey, secretDAL, folderDAL }: TInterpolateSecretArg) => {
const fetchSecretsCrossEnv = () => {
const fetchCache: Record<string, Record<string, string>> = {};
const secretCache: Record<string, Record<string, string>> = {};
const getCacheUniqueKey = (environment: string, secretPath: string) => `${environment}-${secretPath}`;
return async (secRefEnv: string, secRefPath: string[], secRefKey: string) => {
const secRefPathUrl = path.join("/", ...secRefPath);
const uniqKey = `${secRefEnv}-${secRefPathUrl}`;
const fetchSecret = async (environment: string, secretPath: string, secretKey: string) => {
const cacheKey = getCacheUniqueKey(environment, secretPath);
const uniqKey = `${environment}-${cacheKey}`;
if (fetchCache?.[uniqKey]) {
return fetchCache[uniqKey][secRefKey];
}
if (secretCache?.[uniqKey]) {
return secretCache[uniqKey][secretKey] || "";
}
const folder = await folderDAL.findBySecretPath(projectId, secRefEnv, secRefPathUrl);
if (!folder) return "";
const secrets = await secretDAL.findByFolderId(folder.id);
const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
if (!folder) return "";
const secrets = await secretDAL.findByFolderId(folder.id);
const decryptedSec = secrets.reduce<Record<string, string>>((prev, secret) => {
const secretKey = decryptSymmetric128BitHexKeyUTF8({
ciphertext: secret.secretKeyCiphertext,
iv: secret.secretKeyIV,
tag: secret.secretKeyTag,
key: secretEncKey
});
const secretValue = decryptSymmetric128BitHexKeyUTF8({
ciphertext: secret.secretValueCiphertext,
iv: secret.secretValueIV,
tag: secret.secretValueTag,
key: secretEncKey
});
const decryptedSec = secrets.reduce<Record<string, string>>((prev, secret) => {
const decryptedSecretKey = decryptSymmetric128BitHexKeyUTF8({
ciphertext: secret.secretKeyCiphertext,
iv: secret.secretKeyIV,
tag: secret.secretKeyTag,
key: secretEncKey
});
const decryptedSecretValue = decryptSymmetric128BitHexKeyUTF8({
ciphertext: secret.secretValueCiphertext,
iv: secret.secretValueIV,
tag: secret.secretValueTag,
key: secretEncKey
});
// eslint-disable-next-line
prev[secretKey] = secretValue;
return prev;
}, {});
// eslint-disable-next-line
prev[decryptedSecretKey] = decryptedSecretValue;
return prev;
}, {});
fetchCache[uniqKey] = decryptedSec;
secretCache[uniqKey] = decryptedSec;
return fetchCache[uniqKey][secRefKey];
};
return secretCache[uniqKey][secretKey] || "";
};
const recursivelyExpandSecret = async (
expandedSec: Record<string, string>,
interpolatedSec: Record<string, string>,
fetchCrossEnv: (env: string, secPath: string[], secKey: string) => Promise<string>,
recursionChainBreaker: Record<string, boolean>,
key: string
) => {
if (expandedSec?.[key] !== undefined) {
return expandedSec[key];
}
if (recursionChainBreaker?.[key]) {
return "";
}
// eslint-disable-next-line
recursionChainBreaker[key] = true;
const recursivelyExpandSecret = async ({
value,
secretPath,
environment,
depth = 0
}: {
value?: string;
secretPath: string;
environment: string;
depth?: number;
}) => {
if (!value) return "";
if (depth > MAX_SECRET_REFERENCE_DEPTH) return "";
let interpolatedValue = interpolatedSec[key];
if (!interpolatedValue) {
// eslint-disable-next-line no-console
console.error(`Couldn't find referenced value - ${key}`);
return "";
}
const refs = interpolatedValue.match(INTERPOLATION_SYNTAX_REG);
const refs = value.match(INTERPOLATION_SYNTAX_REG);
let expandedValue = value;
if (refs) {
for (const interpolationSyntax of refs) {
const interpolationKey = interpolationSyntax.slice(2, interpolationSyntax.length - 1);
const entities = interpolationKey.trim().split(".");
if (entities.length === 1) {
const val = await recursivelyExpandSecret(
expandedSec,
interpolatedSec,
fetchCrossEnv,
recursionChainBreaker,
interpolationKey
);
if (val) {
interpolatedValue = interpolatedValue.replaceAll(interpolationSyntax, val);
}
const [secretKey] = entities;
// eslint-disable-next-line
continue;
let referenceValue = await fetchSecret(environment, secretPath, secretKey);
if (INTERPOLATION_SYNTAX_REG.test(referenceValue)) {
// eslint-disable-next-line
referenceValue = await recursivelyExpandSecret({
environment,
secretPath,
value: referenceValue,
depth: depth + 1
});
}
const cacheKey = getCacheUniqueKey(environment, secretPath);
secretCache[cacheKey][secretKey] = referenceValue;
expandedValue = expandedValue.replaceAll(interpolationSyntax, referenceValue);
}
if (entities.length > 1) {
const secRefEnv = entities[0];
const secRefPath = entities.slice(1, entities.length - 1);
const secRefKey = entities[entities.length - 1];
const secretReferenceEnvironment = entities[0];
const secretReferencePath = path.join("/", ...entities.slice(1, entities.length - 1));
const secretReferenceKey = entities[entities.length - 1];
const val = await fetchCrossEnv(secRefEnv, secRefPath, secRefKey);
if (val) {
interpolatedValue = interpolatedValue.replaceAll(interpolationSyntax, val);
// eslint-disable-next-line
let referenceValue = await fetchSecret(secretReferenceEnvironment, secretReferencePath, secretReferenceKey);
if (INTERPOLATION_SYNTAX_REG.test(referenceValue)) {
// eslint-disable-next-line
referenceValue = await recursivelyExpandSecret({
environment: secretReferenceEnvironment,
secretPath: secretReferencePath,
value: referenceValue,
depth: depth + 1
});
}
const cacheKey = getCacheUniqueKey(secretReferenceEnvironment, secretReferencePath);
secretCache[cacheKey][secretReferenceKey] = referenceValue;
expandedValue = expandedValue.replaceAll(interpolationSyntax, referenceValue);
}
}
}
// eslint-disable-next-line
expandedSec[key] = interpolatedValue;
return interpolatedValue;
return expandedValue;
};
// used to convert multi line ones to quotes ones with \n
const formatMultiValueEnv = (val?: string) => {
if (!val) return "";
if (!val.match("\n")) return val;
return `"${val.replace(/\n/g, "\\n")}"`;
const expandSecret = async (inputSecret: {
value?: string;
skipMultilineEncoding?: boolean | null;
secretPath: string;
environment: string;
}) => {
if (!inputSecret.value) return inputSecret.value;
const shouldExpand = Boolean(inputSecret.value?.match(INTERPOLATION_SYNTAX_REG));
if (!shouldExpand) return inputSecret.value;
const expandedSecretValue = await recursivelyExpandSecret(inputSecret);
return inputSecret.skipMultilineEncoding ? formatMultiValueEnv(expandedSecretValue) : expandedSecretValue;
};
const expandSecrets = async (
secrets: Record<string, { value: string; comment?: string; skipMultilineEncoding?: boolean | null }>
) => {
const expandedSec: Record<string, string> = {};
const interpolatedSec: Record<string, string> = {};
const crossSecEnvFetch = fetchSecretsCrossEnv();
Object.keys(secrets).forEach((key) => {
if (secrets[key].value.match(INTERPOLATION_SYNTAX_REG)) {
interpolatedSec[key] = secrets[key].value;
} else {
expandedSec[key] = secrets[key].value;
}
});
for (const key of Object.keys(secrets)) {
if (expandedSec?.[key]) {
// should not do multi line encoding if user has set it to skip
// eslint-disable-next-line
secrets[key].value = secrets[key].skipMultilineEncoding
? formatMultiValueEnv(expandedSec[key])
: expandedSec[key];
// eslint-disable-next-line
continue;
}
// this is to avoid recursion loop. So the graph should be direct graph rather than cyclic
// so for any recursion building if there is an entity two times same key meaning it will be looped
const recursionChainBreaker: Record<string, boolean> = {};
const expandedVal = await recursivelyExpandSecret(
expandedSec,
interpolatedSec,
crossSecEnvFetch,
recursionChainBreaker,
key
);
// eslint-disable-next-line
secrets[key].value = secrets[key].skipMultilineEncoding ? formatMultiValueEnv(expandedVal) : expandedVal;
}
return secrets;
};
return expandSecrets;
return expandSecret;
};
export const decryptSecretRaw = (
backend/src/services/secret/secret-queue.ts
+70 -53
View File
@@ -258,6 +258,7 @@ export const secretQueueFactory = ({
const getIntegrationSecretsV2 = async (dto: {
projectId: string;
environment: string;
secretPath: string;
folderId: string;
depth: number;
decryptor: (value: Buffer | null | undefined) => string;
@@ -269,30 +270,36 @@ export const secretQueueFactory = ({
);
return content;
}
// process secrets in current folder
const secrets = await secretV2BridgeDAL.findByFolderId(dto.folderId);
secrets.forEach((secret) => {
const secretKey = secret.key;
const secretValue = dto.decryptor(secret.encryptedValue);
content[secretKey] = { value: secretValue };
if (secret.encryptedComment) {
const commentValue = dto.decryptor(secret.encryptedComment);
content[secretKey].comment = commentValue;
}
content[secretKey].skipMultilineEncoding = Boolean(secret.skipMultilineEncoding);
});
const expandSecretReferences = expandSecretReferencesFactory({
decryptSecretValue: dto.decryptor,
secretDAL: secretV2BridgeDAL,
folderDAL,
projectId: dto.projectId
});
// process secrets in current folder
const secrets = await secretV2BridgeDAL.findByFolderId(dto.folderId);
await Promise.allSettled(
secrets.map(async (secret) => {
const secretKey = secret.key;
const secretValue = dto.decryptor(secret.encryptedValue);
const expandedSecretValue = await expandSecretReferences({
environment: dto.environment,
secretPath: dto.secretPath,
skipMultilineEncoding: secret.skipMultilineEncoding,
value: secretValue
});
content[secretKey] = { value: expandedSecretValue || "" };
if (secret.encryptedComment) {
const commentValue = dto.decryptor(secret.encryptedComment);
content[secretKey].comment = commentValue;
}
content[secretKey].skipMultilineEncoding = Boolean(secret.skipMultilineEncoding);
})
);
await expandSecretReferences(content);
// check if current folder has any imports from other folders
const secretImports = await secretImportDAL.find({ folderId: dto.folderId, isReplication: false });
@@ -329,6 +336,7 @@ export const secretQueueFactory = ({
const getIntegrationSecrets = async (dto: {
projectId: string;
environment: string;
secretPath: string;
folderId: string;
key: string;
depth: number;
@@ -341,46 +349,52 @@ export const secretQueueFactory = ({
return content;
}
// process secrets in current folder
const secrets = await secretDAL.findByFolderId(dto.folderId);
secrets.forEach((secret) => {
const secretKey = decryptSymmetric128BitHexKeyUTF8({
ciphertext: secret.secretKeyCiphertext,
iv: secret.secretKeyIV,
tag: secret.secretKeyTag,
key: dto.key
});
const secretValue = decryptSymmetric128BitHexKeyUTF8({
ciphertext: secret.secretValueCiphertext,
iv: secret.secretValueIV,
tag: secret.secretValueTag,
key: dto.key
});
content[secretKey] = { value: secretValue };
if (secret.secretCommentCiphertext && secret.secretCommentIV && secret.secretCommentTag) {
const commentValue = decryptSymmetric128BitHexKeyUTF8({
ciphertext: secret.secretCommentCiphertext,
iv: secret.secretCommentIV,
tag: secret.secretCommentTag,
key: dto.key
});
content[secretKey].comment = commentValue;
}
content[secretKey].skipMultilineEncoding = Boolean(secret.skipMultilineEncoding);
});
const expandSecrets = interpolateSecrets({
const expandSecretReferences = interpolateSecrets({
projectId: dto.projectId,
secretEncKey: dto.key,
folderDAL,
secretDAL
});
await expandSecrets(content);
// process secrets in current folder
const secrets = await secretDAL.findByFolderId(dto.folderId);
await Promise.allSettled(
secrets.map(async (secret) => {
const secretKey = decryptSymmetric128BitHexKeyUTF8({
ciphertext: secret.secretKeyCiphertext,
iv: secret.secretKeyIV,
tag: secret.secretKeyTag,
key: dto.key
});
const secretValue = decryptSymmetric128BitHexKeyUTF8({
ciphertext: secret.secretValueCiphertext,
iv: secret.secretValueIV,
tag: secret.secretValueTag,
key: dto.key
});
const expandedSecretValue = await expandSecretReferences({
environment: dto.environment,
secretPath: dto.secretPath,
skipMultilineEncoding: secret.skipMultilineEncoding,
value: secretValue
});
content[secretKey] = { value: expandedSecretValue || "" };
if (secret.secretCommentCiphertext && secret.secretCommentIV && secret.secretCommentTag) {
const commentValue = decryptSymmetric128BitHexKeyUTF8({
ciphertext: secret.secretCommentCiphertext,
iv: secret.secretCommentIV,
tag: secret.secretCommentTag,
key: dto.key
});
content[secretKey].comment = commentValue;
}
content[secretKey].skipMultilineEncoding = Boolean(secret.skipMultilineEncoding);
})
);
// check if current folder has any imports from other folders
const secretImport = await secretImportDAL.find({ folderId: dto.folderId, isReplication: false });
@@ -404,7 +418,8 @@ export const secretQueueFactory = ({
projectId: dto.projectId,
folderId: folder.id,
key: dto.key,
depth: dto.depth + 1
depth: dto.depth + 1,
secretPath: dto.secretPath
});
// add the imported secrets to the current folder secrets
@@ -686,6 +701,7 @@ export const secretQueueFactory = ({
projectId,
folderId: folder.id,
depth: 1,
secretPath,
decryptor: (value) => (value ? secretManagerDecryptor({ cipherTextBlob: value }).toString() : "")
})
: await getIntegrationSecrets({
@@ -693,7 +709,8 @@ export const secretQueueFactory = ({
projectId,
folderId: folder.id,
key: botKey as string,
depth: 1
depth: 1,
secretPath
});
for (const integration of toBeSyncedIntegrations) {
backend/src/services/secret/secret-service.ts
+49 -97
View File
@@ -482,7 +482,7 @@ export const secretServiceFactory = ({
projectId,
environmentSlug: folder.environment.slug
});
// TODO(akhilmhdh-pg): licence check, posthog service and snapshot
// TODO(akhilmhdh-pg): license check, posthog service and snapshot
return { ...deletedSecret[0], _id: deletedSecret[0].id, workspace: projectId, environment, secretPath: path };
};
@@ -1047,74 +1047,47 @@ export const secretServiceFactory = ({
};
});
const expandSecret = interpolateSecrets({
folderDAL,
projectId,
secretDAL,
secretEncKey: botKey
});
if (expandSecretReferences) {
const expandSecrets = interpolateSecrets({
folderDAL,
projectId,
secretDAL,
secretEncKey: botKey
});
const batchSecretsExpand = async (
secretBatch: {
secretKey: string;
secretValue: string;
secretComment?: string;
secretPath: string;
skipMultilineEncoding: boolean | null | undefined;
}[]
) => {
// Group secrets by secretPath
const secretsByPath: Record<
string,
{
secretKey: string;
secretValue: string;
secretComment?: string;
skipMultilineEncoding: boolean | null | undefined;
}[]
> = {};
secretBatch.forEach((secret) => {
if (!secretsByPath[secret.secretPath]) {
secretsByPath[secret.secretPath] = [];
}
secretsByPath[secret.secretPath].push(secret);
});
// Expand secrets for each group
for (const secPath in secretsByPath) {
if (!Object.hasOwn(secretsByPath, path)) {
// eslint-disable-next-line no-continue
continue;
}
const secretRecord: Record<
string,
{ value: string; comment?: string; skipMultilineEncoding: boolean | null | undefined }
> = {};
secretsByPath[secPath].forEach((decryptedSecret) => {
secretRecord[decryptedSecret.secretKey] = {
value: decryptedSecret.secretValue,
comment: decryptedSecret.secretComment,
skipMultilineEncoding: decryptedSecret.skipMultilineEncoding
};
});
await expandSecrets(secretRecord);
secretsByPath[secPath].forEach((decryptedSecret) => {
// eslint-disable-next-line no-param-reassign
decryptedSecret.secretValue = secretRecord[decryptedSecret.secretKey].value;
});
}
};
// expand secrets
await batchSecretsExpand(filteredSecrets);
// expand imports by batch
await Promise.all(processedImports.map((processedImport) => batchSecretsExpand(processedImport.secrets)));
const secretsGroupByPath = groupBy(filteredSecrets, (i) => i.secretPath);
await Promise.allSettled(
Object.keys(secretsGroupByPath).map((groupedPath) =>
Promise.allSettled(
secretsGroupByPath[groupedPath].map(async (decryptedSecret, index) => {
const expandedSecretValue = await expandSecret({
value: decryptedSecret.secretValue,
secretPath: groupedPath,
environment,
skipMultilineEncoding: decryptedSecret.skipMultilineEncoding
});
// eslint-disable-next-line no-param-reassign
secretsGroupByPath[groupedPath][index].secretValue = expandedSecretValue || "";
})
)
)
);
await Promise.allSettled(
processedImports.map((processedImport) =>
Promise.allSettled(
processedImport.secrets.map(async (decryptedSecret, index) => {
const expandedSecretValue = await expandSecret({
value: decryptedSecret.secretValue,
secretPath: path,
environment,
skipMultilineEncoding: decryptedSecret.skipMultilineEncoding
});
// eslint-disable-next-line no-param-reassign
processedImport.secrets[index].secretValue = expandedSecretValue || "";
})
)
)
);
}
return {
@@ -1177,40 +1150,19 @@ export const secretServiceFactory = ({
const decryptedSecret = decryptSecretRaw(encryptedSecret, botKey);
if (expandSecretReferences) {
const expandSecrets = interpolateSecrets({
const expandSecret = interpolateSecrets({
folderDAL,
projectId,
secretDAL,
secretEncKey: botKey
});
const expandSingleSecret = async (secret: {
secretKey: string;
secretValue: string;
secretComment?: string;
secretPath: string;
skipMultilineEncoding: boolean | null | undefined;
}) => {
const secretRecord: Record<
string,
{ value: string; comment?: string; skipMultilineEncoding: boolean | null | undefined }
> = {
[secret.secretKey]: {
value: secret.secretValue,
comment: secret.secretComment,
skipMultilineEncoding: secret.skipMultilineEncoding
}
};
await expandSecrets(secretRecord);
// Update the secret with the expanded value
// eslint-disable-next-line no-param-reassign
secret.secretValue = secretRecord[secret.secretKey].value;
};
// Expand the secret
await expandSingleSecret(decryptedSecret);
const expandedSecretValue = await expandSecret({
environment,
secretPath: path,
value: decryptedSecret.secretValue,
skipMultilineEncoding: decryptedSecret.skipMultilineEncoding
});
decryptedSecret.secretValue = expandedSecretValue || "";
}
return decryptedSecret;
backend/src/services/secret/secret-version-dal.ts
+4
View File
@@ -4,6 +4,8 @@ import { TDbClient } from "@app/db";
import { TableName, TSecretVersions, TSecretVersionsUpdate } from "@app/db/schemas";
import { BadRequestError, DatabaseError } from "@app/lib/errors";
import { ormify, selectAllTableCols } from "@app/lib/knex";
import { logger } from "@app/lib/logger";
import { QueueName } from "@app/queue";
export type TSecretVersionDALFactory = ReturnType<typeof secretVersionDALFactory>;
@@ -112,6 +114,7 @@ export const secretVersionDALFactory = (db: TDbClient) => {
};
const pruneExcessVersions = async () => {
logger.info(`${QueueName.DailyResourceCleanUp}: pruning secret version v1 started`);
try {
await db(TableName.SecretVersion)
.with("version_cte", (qb) => {
@@ -137,6 +140,7 @@ export const secretVersionDALFactory = (db: TDbClient) => {
name: "Secret Version Prune"
});
}
logger.info(`${QueueName.DailyResourceCleanUp}: pruning secret version v1 completed`);
};
return {