awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-10-14 06:29:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin' into ldap

Browse Source
This commit is contained in:
Tuan Dang
2024-03-10 12:31:06 -07:00
parent 7594929042 f1b7653a52
commit feb7563eab
59 changed files with 2755 additions and 547 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/db/migrations/20240307232900_integration-last-used.ts
+15
View File
@@ -0,0 +1,15 @@
import { Knex } from "knex";
import { TableName } from "../schemas";
export async function up(knex: Knex): Promise<void> {
await knex.schema.alterTable(TableName.Integration, (t) => {
t.datetime("lastUsed");
});
}
export async function down(knex: Knex): Promise<void> {
await knex.schema.alterTable(TableName.Integration, (t) => {
t.dropColumn("lastUsed");
});
}
backend/src/db/schemas/integrations.ts
+2 -1
View File
@@ -27,7 +27,8 @@ export const IntegrationsSchema = z.object({
envId: z.string().uuid(),
secretPath: z.string().default("/"),
createdAt: z.date(),
updatedAt: z.date()
updatedAt: z.date(),
lastUsed: z.date().nullable().optional()
});
export type TIntegrations = z.infer<typeof IntegrationsSchema>;
backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts
+26 -8
View File
@@ -12,9 +12,11 @@ import { groupBy, pick, unique } from "@app/lib/fn";
import { alphaNumericNanoId } from "@app/lib/nanoid";
import { ActorType } from "@app/services/auth/auth-type";
import { TProjectDALFactory } from "@app/services/project/project-dal";
import { TSecretDALFactory } from "@app/services/secret/secret-dal";
import { TSecretQueueFactory } from "@app/services/secret/secret-queue";
import { TSecretServiceFactory } from "@app/services/secret/secret-service";
import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal";
import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal";
import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";
import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
@@ -44,10 +46,12 @@ type TSecretApprovalRequestServiceFactoryDep = {
secretApprovalRequestSecretDAL: TSecretApprovalRequestSecretDALFactory;
secretApprovalRequestReviewerDAL: TSecretApprovalRequestReviewerDALFactory;
folderDAL: Pick<TSecretFolderDALFactory, "findBySecretPath" | "findById" | "findSecretPathByFolderIds">;
secretTagDAL: Pick<TSecretTagDALFactory, "findManyTagsById">;
secretDAL: TSecretDALFactory;
secretTagDAL: Pick<TSecretTagDALFactory, "findManyTagsById" | "saveTagsToSecret" | "deleteTagsManySecret">;
secretBlindIndexDAL: Pick<TSecretBlindIndexDALFactory, "findOne">;
snapshotService: Pick<TSecretSnapshotServiceFactory, "performSnapshot">;
secretVersionDAL: Pick<TSecretVersionDALFactory, "findLatestVersionMany">;
secretVersionDAL: Pick<TSecretVersionDALFactory, "findLatestVersionMany" | "insertMany">;
secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
projectDAL: Pick<TProjectDALFactory, "checkProjectUpgradeStatus">;
secretService: Pick<
TSecretServiceFactory,
@@ -64,8 +68,10 @@ export type TSecretApprovalRequestServiceFactory = ReturnType<typeof secretAppro
export const secretApprovalRequestServiceFactory = ({
secretApprovalRequestDAL,
secretDAL,
folderDAL,
secretTagDAL,
secretVersionTagDAL,
secretApprovalRequestReviewerDAL,
secretApprovalRequestSecretDAL,
secretBlindIndexDAL,
@@ -335,7 +341,11 @@ export const secretApprovalRequestServiceFactory = ({
tags: el?.tags.map(({ id }) => id),
version: 1,
type: SecretType.Shared
}))
})),
secretDAL,
secretVersionDAL,
secretTagDAL,
secretVersionTagDAL
})
: [];
const updatedSecrets = secretUpdationCommits.length
@@ -367,7 +377,11 @@ export const secretApprovalRequestServiceFactory = ({
"secretBlindIndex"
])
}
}))
})),
secretDAL,
secretVersionDAL,
secretTagDAL,
secretVersionTagDAL
})
: [];
const deletedSecret = secretDeletionCommits.length
@@ -455,7 +469,8 @@ export const secretApprovalRequestServiceFactory = ({
inputSecrets: createdSecrets,
folderId,
isNew: true,
blindIndexCfg
blindIndexCfg,
secretDAL
});
commits.push(
@@ -482,7 +497,8 @@ export const secretApprovalRequestServiceFactory = ({
inputSecrets: updatedSecrets,
folderId,
isNew: false,
blindIndexCfg
blindIndexCfg,
secretDAL
});
// now find any secret that needs to update its name
@@ -492,7 +508,8 @@ export const secretApprovalRequestServiceFactory = ({
inputSecrets: nameUpdatedSecrets,
folderId,
isNew: true,
blindIndexCfg
blindIndexCfg,
secretDAL
});
const secsGroupedByBlindIndex = groupBy(secretsToBeUpdated, (el) => el.secretBlindIndex as string);
@@ -531,7 +548,8 @@ export const secretApprovalRequestServiceFactory = ({
inputSecrets: deletedSecrets,
folderId,
isNew: false,
blindIndexCfg
blindIndexCfg,
secretDAL
});
const secretsGroupedByBlindIndex = groupBy(secrets, (i) => {
if (!i.secretBlindIndex) throw new BadRequestError({ message: "Missing secret blind index" });
backend/src/server/routes/index.ts
+8 -1
View File
@@ -436,7 +436,12 @@ export const registerRoutes = async (
orgDAL,
projectMembershipDAL,
smtpService,
projectDAL
projectDAL,
projectBotDAL,
secretVersionDAL,
secretBlindIndexDAL,
secretTagDAL,
secretVersionTagDAL
});
const secretBlindIndexService = secretBlindIndexServiceFactory({
permissionService,
@@ -460,6 +465,7 @@ export const registerRoutes = async (
const sarService = secretApprovalRequestServiceFactory({
permissionService,
folderDAL,
secretDAL,
secretTagDAL,
secretApprovalRequestSecretDAL: sarSecretDAL,
secretApprovalRequestReviewerDAL: sarReviewerDAL,
@@ -469,6 +475,7 @@ export const registerRoutes = async (
secretApprovalRequestDAL,
secretService,
snapshotService,
secretVersionTagDAL,
secretQueueService
});
const secretRotationQueue = secretRotationQueueFactory({
backend/src/server/routes/v1/integration-auth-router.ts
+31
View File
@@ -513,6 +513,37 @@ export const registerIntegrationAuthRouter = async (server: FastifyZodProvider)
}
});
server.route({
url: "/:integrationAuthId/heroku/pipelines",
method: "GET",
onRequest: verifyAuth([AuthMode.JWT]),
schema: {
params: z.object({
integrationAuthId: z.string().trim()
}),
response: {
200: z.object({
pipelines: z
.object({
app: z.object({ appId: z.string() }),
stage: z.string(),
pipeline: z.object({ name: z.string(), pipelineId: z.string() })
})
.array()
})
}
},
handler: async (req) => {
const pipelines = await server.services.integrationAuth.getHerokuPipelines({
actorId: req.permission.id,
actor: req.permission.type,
actorOrgId: req.permission.orgId,
id: req.params.integrationAuthId
});
return { pipelines };
}
});
server.route({
url: "/:integrationAuthId/railway/environments",
method: "GET",
backend/src/server/routes/v1/integration-router.ts
+1
View File
@@ -32,6 +32,7 @@ export const registerIntegrationRouter = async (server: FastifyZodProvider) => {
.object({
secretPrefix: z.string().optional(),
secretSuffix: z.string().optional(),
initialSyncBehavior: z.string().optional(),
secretGCPLabel: z
.object({
labelName: z.string(),
backend/src/server/routes/v1/secret-folder-router.ts
+3 -3
View File
@@ -120,7 +120,7 @@ export const registerSecretFolderRouter = async (server: FastifyZodProvider) =>
});
server.route({
url: "/:folderId",
url: "/:folderIdOrName",
method: "DELETE",
schema: {
description: "Delete a folder",
@@ -131,7 +131,7 @@ export const registerSecretFolderRouter = async (server: FastifyZodProvider) =>
}
],
params: z.object({
folderId: z.string()
folderIdOrName: z.string()
}),
body: z.object({
workspaceId: z.string().trim(),
@@ -155,7 +155,7 @@ export const registerSecretFolderRouter = async (server: FastifyZodProvider) =>
actorOrgId: req.permission.orgId,
...req.body,
projectId: req.body.workspaceId,
id: req.params.folderId,
idOrName: req.params.folderIdOrName,
path
});
await server.services.auditLog.createAuditLog({
backend/src/services/integration-auth/integration-app-list.ts
+3 -2
View File
@@ -109,7 +109,7 @@ const getAppsGCPSecretManager = async ({ accessToken }: { accessToken: string })
*/
const getAppsHeroku = async ({ accessToken }: { accessToken: string }) => {
const res = (
await request.get<{ name: string }[]>(`${IntegrationUrls.HEROKU_API_URL}/apps`, {
await request.get<{ name: string; id: string }[]>(`${IntegrationUrls.HEROKU_API_URL}/apps`, {
headers: {
Accept: "application/vnd.heroku+json; version=3",
Authorization: `Bearer ${accessToken}`
@@ -118,7 +118,8 @@ const getAppsHeroku = async ({ accessToken }: { accessToken: string }) => {
).data;
const apps = res.map((a) => ({
name: a.name
name: a.name,
appId: a.id
}));
return apps;
backend/src/services/integration-auth/integration-auth-service.ts
+49 -25
View File
@@ -20,9 +20,11 @@ import {
TDeleteIntegrationAuthsDTO,
TGetIntegrationAuthDTO,
TGetIntegrationAuthTeamCityBuildConfigDTO,
THerokuPipelineCoupling,
TIntegrationAuthAppsDTO,
TIntegrationAuthBitbucketWorkspaceDTO,
TIntegrationAuthChecklyGroupsDTO,
TIntegrationAuthHerokuPipelinesDTO,
TIntegrationAuthNorthflankSecretGroupDTO,
TIntegrationAuthQoveryEnvironmentsDTO,
TIntegrationAuthQoveryOrgsDTO,
@@ -576,6 +578,38 @@ export const integrationAuthServiceFactory = ({
return [];
};
const getHerokuPipelines = async ({ id, actor, actorId, actorOrgId }: TIntegrationAuthHerokuPipelinesDTO) => {
const integrationAuth = await integrationAuthDAL.findById(id);
if (!integrationAuth) throw new BadRequestError({ message: "Failed to find integration" });
const { permission } = await permissionService.getProjectPermission(
actor,
actorId,
integrationAuth.projectId,
actorOrgId
);
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Integrations);
const botKey = await projectBotService.getBotKey(integrationAuth.projectId);
const { accessToken } = await getIntegrationAccessToken(integrationAuth, botKey);
const { data } = await request.get<THerokuPipelineCoupling[]>(
`${IntegrationUrls.HEROKU_API_URL}/pipeline-couplings`,
{
headers: {
Accept: "application/vnd.heroku+json; version=3",
Authorization: `Bearer ${accessToken}`,
"Accept-Encoding": "application/json"
}
}
);
return data.map(({ app: { id: appId }, stage, pipeline: { id: pipelineId, name } }) => ({
app: { appId },
stage,
pipeline: { pipelineId, name }
}));
};
const getRailwayEnvironments = async ({ id, actor, actorId, actorOrgId, appId }: TIntegrationAuthRailwayEnvDTO) => {
const integrationAuth = await integrationAuthDAL.findById(id);
if (!integrationAuth) throw new BadRequestError({ message: "Failed to find integration" });
@@ -649,33 +683,21 @@ export const integrationAuthServiceFactory = ({
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Integrations);
const botKey = await projectBotService.getBotKey(integrationAuth.projectId);
const { accessToken } = await getIntegrationAccessToken(integrationAuth, botKey);
if (appId) {
if (appId && appId !== "") {
const query = `
query project($id: String!) {
project(id: $id) {
createdAt
deletedAt
id
description
expiredAt
isPublic
isTempProject
isUpdatable
name
prDeploys
teamId
updatedAt
upstreamUrl
services {
edges {
node {
id
name
}
}
}
query project($id: String!) {
project(id: $id) {
services {
edges {
node {
id
name
}
}
}
}
}
}
`;
const variables = {
@@ -711,6 +733,7 @@ export const integrationAuthServiceFactory = ({
);
return edges.map(({ node: { name, id: serviceId } }) => ({ name, serviceId }));
}
return [];
};
@@ -915,6 +938,7 @@ export const integrationAuthServiceFactory = ({
getQoveryApps,
getQoveryEnvs,
getQoveryJobs,
getHerokuPipelines,
getQoveryOrgs,
getQoveryProjects,
getQoveryContainers,
backend/src/services/integration-auth/integration-auth-types.ts
+10
View File
@@ -62,6 +62,10 @@ export type TIntegrationAuthQoveryScopesDTO = {
environmentId: string;
} & Omit<TProjectPermission, "projectId">;
export type TIntegrationAuthHerokuPipelinesDTO = {
id: string;
} & Omit<TProjectPermission, "projectId">;
export type TIntegrationAuthRailwayEnvDTO = {
id: string;
appId: string;
@@ -129,6 +133,12 @@ export type TNorthflankSecretGroup = {
projectId: string;
};
export type THerokuPipelineCoupling = {
app: { id: string };
stage: string;
pipeline: { id: string; name: string };
};
export type TTeamCityBuildConfig = {
id: string;
name: string;
backend/src/services/integration-auth/integration-list.ts
+6
View File
@@ -37,6 +37,12 @@ export enum IntegrationType {
OAUTH2 = "oauth2"
}
export enum IntegrationInitialSyncBehavior {
OVERWRITE_TARGET = "overwrite-target",
PREFER_TARGET = "prefer-target",
PREFER_SOURCE = "prefer-source"
}
export enum IntegrationUrls {
// integration oauth endpoints
GCP_TOKEN_URL = "https://oauth2.googleapis.com/token",
backend/src/services/integration-auth/integration-sync-secret.ts
+118 -16
View File
@@ -20,11 +20,13 @@ import sodium from "libsodium-wrappers";
import isEqual from "lodash.isequal";
import { z } from "zod";
import { TIntegrationAuths, TIntegrations } from "@app/db/schemas";
import { SecretType, TIntegrationAuths, TIntegrations, TSecrets } from "@app/db/schemas";
import { request } from "@app/lib/config/request";
import { BadRequestError } from "@app/lib/errors";
import { TCreateManySecretsRawFn, TUpdateManySecretsRawFn } from "@app/services/secret/secret-types";
import { Integrations, IntegrationUrls } from "./integration-list";
import { TIntegrationDALFactory } from "../integration/integration-dal";
import { IntegrationInitialSyncBehavior, Integrations, IntegrationUrls } from "./integration-list";
const getSecretKeyValuePair = (secrets: Record<string, { value: string | null; comment?: string } | null>) =>
Object.keys(secrets).reduce<Record<string, string | null | undefined>>((prev, key) => {
@@ -582,11 +584,25 @@ const syncSecretsAWSSecretManager = async ({
* Sync/push [secrets] to Heroku app named [integration.app]
*/
const syncSecretsHeroku = async ({
createManySecretsRawFn,
updateManySecretsRawFn,
integrationDAL,
integration,
secrets,
accessToken
}: {
integration: TIntegrations;
createManySecretsRawFn: (params: TCreateManySecretsRawFn) => Promise<Array<TSecrets & { _id: string }>>;
updateManySecretsRawFn: (params: TUpdateManySecretsRawFn) => Promise<Array<TSecrets & { _id: string }>>;
integrationDAL: Pick<TIntegrationDALFactory, "updateById">;
integration: TIntegrations & {
projectId: string;
environment: {
id: string;
name: string;
slug: string;
};
secretPath: string;
};
secrets: Record<string, { value: string; comment?: string } | null>;
accessToken: string;
}) => {
@@ -600,12 +616,74 @@ const syncSecretsHeroku = async ({
})
).data;
const secretsToAdd: { [key: string]: string } = {};
const secretsToUpdate: { [key: string]: string } = {};
const metadata = z.record(z.any()).parse(integration.metadata);
Object.keys(herokuSecrets).forEach((key) => {
if (!(key in secrets)) {
secrets[key] = null;
}
if (!integration.lastUsed) {
// first time using integration
// -> apply initial sync behavior
switch (metadata.initialSyncBehavior) {
case IntegrationInitialSyncBehavior.OVERWRITE_TARGET: {
if (!(key in secrets)) secrets[key] = null;
break;
}
case IntegrationInitialSyncBehavior.PREFER_TARGET: {
if (!(key in secrets)) {
secretsToAdd[key] = herokuSecrets[key];
} else if (secrets[key]?.value !== herokuSecrets[key]) {
secretsToUpdate[key] = herokuSecrets[key];
}
secrets[key] = {
value: herokuSecrets[key]
};
break;
}
case IntegrationInitialSyncBehavior.PREFER_SOURCE: {
if (!(key in secrets)) {
secrets[key] = herokuSecrets[key];
secretsToAdd[key] = herokuSecrets[key];
}
break;
}
default: {
if (!(key in secrets)) secrets[key] = null;
break;
}
}
} else if (!(key in secrets)) secrets[key] = null;
});
if (Object.keys(secretsToAdd).length) {
await createManySecretsRawFn({
projectId: integration.projectId,
environment: integration.environment.slug,
path: integration.secretPath,
secrets: Object.keys(secretsToAdd).map((key) => ({
secretName: key,
secretValue: secretsToAdd[key],
type: SecretType.Shared,
secretComment: ""
}))
});
}
if (Object.keys(secretsToUpdate).length) {
await updateManySecretsRawFn({
projectId: integration.projectId,
environment: integration.environment.slug,
path: integration.secretPath,
secrets: Object.keys(secretsToUpdate).map((key) => ({
secretName: key,
secretValue: secretsToUpdate[key],
type: SecretType.Shared,
secretComment: ""
}))
});
}
await request.patch(
`${IntegrationUrls.HEROKU_API_URL}/apps/${integration.app}/config-vars`,
getSecretKeyValuePair(secrets),
@@ -617,6 +695,10 @@ const syncSecretsHeroku = async ({
}
}
);
await integrationDAL.updateById(integration.id, {
lastUsed: new Date()
});
};
/**
@@ -1204,21 +1286,21 @@ const syncSecretsRailway = async ({
}
`;
const input = {
projectId: integration.appId,
environmentId: integration.targetEnvironmentId,
...(integration.targetServiceId ? { serviceId: integration.targetServiceId } : {}),
replace: true,
variables: getSecretKeyValuePair(secrets)
const variables = {
input: {
projectId: integration.appId,
environmentId: integration.targetEnvironmentId,
...(integration.targetServiceId ? { serviceId: integration.targetServiceId } : {}),
replace: true,
variables: getSecretKeyValuePair(secrets)
}
};
await request.post(
IntegrationUrls.RAILWAY_API_URL,
{
query,
variables: {
input
}
variables
},
{
headers: {
@@ -2930,8 +3012,14 @@ const syncSecretsHasuraCloud = async ({
/**
* Sync/push [secrets] to [app] in integration named [integration]
*
* Do this in terms of DAL
*
*/
export const syncIntegrationSecrets = async ({
createManySecretsRawFn,
updateManySecretsRawFn,
integrationDAL,
integration,
integrationAuth,
secrets,
@@ -2939,7 +3027,18 @@ export const syncIntegrationSecrets = async ({
accessToken,
appendices
}: {
integration: TIntegrations;
createManySecretsRawFn: (params: TCreateManySecretsRawFn) => Promise<Array<TSecrets & { _id: string }>>;
updateManySecretsRawFn: (params: TUpdateManySecretsRawFn) => Promise<Array<TSecrets & { _id: string }>>;
integrationDAL: Pick<TIntegrationDALFactory, "updateById">;
integration: TIntegrations & {
projectId: string;
environment: {
id: string;
name: string;
slug: string;
};
secretPath: string;
};
integrationAuth: TIntegrationAuths;
secrets: Record<string, { value: string; comment?: string }>;
accessId: string | null;
@@ -2979,6 +3078,9 @@ export const syncIntegrationSecrets = async ({
break;
case Integrations.HEROKU:
await syncSecretsHeroku({
createManySecretsRawFn,
updateManySecretsRawFn,
integrationDAL,
integration,
secrets,
accessToken
backend/src/services/project-bot/project-bot-fns.ts
+36
View File
@@ -0,0 +1,36 @@
import { SecretKeyEncoding } from "@app/db/schemas";
import { decryptAsymmetric, infisicalSymmetricDecrypt } from "@app/lib/crypto/encryption";
import { BadRequestError } from "@app/lib/errors";
import { TProjectBotDALFactory } from "@app/services/project-bot/project-bot-dal";
import { TGetPrivateKeyDTO } from "./project-bot-types";
export const getBotPrivateKey = ({ bot }: TGetPrivateKeyDTO) =>
infisicalSymmetricDecrypt({
keyEncoding: bot.keyEncoding as SecretKeyEncoding,
iv: bot.iv,
tag: bot.tag,
ciphertext: bot.encryptedPrivateKey
});
export const getBotKeyFnFactory = (projectBotDAL: TProjectBotDALFactory) => {
const getBotKeyFn = async (projectId: string) => {
const bot = await projectBotDAL.findOne({ projectId });
if (!bot) throw new BadRequestError({ message: "failed to find bot key" });
if (!bot.isActive) throw new BadRequestError({ message: "Bot is not active" });
if (!bot.encryptedProjectKeyNonce || !bot.encryptedProjectKey)
throw new BadRequestError({ message: "Encryption key missing" });
const botPrivateKey = getBotPrivateKey({ bot });
return decryptAsymmetric({
ciphertext: bot.encryptedProjectKey,
privateKey: botPrivateKey,
nonce: bot.encryptedProjectKeyNonce,
publicKey: bot.sender.publicKey
});
};
return getBotKeyFn;
};
backend/src/services/project-bot/project-bot-service.ts
+7 -25
View File
@@ -1,15 +1,16 @@
import { ForbiddenError } from "@casl/ability";
import { ProjectVersion, SecretKeyEncoding } from "@app/db/schemas";
import { ProjectVersion } from "@app/db/schemas";
import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission";
import { decryptAsymmetric, generateAsymmetricKeyPair } from "@app/lib/crypto";
import { infisicalSymmetricDecrypt, infisicalSymmetricEncypt } from "@app/lib/crypto/encryption";
import { generateAsymmetricKeyPair } from "@app/lib/crypto";
import { infisicalSymmetricEncypt } from "@app/lib/crypto/encryption";
import { BadRequestError } from "@app/lib/errors";
import { TProjectDALFactory } from "../project/project-dal";
import { TProjectBotDALFactory } from "./project-bot-dal";
import { TFindBotByProjectIdDTO, TGetPrivateKeyDTO, TSetActiveStateDTO } from "./project-bot-types";
import { getBotKeyFnFactory, getBotPrivateKey } from "./project-bot-fns";
import { TFindBotByProjectIdDTO, TSetActiveStateDTO } from "./project-bot-types";
type TProjectBotServiceFactoryDep = {
permissionService: Pick<TPermissionServiceFactory, "getProjectPermission">;
@@ -24,29 +25,10 @@ export const projectBotServiceFactory = ({
projectDAL,
permissionService
}: TProjectBotServiceFactoryDep) => {
const getBotPrivateKey = ({ bot }: TGetPrivateKeyDTO) =>
infisicalSymmetricDecrypt({
keyEncoding: bot.keyEncoding as SecretKeyEncoding,
iv: bot.iv,
tag: bot.tag,
ciphertext: bot.encryptedPrivateKey
});
const getBotKeyFn = getBotKeyFnFactory(projectBotDAL);
const getBotKey = async (projectId: string) => {
const bot = await projectBotDAL.findOne({ projectId });
if (!bot) throw new BadRequestError({ message: "failed to find bot key" });
if (!bot.isActive) throw new BadRequestError({ message: "Bot is not active" });
if (!bot.encryptedProjectKeyNonce || !bot.encryptedProjectKey)
throw new BadRequestError({ message: "Encryption key missing" });
const botPrivateKey = getBotPrivateKey({ bot });
return decryptAsymmetric({
ciphertext: bot.encryptedProjectKey,
privateKey: botPrivateKey,
nonce: bot.encryptedProjectKeyNonce,
publicKey: bot.sender.publicKey
});
return getBotKeyFn(projectId);
};
const findBotByProjectId = async ({
backend/src/services/secret-folder/secret-folder-service.ts
+6 -3
View File
@@ -1,6 +1,6 @@
import { ForbiddenError, subject } from "@casl/ability";
import path from "path";
import { v4 as uuidv4 } from "uuid";
import { v4 as uuidv4, validate as uuidValidate } from "uuid";
import { TSecretFoldersInsert } from "@app/db/schemas";
import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
@@ -164,7 +164,7 @@ export const secretFolderServiceFactory = ({
actorOrgId,
environment,
path: secretPath,
id
idOrName
}: TDeleteFolderDTO) => {
const { permission } = await permissionService.getProjectPermission(actor, actorId, projectId, actorOrgId);
ForbiddenError.from(permission).throwUnlessCan(
@@ -179,7 +179,10 @@ export const secretFolderServiceFactory = ({
const parentFolder = await folderDAL.findBySecretPath(projectId, environment, secretPath, tx);
if (!parentFolder) throw new BadRequestError({ message: "Secret path not found" });
const [doc] = await folderDAL.delete({ envId: env.id, id, parentId: parentFolder.id }, tx);
const [doc] = await folderDAL.delete(
{ envId: env.id, [uuidValidate(idOrName) ? "id" : "name"]: idOrName, parentId: parentFolder.id },
tx
);
if (!doc) throw new BadRequestError({ message: "Folder not found", name: "Delete folder" });
return doc;
});
backend/src/services/secret-folder/secret-folder-types.ts
+1 -1
View File
@@ -16,7 +16,7 @@ export type TUpdateFolderDTO = {
export type TDeleteFolderDTO = {
environment: string;
path: string;
id: string;
idOrName: string;
} & TProjectPermission;
export type TGetFolderDTO = {
backend/src/services/secret/secret-fns.ts
+421 -2
View File
@@ -1,12 +1,35 @@
/* eslint-disable no-await-in-loop */
import path from "path";
import { SecretKeyEncoding, TSecretBlindIndexes, TSecrets } from "@app/db/schemas";
import {
SecretEncryptionAlgo,
SecretKeyEncoding,
SecretType,
TableName,
TSecretBlindIndexes,
TSecrets
} from "@app/db/schemas";
import { getConfig } from "@app/lib/config/env";
import { buildSecretBlindIndexFromName, decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
import {
buildSecretBlindIndexFromName,
decryptSymmetric128BitHexKeyUTF8,
encryptSymmetric128BitHexKeyUTF8
} from "@app/lib/crypto";
import { BadRequestError } from "@app/lib/errors";
import { groupBy, unique } from "@app/lib/fn";
import { getBotKeyFnFactory } from "../project-bot/project-bot-fns";
import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
import { TSecretDALFactory } from "./secret-dal";
import {
TCreateManySecretsRawFn,
TCreateManySecretsRawFnFactory,
TFnSecretBlindIndexCheck,
TFnSecretBulkInsert,
TFnSecretBulkUpdate,
TUpdateManySecretsRawFn,
TUpdateManySecretsRawFnFactory
} from "./secret-types";
export const generateSecretBlindIndexBySalt = async (secretName: string, secretBlindIndexDoc: TSecretBlindIndexes) => {
const appCfg = getConfig();
@@ -228,3 +251,399 @@ export const decryptSecretRaw = (secret: TSecrets & { workspace: string; environ
user: secret.userId
};
};
/**
* Checks and handles secrets using a blind index method.
* The function generates mappings between secret names and their blind indexes, validates user IDs for personal secrets, and retrieves secrets from the database based on their blind indexes.
* For new secrets (isNew = true), it ensures they don't already exist in the database.
* For existing secrets, it verifies their presence in the database.
* If discrepancies are found, errors are thrown. The function returns mappings and the fetched secrets.
*/
export const fnSecretBlindIndexCheck = async ({
inputSecrets,
folderId,
isNew,
userId,
blindIndexCfg,
secretDAL
}: TFnSecretBlindIndexCheck) => {
const blindIndex2KeyName: Record<string, string> = {}; // used at audit log point
const keyName2BlindIndex = await Promise.all(
inputSecrets.map(({ secretName }) => generateSecretBlindIndexBySalt(secretName, blindIndexCfg))
).then((blindIndexes) =>
blindIndexes.reduce<Record<string, string>>((prev, curr, i) => {
// eslint-disable-next-line
prev[inputSecrets[i].secretName] = curr;
blindIndex2KeyName[curr] = inputSecrets[i].secretName;
return prev;
}, {})
);
if (inputSecrets.some(({ type }) => type === SecretType.Personal) && !userId) {
throw new BadRequestError({ message: "Missing user id for personal secret" });
}
const secrets = await secretDAL.findByBlindIndexes(
folderId,
inputSecrets.map(({ secretName, type }) => ({
blindIndex: keyName2BlindIndex[secretName],
type: type || SecretType.Shared
})),
userId
);
if (isNew) {
if (secrets.length) throw new BadRequestError({ message: "Secret already exist" });
} else {
const secretKeysInDB = unique(secrets, (el) => el.secretBlindIndex as string).map(
(el) => blindIndex2KeyName[el.secretBlindIndex as string]
);
const hasUnknownSecretsProvided = secretKeysInDB.length !== inputSecrets.length;
if (hasUnknownSecretsProvided) {
const keysMissingInDB = Object.keys(keyName2BlindIndex).filter((key) => !secretKeysInDB.includes(key));
throw new BadRequestError({
message: `Secret not found: blind index ${keysMissingInDB.join(",")}`
});
}
}
return { blindIndex2KeyName, keyName2BlindIndex, secrets };
};
// these functions are special functions shared by a couple of resources
// used by secret approval, rotation or anywhere in which secret needs to modified
export const fnSecretBulkInsert = async ({
// TODO: Pick types here
folderId,
inputSecrets,
secretDAL,
secretVersionDAL,
secretTagDAL,
secretVersionTagDAL,
tx
}: TFnSecretBulkInsert) => {
const newSecrets = await secretDAL.insertMany(
inputSecrets.map(({ tags, ...el }) => ({ ...el, folderId })),
tx
);
const newSecretGroupByBlindIndex = groupBy(newSecrets, (item) => item.secretBlindIndex as string);
const newSecretTags = inputSecrets.flatMap(({ tags: secretTags = [], secretBlindIndex }) =>
secretTags.map((tag) => ({
[`${TableName.SecretTag}Id` as const]: tag,
[`${TableName.Secret}Id` as const]: newSecretGroupByBlindIndex[secretBlindIndex as string][0].id
}))
);
const secretVersions = await secretVersionDAL.insertMany(
inputSecrets.map(({ tags, ...el }) => ({
...el,
folderId,
secretId: newSecretGroupByBlindIndex[el.secretBlindIndex as string][0].id
})),
tx
);
if (newSecretTags.length) {
const secTags = await secretTagDAL.saveTagsToSecret(newSecretTags, tx);
const secVersionsGroupBySecId = groupBy(secretVersions, (i) => i.secretId);
const newSecretVersionTags = secTags.flatMap(({ secretsId, secret_tagsId }) => ({
[`${TableName.SecretVersion}Id` as const]: secVersionsGroupBySecId[secretsId][0].id,
[`${TableName.SecretTag}Id` as const]: secret_tagsId
}));
await secretVersionTagDAL.insertMany(newSecretVersionTags, tx);
}
return newSecrets.map((secret) => ({ ...secret, _id: secret.id }));
};
export const fnSecretBulkUpdate = async ({
tx,
inputSecrets,
folderId,
projectId,
secretDAL,
secretVersionDAL,
secretTagDAL,
secretVersionTagDAL
}: TFnSecretBulkUpdate) => {
const newSecrets = await secretDAL.bulkUpdate(
inputSecrets.map(({ filter, data: { tags, ...data } }) => ({
filter: { ...filter, folderId },
data
})),
tx
);
const secretVersions = await secretVersionDAL.insertMany(
newSecrets.map(({ id, createdAt, updatedAt, ...el }) => ({
...el,
secretId: id
})),
tx
);
const secsUpdatedTag = inputSecrets.flatMap(({ data: { tags } }, i) =>
tags !== undefined ? { tags, secretId: newSecrets[i].id } : []
);
if (secsUpdatedTag.length) {
await secretTagDAL.deleteTagsManySecret(
projectId,
secsUpdatedTag.map(({ secretId }) => secretId),
tx
);
const newSecretTags = secsUpdatedTag.flatMap(({ tags: secretTags = [], secretId }) =>
secretTags.map((tag) => ({
[`${TableName.SecretTag}Id` as const]: tag,
[`${TableName.Secret}Id` as const]: secretId
}))
);
if (newSecretTags.length) {
const secTags = await secretTagDAL.saveTagsToSecret(newSecretTags, tx);
const secVersionsGroupBySecId = groupBy(secretVersions, (i) => i.secretId);
const newSecretVersionTags = secTags.flatMap(({ secretsId, secret_tagsId }) => ({
[`${TableName.SecretVersion}Id` as const]: secVersionsGroupBySecId[secretsId][0].id,
[`${TableName.SecretTag}Id` as const]: secret_tagsId
}));
await secretVersionTagDAL.insertMany(newSecretVersionTags, tx);
}
}
return newSecrets.map((secret) => ({ ...secret, _id: secret.id }));
};
export const createManySecretsRawFnFactory = ({
projectDAL,
projectBotDAL,
secretDAL,
secretVersionDAL,
secretBlindIndexDAL,
secretTagDAL,
secretVersionTagDAL,
folderDAL
}: TCreateManySecretsRawFnFactory) => {
const getBotKeyFn = getBotKeyFnFactory(projectBotDAL);
const createManySecretsRawFn = async ({
projectId,
environment,
path: secretPath,
secrets,
userId
}: TCreateManySecretsRawFn) => {
const botKey = await getBotKeyFn(projectId);
if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" });
await projectDAL.checkProjectUpgradeStatus(projectId);
const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Create secret" });
const folderId = folder.id;
const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId });
if (!blindIndexCfg) throw new BadRequestError({ message: "Blind index not found", name: "Create secret" });
// insert operation
const { keyName2BlindIndex } = await fnSecretBlindIndexCheck({
inputSecrets: secrets,
folderId,
isNew: true,
blindIndexCfg,
secretDAL
});
const inputSecrets = await Promise.all(
secrets.map(async (secret) => {
const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey);
const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey);
const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey);
if (secret.type === SecretType.Personal) {
if (!userId) throw new BadRequestError({ message: "Missing user id for personal secret" });
const sharedExist = await secretDAL.findOne({
secretBlindIndex: keyName2BlindIndex[secret.secretName],
folderId,
type: SecretType.Shared
});
if (!sharedExist)
throw new BadRequestError({
message: "Failed to create personal secret override for no corresponding shared secret"
});
}
const tags = secret.tags ? await secretTagDAL.findManyTagsById(projectId, secret.tags) : [];
if ((secret.tags || []).length !== tags.length) throw new BadRequestError({ message: "Tag not found" });
return {
type: secret.type,
userId: secret.type === SecretType.Personal ? userId : null,
secretName: secret.secretName,
secretKeyCiphertext: secretKeyEncrypted.ciphertext,
secretKeyIV: secretKeyEncrypted.iv,
secretKeyTag: secretKeyEncrypted.tag,
secretValueCiphertext: secretValueEncrypted.ciphertext,
secretValueIV: secretValueEncrypted.iv,
secretValueTag: secretValueEncrypted.tag,
secretCommentCiphertext: secretCommentEncrypted.ciphertext,
secretCommentIV: secretCommentEncrypted.iv,
secretCommentTag: secretCommentEncrypted.tag,
skipMultilineEncoding: secret.skipMultilineEncoding,
tags: secret.tags
};
})
);
const newSecrets = await secretDAL.transaction(async (tx) =>
fnSecretBulkInsert({
inputSecrets: inputSecrets.map(({ secretName, ...el }) => ({
...el,
version: 0,
secretBlindIndex: keyName2BlindIndex[secretName],
algorithm: SecretEncryptionAlgo.AES_256_GCM,
keyEncoding: SecretKeyEncoding.UTF8
})),
folderId,
secretDAL,
secretVersionDAL,
secretTagDAL,
secretVersionTagDAL,
tx
})
);
return newSecrets;
};
return createManySecretsRawFn;
};
export const updateManySecretsRawFnFactory = ({
projectDAL,
projectBotDAL,
secretDAL,
secretVersionDAL,
secretBlindIndexDAL,
secretTagDAL,
secretVersionTagDAL,
folderDAL
}: TUpdateManySecretsRawFnFactory) => {
const getBotKeyFn = getBotKeyFnFactory(projectBotDAL);
const updateManySecretsRawFn = async ({
projectId,
environment,
path: secretPath,
secrets, // consider accepting instead ciphertext secrets
userId
}: TUpdateManySecretsRawFn): Promise<Array<TSecrets & { _id: string }>> => {
const botKey = await getBotKeyFn(projectId);
if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" });
await projectDAL.checkProjectUpgradeStatus(projectId);
const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Update secret" });
const folderId = folder.id;
const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId });
if (!blindIndexCfg) throw new BadRequestError({ message: "Blind index not found", name: "Update secret" });
const { keyName2BlindIndex } = await fnSecretBlindIndexCheck({
inputSecrets: secrets,
folderId,
isNew: false,
blindIndexCfg,
secretDAL,
userId
});
const inputSecrets = await Promise.all(
secrets.map(async (secret) => {
if (secret.newSecretName === "") {
throw new BadRequestError({ message: "New secret name cannot be empty" });
}
const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey);
const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey);
const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey);
if (secret.type === SecretType.Personal) {
if (!userId) throw new BadRequestError({ message: "Missing user id for personal secret" });
const sharedExist = await secretDAL.findOne({
secretBlindIndex: keyName2BlindIndex[secret.secretName],
folderId,
type: SecretType.Shared
});
if (!sharedExist)
throw new BadRequestError({
message: "Failed to update personal secret override for no corresponding shared secret"
});
if (secret.newSecretName)
throw new BadRequestError({ message: "Personal secret cannot change the key name" });
}
const tags = secret.tags ? await secretTagDAL.findManyTagsById(projectId, secret.tags) : [];
if ((secret.tags || []).length !== tags.length) throw new BadRequestError({ message: "Tag not found" });
return {
type: secret.type,
userId: secret.type === SecretType.Personal ? userId : null,
secretName: secret.secretName,
newSecretName: secret.newSecretName,
secretKeyCiphertext: secretKeyEncrypted.ciphertext,
secretKeyIV: secretKeyEncrypted.iv,
secretKeyTag: secretKeyEncrypted.tag,
secretValueCiphertext: secretValueEncrypted.ciphertext,
secretValueIV: secretValueEncrypted.iv,
secretValueTag: secretValueEncrypted.tag,
secretCommentCiphertext: secretCommentEncrypted.ciphertext,
secretCommentIV: secretCommentEncrypted.iv,
secretCommentTag: secretCommentEncrypted.tag,
skipMultilineEncoding: secret.skipMultilineEncoding,
tags: secret.tags
};
})
);
const tagIds = inputSecrets.flatMap(({ tags = [] }) => tags);
const tags = tagIds.length ? await secretTagDAL.findManyTagsById(projectId, tagIds) : [];
if (tagIds.length !== tags.length) throw new BadRequestError({ message: "Tag not found" });
// now find any secret that needs to update its name
// same process as above
const nameUpdatedSecrets = inputSecrets.filter(({ newSecretName }) => Boolean(newSecretName));
const { keyName2BlindIndex: newKeyName2BlindIndex } = await fnSecretBlindIndexCheck({
inputSecrets: nameUpdatedSecrets,
folderId,
isNew: true,
blindIndexCfg,
secretDAL
});
const updatedSecrets = await secretDAL.transaction(async (tx) =>
fnSecretBulkUpdate({
folderId,
projectId,
tx,
inputSecrets: inputSecrets.map(({ secretName, newSecretName, ...el }) => ({
filter: { secretBlindIndex: keyName2BlindIndex[secretName], type: SecretType.Shared },
data: {
...el,
folderId,
secretBlindIndex:
newSecretName && newKeyName2BlindIndex[newSecretName]
? newKeyName2BlindIndex[newSecretName]
: keyName2BlindIndex[secretName],
algorithm: SecretEncryptionAlgo.AES_256_GCM,
keyEncoding: SecretKeyEncoding.UTF8
}
})),
secretDAL,
secretVersionDAL,
secretTagDAL,
secretVersionTagDAL
})
);
return updatedSecrets;
};
return updateManySecretsRawFn;
};
backend/src/services/secret/secret-queue.ts
+46 -5
View File
@@ -6,6 +6,12 @@ import { BadRequestError } from "@app/lib/errors";
import { isSamePath } from "@app/lib/fn";
import { logger } from "@app/lib/logger";
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
import { TProjectBotDALFactory } from "@app/services/project-bot/project-bot-dal";
import { createManySecretsRawFnFactory, updateManySecretsRawFnFactory } from "@app/services/secret/secret-fns";
import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal";
import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal";
import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
import { TIntegrationDALFactory } from "../integration/integration-dal";
import { TIntegrationAuthServiceFactory } from "../integration-auth/integration-auth-service";
@@ -29,18 +35,23 @@ export type TSecretQueueFactory = ReturnType<typeof secretQueueFactory>;
type TSecretQueueFactoryDep = {
queueService: TQueueServiceFactory;
integrationDAL: Pick<TIntegrationDALFactory, "findByProjectIdV2">;
integrationDAL: Pick<TIntegrationDALFactory, "findByProjectIdV2" | "updateById">;
projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
integrationAuthService: Pick<TIntegrationAuthServiceFactory, "getIntegrationAccessToken">;
folderDAL: Pick<TSecretFolderDALFactory, "findBySecretPath" | "findByManySecretPath">;
secretDAL: Pick<TSecretDALFactory, "findByFolderId" | "find">;
folderDAL: TSecretFolderDALFactory;
secretDAL: TSecretDALFactory;
secretImportDAL: Pick<TSecretImportDALFactory, "find">;
webhookDAL: Pick<TWebhookDALFactory, "findAllWebhooks" | "transaction" | "update" | "bulkUpdate">;
projectEnvDAL: Pick<TProjectEnvDALFactory, "findOne">;
projectDAL: Pick<TProjectDALFactory, "findById">;
projectDAL: TProjectDALFactory;
projectBotDAL: TProjectBotDALFactory;
projectMembershipDAL: Pick<TProjectMembershipDALFactory, "findAllProjectMembers">;
smtpService: TSmtpService;
orgDAL: Pick<TOrgDALFactory, "findOrgByProjectId">;
secretVersionDAL: TSecretVersionDALFactory;
secretBlindIndexDAL: TSecretBlindIndexDALFactory;
secretTagDAL: TSecretTagDALFactory;
secretVersionTagDAL: TSecretVersionTagDALFactory;
};
export type TGetSecrets = {
@@ -62,8 +73,35 @@ export const secretQueueFactory = ({
orgDAL,
smtpService,
projectDAL,
projectMembershipDAL
projectBotDAL,
projectMembershipDAL,
secretVersionDAL,
secretBlindIndexDAL,
secretTagDAL,
secretVersionTagDAL
}: TSecretQueueFactoryDep) => {
const createManySecretsRawFn = createManySecretsRawFnFactory({
projectDAL,
projectBotDAL,
secretDAL,
secretVersionDAL,
secretBlindIndexDAL,
secretTagDAL,
secretVersionTagDAL,
folderDAL
});
const updateManySecretsRawFn = updateManySecretsRawFnFactory({
projectDAL,
projectBotDAL,
secretDAL,
secretVersionDAL,
secretBlindIndexDAL,
secretTagDAL,
secretVersionTagDAL,
folderDAL
});
const syncIntegrations = async (dto: TGetSecrets) => {
await queueService.queue(QueueName.IntegrationSync, QueueJobs.IntegrationSync, dto, {
attempts: 5,
@@ -307,6 +345,9 @@ export const secretQueueFactory = ({
}
await syncIntegrationSecrets({
createManySecretsRawFn,
updateManySecretsRawFn,
integrationDAL,
integration,
integrationAuth,
secrets: Object.keys(suffixedSecrets).length !== 0 ? suffixedSecrets : secrets,
backend/src/services/secret/secret-service.ts
+38 -153
View File
@@ -1,13 +1,13 @@
import { ForbiddenError, subject } from "@casl/ability";
import { SecretEncryptionAlgo, SecretKeyEncoding, SecretsSchema, SecretType, TableName } from "@app/db/schemas";
import { SecretEncryptionAlgo, SecretKeyEncoding, SecretsSchema, SecretType } from "@app/db/schemas";
import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission";
import { TSecretSnapshotServiceFactory } from "@app/ee/services/secret-snapshot/secret-snapshot-service";
import { getConfig } from "@app/lib/config/env";
import { buildSecretBlindIndexFromName, encryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
import { BadRequestError } from "@app/lib/errors";
import { groupBy, pick, unique } from "@app/lib/fn";
import { groupBy, pick } from "@app/lib/fn";
import { logger } from "@app/lib/logger";
import { ActorType } from "../auth/auth-type";
@@ -19,7 +19,7 @@ import { TSecretImportDALFactory } from "../secret-import/secret-import-dal";
import { fnSecretsFromImports } from "../secret-import/secret-import-fns";
import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal";
import { TSecretDALFactory } from "./secret-dal";
import { decryptSecretRaw, generateSecretBlindIndexBySalt } from "./secret-fns";
import { decryptSecretRaw, fnSecretBlindIndexCheck, fnSecretBulkInsert, fnSecretBulkUpdate } from "./secret-fns";
import { TSecretQueueFactory } from "./secret-queue";
import {
TCreateBulkSecretDTO,
@@ -28,11 +28,8 @@ import {
TDeleteBulkSecretDTO,
TDeleteSecretDTO,
TDeleteSecretRawDTO,
TFnSecretBlindIndexCheck,
TFnSecretBlindIndexCheckV2,
TFnSecretBulkDelete,
TFnSecretBulkInsert,
TFnSecretBulkUpdate,
TGetASecretDTO,
TGetASecretRawDTO,
TGetSecretsDTO,
@@ -95,85 +92,6 @@ export const secretServiceFactory = ({
return secretBlindIndex;
};
// these functions are special functions shared by a couple of resources
// used by secret approval, rotation or anywhere in which secret needs to modified
const fnSecretBulkInsert = async ({ folderId, inputSecrets, tx }: TFnSecretBulkInsert) => {
const newSecrets = await secretDAL.insertMany(
inputSecrets.map(({ tags, ...el }) => ({ ...el, folderId })),
tx
);
const newSecretGroupByBlindIndex = groupBy(newSecrets, (item) => item.secretBlindIndex as string);
const newSecretTags = inputSecrets.flatMap(({ tags: secretTags = [], secretBlindIndex }) =>
secretTags.map((tag) => ({
[`${TableName.SecretTag}Id` as const]: tag,
[`${TableName.Secret}Id` as const]: newSecretGroupByBlindIndex[secretBlindIndex as string][0].id
}))
);
const secretVersions = await secretVersionDAL.insertMany(
inputSecrets.map(({ tags, ...el }) => ({
...el,
folderId,
secretId: newSecretGroupByBlindIndex[el.secretBlindIndex as string][0].id
})),
tx
);
if (newSecretTags.length) {
const secTags = await secretTagDAL.saveTagsToSecret(newSecretTags, tx);
const secVersionsGroupBySecId = groupBy(secretVersions, (i) => i.secretId);
const newSecretVersionTags = secTags.flatMap(({ secretsId, secret_tagsId }) => ({
[`${TableName.SecretVersion}Id` as const]: secVersionsGroupBySecId[secretsId][0].id,
[`${TableName.SecretTag}Id` as const]: secret_tagsId
}));
await secretVersionTagDAL.insertMany(newSecretVersionTags, tx);
}
return newSecrets.map((secret) => ({ ...secret, _id: secret.id }));
};
const fnSecretBulkUpdate = async ({ tx, inputSecrets, folderId, projectId }: TFnSecretBulkUpdate) => {
const newSecrets = await secretDAL.bulkUpdate(
inputSecrets.map(({ filter, data: { tags, ...data } }) => ({
filter: { ...filter, folderId },
data
})),
tx
);
const secretVersions = await secretVersionDAL.insertMany(
newSecrets.map(({ id, createdAt, updatedAt, ...el }) => ({
...el,
secretId: id
})),
tx
);
const secsUpdatedTag = inputSecrets.flatMap(({ data: { tags } }, i) =>
tags !== undefined ? { tags, secretId: newSecrets[i].id } : []
);
if (secsUpdatedTag.length) {
await secretTagDAL.deleteTagsManySecret(
projectId,
secsUpdatedTag.map(({ secretId }) => secretId),
tx
);
const newSecretTags = secsUpdatedTag.flatMap(({ tags: secretTags = [], secretId }) =>
secretTags.map((tag) => ({
[`${TableName.SecretTag}Id` as const]: tag,
[`${TableName.Secret}Id` as const]: secretId
}))
);
if (newSecretTags.length) {
const secTags = await secretTagDAL.saveTagsToSecret(newSecretTags, tx);
const secVersionsGroupBySecId = groupBy(secretVersions, (i) => i.secretId);
const newSecretVersionTags = secTags.flatMap(({ secretsId, secret_tagsId }) => ({
[`${TableName.SecretVersion}Id` as const]: secVersionsGroupBySecId[secretsId][0].id,
[`${TableName.SecretTag}Id` as const]: secret_tagsId
}));
await secretVersionTagDAL.insertMany(newSecretVersionTags, tx);
}
}
return newSecrets.map((secret) => ({ ...secret, _id: secret.id }));
};
const fnSecretBulkDelete = async ({ folderId, inputSecrets, tx, actorId }: TFnSecretBulkDelete) => {
const deletedSecrets = await secretDAL.deleteMany(
inputSecrets.map(({ type, secretBlindIndex }) => ({
@@ -202,63 +120,6 @@ export const secretServiceFactory = ({
return deletedSecrets;
};
/**
* Checks and handles secrets using a blind index method.
* The function generates mappings between secret names and their blind indexes, validates user IDs for personal secrets, and retrieves secrets from the database based on their blind indexes.
* For new secrets (isNew = true), it ensures they don't already exist in the database.
* For existing secrets, it verifies their presence in the database.
* If discrepancies are found, errors are thrown. The function returns mappings and the fetched secrets.
*/
const fnSecretBlindIndexCheck = async ({
inputSecrets,
folderId,
isNew,
userId,
blindIndexCfg
}: TFnSecretBlindIndexCheck) => {
const blindIndex2KeyName: Record<string, string> = {}; // used at audit log point
const keyName2BlindIndex = await Promise.all(
inputSecrets.map(({ secretName }) => generateSecretBlindIndexBySalt(secretName, blindIndexCfg))
).then((blindIndexes) =>
blindIndexes.reduce<Record<string, string>>((prev, curr, i) => {
// eslint-disable-next-line
prev[inputSecrets[i].secretName] = curr;
blindIndex2KeyName[curr] = inputSecrets[i].secretName;
return prev;
}, {})
);
if (inputSecrets.some(({ type }) => type === SecretType.Personal) && !userId) {
throw new BadRequestError({ message: "Missing user id for personal secret" });
}
const secrets = await secretDAL.findByBlindIndexes(
folderId,
inputSecrets.map(({ secretName, type }) => ({
blindIndex: keyName2BlindIndex[secretName],
type: type || SecretType.Shared
})),
userId
);
if (isNew) {
if (secrets.length) throw new BadRequestError({ message: "Secret already exist" });
} else {
const secretKeysInDB = unique(secrets, (el) => el.secretBlindIndex as string).map(
(el) => blindIndex2KeyName[el.secretBlindIndex as string]
);
const hasUnknownSecretsProvided = secretKeysInDB.length !== inputSecrets.length;
if (hasUnknownSecretsProvided) {
const keysMissingInDB = Object.keys(keyName2BlindIndex).filter((key) => !secretKeysInDB.includes(key));
throw new BadRequestError({
message: `Secret not found: blind index ${keysMissingInDB.join(",")}`
});
}
}
return { blindIndex2KeyName, keyName2BlindIndex, secrets };
};
// this is used when secret blind index already exist
// mainly for secret approval
const fnSecretBlindIndexCheckV2 = async ({ inputSecrets, folderId, userId }: TFnSecretBlindIndexCheckV2) => {
@@ -311,7 +172,8 @@ export const secretServiceFactory = ({
folderId,
isNew: true,
userId: actorId,
blindIndexCfg
blindIndexCfg,
secretDAL
});
// if user creating personal check its shared also exist
@@ -348,6 +210,10 @@ export const secretServiceFactory = ({
tags: inputSecret.tags
}
],
secretDAL,
secretVersionDAL,
secretTagDAL,
secretVersionTagDAL,
tx
})
);
@@ -395,7 +261,8 @@ export const secretServiceFactory = ({
folderId,
isNew: false,
blindIndexCfg,
userId: actorId
userId: actorId,
secretDAL
});
if (inputSecret.newSecretName && inputSecret.type === SecretType.Personal) {
throw new BadRequestError({ message: "Personal secret cannot change the key name" });
@@ -407,7 +274,8 @@ export const secretServiceFactory = ({
inputSecrets: [{ secretName: inputSecret.newSecretName }],
folderId,
isNew: true,
blindIndexCfg
blindIndexCfg,
secretDAL
});
newSecretNameBlindIndex = kN2NewBlindIndex[inputSecret.newSecretName];
}
@@ -454,6 +322,10 @@ export const secretServiceFactory = ({
}
}
],
secretDAL,
secretVersionDAL,
secretTagDAL,
secretVersionTagDAL,
tx
})
);
@@ -496,7 +368,8 @@ export const secretServiceFactory = ({
inputSecrets: [{ secretName: inputSecret.secretName }],
folderId,
isNew: false,
blindIndexCfg
blindIndexCfg,
secretDAL
});
const deletedSecret = await secretDAL.transaction(async (tx) =>
@@ -679,13 +552,14 @@ export const secretServiceFactory = ({
const folderId = folder.id;
const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId });
if (!blindIndexCfg) throw new BadRequestError({ message: "Blind index not found", name: "Update secret" });
if (!blindIndexCfg) throw new BadRequestError({ message: "Blind index not found", name: "Create secret" });
const { keyName2BlindIndex } = await fnSecretBlindIndexCheck({
inputSecrets,
folderId,
isNew: true,
blindIndexCfg
blindIndexCfg,
secretDAL
});
// get all tags
@@ -704,6 +578,10 @@ export const secretServiceFactory = ({
keyEncoding: SecretKeyEncoding.UTF8
})),
folderId,
secretDAL,
secretVersionDAL,
secretTagDAL,
secretVersionTagDAL,
tx
})
);
@@ -732,7 +610,7 @@ export const secretServiceFactory = ({
await projectDAL.checkProjectUpgradeStatus(projectId);
const folder = await folderDAL.findBySecretPath(projectId, environment, path);
if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Create secret" });
if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Update secret" });
const folderId = folder.id;
const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId });
@@ -742,7 +620,8 @@ export const secretServiceFactory = ({
inputSecrets,
folderId,
isNew: false,
blindIndexCfg
blindIndexCfg,
secretDAL
});
// now find any secret that needs to update its name
@@ -752,7 +631,8 @@ export const secretServiceFactory = ({
inputSecrets: nameUpdatedSecrets,
folderId,
isNew: true,
blindIndexCfg
blindIndexCfg,
secretDAL
});
// get all tags
@@ -777,7 +657,11 @@ export const secretServiceFactory = ({
algorithm: SecretEncryptionAlgo.AES_256_GCM,
keyEncoding: SecretKeyEncoding.UTF8
}
}))
})),
secretDAL,
secretVersionDAL,
secretTagDAL,
secretVersionTagDAL
})
);
@@ -815,7 +699,8 @@ export const secretServiceFactory = ({
inputSecrets,
folderId,
isNew: false,
blindIndexCfg
blindIndexCfg,
secretDAL
});
const secretsDeleted = await secretDAL.transaction(async (tx) =>
backend/src/services/secret/secret-types.ts
+80
View File
@@ -2,6 +2,14 @@ import { Knex } from "knex";
import { SecretType, TSecretBlindIndexes, TSecrets, TSecretsInsert, TSecretsUpdate } from "@app/db/schemas";
import { TProjectPermission } from "@app/lib/types";
import { TProjectDALFactory } from "@app/services/project/project-dal";
import { TProjectBotDALFactory } from "@app/services/project-bot/project-bot-dal";
import { TSecretDALFactory } from "@app/services/secret/secret-dal";
import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal";
import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal";
import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";
import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
type TPartialSecret = Pick<TSecrets, "id" | "secretReminderRepeatDays" | "secretReminderNote">;
@@ -181,12 +189,20 @@ export type TFnSecretBulkInsert = {
folderId: string;
tx?: Knex;
inputSecrets: Array<Omit<TSecretsInsert, "folderId"> & { tags?: string[] }>;
secretDAL: Pick<TSecretDALFactory, "insertMany">;
secretVersionDAL: Pick<TSecretVersionDALFactory, "insertMany">;
secretTagDAL: Pick<TSecretTagDALFactory, "saveTagsToSecret">;
secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
};
export type TFnSecretBulkUpdate = {
folderId: string;
projectId: string;
inputSecrets: { filter: Partial<TSecrets>; data: TSecretsUpdate & { tags?: string[] } }[];
secretDAL: Pick<TSecretDALFactory, "bulkUpdate">;
secretVersionDAL: Pick<TSecretVersionDALFactory, "insertMany">;
secretTagDAL: Pick<TSecretTagDALFactory, "saveTagsToSecret" | "deleteTagsManySecret">;
secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
tx?: Knex;
};
@@ -204,6 +220,7 @@ export type TFnSecretBlindIndexCheck = {
blindIndexCfg: TSecretBlindIndexes;
inputSecrets: Array<{ secretName: string; type?: SecretType }>;
isNew: boolean;
secretDAL: Pick<TSecretDALFactory, "findByBlindIndexes">;
};
// when blind index is already present
@@ -229,3 +246,66 @@ export type TRemoveSecretReminderDTO = {
secretId: string;
repeatDays: number;
};
// ---
export type TCreateManySecretsRawFnFactory = {
projectDAL: TProjectDALFactory;
projectBotDAL: TProjectBotDALFactory;
secretDAL: TSecretDALFactory;
secretVersionDAL: TSecretVersionDALFactory;
secretBlindIndexDAL: TSecretBlindIndexDALFactory;
secretTagDAL: TSecretTagDALFactory;
secretVersionTagDAL: TSecretVersionTagDALFactory;
folderDAL: TSecretFolderDALFactory;
};
export type TCreateManySecretsRawFn = {
projectId: string;
environment: string;
path: string;
secrets: {
secretName: string;
secretValue: string;
type: SecretType;
secretComment?: string;
skipMultilineEncoding?: boolean;
tags?: string[];
metadata?: {
source?: string;
};
}[];
userId?: string; // only relevant for personal secret(s)
};
export type TUpdateManySecretsRawFnFactory = {
projectDAL: TProjectDALFactory;
projectBotDAL: TProjectBotDALFactory;
secretDAL: TSecretDALFactory;
secretVersionDAL: TSecretVersionDALFactory;
secretBlindIndexDAL: TSecretBlindIndexDALFactory;
secretTagDAL: TSecretTagDALFactory;
secretVersionTagDAL: TSecretVersionTagDALFactory;
folderDAL: TSecretFolderDALFactory;
};
export type TUpdateManySecretsRawFn = {
projectId: string;
environment: string;
path: string;
secrets: {
secretName: string;
newSecretName?: string;
secretValue: string;
type: SecretType;
secretComment?: string;
skipMultilineEncoding?: boolean;
secretReminderRepeatDays?: number | null;
secretReminderNote?: string | null;
tags?: string[];
metadata?: {
source?: string;
};
}[];
userId?: string;
};