awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-10-14 05:29:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(secrets-ui): Improve SecretDashboard table to support multiple changes on a single commit

Browse Source
This commit is contained in:
Carlos Monastyrski
2025-07-10 11:17:45 -03:00
parent 78da7ec343
commit 246fe81134
24 changed files with 3177 additions and 499 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/ee/routes/v1/pit-router.ts
+134 -1
View File
@@ -3,11 +3,14 @@ import { z } from "zod";
import { EventType } from "@app/ee/services/audit-log/audit-log-types";
import { removeTrailingSlash } from "@app/lib/fn";
import { readLimit } from "@app/server/config/rateLimiter";
import { isValidFolderName } from "@app/lib/validator";
import { readLimit, secretsLimit } from "@app/server/config/rateLimiter";
import { SecretNameSchema } from "@app/server/lib/schemas";
import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
import { booleanSchema } from "@app/server/routes/sanitizedSchemas";
import { AuthMode } from "@app/services/auth/auth-type";
import { commitChangesResponseSchema, resourceChangeSchema } from "@app/services/folder-commit/folder-commit-schemas";
import { ResourceMetadataSchema } from "@app/services/resource-metadata/resource-metadata-schema";
const commitHistoryItemSchema = z.object({
id: z.string(),
@@ -413,4 +416,134 @@ export const registerPITRouter = async (server: FastifyZodProvider) => {
return result;
}
});
server.route({
method: "POST",
url: "/batch/commit",
config: {
rateLimit: secretsLimit
},
schema: {
hide: true,
description: "Commit changes",
security: [
{
bearerAuth: []
}
],
body: z.object({
projectSlug: z.string().trim().optional(),
workspaceId: z.string().trim().optional(),
environment: z.string().trim(),
secretPath: z.string().trim().default("/").transform(removeTrailingSlash),
message: z.string().trim(),
changes: z.object({
secrets: z.object({
create: z
.array(
z.object({
secretKey: SecretNameSchema,
secretValue: z.string().transform((val) => (val.at(-1) === "\n" ? `${val.trim()}\n` : val.trim())),
secretComment: z.string().trim().optional().default(""),
skipMultilineEncoding: z.boolean().optional(),
metadata: z.record(z.string()).optional(),
secretMetadata: ResourceMetadataSchema.optional(),
tagIds: z.string().array().optional()
})
)
.optional(),
update: z
.array(
z.object({
secretKey: SecretNameSchema,
newSecretName: SecretNameSchema.optional(),
secretValue: z
.string()
.transform((val) => (val.at(-1) === "\n" ? `${val.trim()}\n` : val.trim()))
.optional(),
secretComment: z.string().trim().optional().default(""),
skipMultilineEncoding: z.boolean().optional(),
metadata: z.record(z.string()).optional(),
secretMetadata: ResourceMetadataSchema.optional(),
tagIds: z.string().array().optional()
})
)
.optional(),
delete: z
.array(
z.object({
secretKey: SecretNameSchema
})
)
.optional()
}),
folders: z.object({
create: z
.array(
z.object({
folderName: z
.string()
.trim()
.refine((name) => isValidFolderName(name), {
message: "Invalid folder name. Only alphanumeric characters, dashes, and underscores are allowed."
}),
description: z.string().optional()
})
)
.optional(),
update: z
.array(
z.object({
folderName: z
.string()
.trim()
.refine((name) => isValidFolderName(name), {
message: "Invalid folder name. Only alphanumeric characters, dashes, and underscores are allowed."
}),
description: z.string().optional(),
id: z.string()
})
)
.optional(),
delete: z
.array(
z.object({
folderName: z
.string()
.trim()
.refine((name) => isValidFolderName(name), {
message: "Invalid folder name. Only alphanumeric characters, dashes, and underscores are allowed."
}),
id: z.string()
})
)
.optional()
})
})
}),
response: {
200: z.object({
message: z.string()
})
}
},
onRequest: verifyAuth([AuthMode.JWT]),
handler: async (req) => {
await server.services.pit.processNewCommitRaw({
actorId: req.permission.id,
actor: req.permission.type,
actorOrgId: req.permission.orgId,
actorAuthMethod: req.permission.authMethod,
projectId: req.body.workspaceId,
environment: req.body.environment,
secretPath: req.body.secretPath,
message: req.body.message,
changes: {
secrets: req.body.changes.secrets,
folders: req.body.changes.folders
}
});
return { message: "success" };
}
});
};
backend/src/ee/services/pit/pit-service.ts
+279 -6
View File
@@ -2,28 +2,50 @@
import { ForbiddenError } from "@casl/ability";
import { ProjectPermissionCommitsActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission";
import { NotFoundError } from "@app/lib/errors";
import { BadRequestError, NotFoundError } from "@app/lib/errors";
import { logger } from "@app/lib/logger";
import { ActorAuthMethod, ActorType } from "@app/services/auth/auth-type";
import { ResourceType, TFolderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service";
import { TFolderCommitDALFactory } from "@app/services/folder-commit/folder-commit-dal";
import {
ResourceType,
TCreateCommitChangeDTO,
TFolderCommitServiceFactory
} from "@app/services/folder-commit/folder-commit-service";
import {
isFolderCommitChange,
isSecretCommitChange
} from "@app/services/folder-commit-changes/folder-commit-changes-dal";
import { TProjectDALFactory } from "@app/services/project/project-dal";
import { TProjectEnvDALFactory } from "@app/services/project-env/project-env-dal";
import { TSecretServiceFactory } from "@app/services/secret/secret-service";
import { SecretProtectionType, TProcessNewCommitRawDTO } from "@app/services/secret/secret-types";
import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";
import { TSecretFolderServiceFactory } from "@app/services/secret-folder/secret-folder-service";
import { TSecretV2BridgeServiceFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-service";
import { SecretOperations, SecretUpdateMode } from "@app/services/secret-v2-bridge/secret-v2-bridge-types";
import { TPermissionServiceFactory } from "../permission/permission-service-types";
import { TSecretApprovalPolicyServiceFactory } from "../secret-approval-policy/secret-approval-policy-service";
import { TSecretApprovalRequestServiceFactory } from "../secret-approval-request/secret-approval-request-service";
type TPitServiceFactoryDep = {
folderCommitService: TFolderCommitServiceFactory;
secretService: Pick<TSecretServiceFactory, "getSecretVersionsV2ByIds" | "getChangeVersions">;
folderService: Pick<TSecretFolderServiceFactory, "getFolderById" | "getFolderVersions">;
folderService: Pick<
TSecretFolderServiceFactory,
"getFolderById" | "getFolderVersions" | "createManyFolders" | "updateManyFolders" | "deleteManyFolders"
>;
permissionService: Pick<TPermissionServiceFactory, "getProjectPermission">;
folderDAL: Pick<TSecretFolderDALFactory, "findSecretPathByFolderIds">;
folderDAL: Pick<TSecretFolderDALFactory, "findSecretPathByFolderIds" | "findBySecretPath">;
projectEnvDAL: Pick<TProjectEnvDALFactory, "findOne">;
secretApprovalRequestService: Pick<
TSecretApprovalRequestServiceFactory,
"generateSecretApprovalRequest" | "generateSecretApprovalRequestV2Bridge"
>;
secretApprovalPolicyService: Pick<TSecretApprovalPolicyServiceFactory, "getSecretApprovalPolicy">;
projectDAL: Pick<TProjectDALFactory, "checkProjectUpgradeStatus" | "findProjectBySlug" | "findById">;
secretV2BridgeService: TSecretV2BridgeServiceFactory;
folderCommitDAL: Pick<TFolderCommitDALFactory, "transaction">;
};
export type TPitServiceFactory = ReturnType<typeof pitServiceFactory>;
@@ -34,7 +56,12 @@ export const pitServiceFactory = ({
folderService,
permissionService,
folderDAL,
projectEnvDAL
projectEnvDAL,
secretApprovalRequestService,
secretApprovalPolicyService,
projectDAL,
secretV2BridgeService,
folderCommitDAL
}: TPitServiceFactoryDep) => {
const getCommitsCount = async ({
actor,
@@ -471,6 +498,251 @@ export const pitServiceFactory = ({
});
};
const processNewCommitRaw = async ({
actorId,
projectSlug,
projectId: optionalProjectId,
environment,
actor,
actorOrgId,
actorAuthMethod,
secretPath,
message,
changes = {
secrets: {
create: [],
update: [],
delete: []
},
folders: {
create: [],
update: [],
delete: []
}
}
}: {
actorId: string;
projectSlug?: string;
projectId?: string;
environment: string;
actor: ActorType;
actorOrgId: string;
actorAuthMethod: ActorAuthMethod;
secretPath: string;
message: string;
changes: TProcessNewCommitRawDTO;
}) => {
if (!projectSlug && !optionalProjectId)
throw new BadRequestError({ message: "Must provide either project slug or projectId" });
let projectId = optionalProjectId as string;
if (!optionalProjectId && projectSlug) {
const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId);
if (!project) throw new NotFoundError({ message: `Project with slug '${projectSlug}' not found` });
projectId = project.id;
}
const policy =
actor === ActorType.USER
? await secretApprovalPolicyService.getSecretApprovalPolicy(projectId, environment, secretPath)
: undefined;
const project = await projectDAL.findById(projectId);
if (project.enforceCapitalization) {
const caseViolatingSecretKeys = [
// Check create operations
...(changes.secrets?.create
?.filter((sec) => sec.secretKey !== sec.secretKey.toUpperCase())
.map((sec) => sec.secretKey) ?? []),
// Check update operations
...(changes.secrets?.update
?.filter(
(sec) =>
sec.secretKey !== sec.secretKey.toUpperCase() ||
(sec.newSecretKey && sec.newSecretKey !== sec.newSecretKey.toUpperCase())
)
.map((sec) => sec.secretKey) ?? [])
];
if (caseViolatingSecretKeys.length) {
throw new BadRequestError({
message: `Secret names must be in UPPERCASE per project requirements: ${caseViolatingSecretKeys.join(
", "
)}. You can disable this requirement in project settings`
});
}
}
await folderCommitDAL.transaction(async (trx) => {
const targetFolder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
if (!targetFolder)
throw new NotFoundError({
message: `Folder with path '${secretPath}' in environment with slug '${environment}' not found`,
name: "CreateManySecret"
});
const commitChanges: TCreateCommitChangeDTO[] = [];
if ((changes.folders?.create?.length ?? 0) > 0) {
await folderService.createManyFolders({
projectId,
actor,
actorId,
actorOrgId,
actorAuthMethod,
folders:
changes.folders?.create?.map((folder) => ({
name: folder.folderName,
environment,
path: secretPath,
description: folder.description
})) ?? [],
tx: trx,
commitChanges
});
}
if ((changes.folders?.update?.length ?? 0) > 0) {
await folderService.updateManyFolders({
projectId,
projectSlug: projectSlug || "",
actor,
actorId,
actorOrgId,
actorAuthMethod,
folders:
changes.folders?.update?.map((folder) => ({
environment,
path: secretPath,
id: folder.id,
name: folder.folderName,
description: folder.description
})) ?? [],
tx: trx,
commitChanges
});
}
if ((changes.folders?.delete?.length ?? 0) > 0) {
await folderService.deleteManyFolders({
projectId,
actor,
actorId,
actorOrgId,
actorAuthMethod,
folders:
changes.folders?.delete?.map((folder) => ({
environment,
path: secretPath,
idOrName: folder.id
})) ?? [],
tx: trx,
commitChanges
});
}
if (policy) {
const approval = await secretApprovalRequestService.generateSecretApprovalRequestV2Bridge({
policy,
secretPath,
environment,
projectId,
actor,
actorId,
actorOrgId,
actorAuthMethod,
data: {
[SecretOperations.Create]:
changes.secrets?.create?.map((el) => ({
tagIds: el.tagIds,
secretValue: el.secretValue,
secretComment: el.secretComment,
metadata: el.metadata,
skipMultilineEncoding: el.skipMultilineEncoding,
secretKey: el.secretKey,
secretMetadata: el.secretMetadata
})) ?? [],
[SecretOperations.Update]:
changes.secrets?.update?.map((el) => ({
tagIds: el.tagIds,
secretValue: el.secretValue,
secretComment: el.secretComment,
metadata: el.metadata,
skipMultilineEncoding: el.skipMultilineEncoding,
secretKey: el.secretKey,
secretMetadata: el.secretMetadata
})) ?? [],
[SecretOperations.Delete]:
changes.secrets?.delete?.map((el) => ({
secretKey: el.secretKey
})) ?? []
}
});
return { type: SecretProtectionType.Approval as const, approval };
}
if ((changes.secrets?.create?.length ?? 0) > 0) {
await secretV2BridgeService.createManySecret({
secretPath,
environment,
projectId,
actorAuthMethod,
actorOrgId,
actor,
actorId,
secrets: changes.secrets?.create ?? [],
tx: trx,
commitChanges
});
}
if ((changes.secrets?.update?.length ?? 0) > 0) {
await secretV2BridgeService.updateManySecret({
secretPath,
environment,
projectId,
actorAuthMethod,
actorOrgId,
actor,
actorId,
secrets: changes.secrets?.update ?? [],
mode: SecretUpdateMode.FailOnNotFound,
tx: trx,
commitChanges
});
}
if ((changes.secrets?.delete?.length ?? 0) > 0) {
await secretV2BridgeService.deleteManySecret({
secretPath,
environment,
projectId,
actorAuthMethod,
actorOrgId,
actor,
actorId,
secrets: changes.secrets?.delete ?? [],
tx: trx,
commitChanges
});
}
if (commitChanges?.length > 0) {
await folderCommitService.createCommit(
{
actor: {
type: actor || ActorType.PLATFORM,
metadata: {
id: actorId
}
},
message,
folderId: targetFolder.id,
changes: commitChanges
},
trx
);
}
});
};
return {
getCommitsCount,
getCommitsForFolder,
@@ -478,6 +750,7 @@ export const pitServiceFactory = ({
compareCommitChanges,
rollbackToCommit,
revertCommit,
getFolderStateAtCommit
getFolderStateAtCommit,
processNewCommitRaw
};
};
backend/src/server/routes/index.ts
+6 -1
View File
@@ -1536,7 +1536,12 @@ export const registerRoutes = async (
folderService,
permissionService,
folderDAL,
projectEnvDAL
projectEnvDAL,
secretApprovalRequestService,
secretApprovalPolicyService,
projectDAL,
secretV2BridgeService,
folderCommitDAL
});
const identityOidcAuthService = identityOidcAuthServiceFactory({
backend/src/services/folder-commit/folder-commit-service.ts
+19 -17
View File
@@ -47,6 +47,14 @@ export enum ResourceType {
FOLDER = "folder"
}
export type TCreateCommitChangeDTO = {
type: string;
secretVersionId?: string;
folderVersionId?: string;
isUpdate?: boolean;
folderId?: string;
};
type TCreateCommitDTO = {
actor: {
type: string;
@@ -57,13 +65,7 @@ type TCreateCommitDTO = {
};
message?: string;
folderId: string;
changes: {
type: string;
secretVersionId?: string;
folderVersionId?: string;
isUpdate?: boolean;
folderId?: string;
}[];
changes: TCreateCommitChangeDTO[];
omitIgnoreFilter?: boolean;
};
@@ -895,16 +897,16 @@ export const folderCommitServiceFactory = ({
}
}
const newCommit = await folderCommitDAL.create(
{
actorMetadata: metadata,
actorType: data.actor.type,
message: data.message,
folderId: data.folderId,
envId: folder.envId
},
tx
);
const newCommit = await folderCommitDAL.create(
{
actorMetadata: metadata,
actorType: data.actor.type,
message: data.message,
folderId: data.folderId,
envId: folder.envId
},
tx
);
const batchSize = 500;
const chunks = chunkArray(changes, batchSize);
backend/src/services/secret-folder/secret-folder-service.ts
+419 -35
View File
@@ -1,4 +1,6 @@
/* eslint-disable no-await-in-loop */
import { ForbiddenError, subject } from "@casl/ability";
import { Knex } from "knex";
import path from "path";
import { v4 as uuidv4, validate as uuidValidate } from "uuid";
@@ -12,14 +14,21 @@ import { BadRequestError, NotFoundError } from "@app/lib/errors";
import { OrderByDirection, OrgServiceActor } from "@app/lib/types";
import { buildFolderPath } from "@app/services/secret-folder/secret-folder-fns";
import { ChangeType, CommitType, TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service";
import {
ChangeType,
CommitType,
TCreateCommitChangeDTO,
TFolderCommitServiceFactory
} from "../folder-commit/folder-commit-service";
import { TProjectDALFactory } from "../project/project-dal";
import { TProjectEnvDALFactory } from "../project-env/project-env-dal";
import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal";
import { TSecretFolderDALFactory } from "./secret-folder-dal";
import {
TCreateFolderDTO,
TCreateManyFoldersDTO,
TDeleteFolderDTO,
TDeleteManyFoldersDTO,
TGetFolderByIdDTO,
TGetFolderDTO,
TGetFoldersDeepByEnvsDTO,
@@ -236,19 +245,26 @@ export const secretFolderServiceFactory = ({
actor,
actorId,
projectSlug,
projectId: providedProjectId,
actorAuthMethod,
actorOrgId,
folders
}: TUpdateManyFoldersDTO) => {
const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId);
if (!project) {
throw new NotFoundError({ message: `Project with slug '${projectSlug}' not found` });
folders,
tx: providedTx,
commitChanges
}: TUpdateManyFoldersDTO & { tx?: Knex; commitChanges?: TCreateCommitChangeDTO[]; projectId?: string }) => {
let projectId = providedProjectId;
if (!projectId) {
const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId);
if (!project) {
throw new NotFoundError({ message: `Project with slug '${projectSlug}' not found` });
}
projectId = project.id;
}
const { permission } = await permissionService.getProjectPermission({
actor,
actorId,
projectId: project.id,
projectId,
actorAuthMethod,
actorOrgId
});
@@ -260,12 +276,12 @@ export const secretFolderServiceFactory = ({
);
});
const result = await folderDAL.transaction(async (tx) =>
Promise.all(
const executeBulkUpdate = async (tx: Knex) => {
return Promise.all(
folders.map(async (newFolder) => {
const { environment, path: secretPath, id, name, description } = newFolder;
const parentFolder = await folderDAL.findBySecretPath(project.id, environment, secretPath);
const parentFolder = await folderDAL.findBySecretPath(projectId as string, environment, secretPath, tx);
if (!parentFolder) {
throw new NotFoundError({
message: `Folder with path '${secretPath}' in environment with slug '${environment}' not found`,
@@ -273,10 +289,10 @@ export const secretFolderServiceFactory = ({
});
}
const env = await projectEnvDAL.findOne({ projectId: project.id, slug: environment });
const env = await projectEnvDAL.findOne({ projectId, slug: environment });
if (!env) {
throw new NotFoundError({
message: `Environment with slug '${environment}' in project with ID '${project.id}' not found`,
message: `Environment with slug '${environment}' in project with ID '${projectId}' not found`,
name: "UpdateManyFolders"
});
}
@@ -323,26 +339,34 @@ export const secretFolderServiceFactory = ({
},
tx
);
await folderCommitService.createCommit(
{
actor: {
type: actor,
metadata: {
id: actorId
}
if (commitChanges) {
commitChanges.push({
type: CommitType.ADD,
isUpdate: true,
folderVersionId: folderVersion.id
});
} else {
await folderCommitService.createCommit(
{
actor: {
type: actor,
metadata: {
id: actorId
}
},
message: "Folder updated",
folderId: parentFolder.id,
changes: [
{
type: CommitType.ADD,
isUpdate: true,
folderVersionId: folderVersion.id
}
]
},
message: "Folder updated",
folderId: parentFolder.id,
changes: [
{
type: CommitType.ADD,
isUpdate: true,
folderVersionId: folderVersion.id
}
]
},
tx
);
tx
);
}
if (!doc) {
throw new NotFoundError({
message: `Failed to update folder with id '${id}', not found`,
@@ -352,13 +376,16 @@ export const secretFolderServiceFactory = ({
return { oldFolder: folder, newFolder: doc };
})
)
);
);
};
// Execute with provided transaction or create new one
const result = providedTx ? await executeBulkUpdate(providedTx) : await folderDAL.transaction(executeBulkUpdate);
await Promise.all(result.map(async (res) => snapshotService.performSnapshot(res.newFolder.parentId as string)));
return {
projectId: project.id,
projectId,
newFolders: result.map((res) => res.newFolder),
oldFolders: result.map((res) => res.oldFolder)
};
@@ -974,6 +1001,361 @@ export const secretFolderServiceFactory = ({
}));
};
const createManyFolders = async ({
projectId,
actor,
actorId,
actorAuthMethod,
actorOrgId,
folders,
tx: providedTx,
commitChanges
}: TCreateManyFoldersDTO & { tx?: Knex; commitChanges?: TCreateCommitChangeDTO[] }) => {
const { permission } = await permissionService.getProjectPermission({
actor,
actorId,
projectId,
actorAuthMethod,
actorOrgId
});
folders.forEach(({ environment, path: secretPath }) => {
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionActions.Create,
subject(ProjectPermissionSub.SecretFolders, { environment, secretPath })
);
});
const foldersByEnv = folders.reduce(
(acc, folder) => {
if (!acc[folder.environment]) {
acc[folder.environment] = [];
}
acc[folder.environment].push(folder);
return acc;
},
{} as Record<string, typeof folders>
);
const executeBulkCreate = async (tx: Knex) => {
const createdFolders = [];
for (const [environment, envFolders] of Object.entries(foldersByEnv)) {
const env = await projectEnvDAL.findOne({ projectId, slug: environment });
if (!env) {
throw new NotFoundError({
message: `Environment with slug '${environment}' in project with ID '${projectId}' not found`
});
}
await tx.raw("SELECT pg_advisory_xact_lock(?)", [PgSqlLock.CreateFolder(env.id, env.projectId)]);
for (const folderSpec of envFolders) {
const { name, path: secretPath, description } = folderSpec;
const pathWithFolder = path.join(secretPath, name);
const parentFolder = await folderDAL.findClosestFolder(projectId, environment, pathWithFolder, tx);
if (!parentFolder) {
throw new NotFoundError({
message: `Parent folder for path '${pathWithFolder}' not found`
});
}
// Check if the exact folder already exists
const existingFolder = await folderDAL.findOne(
{
envId: env.id,
parentId: parentFolder.id,
name,
isReserved: false
},
tx
);
if (existingFolder) {
createdFolders.push(existingFolder);
// eslint-disable-next-line no-continue
continue;
}
// Handle exact folder case
if (parentFolder.path === pathWithFolder) {
createdFolders.push(parentFolder);
// eslint-disable-next-line no-continue
continue;
}
let currentParentId = parentFolder.id;
// Build the full path we need by processing each segment
if (parentFolder.path !== secretPath) {
const missingSegments = secretPath.substring(parentFolder.path.length).split("/").filter(Boolean);
const newFolders: TSecretFoldersInsert[] = [];
for (const segment of missingSegments) {
const existingSegment = await folderDAL.findOne(
{
name: segment,
parentId: currentParentId,
envId: env.id,
isReserved: false
},
tx
);
if (existingSegment) {
currentParentId = existingSegment.id;
} else {
const newFolder = {
name: segment,
parentId: currentParentId,
id: uuidv4(),
envId: env.id,
version: 1
};
currentParentId = newFolder.id;
newFolders.push(newFolder);
}
}
if (newFolders.length) {
const docs = await folderDAL.insertMany(newFolders, tx);
const folderVersions = await folderVersionDAL.insertMany(
docs.map((doc) => ({
name: doc.name,
envId: doc.envId,
version: doc.version,
folderId: doc.id,
description: doc.description
})),
tx
);
await folderCommitService.createCommit(
{
actor: {
type: actor,
metadata: {
id: actorId
}
},
message: "Folders created (batch)",
folderId: currentParentId,
changes: folderVersions.map((fv) => ({
type: CommitType.ADD,
folderVersionId: fv.id
}))
},
tx
);
}
}
// Create the target folder
const doc = await folderDAL.create(
{ name, envId: env.id, version: 1, parentId: currentParentId, description },
tx
);
const folderVersion = await folderVersionDAL.create(
{
name: doc.name,
envId: doc.envId,
version: doc.version,
folderId: doc.id,
description: doc.description
},
tx
);
if (commitChanges) {
commitChanges.push({
type: CommitType.ADD,
folderVersionId: folderVersion.id
});
} else {
await folderCommitService.createCommit(
{
actor: {
type: actor,
metadata: {
id: actorId
}
},
message: "Folder created (batch)",
folderId: doc.id,
changes: [
{
type: CommitType.ADD,
folderVersionId: folderVersion.id
}
]
},
tx
);
}
createdFolders.push(doc);
}
}
return createdFolders;
};
const result = providedTx ? await executeBulkCreate(providedTx) : await folderDAL.transaction(executeBulkCreate);
const uniqueParentIds = [...new Set(result.map((folder) => folder.parentId).filter(Boolean))];
await Promise.all(uniqueParentIds.map((parentId) => snapshotService.performSnapshot(parentId as string)));
return {
folders: result,
count: result.length
};
};
const deleteManyFolders = async ({
projectId,
actor,
actorId,
actorOrgId,
actorAuthMethod,
folders,
tx: providedTx,
commitChanges
}: TDeleteManyFoldersDTO & { tx?: Knex; commitChanges?: TCreateCommitChangeDTO[] }) => {
const { permission } = await permissionService.getProjectPermission({
actor,
actorId,
projectId,
actorAuthMethod,
actorOrgId
});
folders.forEach(({ environment, path: secretPath }) => {
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionActions.Delete,
subject(ProjectPermissionSub.SecretFolders, { environment, secretPath })
);
});
const foldersByEnv = folders.reduce(
(acc, folder) => {
if (!acc[folder.environment]) {
acc[folder.environment] = [];
}
acc[folder.environment].push(folder);
return acc;
},
{} as Record<string, typeof folders>
);
const executeBulkDelete = async (tx: Knex) => {
const deletedFolders = [];
for (const [environment, envFolders] of Object.entries(foldersByEnv)) {
const env = await projectEnvDAL.findOne({ projectId, slug: environment });
if (!env) {
throw new NotFoundError({
message: `Environment with slug '${environment}' not found`
});
}
for (const folderSpec of envFolders) {
const { path: secretPath, idOrName } = folderSpec;
const parentFolder = await folderDAL.findBySecretPath(projectId, environment, secretPath, tx);
if (!parentFolder) {
throw new NotFoundError({
message: `Folder with path '${secretPath}' in environment with slug '${environment}' not found`
});
}
await $checkFolderPolicy({ projectId, env, parentId: parentFolder.id, idOrName });
let folderToDelete = await folderDAL
.findOne({
envId: env.id,
name: idOrName,
parentId: parentFolder.id,
isReserved: false
})
.catch(() => null);
if (!folderToDelete && uuidValidate(idOrName)) {
folderToDelete = await folderDAL
.findOne({
envId: env.id,
id: idOrName,
parentId: parentFolder.id,
isReserved: false
})
.catch(() => null);
}
if (!folderToDelete) {
throw new NotFoundError({
message: `Folder with ID/name '${idOrName}' not found`
});
}
const [doc] = await folderDAL.delete(
{
envId: env.id,
id: folderToDelete.id,
parentId: parentFolder.id,
isReserved: false
},
tx
);
const folderVersions = await folderVersionDAL.findLatestFolderVersions([doc.id], tx);
if (commitChanges) {
commitChanges.push({
type: CommitType.DELETE,
folderVersionId: folderVersions[doc.id].id,
folderId: doc.id
});
} else {
await folderCommitService.createCommit(
{
actor: {
type: actor,
metadata: {
id: actorId
}
},
message: "Folder deleted (batch)",
folderId: parentFolder.id,
changes: [
{
type: CommitType.DELETE,
folderVersionId: folderVersions[doc.id].id,
folderId: doc.id
}
]
},
tx
);
}
deletedFolders.push(doc);
}
}
return deletedFolders;
};
const result = providedTx ? await executeBulkDelete(providedTx) : await folderDAL.transaction(executeBulkDelete);
const uniqueParentIds = [...new Set(result.map((folder) => folder.parentId).filter(Boolean))];
await Promise.all(uniqueParentIds.map((parentId) => snapshotService.performSnapshot(parentId as string)));
return {
folders: result,
count: result.length
};
};
return {
createFolder,
updateFolder,
@@ -986,6 +1368,8 @@ export const secretFolderServiceFactory = ({
getFoldersDeepByEnvs,
getProjectEnvironmentsFolders,
getFolderVersionsByIds,
getFolderVersions
getFolderVersions,
createManyFolders,
deleteManyFolders
};
};
backend/src/services/secret-folder/secret-folder-types.ts
+29
View File
@@ -1,6 +1,8 @@
import { OrderByDirection, TProjectPermission } from "@app/lib/types";
import { SecretsOrderBy } from "@app/services/secret/secret-types";
import { ActorAuthMethod, ActorType } from "../auth/auth-type";
export enum ReservedFolders {
SecretReplication = "__reserve_replication_"
}
@@ -62,3 +64,30 @@ export type TGetFoldersDeepByEnvsDTO = {
export type TFindFoldersDeepByParentIdsDTO = {
parentIds: string[];
};
export type TCreateManyFoldersDTO = {
projectId: string;
actor: ActorType;
actorId: string;
actorAuthMethod: ActorAuthMethod;
actorOrgId?: string;
folders: Array<{
name: string;
environment: string;
path: string;
description?: string;
}>;
};
export type TDeleteManyFoldersDTO = {
projectId: string;
actor: ActorType;
actorId: string;
actorAuthMethod: ActorAuthMethod;
actorOrgId?: string;
folders: Array<{
environment: string;
path: string;
idOrName: string;
}>;
};
backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts
+61 -46
View File
@@ -67,6 +67,7 @@ export const getAllSecretReferences = (maybeSecretReference: string) => {
export const fnSecretBulkInsert = async ({
// TODO: Pick types here
folderId,
commitChanges,
orgId,
inputSecrets,
secretDAL,
@@ -134,28 +135,32 @@ export const fnSecretBulkInsert = async ({
tx
);
const commitChanges = secretVersions
const changes = secretVersions
.filter(({ type }) => type === SecretType.Shared)
.map((sv) => ({
type: CommitType.ADD,
secretVersionId: sv.id
}));
if (commitChanges.length > 0) {
await folderCommitService.createCommit(
{
actor: {
type: actorType || ActorType.PLATFORM,
metadata: {
id: actor?.actorId
}
if (changes.length > 0) {
if (commitChanges) {
commitChanges.push(...changes);
} else {
await folderCommitService.createCommit(
{
actor: {
type: actorType || ActorType.PLATFORM,
metadata: {
id: actor?.actorId
}
},
message: "Secret Created",
folderId,
changes
},
message: "Secret Created",
folderId,
changes: commitChanges
},
tx
);
tx
);
}
}
await secretDAL.upsertSecretReferences(
@@ -209,6 +214,7 @@ export const fnSecretBulkUpdate = async ({
tx,
inputSecrets,
folderId,
commitChanges,
orgId,
secretDAL,
secretVersionDAL,
@@ -359,28 +365,32 @@ export const fnSecretBulkUpdate = async ({
{ tx }
);
const commitChanges = secretVersions
const changes = secretVersions
.filter(({ type }) => type === SecretType.Shared)
.map((sv) => ({
type: CommitType.ADD,
isUpdate: true,
secretVersionId: sv.id
}));
if (commitChanges.length > 0) {
await folderCommitService.createCommit(
{
actor: {
type: actorType || ActorType.PLATFORM,
metadata: {
id: actor?.actorId
}
if (changes.length > 0) {
if (commitChanges) {
commitChanges.push(...changes);
} else {
await folderCommitService.createCommit(
{
actor: {
type: actorType || ActorType.PLATFORM,
metadata: {
id: actor?.actorId
}
},
message: "Secret Updated",
folderId,
changes
},
message: "Secret Updated",
folderId,
changes: commitChanges
},
tx
);
tx
);
}
}
return secretsWithTags.map((secret) => ({ ...secret, _id: secret.id }));
@@ -395,7 +405,8 @@ export const fnSecretBulkDelete = async ({
secretDAL,
secretQueueService,
folderCommitService,
secretVersionDAL
secretVersionDAL,
commitChanges
}: TFnSecretBulkDelete) => {
const deletedSecrets = await secretDAL.deleteMany(
inputSecrets.map(({ type, secretKey }) => ({
@@ -421,27 +432,31 @@ export const fnSecretBulkDelete = async ({
tx
);
const commitChanges = deletedSecrets
const changes = deletedSecrets
.filter(({ type }) => type === SecretType.Shared)
.map(({ id }) => ({
type: CommitType.DELETE,
secretVersionId: secretVersions[id].id
}));
if (commitChanges.length > 0) {
await folderCommitService.createCommit(
{
actor: {
type: actorType || ActorType.PLATFORM,
metadata: {
id: actorId
}
if (changes.length > 0) {
if (commitChanges) {
commitChanges.push(...changes);
} else {
await folderCommitService.createCommit(
{
actor: {
type: actorType || ActorType.PLATFORM,
metadata: {
id: actorId
}
},
message: "Secret Deleted",
folderId,
changes
},
message: "Secret Deleted",
folderId,
changes: commitChanges
},
tx
);
tx
);
}
}
return deletedSecrets;
backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts
+68 -42
View File
@@ -28,7 +28,7 @@ import { logger } from "@app/lib/logger";
import { alphaNumericNanoId } from "@app/lib/nanoid";
import { ActorType } from "../auth/auth-type";
import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service";
import { TCreateCommitChangeDTO, TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service";
import { TKmsServiceFactory } from "../kms/kms-service";
import { KmsDataKey } from "../kms/kms-types";
import { TProjectEnvDALFactory } from "../project-env/project-env-dal";
@@ -1474,8 +1474,10 @@ export const secretV2BridgeServiceFactory = ({
actorOrgId,
environment,
projectId,
secrets: inputSecrets
}: TCreateManySecretDTO) => {
secrets: inputSecrets,
tx: providedTx,
commitChanges
}: TCreateManySecretDTO & { tx?: Knex; commitChanges?: TCreateCommitChangeDTO[] }) => {
const { permission } = await permissionService.getProjectPermission({
actor,
actorId,
@@ -1558,8 +1560,8 @@ export const secretV2BridgeServiceFactory = ({
const { encryptor: secretManagerEncryptor, decryptor: secretManagerDecryptor } =
await kmsService.createCipherPairWithDataKey({ type: KmsDataKey.SecretManager, projectId });
const newSecrets = await secretDAL.transaction(async (tx) =>
fnSecretBulkInsert({
const executeBulkInsert = async (tx: Knex) => {
return fnSecretBulkInsert({
inputSecrets: inputSecrets.map((el) => {
const references = secretReferencesGroupByInputSecretKey[el.secretKey]?.nestedReferences;
@@ -1581,6 +1583,7 @@ export const secretV2BridgeServiceFactory = ({
};
}),
folderId,
commitChanges,
orgId: actorOrgId,
secretDAL,
resourceMetadataDAL,
@@ -1593,8 +1596,13 @@ export const secretV2BridgeServiceFactory = ({
actorId
},
tx
})
);
});
};
const newSecrets = providedTx
? await executeBulkInsert(providedTx)
: await secretDAL.transaction(executeBulkInsert);
await secretDAL.invalidateSecretCacheByProjectId(projectId);
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({
@@ -1641,8 +1649,10 @@ export const secretV2BridgeServiceFactory = ({
projectId,
secretPath: defaultSecretPath = "/",
secrets: inputSecrets,
mode: updateMode
}: TUpdateManySecretDTO) => {
mode: updateMode,
tx: providedTx,
commitChanges
}: TUpdateManySecretDTO & { tx?: Knex; commitChanges?: TCreateCommitChangeDTO[] }) => {
const { permission } = await permissionService.getProjectPermission({
actor,
actorId,
@@ -1671,18 +1681,20 @@ export const secretV2BridgeServiceFactory = ({
const { encryptor: secretManagerEncryptor, decryptor: secretManagerDecryptor } =
await kmsService.createCipherPairWithDataKey({ type: KmsDataKey.SecretManager, projectId });
const updatedSecrets: Array<
TSecretsV2 & {
secretPath: string;
tags: {
id: string;
slug: string;
color?: string | null;
name: string;
}[];
}
> = [];
await secretDAL.transaction(async (tx) => {
// Function to execute the bulk update operation
const executeBulkUpdate = async (tx: Knex) => {
const updatedSecrets: Array<
TSecretsV2 & {
secretPath: string;
tags: {
id: string;
slug: string;
color?: string | null;
name: string;
}[];
}
> = [];
for await (const folder of folders) {
if (!folder) throw new NotFoundError({ message: "Folder not found" });
@@ -1801,7 +1813,7 @@ export const secretV2BridgeServiceFactory = ({
{
operator: "eq",
field: `${TableName.SecretV2}.key` as "key",
value: el.secretKey
value: el.newSecretName as string
},
{
operator: "eq",
@@ -1855,6 +1867,7 @@ export const secretV2BridgeServiceFactory = ({
orgId: actorOrgId,
folderCommitService,
tx,
commitChanges,
inputSecrets: secretsToUpdate.map((el) => {
const originalSecret = secretsToUpdateInDBGroupedByKey[el.secretKey][0];
const encryptedValue =
@@ -1934,7 +1947,13 @@ export const secretV2BridgeServiceFactory = ({
updatedSecrets.push(...bulkInsertedSecrets.map((el) => ({ ...el, secretPath: folder.path })));
}
}
});
return updatedSecrets;
};
const updatedSecrets = providedTx
? await executeBulkUpdate(providedTx)
: await secretDAL.transaction(executeBulkUpdate);
await secretDAL.invalidateSecretCacheByProjectId(projectId);
await Promise.allSettled(folders.map((el) => (el?.id ? snapshotService.performSnapshot(el.id) : undefined)));
@@ -1991,8 +2010,10 @@ export const secretV2BridgeServiceFactory = ({
actor,
actorId,
actorAuthMethod,
actorOrgId
}: TDeleteManySecretDTO) => {
actorOrgId,
tx: providedTx,
commitChanges
}: TDeleteManySecretDTO & { tx?: Knex; commitChanges?: TCreateCommitChangeDTO[] }) => {
const { permission } = await permissionService.getProjectPermission({
actor,
actorId,
@@ -2051,24 +2072,29 @@ export const secretV2BridgeServiceFactory = ({
);
});
const executeBulkDelete = async (tx: Knex) => {
return fnSecretBulkDelete({
secretDAL,
secretQueueService,
folderCommitService,
secretVersionDAL,
inputSecrets: inputSecrets.map(({ type, secretKey }) => ({
secretKey,
type: type || SecretType.Shared
})),
projectId,
folderId,
actorId,
actorType: actor,
commitChanges,
tx
});
};
try {
const secretsDeleted = await secretDAL.transaction(async (tx) =>
fnSecretBulkDelete({
secretDAL,
secretQueueService,
folderCommitService,
secretVersionDAL,
inputSecrets: inputSecrets.map(({ type, secretKey }) => ({
secretKey,
type: type || SecretType.Shared
})),
projectId,
folderId,
actorId,
actorType: actor,
tx
})
);
const secretsDeleted = providedTx
? await executeBulkDelete(providedTx)
: await secretDAL.transaction(executeBulkDelete);
await secretDAL.invalidateSecretCacheByProjectId(projectId);
await snapshotService.performSnapshot(folderId);
backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts
+4 -1
View File
@@ -8,7 +8,7 @@ import { SecretsOrderBy } from "@app/services/secret/secret-types";
import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";
import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service";
import { TCreateCommitChangeDTO, TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service";
import { TResourceMetadataDALFactory } from "../resource-metadata/resource-metadata-dal";
import { ResourceMetadataDTO } from "../resource-metadata/resource-metadata-schema";
import { TSecretV2BridgeDALFactory } from "./secret-v2-bridge-dal";
@@ -167,6 +167,7 @@ export type TFnSecretBulkInsert = {
folderId: string;
orgId: string;
tx?: Knex;
commitChanges?: TCreateCommitChangeDTO[];
inputSecrets: Array<
Omit<TSecretsV2Insert, "folderId"> & {
tagIds?: string[];
@@ -214,6 +215,7 @@ export type TFnSecretBulkUpdate = {
actorId?: string;
};
tx?: Knex;
commitChanges?: TCreateCommitChangeDTO[];
};
export type TFnSecretBulkDelete = {
@@ -223,6 +225,7 @@ export type TFnSecretBulkDelete = {
actorId: string;
actorType?: string;
tx?: Knex;
commitChanges?: TCreateCommitChangeDTO[];
secretDAL: Pick<TSecretV2BridgeDALFactory, "deleteMany">;
secretQueueService: {
removeSecretReminder: (data: TRemoveSecretReminderDTO, tx?: Knex) => Promise<void>;
backend/src/services/secret/secret-types.ts
+30
View File
@@ -544,3 +544,33 @@ export enum SecretProtectionType {
}
export type TStartSecretsV2MigrationDTO = TProjectPermission;
export type TProcessNewCommitRawDTO = {
secrets: {
create?: {
secretKey: string;
secretValue: string;
secretComment?: string;
skipMultilineEncoding?: boolean;
tagIds?: string[];
secretMetadata?: ResourceMetadataDTO;
metadata?: { source?: string };
}[];
update?: {
secretKey: string;
newSecretKey?: string;
secretValue?: string;
secretComment?: string;
skipMultilineEncoding?: boolean;
tagIds?: string[];
secretMetadata?: ResourceMetadataDTO;
metadata?: { source?: string };
}[];
delete?: { secretKey: string }[];
};
folders: {
create?: { folderName: string; description?: string }[];
update?: { folderName: string; description?: string; id: string }[];
delete?: { folderName: string; id: string }[];
};
};