PIT: address PR suggestions

This commit is contained in:
carlosmonastyrski
2025-05-21 19:42:09 -03:00
parent 2493bbbc97
commit f06004370d
31 changed files with 862 additions and 576 deletions
+7
View File
@@ -84,6 +84,10 @@ const getZodDefaultValue = (type: unknown, value: string | number | boolean | Ob
} }
}; };
const bigIntegerColumns = {
"folder_commits": ["commitId"]
}
const main = async () => { const main = async () => {
const tables = ( const tables = (
await db("information_schema.tables") await db("information_schema.tables")
@@ -108,6 +112,9 @@ const main = async () => {
const columnName = columnNames[colNum]; const columnName = columnNames[colNum];
const colInfo = columns[columnName]; const colInfo = columns[columnName];
let ztype = getZodPrimitiveType(colInfo.type); let ztype = getZodPrimitiveType(colInfo.type);
if (bigIntegerColumns[tableName]?.includes(columnName)) {
ztype = "z.coerce.bigint()";
}
if (["zodBuffer"].includes(ztype)) { if (["zodBuffer"].includes(ztype)) {
zodImportSet.add(ztype); zodImportSet.add(ztype);
} }
@@ -101,12 +101,29 @@ export async function up(knex: Knex): Promise<void> {
}); });
} }
await createOnUpdateTrigger(knex, TableName.FolderCommit); if (!hasFolderCommitTable) {
await createOnUpdateTrigger(knex, TableName.FolderCommitChanges); await createOnUpdateTrigger(knex, TableName.FolderCommit);
await createOnUpdateTrigger(knex, TableName.FolderCheckpoint); }
await createOnUpdateTrigger(knex, TableName.FolderCheckpointResources);
await createOnUpdateTrigger(knex, TableName.FolderTreeCheckpoint); if (!hasFolderCommitChangesTable) {
await createOnUpdateTrigger(knex, TableName.FolderTreeCheckpointResources); await createOnUpdateTrigger(knex, TableName.FolderCommitChanges);
}
if (!hasFolderCheckpointTable) {
await createOnUpdateTrigger(knex, TableName.FolderCheckpoint);
}
if (!hasFolderCheckpointResourcesTable) {
await createOnUpdateTrigger(knex, TableName.FolderCheckpointResources);
}
if (!hasFolderTreeCheckpointTable) {
await createOnUpdateTrigger(knex, TableName.FolderTreeCheckpoint);
}
if (!hasFolderTreeCheckpointResourcesTable) {
await createOnUpdateTrigger(knex, TableName.FolderTreeCheckpointResources);
}
} }
export async function down(knex: Knex): Promise<void> { export async function down(knex: Knex): Promise<void> {
@@ -11,12 +11,10 @@ export async function up(knex: Knex): Promise<void> {
const keyStore = inMemoryKeyStore(); const keyStore = inMemoryKeyStore();
const { folderCommitService } = await getMigrationPITServices({ db: knex, keyStore }); const { folderCommitService } = await getMigrationPITServices({ db: knex, keyStore });
const projects = await knex(TableName.Project).where({ version: 3, type: ProjectType.SecretManager }).select("id"); const projects = await knex(TableName.Project).where({ version: 3, type: ProjectType.SecretManager }).select("id");
await knex.transaction(async (tx) => { for (const project of projects) {
for (const project of projects) { // eslint-disable-next-line no-await-in-loop
// eslint-disable-next-line no-await-in-loop await folderCommitService.initializeProject(project.id);
await folderCommitService.initializeProject(project.id, tx); }
}
});
} }
} }
+1 -1
View File
@@ -9,7 +9,7 @@ import { TImmutableDBKeys } from "./models";
export const FolderCommitsSchema = z.object({ export const FolderCommitsSchema = z.object({
id: z.string().uuid(), id: z.string().uuid(),
commitId: z.coerce.number(), commitId: z.coerce.bigint(),
actorMetadata: z.unknown(), actorMetadata: z.unknown(),
actorType: z.string(), actorType: z.string(),
message: z.string().nullable().optional(), message: z.string().nullable().optional(),
+152 -177
View File
@@ -1,14 +1,18 @@
/* eslint-disable @typescript-eslint/no-base-to-string */ /* eslint-disable @typescript-eslint/no-base-to-string */
import { ForbiddenError } from "@casl/ability";
import { z } from "zod"; import { z } from "zod";
import { ActionProjectType } from "@app/db/schemas";
import { EventType } from "@app/ee/services/audit-log/audit-log-types"; import { EventType } from "@app/ee/services/audit-log/audit-log-types";
import { ProjectPermissionCommitsActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission";
import { NotFoundError } from "@app/lib/errors"; import { NotFoundError } from "@app/lib/errors";
import { removeTrailingSlash } from "@app/lib/fn"; import { removeTrailingSlash } from "@app/lib/fn";
import { readLimit } from "@app/server/config/rateLimiter"; import { readLimit } from "@app/server/config/rateLimiter";
import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
import { booleanSchema } from "@app/server/routes/sanitizedSchemas"; import { booleanSchema } from "@app/server/routes/sanitizedSchemas";
import { AuthMode } from "@app/services/auth/auth-type"; import { AuthMode } from "@app/services/auth/auth-type";
import { ChangeType } from "@app/services/folder-commit/folder-commit-service"; import { ChangeType, ResourceChange } from "@app/services/folder-commit/folder-commit-service";
import { commitChangesResponseSchema } from "@app/services/folder-commit/folder-commit-types";
const commitHistoryItemSchema = z.object({ const commitHistoryItemSchema = z.object({
id: z.string(), id: z.string(),
@@ -21,19 +25,6 @@ const commitHistoryItemSchema = z.object({
envId: z.string() envId: z.string()
}); });
const versionSchema = z.object({
secretKey: z.string().optional(),
secretComment: z.string().optional().nullable(),
skipMultilineEncoding: z.boolean().optional().nullable(),
secretReminderRepeatDays: z.number().optional().nullable(),
secretReminderNote: z.string().optional().nullable(),
metadata: z.unknown().optional().nullable(),
tags: z.array(z.string()).optional().nullable(),
secretReminderRecipients: z.array(z.any()).optional().nullable(),
secretValue: z.string().optional().nullable(),
name: z.string().optional().nullable()
});
const folderStateSchema = z.array( const folderStateSchema = z.array(
z.object({ z.object({
type: z.string(), type: z.string(),
@@ -50,17 +41,15 @@ export const registerPITRouter = async (server: FastifyZodProvider) => {
// Get commits count for a folder // Get commits count for a folder
server.route({ server.route({
method: "GET", method: "GET",
url: "/commits/count/:workspaceId", url: "/commits/count",
config: { config: {
rateLimit: readLimit rateLimit: readLimit
}, },
schema: { schema: {
params: z.object({
workspaceId: z.string().trim()
}),
querystring: z.object({ querystring: z.object({
environment: z.string().trim(), environment: z.string().trim(),
path: z.string().trim().default("/").transform(removeTrailingSlash) path: z.string().trim().default("/").transform(removeTrailingSlash),
workspaceId: z.string().trim()
}), }),
response: { response: {
200: z.object({ 200: z.object({
@@ -76,14 +65,14 @@ export const registerPITRouter = async (server: FastifyZodProvider) => {
actorId: req.permission?.id, actorId: req.permission?.id,
actorOrgId: req.permission?.orgId, actorOrgId: req.permission?.orgId,
actorAuthMethod: req.permission?.authMethod, actorAuthMethod: req.permission?.authMethod,
projectId: req.params.workspaceId, projectId: req.query.workspaceId,
environment: req.query.environment, environment: req.query.environment,
path: req.query.path path: req.query.path
}); });
await server.services.auditLog.createAuditLog({ await server.services.auditLog.createAuditLog({
...req.auditLogInfo, ...req.auditLogInfo,
projectId: req.params.workspaceId, projectId: req.query.workspaceId,
event: { event: {
type: EventType.GET_PROJECT_PIT_COMMIT_COUNT, type: EventType.GET_PROJECT_PIT_COMMIT_COUNT,
metadata: { metadata: {
@@ -101,17 +90,15 @@ export const registerPITRouter = async (server: FastifyZodProvider) => {
// Get all commits for a folder // Get all commits for a folder
server.route({ server.route({
method: "GET", method: "GET",
url: "/commits/:workspaceId", url: "/commits",
config: { config: {
rateLimit: readLimit rateLimit: readLimit
}, },
schema: { schema: {
params: z.object({
workspaceId: z.string().trim()
}),
querystring: z.object({ querystring: z.object({
environment: z.string().trim(), environment: z.string().trim(),
path: z.string().trim().default("/").transform(removeTrailingSlash) path: z.string().trim().default("/").transform(removeTrailingSlash),
workspaceId: z.string().trim()
}), }),
response: { response: {
200: commitHistoryItemSchema.array() 200: commitHistoryItemSchema.array()
@@ -124,14 +111,14 @@ export const registerPITRouter = async (server: FastifyZodProvider) => {
actorId: req.permission?.id, actorId: req.permission?.id,
actorOrgId: req.permission?.orgId, actorOrgId: req.permission?.orgId,
actorAuthMethod: req.permission?.authMethod, actorAuthMethod: req.permission?.authMethod,
projectId: req.params.workspaceId, projectId: req.query.workspaceId,
environment: req.query.environment, environment: req.query.environment,
path: req.query.path path: req.query.path
}); });
await server.services.auditLog.createAuditLog({ await server.services.auditLog.createAuditLog({
...req.auditLogInfo, ...req.auditLogInfo,
projectId: req.params.workspaceId, projectId: req.query.workspaceId,
event: { event: {
type: EventType.GET_PROJECT_PIT_COMMITS, type: EventType.GET_PROJECT_PIT_COMMITS,
metadata: { metadata: {
@@ -149,72 +136,75 @@ export const registerPITRouter = async (server: FastifyZodProvider) => {
} }
}); });
const getChangeVersions = async (
change: ResourceChange,
previousVersion: string,
actorId: string,
actor: string,
actorOrgId: string,
actorAuthMethod: string,
folderId: string
) => {
if (change.secretVersion) {
const currentVersion = change.secretVersion || "1";
const secretId = change.secretId ? change.secretId : change.id;
// eslint-disable-next-line no-await-in-loop
const versions = await server.services.secret.getSecretVersionsV2ByIds({
actorId,
actor,
actorOrgId,
actorAuthMethod,
secretId,
// if it's update add also the previous secretversionid
secretVersions:
change.isUpdate || change.changeType === ChangeType.UPDATE
? [currentVersion, previousVersion]
: [currentVersion],
folderId
});
return versions?.map((v) => ({
secretKey: v.secretKey,
secretComment: v.secretComment,
skipMultilineEncoding: v.skipMultilineEncoding,
secretReminderRepeatDays: v.secretReminderRepeatDays,
secretReminderNote: v.secretReminderNote,
metadata: v.metadata,
tags: v.tags?.map((t) => t.name),
secretReminderRecipients: v.secretReminderRecipients?.map((r) => r.toString()),
secretValue: v.secretValue
}));
}
};
const getFolderVersions = async (change: ResourceChange, fromVersion: string, folderId: string) => {
const currentVersion = change.folderVersion || "1";
// eslint-disable-next-line no-await-in-loop
const versions = await server.services.folder.getFolderVersionsByIds({
folderId,
folderVersions:
change.isUpdate || change.changeType === ChangeType.UPDATE ? [currentVersion, fromVersion] : [currentVersion]
});
return versions.map((v) => ({
name: v.name
}));
};
// Get commit changes for a specific commit // Get commit changes for a specific commit
server.route({ server.route({
method: "GET", method: "GET",
url: "/commits/:workspaceId/:commitId/changes", url: "/commits/:commitId/changes",
config: { config: {
rateLimit: readLimit rateLimit: readLimit
}, },
schema: { schema: {
params: z.object({ params: z.object({
workspaceId: z.string().trim(),
commitId: z.string().trim() commitId: z.string().trim()
}), }),
querystring: z.object({
workspaceId: z.string().trim()
}),
response: { response: {
200: z.object({ 200: commitChangesResponseSchema
changes: z.object({
id: z.string(),
commitId: z.string(),
actorMetadata: z
.union([
z.object({
id: z.string().optional(),
name: z.string().optional()
}),
z.unknown()
])
.optional(),
actorType: z.string(),
message: z.string().optional().nullable(),
folderId: z.string(),
envId: z.string(),
createdAt: z.string().or(z.date()),
updatedAt: z.string().or(z.date()),
changes: z.array(
z.object({
id: z.string(),
folderCommitId: z.string(),
changeType: z.string(),
isUpdate: z.boolean().optional(),
secretVersionId: z.string().optional().nullable(),
folderVersionId: z.string().optional().nullable(),
// Fix these two fields to accept either string or Date objects
createdAt: z.union([z.string(), z.date()]),
updatedAt: z.union([z.string(), z.date()]),
folderName: z.string().optional().nullable(),
folderChangeId: z.string().optional().nullable(),
folderVersion: z.union([z.string(), z.number()]).optional().nullable(),
secretKey: z.string().optional().nullable(),
secretVersion: z.union([z.string(), z.number()]).optional().nullable(),
secretId: z.string().optional().nullable(),
actorMetadata: z
.union([
z.object({
id: z.string().optional(),
name: z.string().optional()
}),
z.unknown()
])
.optional(),
actorType: z.string().optional(),
message: z.string().optional().nullable(),
folderId: z.string().optional().nullable(),
versions: z.array(versionSchema).optional()
})
)
})
})
} }
}, },
onRequest: verifyAuth([AuthMode.JWT]), onRequest: verifyAuth([AuthMode.JWT]),
@@ -224,53 +214,36 @@ export const registerPITRouter = async (server: FastifyZodProvider) => {
actorId: req.permission?.id, actorId: req.permission?.id,
actorOrgId: req.permission?.orgId, actorOrgId: req.permission?.orgId,
actorAuthMethod: req.permission?.authMethod, actorAuthMethod: req.permission?.authMethod,
projectId: req.params.workspaceId, projectId: req.query.workspaceId,
commitId: req.params.commitId commitId: req.params.commitId
}); });
for (const change of changes.changes) { for (const change of changes.changes) {
if (change.secretVersionId) { if (change.secretVersionId) {
const currentVersion = change.secretVersion || "1"; change.objectType = "secret";
const previousVersion = (Number.parseInt(currentVersion, 10) - 1).toString();
if (change.secretId) {
// eslint-disable-next-line no-await-in-loop
const versions = await server.services.secret.getSecretVersionsV2ByIds({
actorId: req.permission?.id,
actor: req.permission?.type,
actorOrgId: req.permission?.orgId,
actorAuthMethod: req.permission?.authMethod,
secretId: change.secretId,
secretVersions: change.isUpdate ? [currentVersion, previousVersion] : [currentVersion],
folderId: change.folderId
});
change.versions = versions?.map((v) => ({
secretKey: v.secretKey,
secretComment: v.secretComment,
skipMultilineEncoding: v.skipMultilineEncoding,
secretReminderRepeatDays: v.secretReminderRepeatDays,
secretReminderNote: v.secretReminderNote,
metadata: v.secretMetadata,
tags: v.tags?.map((t) => t.name),
secretReminderRecipients: v.secretReminderRecipients?.map((r) => r.toString()),
secretValue: v.secretValue
}));
}
} else if (change.folderVersionId && change.folderChangeId) {
const currentVersion = change.folderVersion || "1";
const previousVersion = (Number.parseInt(currentVersion, 10) - 1).toString();
// eslint-disable-next-line no-await-in-loop // eslint-disable-next-line no-await-in-loop
const versions = await server.services.folder.getFolderVersionsByIds({ change.versions = await getChangeVersions(
folderId: change.folderChangeId, change,
folderVersions: change.isUpdate ? [currentVersion, previousVersion] : [currentVersion] (Number.parseInt(change.secretVersion, 10) - 1).toString(),
}); req.permission?.id,
change.versions = versions.map((v) => ({ req.permission?.type,
name: v.name req.permission?.orgId,
})); req.permission?.authMethod,
change.folderId
);
} else if (change.folderVersionId && change.folderChangeId) {
change.objectType = "folder";
// eslint-disable-next-line no-await-in-loop
change.versions = await getFolderVersions(
change,
(Number.parseInt(change.folderVersion, 10) - 1).toString(),
change.folderChangeId
);
} }
} }
await server.services.auditLog.createAuditLog({ await server.services.auditLog.createAuditLog({
...req.auditLogInfo, ...req.auditLogInfo,
projectId: req.params.workspaceId, projectId: req.query.workspaceId,
event: { event: {
type: EventType.GET_PROJECT_PIT_COMMIT_CHANGES, type: EventType.GET_PROJECT_PIT_COMMIT_CHANGES,
metadata: { metadata: {
@@ -292,20 +265,20 @@ export const registerPITRouter = async (server: FastifyZodProvider) => {
// Retrieve rollback changes for a commit // Retrieve rollback changes for a commit
server.route({ server.route({
method: "GET", method: "GET",
url: "/commits/:workspaceId/:commitId/compare", url: "/commits/:commitId/compare",
config: { config: {
rateLimit: readLimit rateLimit: readLimit
}, },
schema: { schema: {
params: z.object({ params: z.object({
workspaceId: z.string().trim(),
commitId: z.string().trim() commitId: z.string().trim()
}), }),
querystring: z.object({ querystring: z.object({
folderId: z.string().trim(), folderId: z.string().trim(),
envId: z.string().trim(), envId: z.string().trim(),
deepRollback: booleanSchema.default(false), deepRollback: booleanSchema.default(false),
secretPath: z.string().trim().default("/").transform(removeTrailingSlash) secretPath: z.string().trim().default("/").transform(removeTrailingSlash),
workspaceId: z.string().trim()
}), }),
response: { response: {
200: z.array( 200: z.array(
@@ -326,7 +299,7 @@ export const registerPITRouter = async (server: FastifyZodProvider) => {
actorId: req.permission?.id, actorId: req.permission?.id,
actorOrgId: req.permission?.orgId, actorOrgId: req.permission?.orgId,
actorAuthMethod: req.permission?.authMethod, actorAuthMethod: req.permission?.authMethod,
projectId: req.params.workspaceId projectId: req.query.workspaceId
}); });
if (!latestCommit) { if (!latestCommit) {
throw new NotFoundError({ message: "Latest commit not found" }); throw new NotFoundError({ message: "Latest commit not found" });
@@ -337,9 +310,7 @@ export const registerPITRouter = async (server: FastifyZodProvider) => {
diffs = await server.services.folderCommit.deepCompareFolder({ diffs = await server.services.folderCommit.deepCompareFolder({
targetCommitId: req.params.commitId, targetCommitId: req.params.commitId,
envId: req.query.envId, envId: req.query.envId,
actorId: req.permission?.id, projectId: req.query.workspaceId
actorType: req.permission?.type,
projectId: req.params.workspaceId
}); });
} else { } else {
const folder = await server.services.folder.getFolderById({ const folder = await server.services.folder.getFolderById({
@@ -365,51 +336,27 @@ export const registerPITRouter = async (server: FastifyZodProvider) => {
for (const diff of diffs) { for (const diff of diffs) {
for (const change of diff.changes) { for (const change of diff.changes) {
if (change.secretKey) { if (change.secretKey) {
const currentVersion = change.secretVersion || "1";
const previousVersion = change.fromVersion || "1";
// eslint-disable-next-line no-await-in-loop // eslint-disable-next-line no-await-in-loop
const versions = await server.services.secret.getSecretVersionsV2ByIds({ change.versions = await getChangeVersions(
actorId: req.permission?.id, change,
actor: req.permission?.type, change.fromVersion || "1",
actorOrgId: req.permission?.orgId, req.permission?.id,
actorAuthMethod: req.permission?.authMethod, req.permission?.type,
secretId: change.id, req.permission?.orgId,
// if it's update add also the previous secretversionid req.permission?.authMethod,
secretVersions: diff.folderId
change.changeType === ChangeType.UPDATE ? [currentVersion, previousVersion] : [currentVersion], );
folderId: req.query.folderId
});
change.versions = versions?.map((v) => ({
secretKey: v.secretKey,
secretComment: v.secretComment,
skipMultilineEncoding: v.skipMultilineEncoding,
secretReminderRepeatDays: v.secretReminderRepeatDays,
secretReminderNote: v.secretReminderNote,
metadata: v.metadata,
tags: v.tags?.map((t) => t.name),
secretReminderRecipients: v.secretReminderRecipients?.map((r) => r.toString()),
secretValue: v.secretValue
}));
} }
if (change.folderVersion) { if (change.folderVersion) {
const currentVersion = change.folderVersion || "1";
const previousVersion = change.fromVersion || "1";
// eslint-disable-next-line no-await-in-loop // eslint-disable-next-line no-await-in-loop
const versions = await server.services.folder.getFolderVersionsByIds({ change.versions = await getFolderVersions(change, change.fromVersion || "1", change.id);
folderId: change.id,
folderVersions:
change.changeType === ChangeType.UPDATE ? [currentVersion, previousVersion] : [currentVersion]
});
change.versions = versions.map((v) => ({
name: v.name
}));
} }
} }
} }
await server.services.auditLog.createAuditLog({ await server.services.auditLog.createAuditLog({
...req.auditLogInfo, ...req.auditLogInfo,
projectId: req.params.workspaceId, projectId: req.query.workspaceId,
event: { event: {
type: EventType.PIT_COMPARE_FOLDER_STATES, type: EventType.PIT_COMPARE_FOLDER_STATES,
metadata: { metadata: {
@@ -428,20 +375,20 @@ export const registerPITRouter = async (server: FastifyZodProvider) => {
// Rollback to a previous commit // Rollback to a previous commit
server.route({ server.route({
method: "POST", method: "POST",
url: "/commits/:workspaceId/:commitId/rollback", url: "/commits/:commitId/rollback",
config: { config: {
rateLimit: readLimit rateLimit: readLimit
}, },
schema: { schema: {
params: z.object({ params: z.object({
workspaceId: z.string().trim(),
commitId: z.string().trim() commitId: z.string().trim()
}), }),
body: z.object({ body: z.object({
folderId: z.string().trim(), folderId: z.string().trim(),
deepRollback: z.boolean().default(false), deepRollback: z.boolean().default(false),
message: z.string().trim().optional(), message: z.string().trim().optional(),
envId: z.string().trim() envId: z.string().trim(),
workspaceId: z.string().trim()
}), }),
response: { response: {
200: z.object({ 200: z.object({
@@ -454,13 +401,26 @@ export const registerPITRouter = async (server: FastifyZodProvider) => {
}, },
onRequest: verifyAuth([AuthMode.JWT]), onRequest: verifyAuth([AuthMode.JWT]),
handler: async (req) => { handler: async (req) => {
const { permission } = await server.services.permission.getProjectPermission({
actor: req.permission?.type,
actorId: req.permission?.id,
projectId: req.body.workspaceId,
actorAuthMethod: req.permission?.authMethod,
actorOrgId: req.permission?.orgId,
actionProjectType: ActionProjectType.SecretManager
});
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionCommitsActions.PerformRollback,
ProjectPermissionSub.Commits
);
const latestCommit = await server.services.folderCommit.getLatestCommit({ const latestCommit = await server.services.folderCommit.getLatestCommit({
folderId: req.body.folderId, folderId: req.body.folderId,
actor: req.permission?.type, actor: req.permission?.type,
actorId: req.permission?.id, actorId: req.permission?.id,
actorOrgId: req.permission?.orgId, actorOrgId: req.permission?.orgId,
actorAuthMethod: req.permission?.authMethod, actorAuthMethod: req.permission?.authMethod,
projectId: req.params.workspaceId projectId: req.body.workspaceId
}); });
if (!latestCommit) { if (!latestCommit) {
throw new NotFoundError({ message: "Latest commit not found" }); throw new NotFoundError({ message: "Latest commit not found" });
@@ -472,7 +432,7 @@ export const registerPITRouter = async (server: FastifyZodProvider) => {
req.body.envId, req.body.envId,
req.permission.id, req.permission.id,
req.permission.type, req.permission.type,
req.params.workspaceId req.body.workspaceId
); );
return { success: true }; return { success: true };
} }
@@ -489,13 +449,13 @@ export const registerPITRouter = async (server: FastifyZodProvider) => {
message: req.body.message || "Rollback to previous commit" message: req.body.message || "Rollback to previous commit"
}, },
folderId: req.body.folderId, folderId: req.body.folderId,
projectId: req.params.workspaceId, projectId: req.body.workspaceId,
reconstructNewFolders: req.body.deepRollback reconstructNewFolders: req.body.deepRollback
}); });
await server.services.auditLog.createAuditLog({ await server.services.auditLog.createAuditLog({
...req.auditLogInfo, ...req.auditLogInfo,
projectId: req.params.workspaceId, projectId: req.body.workspaceId,
event: { event: {
type: EventType.PIT_ROLLBACK_COMMIT, type: EventType.PIT_ROLLBACK_COMMIT,
metadata: { metadata: {
@@ -520,15 +480,17 @@ export const registerPITRouter = async (server: FastifyZodProvider) => {
// Revert commit // Revert commit
server.route({ server.route({
method: "POST", method: "POST",
url: "/commits/:workspaceId/:commitId/revert", url: "/commits/:commitId/revert",
config: { config: {
rateLimit: readLimit rateLimit: readLimit
}, },
schema: { schema: {
params: z.object({ params: z.object({
workspaceId: z.string().trim(),
commitId: z.string().trim() commitId: z.string().trim()
}), }),
querystring: z.object({
workspaceId: z.string().trim()
}),
response: { response: {
200: z.object({ 200: z.object({
success: z.boolean(), success: z.boolean(),
@@ -547,12 +509,12 @@ export const registerPITRouter = async (server: FastifyZodProvider) => {
actorId: req.permission?.id, actorId: req.permission?.id,
actorAuthMethod: req.permission?.authMethod, actorAuthMethod: req.permission?.authMethod,
actorOrgId: req.permission?.orgId, actorOrgId: req.permission?.orgId,
projectId: req.params.workspaceId projectId: req.query.workspaceId
}); });
await server.services.auditLog.createAuditLog({ await server.services.auditLog.createAuditLog({
...req.auditLogInfo, ...req.auditLogInfo,
projectId: req.params.workspaceId, projectId: req.query.workspaceId,
event: { event: {
type: EventType.PIT_REVERT_COMMIT, type: EventType.PIT_REVERT_COMMIT,
metadata: { metadata: {
@@ -570,17 +532,17 @@ export const registerPITRouter = async (server: FastifyZodProvider) => {
// Folder state at commit // Folder state at commit
server.route({ server.route({
method: "GET", method: "GET",
url: "/commits/:workspaceId/:commitId", url: "/commits/:commitId",
config: { config: {
rateLimit: readLimit rateLimit: readLimit
}, },
schema: { schema: {
params: z.object({ params: z.object({
workspaceId: z.string().trim(),
commitId: z.string().trim() commitId: z.string().trim()
}), }),
querystring: z.object({ querystring: z.object({
folderId: z.string().trim() folderId: z.string().trim(),
workspaceId: z.string().trim()
}), }),
response: { response: {
200: folderStateSchema 200: folderStateSchema
@@ -588,11 +550,24 @@ export const registerPITRouter = async (server: FastifyZodProvider) => {
}, },
onRequest: verifyAuth([AuthMode.JWT]), onRequest: verifyAuth([AuthMode.JWT]),
handler: async (req) => { handler: async (req) => {
const { permission } = await server.services.permission.getProjectPermission({
actor: req.permission?.type,
actorId: req.permission?.id,
projectId: req.query.workspaceId,
actorAuthMethod: req.permission?.authMethod,
actorOrgId: req.permission?.orgId,
actionProjectType: ActionProjectType.SecretManager
});
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionCommitsActions.Read,
ProjectPermissionSub.Commits
);
const response = await server.services.folderCommit.reconstructFolderState(req.params.commitId); const response = await server.services.folderCommit.reconstructFolderState(req.params.commitId);
await server.services.auditLog.createAuditLog({ await server.services.auditLog.createAuditLog({
...req.auditLogInfo, ...req.auditLogInfo,
projectId: req.params.workspaceId, projectId: req.query.workspaceId,
event: { event: {
type: EventType.PIT_GET_FOLDER_STATE, type: EventType.PIT_GET_FOLDER_STATE,
metadata: { metadata: {
+27 -27
View File
@@ -17,44 +17,44 @@ export const getDefaultOnPremFeatures = (): TFeatureSet => ({
environmentsUsed: 0, environmentsUsed: 0,
identityLimit: null, identityLimit: null,
identitiesUsed: 0, identitiesUsed: 0,
dynamicSecret: true, dynamicSecret: false,
secretVersioning: true, secretVersioning: true,
pitRecovery: true, pitRecovery: false,
ipAllowlisting: true, ipAllowlisting: false,
rbac: true, rbac: false,
githubOrgSync: true, githubOrgSync: false,
customRateLimits: true, customRateLimits: false,
customAlerts: true, customAlerts: false,
secretAccessInsights: true, secretAccessInsights: false,
auditLogs: true, auditLogs: false,
auditLogsRetentionDays: 0, auditLogsRetentionDays: 0,
auditLogStreams: true, auditLogStreams: false,
auditLogStreamLimit: 3, auditLogStreamLimit: 3,
samlSSO: true, samlSSO: false,
hsm: true, hsm: false,
oidcSSO: true, oidcSSO: false,
scim: true, scim: false,
ldap: true, ldap: false,
groups: true, groups: false,
status: null, status: null,
trial_end: null, trial_end: null,
has_used_trial: true, has_used_trial: true,
secretApproval: true, secretApproval: false,
secretRotation: true, secretRotation: false,
caCrl: true, caCrl: false,
instanceUserManagement: true, instanceUserManagement: false,
externalKms: true, externalKms: false,
rateLimits: { rateLimits: {
readLimit: 60, readLimit: 60,
writeLimit: 200, writeLimit: 200,
secretsLimit: 40 secretsLimit: 40
}, },
pkiEst: true, pkiEst: false,
enforceMfa: true, enforceMfa: false,
projectTemplates: true, projectTemplates: false,
kmip: true, kmip: false,
gateway: true, gateway: false,
sshHostGroups: true sshHostGroups: false
}); });
export const setupLicenseRequestWithStore = (baseURL: string, refreshUrl: string, licenseKey: string) => { export const setupLicenseRequestWithStore = (baseURL: string, refreshUrl: string, licenseKey: string) => {
@@ -14,7 +14,7 @@ import { logger } from "@app/lib/logger";
import { alphaNumericNanoId } from "@app/lib/nanoid"; import { alphaNumericNanoId } from "@app/lib/nanoid";
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
import { ActorType } from "@app/services/auth/auth-type"; import { ActorType } from "@app/services/auth/auth-type";
import { TFolderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; import { CommitType, TFolderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service";
import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { TKmsServiceFactory } from "@app/services/kms/kms-service";
import { KmsDataKey } from "@app/services/kms/kms-types"; import { KmsDataKey } from "@app/services/kms/kms-types";
import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service"; import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service";
@@ -350,7 +350,7 @@ export const secretRotationQueueFactory = ({
message: "Changed by Secret rotation", message: "Changed by Secret rotation",
folderId: secretVersions[0].folderId, folderId: secretVersions[0].folderId,
changes: secretVersions.map((sv) => ({ changes: secretVersions.map((sv) => ({
type: "add", type: CommitType.ADD,
isUpdate: true, isUpdate: true,
secretVersionId: sv.id secretVersionId: sv.id
})) }))
+1
View File
@@ -27,6 +27,7 @@ export const KeyStorePrefixes = {
KmsOrgDataKeyCreation: "kms-org-data-key-creation-lock", KmsOrgDataKeyCreation: "kms-org-data-key-creation-lock",
WaitUntilReadyKmsOrgKeyCreation: "wait-until-ready-kms-org-key-creation-", WaitUntilReadyKmsOrgKeyCreation: "wait-until-ready-kms-org-key-creation-",
WaitUntilReadyKmsOrgDataKeyCreation: "wait-until-ready-kms-org-data-key-creation-", WaitUntilReadyKmsOrgDataKeyCreation: "wait-until-ready-kms-org-data-key-creation-",
FolderTreeCheckpoint: (envId: string) => `folder-tree-checkpoint-${envId}`,
WaitUntilReadyProjectEnvironmentOperation: (projectId: string) => WaitUntilReadyProjectEnvironmentOperation: (projectId: string) =>
`wait-until-ready-project-environments-operation-${projectId}`, `wait-until-ready-project-environments-operation-${projectId}`,
+2 -2
View File
@@ -230,8 +230,8 @@ const envSchema = z
DATADOG_HOSTNAME: zpStr(z.string().optional()), DATADOG_HOSTNAME: zpStr(z.string().optional()),
// PIT // PIT
PIT_CHECKPOINT_WINDOW: zpStr(z.string().optional().default("10")), PIT_CHECKPOINT_WINDOW: zpStr(z.string().optional().default("30")),
PIT_TREE_CHECKPOINT_WINDOW: zpStr(z.string().optional().default("100")), PIT_TREE_CHECKPOINT_WINDOW: zpStr(z.string().optional().default("30")),
/* CORS ----------------------------------------------------------------------------- */ /* CORS ----------------------------------------------------------------------------- */
+1
View File
@@ -577,6 +577,7 @@ export const registerRoutes = async (
const folderCommitQueueService = folderCommitQueueServiceFactory({ const folderCommitQueueService = folderCommitQueueServiceFactory({
queueService, queueService,
folderTreeCheckpointDAL, folderTreeCheckpointDAL,
keyStore,
folderTreeCheckpointResourcesDAL, folderTreeCheckpointResourcesDAL,
folderCommitDAL, folderCommitDAL,
folderDAL folderDAL
@@ -10,7 +10,7 @@ import { chunkArray } from "@app/lib/fn";
import { logger } from "@app/lib/logger"; import { logger } from "@app/lib/logger";
import { alphaNumericNanoId } from "@app/lib/nanoid"; import { alphaNumericNanoId } from "@app/lib/nanoid";
import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; import { CommitType, TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service";
import { TKmsServiceFactory } from "../kms/kms-service"; import { TKmsServiceFactory } from "../kms/kms-service";
import { KmsDataKey } from "../kms/kms-types"; import { KmsDataKey } from "../kms/kms-types";
import { TProjectDALFactory } from "../project/project-dal"; import { TProjectDALFactory } from "../project/project-dal";
@@ -627,7 +627,7 @@ export const importDataIntoInfisicalFn = async ({
folderId: parentEnv.rootFolderId, folderId: parentEnv.rootFolderId,
changes: [ changes: [
{ {
type: "add", type: CommitType.ADD,
folderVersionId: newFolderVersion.id folderVersionId: newFolderVersion.id
} }
] ]
@@ -21,7 +21,8 @@ export const folderCheckpointDALFactory = (db: TDbClient) => {
const findByCommitId = async (folderCommitId: string, tx?: Knex): Promise<TFolderCheckpoints | undefined> => { const findByCommitId = async (folderCommitId: string, tx?: Knex): Promise<TFolderCheckpoints | undefined> => {
try { try {
const doc = await (tx || db.replicaNode())<TFolderCheckpoints>(TableName.FolderCheckpoint) const doc = await (tx || db.replicaNode())<TFolderCheckpoints>(TableName.FolderCheckpoint)
.where({ folderCommitId }) // eslint-disable-next-line @typescript-eslint/no-misused-promises
.where(buildFindFilter({ folderCommitId }, TableName.FolderCheckpoint))
.select(selectAllTableCols(TableName.FolderCheckpoint)) .select(selectAllTableCols(TableName.FolderCheckpoint))
.first(); .first();
return doc; return doc;
@@ -87,20 +88,11 @@ export const folderCheckpointDALFactory = (db: TDbClient) => {
}; };
const findNearestCheckpoint = async ( const findNearestCheckpoint = async (
folderCommitId: string, folderCommitId: bigint,
folderId: string,
tx?: Knex tx?: Knex
): Promise<(CheckpointWithCommitInfo & { commitId: number }) | undefined> => { ): Promise<(CheckpointWithCommitInfo & { commitId: bigint }) | undefined> => {
try { try {
// First, get the commit info to find the folder ID and commit sequence number
const commit = await (tx || db.replicaNode())(TableName.FolderCommit)
.where({ id: folderCommitId })
.select("commitId", "folderId")
.first();
if (!commit) {
return undefined;
}
// Get the checkpoint with the highest commitId that's still less than or equal to our commit // Get the checkpoint with the highest commitId that's still less than or equal to our commit
const nearestCheckpoint = await (tx || db.replicaNode())(TableName.FolderCheckpoint) const nearestCheckpoint = await (tx || db.replicaNode())(TableName.FolderCheckpoint)
.join<TFolderCommits>( .join<TFolderCommits>(
@@ -108,8 +100,8 @@ export const folderCheckpointDALFactory = (db: TDbClient) => {
`${TableName.FolderCheckpoint}.folderCommitId`, `${TableName.FolderCheckpoint}.folderCommitId`,
`${TableName.FolderCommit}.id` `${TableName.FolderCommit}.id`
) )
.where(`${TableName.FolderCommit}.folderId`, "=", commit.folderId) .where(`${TableName.FolderCommit}.folderId`, "=", folderId)
.where(`${TableName.FolderCommit}.commitId`, "<=", commit.commitId) .where(`${TableName.FolderCommit}.commitId`, "<=", folderCommitId)
.select(selectAllTableCols(TableName.FolderCheckpoint)) .select(selectAllTableCols(TableName.FolderCheckpoint))
.select( .select(
db.ref("actorMetadata").withSchema(TableName.FolderCommit), db.ref("actorMetadata").withSchema(TableName.FolderCommit),
@@ -121,7 +113,6 @@ export const folderCheckpointDALFactory = (db: TDbClient) => {
) )
.orderBy(`${TableName.FolderCommit}.commitId`, "desc") .orderBy(`${TableName.FolderCommit}.commitId`, "desc")
.first(); .first();
return nearestCheckpoint; return nearestCheckpoint;
} catch (error) { } catch (error) {
throw new DatabaseError({ error, name: "FindNearestCheckpoint" }); throw new DatabaseError({ error, name: "FindNearestCheckpoint" });
@@ -5,11 +5,12 @@ import {
TableName, TableName,
TFolderCommitChanges, TFolderCommitChanges,
TFolderCommits, TFolderCommits,
TProjectEnvironments,
TSecretFolderVersions, TSecretFolderVersions,
TSecretVersionsV2 TSecretVersionsV2
} from "@app/db/schemas"; } from "@app/db/schemas";
import { DatabaseError } from "@app/lib/errors"; import { DatabaseError } from "@app/lib/errors";
import { ormify, selectAllTableCols } from "@app/lib/knex"; import { buildFindFilter, ormify, selectAllTableCols } from "@app/lib/knex";
export type TFolderCommitChangesDALFactory = ReturnType<typeof folderCommitChangesDALFactory>; export type TFolderCommitChangesDALFactory = ReturnType<typeof folderCommitChangesDALFactory>;
@@ -41,10 +42,15 @@ type CommitChangeWithCommitInfo = TFolderCommitChanges & {
export const folderCommitChangesDALFactory = (db: TDbClient) => { export const folderCommitChangesDALFactory = (db: TDbClient) => {
const folderCommitChangesOrm = ormify(db, TableName.FolderCommitChanges); const folderCommitChangesOrm = ormify(db, TableName.FolderCommitChanges);
const findByCommitId = async (folderCommitId: string, tx?: Knex): Promise<CommitChangeWithCommitInfo[]> => { const findByCommitId = async (
folderCommitId: string,
projectId: string,
tx?: Knex
): Promise<CommitChangeWithCommitInfo[]> => {
try { try {
const docs = await (tx || db.replicaNode())<TFolderCommitChanges>(TableName.FolderCommitChanges) const docs = await (tx || db.replicaNode())<TFolderCommitChanges>(TableName.FolderCommitChanges)
.where({ folderCommitId }) // eslint-disable-next-line @typescript-eslint/no-misused-promises
.where(buildFindFilter({ folderCommitId }, TableName.FolderCommitChanges))
.leftJoin<TFolderCommits>( .leftJoin<TFolderCommits>(
TableName.FolderCommit, TableName.FolderCommit,
`${TableName.FolderCommitChanges}.folderCommitId`, `${TableName.FolderCommitChanges}.folderCommitId`,
@@ -60,6 +66,16 @@ export const folderCommitChangesDALFactory = (db: TDbClient) => {
`${TableName.FolderCommitChanges}.folderVersionId`, `${TableName.FolderCommitChanges}.folderVersionId`,
`${TableName.SecretFolderVersion}.id` `${TableName.SecretFolderVersion}.id`
) )
.leftJoin<TProjectEnvironments>(
TableName.Environment,
`${TableName.FolderCommit}.envId`,
`${TableName.Environment}.id`
)
.where((qb) => {
if (projectId) {
void qb.where(`${TableName.Environment}.projectId`, "=", projectId);
}
})
.select(selectAllTableCols(TableName.FolderCommitChanges)) .select(selectAllTableCols(TableName.FolderCommitChanges))
.select( .select(
db.ref("name").withSchema(TableName.SecretFolderVersion).as("folderName"), db.ref("name").withSchema(TableName.SecretFolderVersion).as("folderName"),
@@ -90,7 +106,8 @@ export const folderCommitChangesDALFactory = (db: TDbClient) => {
TFolderCommitChanges & TFolderCommitChanges &
Pick<TFolderCommits, "actorMetadata" | "actorType" | "message" | "createdAt" | "folderId"> Pick<TFolderCommits, "actorMetadata" | "actorType" | "message" | "createdAt" | "folderId">
>(TableName.FolderCommitChanges) >(TableName.FolderCommitChanges)
.where({ secretVersionId }) // eslint-disable-next-line @typescript-eslint/no-misused-promises
.where(buildFindFilter({ secretVersionId }, TableName.FolderCommitChanges))
.select(selectAllTableCols(TableName.FolderCommitChanges)) .select(selectAllTableCols(TableName.FolderCommitChanges))
.join(TableName.FolderCommit, `${TableName.FolderCommitChanges}.folderCommitId`, `${TableName.FolderCommit}.id`) .join(TableName.FolderCommit, `${TableName.FolderCommitChanges}.folderCommitId`, `${TableName.FolderCommit}.id`)
.select( .select(
@@ -112,7 +129,8 @@ export const folderCommitChangesDALFactory = (db: TDbClient) => {
TFolderCommitChanges & TFolderCommitChanges &
Pick<TFolderCommits, "actorMetadata" | "actorType" | "message" | "createdAt" | "folderId"> Pick<TFolderCommits, "actorMetadata" | "actorType" | "message" | "createdAt" | "folderId">
>(TableName.FolderCommitChanges) >(TableName.FolderCommitChanges)
.where({ folderVersionId }) // eslint-disable-next-line @typescript-eslint/no-misused-promises
.where(buildFindFilter({ folderVersionId }, TableName.FolderCommitChanges))
.select(selectAllTableCols(TableName.FolderCommitChanges)) .select(selectAllTableCols(TableName.FolderCommitChanges))
.join(TableName.FolderCommit, `${TableName.FolderCommitChanges}.folderCommitId`, `${TableName.FolderCommit}.id`) .join(TableName.FolderCommit, `${TableName.FolderCommitChanges}.folderCommitId`, `${TableName.FolderCommit}.id`)
.select( .select(
@@ -5,6 +5,7 @@ import {
TableName, TableName,
TFolderCommitChanges, TFolderCommitChanges,
TFolderCommits, TFolderCommits,
TProjectEnvironments,
TSecretFolderVersions, TSecretFolderVersions,
TSecretVersionsV2 TSecretVersionsV2
} from "@app/db/schemas"; } from "@app/db/schemas";
@@ -54,10 +55,20 @@ export const folderCommitDALFactory = (db: TDbClient) => {
} }
}; };
const findLatestCommit = async (folderId: string, tx?: Knex): Promise<TFolderCommits | undefined> => { const findLatestCommit = async (
folderId: string,
projectId?: string,
tx?: Knex
): Promise<TFolderCommits | undefined> => {
try { try {
const doc = await (tx || db.replicaNode())(TableName.FolderCommit) const doc = await (tx || db.replicaNode())(TableName.FolderCommit)
.where({ folderId }) .where({ folderId })
.leftJoin(TableName.Environment, `${TableName.FolderCommit}.envId`, `${TableName.Environment}.id`)
.where((qb) => {
if (projectId) {
void qb.where(`${TableName.Environment}.projectId`, "=", projectId);
}
})
.select(selectAllTableCols(TableName.FolderCommit)) .select(selectAllTableCols(TableName.FolderCommit))
.orderBy("commitId", "desc") .orderBy("commitId", "desc")
.first(); .first();
@@ -178,8 +189,8 @@ export const folderCommitDALFactory = (db: TDbClient) => {
const findCommitsToRecreate = async ( const findCommitsToRecreate = async (
folderId: string, folderId: string,
targetCommitNumber: number, targetCommitNumber: bigint,
checkpointCommitNumber: number, checkpointCommitNumber: bigint,
tx?: Knex tx?: Knex
): Promise< ): Promise<
(TFolderCommits & { (TFolderCommits & {
@@ -213,7 +224,7 @@ export const folderCommitDALFactory = (db: TDbClient) => {
// Get all changes for these commits in a single query // Get all changes for these commits in a single query
const allChanges = await (tx || db.replicaNode())(TableName.FolderCommitChanges) const allChanges = await (tx || db.replicaNode())(TableName.FolderCommitChanges)
.whereIn("folderCommitId", commitIds) .whereIn(`${TableName.FolderCommitChanges}.folderCommitId`, commitIds)
.leftJoin<TSecretVersionsV2>( .leftJoin<TSecretVersionsV2>(
TableName.SecretVersionV2, TableName.SecretVersionV2,
`${TableName.FolderCommitChanges}.secretVersionId`, `${TableName.FolderCommitChanges}.secretVersionId`,
@@ -291,7 +302,6 @@ export const folderCommitDALFactory = (db: TDbClient) => {
endCommitId, endCommitId,
tx tx
}: { }: {
folderId?: string;
envId?: string; envId?: string;
startCommitId?: string; startCommitId?: string;
endCommitId?: string; endCommitId?: string;
@@ -323,7 +333,6 @@ export const folderCommitDALFactory = (db: TDbClient) => {
startCommitId, startCommitId,
tx tx
}: { }: {
folderId?: string;
envId?: string; envId?: string;
startCommitId?: string; startCommitId?: string;
tx?: Knex; tx?: Knex;
@@ -346,7 +355,11 @@ export const folderCommitDALFactory = (db: TDbClient) => {
} }
}; };
const findPreviousCommitTo = async (folderId: string, commitId: string, tx?: Knex): Promise<TFolderCommits | undefined> => { const findPreviousCommitTo = async (
folderId: string,
commitId: string,
tx?: Knex
): Promise<TFolderCommits | undefined> => {
try { try {
const doc = await (tx || db.replicaNode())(TableName.FolderCommit) const doc = await (tx || db.replicaNode())(TableName.FolderCommit)
.where({ folderId }) .where({ folderId })
@@ -360,6 +373,30 @@ export const folderCommitDALFactory = (db: TDbClient) => {
} }
}; };
const findById = async (id: string, tx?: Knex, projectId?: string): Promise<TFolderCommits> => {
try {
const doc = await (tx || db.replicaNode())(TableName.FolderCommit)
// eslint-disable-next-line @typescript-eslint/no-misused-promises
.where(buildFindFilter({ id }, TableName.FolderCommit))
.leftJoin<TProjectEnvironments>(
TableName.Environment,
`${TableName.FolderCommit}.envId`,
`${TableName.Environment}.id`
)
.where((qb) => {
if (projectId) {
void qb.where(`${TableName.Environment}.projectId`, "=", projectId);
}
})
.select(selectAllTableCols(TableName.FolderCommit))
.orderBy("commitId", "desc")
.first();
return doc;
} catch (error) {
throw new DatabaseError({ error, name: "FindById" });
}
};
return { return {
...restOfOrm, ...restOfOrm,
findByFolderId, findByFolderId,
@@ -373,6 +410,7 @@ export const folderCommitDALFactory = (db: TDbClient) => {
getEnvNumberOfCommitsSince, getEnvNumberOfCommitsSince,
findLatestCommitByFolderIds, findLatestCommitByFolderIds,
findAllFolderCommitsAfter, findAllFolderCommitsAfter,
findPreviousCommitTo findPreviousCommitTo,
findById
}; };
}; };
@@ -1,6 +1,7 @@
import { Knex } from "knex"; import { Knex } from "knex";
import { TSecretFolders } from "@app/db/schemas"; import { TSecretFolders } from "@app/db/schemas";
import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore";
import { getConfig } from "@app/lib/config/env"; import { getConfig } from "@app/lib/config/env";
import { logger } from "@app/lib/logger"; import { logger } from "@app/lib/logger";
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
@@ -10,8 +11,16 @@ import { TFolderTreeCheckpointResourcesDALFactory } from "../folder-tree-checkpo
import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
import { TFolderCommitDALFactory } from "./folder-commit-dal"; import { TFolderCommitDALFactory } from "./folder-commit-dal";
// Define types for job data
type TCreateFolderTreeCheckpointDTO = {
envId: string;
failedToAcquireLockCount?: number;
folderCommitId?: string;
};
type TFolderCommitQueueServiceFactoryDep = { type TFolderCommitQueueServiceFactoryDep = {
queueService: TQueueServiceFactory; queueService: TQueueServiceFactory;
keyStore: Pick<TKeyStoreFactory, "acquireLock" | "getItem" | "deleteItem">;
folderTreeCheckpointDAL: Pick< folderTreeCheckpointDAL: Pick<
TFolderTreeCheckpointDALFactory, TFolderTreeCheckpointDALFactory,
"create" | "findLatestByEnvId" | "findNearestCheckpoint" "create" | "findLatestByEnvId" | "findNearestCheckpoint"
@@ -31,6 +40,7 @@ export type TFolderCommitQueueServiceFactory = ReturnType<typeof folderCommitQue
export const folderCommitQueueServiceFactory = ({ export const folderCommitQueueServiceFactory = ({
queueService, queueService,
keyStore,
folderTreeCheckpointDAL, folderTreeCheckpointDAL,
folderTreeCheckpointResourcesDAL, folderTreeCheckpointResourcesDAL,
folderCommitDAL, folderCommitDAL,
@@ -38,48 +48,38 @@ export const folderCommitQueueServiceFactory = ({
}: TFolderCommitQueueServiceFactoryDep) => { }: TFolderCommitQueueServiceFactoryDep) => {
const appCfg = getConfig(); const appCfg = getConfig();
const scheduleTreeCheckpoint = async (envId: string) => { // Helper function to calculate delay for requeuing
await queueService.queue( const getRequeueDelay = (failureCount?: number) => {
QueueName.FolderTreeCheckpoint, if (!failureCount) return 0;
QueueJobs.CreateFolderTreeCheckpoint,
{ envId }, const baseDelay = 5000;
{ const maxDelay = 30000;
jobId: `${envId}`,
backoff: { const delay = Math.min(baseDelay * 2 ** failureCount, maxDelay);
type: "exponential", const jitter = delay * (0.5 + Math.random() * 0.5);
delay: 3000
}, return jitter;
removeOnFail: {
count: 3
},
removeOnComplete: true
}
);
}; };
const schedulePeriodicTreeCheckpoint = async (envId: string, intervalMs: number) => { const scheduleTreeCheckpoint = async (payload: TCreateFolderTreeCheckpointDTO) => {
await queueService.queue( const { envId, failedToAcquireLockCount = 0 } = payload;
QueueName.FolderTreeCheckpoint,
QueueJobs.CreateFolderTreeCheckpoint,
{ envId },
{
jobId: `periodic-${envId}`,
repeat: {
every: intervalMs
},
backoff: {
type: "exponential",
delay: 3000
},
removeOnFail: false,
removeOnComplete: false
}
);
};
const cancelScheduledTreeCheckpoint = async (envId: string) => { // Create a unique jobId for each retry to prevent conflicts
await queueService.stopJobById(QueueName.FolderTreeCheckpoint, envId); const jobId =
await queueService.stopRepeatableJobByJobId(QueueName.FolderTreeCheckpoint, `periodic-${envId}`); failedToAcquireLockCount > 0 ? `${envId}-retry-${failedToAcquireLockCount}-${Date.now()}` : `${envId}`;
await queueService.queue(QueueName.FolderTreeCheckpoint, QueueJobs.CreateFolderTreeCheckpoint, payload, {
jobId,
delay: getRequeueDelay(failedToAcquireLockCount),
backoff: {
type: "exponential",
delay: 3000
},
removeOnFail: {
count: 3
},
removeOnComplete: true
});
}; };
// Sort folders by hierarchy (copied from the source code) // Sort folders by hierarchy (copied from the source code)
@@ -121,66 +121,152 @@ export const folderCommitQueueServiceFactory = ({
return result; return result;
}; };
const createFolderTreeCheckpoint = async (envId: string, folderCommitId?: string, tx?: Knex) => { const createFolderTreeCheckpoint = async (jobData: TCreateFolderTreeCheckpointDTO, tx?: Knex) => {
logger.info("Folder tree checkpoint creation started:", envId); const { envId, folderCommitId, failedToAcquireLockCount = 0 } = jobData;
const latestTreeCheckpoint = await folderTreeCheckpointDAL.findLatestByEnvId(envId, tx); logger.info(`Folder tree checkpoint creation started [envId=${envId}] [attempt=${failedToAcquireLockCount + 1}]`);
let latestCommit; // First, try to clear any stale locks before attempting to acquire
if (folderCommitId) { if (failedToAcquireLockCount > 1) {
latestCommit = await folderCommitDAL.findById(folderCommitId, tx); try {
} else { await keyStore.deleteItem(KeyStorePrefixes.FolderTreeCheckpoint(envId));
latestCommit = await folderCommitDAL.findLatestEnvCommit(envId, tx); logger.info(`Cleared potential stale lock for envId ${envId} before attempt ${failedToAcquireLockCount + 1}`);
} } catch (error) {
if (!latestCommit) { // This is fine if it fails, we'll still try to acquire the lock
logger.info(`Latest commit ID not found for envId ${envId}`); logger.info(`No stale lock found for envId ${envId}`);
return;
}
const latestCommitId = latestCommit.id;
if (latestTreeCheckpoint) {
const commitsSinceLastCheckpoint = await folderCommitDAL.getEnvNumberOfCommitsSince(
envId,
latestTreeCheckpoint.folderCommitId,
tx
);
if (commitsSinceLastCheckpoint < Number(appCfg.PIT_TREE_CHECKPOINT_WINDOW)) {
logger.info(
`Commits since last checkpoint ${commitsSinceLastCheckpoint} is less than ${appCfg.PIT_TREE_CHECKPOINT_WINDOW}`
);
return;
} }
} }
const folders = await folderDAL.findByEnvId(envId, tx); let lock: Awaited<ReturnType<typeof keyStore.acquireLock>> | undefined;
const sortedFolders = sortFoldersByHierarchy(folders);
const filteredFoldersIds = sortedFolders.filter((folder) => !folder.isReserved).map((folder) => folder.id);
const folderCommits = await folderCommitDAL.findMultipleLatestCommits(filteredFoldersIds, tx); try {
const folderTreeCheckpoint = await folderTreeCheckpointDAL.create( // Attempt to acquire the lock with a shorter timeout for first attempts
{ const timeout = failedToAcquireLockCount > 3 ? 60 * 1000 : 15 * 1000;
folderCommitId: latestCommitId
},
tx
);
await folderTreeCheckpointResourcesDAL.insertMany( logger.info(`Attempting to acquire lock for envId=${envId} with timeout ${timeout}ms`);
folderCommits.map((folderCommit) => ({
folderTreeCheckpointId: folderTreeCheckpoint.id,
folderId: folderCommit.folderId,
folderCommitId: folderCommit.id
})),
tx
);
logger.info("Folder tree checkpoint created successfully:", folderTreeCheckpoint.id); lock = await keyStore.acquireLock([KeyStorePrefixes.FolderTreeCheckpoint(envId)], timeout);
logger.info(`Successfully acquired lock for envId=${envId}`);
} catch (e) {
logger.info(
`Failed to acquire lock for folder tree checkpoint [envId=${envId}] [attempt=${failedToAcquireLockCount + 1}]`
);
// Requeue with incremented failure count if under max attempts
if (failedToAcquireLockCount < 10) {
// Force a delay between retries
const nextRetryCount = failedToAcquireLockCount + 1;
logger.info(`Scheduling retry #${nextRetryCount} for folder tree checkpoint [envId=${envId}]`);
// Create a new job with incremented counter
await scheduleTreeCheckpoint({
envId,
folderCommitId,
failedToAcquireLockCount: nextRetryCount
});
} else {
// Max retries reached
logger.error(`Maximum lock acquisition attempts (10) reached for envId ${envId}. Giving up.`);
// Try to force-clear the lock for next time
try {
await keyStore.deleteItem(KeyStorePrefixes.FolderTreeCheckpoint(envId));
} catch (clearError) {
logger.error(clearError, `Failed to clear lock after maximum retries for envId=${envId}`);
}
}
return;
}
if (!lock) {
logger.error(`Lock is undefined after acquisition for envId=${envId}. This should never happen.`);
return;
}
try {
logger.info(`Processing tree checkpoint data for envId=${envId}`);
const latestTreeCheckpoint = await folderTreeCheckpointDAL.findLatestByEnvId(envId, tx);
let latestCommit;
if (folderCommitId) {
latestCommit = await folderCommitDAL.findById(folderCommitId, tx);
} else {
latestCommit = await folderCommitDAL.findLatestEnvCommit(envId, tx);
}
if (!latestCommit) {
logger.info(`Latest commit ID not found for envId ${envId}`);
return;
}
const latestCommitId = latestCommit.id;
if (latestTreeCheckpoint) {
const commitsSinceLastCheckpoint = await folderCommitDAL.getEnvNumberOfCommitsSince(
envId,
latestTreeCheckpoint.folderCommitId,
tx
);
if (commitsSinceLastCheckpoint < Number(appCfg.PIT_TREE_CHECKPOINT_WINDOW)) {
logger.info(
`Commits since last checkpoint ${commitsSinceLastCheckpoint} is less than ${appCfg.PIT_TREE_CHECKPOINT_WINDOW}`
);
return;
}
}
const folders = await folderDAL.findByEnvId(envId, tx);
const sortedFolders = sortFoldersByHierarchy(folders);
const filteredFoldersIds = sortedFolders.filter((folder) => !folder.isReserved).map((folder) => folder.id);
const folderCommits = await folderCommitDAL.findMultipleLatestCommits(filteredFoldersIds, tx);
const folderTreeCheckpoint = await folderTreeCheckpointDAL.create(
{
folderCommitId: latestCommitId
},
tx
);
await folderTreeCheckpointResourcesDAL.insertMany(
folderCommits.map((folderCommit) => ({
folderTreeCheckpointId: folderTreeCheckpoint.id,
folderId: folderCommit.folderId,
folderCommitId: folderCommit.id
})),
tx
);
logger.info(`Folder tree checkpoint created successfully: ${folderTreeCheckpoint.id}`);
} catch (error) {
logger.error(error, `Error processing folder tree checkpoint [envId=${envId}]`);
throw error;
} finally {
// Always release the lock
try {
if (lock) {
await lock.release();
logger.info(`Released lock for folder tree checkpoint [envId=${envId}]`);
} else {
logger.error(`No lock to release for envId=${envId}. This should never happen.`);
}
} catch (releaseError) {
logger.error(releaseError, `Error releasing lock for folder tree checkpoint [envId=${envId}]`);
// Try to force delete the lock if release fails
try {
await keyStore.deleteItem(KeyStorePrefixes.FolderTreeCheckpoint(envId));
logger.info(`Force deleted lock after release failure for envId=${envId}`);
} catch (deleteError) {
logger.error(deleteError, `Failed to force delete lock after release failure for envId=${envId}`);
}
}
}
}; };
queueService.start(QueueName.FolderTreeCheckpoint, async (job) => { queueService.start(QueueName.FolderTreeCheckpoint, async (job) => {
try { try {
if (job.name === QueueJobs.CreateFolderTreeCheckpoint) { if (job.name === QueueJobs.CreateFolderTreeCheckpoint) {
const { envId } = job.data as { envId: string }; const jobData = job.data as TCreateFolderTreeCheckpointDTO;
await createFolderTreeCheckpoint(envId); await createFolderTreeCheckpoint(jobData);
} }
} catch (error) { } catch (error) {
logger.error(error, "Error creating folder tree checkpoint:"); logger.error(error, "Error creating folder tree checkpoint:");
@@ -189,9 +275,8 @@ export const folderCommitQueueServiceFactory = ({
}); });
return { return {
scheduleTreeCheckpoint, scheduleTreeCheckpoint: (envId: string) => scheduleTreeCheckpoint({ envId }),
schedulePeriodicTreeCheckpoint, createFolderTreeCheckpoint: (envId: string, folderCommitId?: string, tx?: Knex) =>
cancelScheduledTreeCheckpoint, createFolderTreeCheckpoint({ envId, folderCommitId }, tx)
createFolderTreeCheckpoint
}; };
}; };
@@ -5,7 +5,12 @@ import { TSecretFolderVersions, TSecretVersionsV2 } from "@app/db/schemas";
import { BadRequestError, NotFoundError } from "@app/lib/errors"; import { BadRequestError, NotFoundError } from "@app/lib/errors";
import { ActorType } from "../auth/auth-type"; import { ActorType } from "../auth/auth-type";
import { ChangeType, folderCommitServiceFactory, TFolderCommitServiceFactory } from "./folder-commit-service"; import {
ChangeType,
CommitType,
folderCommitServiceFactory,
TFolderCommitServiceFactory
} from "./folder-commit-service";
// Mock config // Mock config
vi.mock("@app/lib/config/env", () => ({ vi.mock("@app/lib/config/env", () => ({
@@ -204,7 +209,7 @@ describe("folderCommitServiceFactory", () => {
folderId: folderData.id, folderId: folderData.id,
changes: [ changes: [
{ {
type: "add", type: CommitType.ADD,
secretVersionId: "secret-version-1" secretVersionId: "secret-version-1"
} }
] ]
@@ -264,7 +269,7 @@ describe("folderCommitServiceFactory", () => {
folderId: folderData.id, folderId: folderData.id,
changes: [ changes: [
{ {
type: "add", type: CommitType.ADD,
folderVersionId: "folder-version-1" folderVersionId: "folder-version-1"
} }
] ]
@@ -331,7 +336,7 @@ describe("folderCommitServiceFactory", () => {
const data = { const data = {
folderCommitId: commitData.id, folderCommitId: commitData.id,
changeType: "add", changeType: CommitType.ADD,
secretVersionId: "secret-version-1" secretVersionId: "secret-version-1"
}; };
@@ -348,7 +353,7 @@ describe("folderCommitServiceFactory", () => {
// Arrange // Arrange
const data = { const data = {
folderCommitId: "commit-id", folderCommitId: "commit-id",
changeType: "add" changeType: CommitType.ADD
}; };
// Act & Assert // Act & Assert
@@ -361,7 +366,7 @@ describe("folderCommitServiceFactory", () => {
const data = { const data = {
folderCommitId: "non-existent-commit", folderCommitId: "non-existent-commit",
changeType: "add", changeType: CommitType.ADD,
secretVersionId: "secret-version-1" secretVersionId: "secret-version-1"
}; };
@@ -32,6 +32,11 @@ export enum ChangeType {
CREATE = "create" CREATE = "create"
} }
export enum CommitType {
ADD = "add",
DELETE = "delete"
}
enum ResourceType { enum ResourceType {
SECRET = "secret", SECRET = "secret",
FOLDER = "folder" FOLDER = "folder"
@@ -68,11 +73,12 @@ export type ResourceChange = {
versionId: string; versionId: string;
oldVersionId?: string; oldVersionId?: string;
changeType: ChangeType; changeType: ChangeType;
commitId: number; commitId: bigint;
createdAt?: Date; createdAt?: Date;
parentId?: string; parentId?: string;
secretKey?: string; secretKey?: string;
secretVersion?: string; secretVersion?: string;
secretId?: string;
folderName?: string; folderName?: string;
folderVersion?: string; folderVersion?: string;
fromVersion?: string; fromVersion?: string;
@@ -209,7 +215,7 @@ export const folderCommitServiceFactory = ({
const latestCheckpoint = await folderCheckpointDAL.findLatestByFolderId(folderId, tx); const latestCheckpoint = await folderCheckpointDAL.findLatestByFolderId(folderId, tx);
if (!latestCommitId) { if (!latestCommitId) {
const latestCommit = await folderCommitDAL.findLatestCommit(folderId, tx); const latestCommit = await folderCommitDAL.findLatestCommit(folderId, undefined, tx);
if (!latestCommit) { if (!latestCommit) {
throw new BadRequestError({ message: "Latest commit ID not found" }); throw new BadRequestError({ message: "Latest commit ID not found" });
} }
@@ -264,7 +270,11 @@ export const folderCommitServiceFactory = ({
throw new NotFoundError({ message: `Commit with ID ${folderCommitId} not found` }); throw new NotFoundError({ message: `Commit with ID ${folderCommitId} not found` });
} }
const nearestCheckpoint = await folderCheckpointDAL.findNearestCheckpoint(folderCommitId, tx); const nearestCheckpoint = await folderCheckpointDAL.findNearestCheckpoint(
targetCommit.commitId,
targetCommit.folderId,
tx
);
if (!nearestCheckpoint) { if (!nearestCheckpoint) {
throw new NotFoundError({ message: `Nearest checkpoint not found for commit ${folderCommitId}` }); throw new NotFoundError({ message: `Nearest checkpoint not found for commit ${folderCommitId}` });
} }
@@ -288,7 +298,7 @@ export const folderCommitServiceFactory = ({
checkpointResources.forEach((resource) => { checkpointResources.forEach((resource) => {
if (resource.secretVersionId && resource.referencedSecretId) { if (resource.secretVersionId && resource.referencedSecretId) {
folderState[`secret-${resource.referencedSecretId}`] = { folderState[`secret-${resource.referencedSecretId}`] = {
type: "secret", type: ResourceType.SECRET,
id: resource.referencedSecretId, id: resource.referencedSecretId,
versionId: resource.secretVersionId, versionId: resource.secretVersionId,
secretKey: resource.secretKey, secretKey: resource.secretKey,
@@ -296,7 +306,7 @@ export const folderCommitServiceFactory = ({
}; };
} else if (resource.folderVersionId && resource.referencedFolderId) { } else if (resource.folderVersionId && resource.referencedFolderId) {
folderState[`folder-${resource.referencedFolderId}`] = { folderState[`folder-${resource.referencedFolderId}`] = {
type: "folder", type: ResourceType.FOLDER,
id: resource.referencedFolderId, id: resource.referencedFolderId,
versionId: resource.folderVersionId, versionId: resource.folderVersionId,
folderName: resource.folderName, folderName: resource.folderName,
@@ -323,7 +333,7 @@ export const folderCommitServiceFactory = ({
if (change.changeType.toLowerCase() === "add") { if (change.changeType.toLowerCase() === "add") {
folderState[key] = { folderState[key] = {
type: "secret", type: ResourceType.SECRET,
id: change.referencedSecretId, id: change.referencedSecretId,
versionId: change.secretVersionId, versionId: change.secretVersionId,
secretKey: change.secretKey, secretKey: change.secretKey,
@@ -337,7 +347,7 @@ export const folderCommitServiceFactory = ({
if (change.changeType.toLowerCase() === "add") { if (change.changeType.toLowerCase() === "add") {
folderState[key] = { folderState[key] = {
type: "folder", type: ResourceType.FOLDER,
id: change.referencedFolderId, id: change.referencedFolderId,
versionId: change.folderVersionId, versionId: change.folderVersionId,
folderName: change.folderName, folderName: change.folderName,
@@ -902,12 +912,14 @@ export const folderCommitServiceFactory = ({
const secretVersions = await secretVersionV2BridgeDAL.findByIdsWithLatestVersion( const secretVersions = await secretVersionV2BridgeDAL.findByIdsWithLatestVersion(
folderId, folderId,
secretChanges.map((diff) => diff.id), secretChanges.map((diff) => diff.id),
secretChanges.map((diff) => diff.versionId) secretChanges.map((diff) => diff.versionId),
tx
); );
const folderVersions = await folderVersionDAL.findByIdsWithLatestVersion( const folderVersions = await folderVersionDAL.findByIdsWithLatestVersion(
folderChanges.map((diff) => diff.id), folderChanges.map((diff) => diff.id),
folderChanges.map((diff) => diff.versionId) folderChanges.map((diff) => diff.versionId),
tx
); );
// Process changes in parallel // Process changes in parallel
@@ -1072,8 +1084,8 @@ export const folderCommitServiceFactory = ({
actorOrgId, actorOrgId,
projectId projectId
}); });
const changes = await folderCommitChangesDAL.findByCommitId(commitId); const changes = await folderCommitChangesDAL.findByCommitId(commitId, projectId);
const commit = await folderCommitDAL.findById(commitId); const commit = await folderCommitDAL.findById(commitId, undefined, projectId);
return { ...commit, changes }; return { ...commit, changes };
}; };
@@ -1244,8 +1256,6 @@ export const folderCommitServiceFactory = ({
}: { }: {
targetCommitId: string; targetCommitId: string;
envId: string; envId: string;
actorId: string;
actorType: ActorType;
projectId: string; projectId: string;
tx?: Knex; tx?: Knex;
}) => { }) => {
@@ -1254,7 +1264,7 @@ export const folderCommitServiceFactory = ({
throw new NotFoundError({ message: `No commit found for commit ID ${targetCommitId}` }); throw new NotFoundError({ message: `No commit found for commit ID ${targetCommitId}` });
} }
const checkpoint = await folderTreeCheckpointDAL.findNearestCheckpoint(targetCommitId, envId, tx); const checkpoint = await folderTreeCheckpointDAL.findNearestCheckpoint(targetCommit.commitId, envId, tx);
if (!checkpoint) { if (!checkpoint) {
throw new NotFoundError({ message: `No checkpoint found for commit ID ${targetCommitId}` }); throw new NotFoundError({ message: `No checkpoint found for commit ID ${targetCommitId}` });
} }
@@ -1267,7 +1277,7 @@ export const folderCommitServiceFactory = ({
}); });
// Group commits by folderId and keep only the latest // Group commits by folderId and keep only the latest
const folderGroups = new Map<string, { commitId: number; id: string }>(); const folderGroups = new Map<string, { commitId: bigint; id: string }>();
if (folderCheckpointCommits && folderCheckpointCommits.length > 0) { if (folderCheckpointCommits && folderCheckpointCommits.length > 0) {
for (const commit of folderCheckpointCommits) { for (const commit of folderCheckpointCommits) {
@@ -1358,7 +1368,7 @@ export const folderCommitServiceFactory = ({
throw new NotFoundError({ message: `No commit found for commit ID ${targetCommitId}` }); throw new NotFoundError({ message: `No commit found for commit ID ${targetCommitId}` });
} }
const checkpoint = await folderTreeCheckpointDAL.findNearestCheckpoint(targetCommitId, envId, tx); const checkpoint = await folderTreeCheckpointDAL.findNearestCheckpoint(targetCommit.commitId, envId, tx);
if (!checkpoint) { if (!checkpoint) {
throw new NotFoundError({ message: `No checkpoint found for commit ID ${targetCommitId}` }); throw new NotFoundError({ message: `No checkpoint found for commit ID ${targetCommitId}` });
} }
@@ -1371,7 +1381,7 @@ export const folderCommitServiceFactory = ({
}); });
// Group commits by folderId and keep only the latest // Group commits by folderId and keep only the latest
const folderGroups = new Map<string, { commitId: number; id: string }>(); const folderGroups = new Map<string, { commitId: bigint; id: string }>();
if (folderCheckpointCommits && folderCheckpointCommits.length > 0) { if (folderCheckpointCommits && folderCheckpointCommits.length > 0) {
for (const commit of folderCheckpointCommits) { for (const commit of folderCheckpointCommits) {
@@ -1421,9 +1431,9 @@ export const folderCommitServiceFactory = ({
const foldersToDelete = new Set<string>(); const foldersToDelete = new Set<string>();
// Process all DELETE operations to build a complete set of folders to be deleted // Process all DELETE operations to build a complete set of folders to be deleted
for (const [folderId, changes] of folderDiffs.entries()) { for (const changes of folderDiffs.values()) {
for (const change of changes) { for (const change of changes) {
if (change.changeType === ChangeType.DELETE && change.type === "folder") { if (change.changeType === ChangeType.DELETE && change.type === ResourceType.FOLDER) {
foldersToDelete.add(change.id); foldersToDelete.add(change.id);
} }
} }
@@ -1483,7 +1493,7 @@ export const folderCommitServiceFactory = ({
actorOrgId, actorOrgId,
projectId projectId
}); });
return folderCommitDAL.findLatestCommit(folderId); return folderCommitDAL.findLatestCommit(folderId, projectId);
}; };
/** /**
@@ -1506,6 +1516,19 @@ export const folderCommitServiceFactory = ({
projectId: string; projectId: string;
message?: string; message?: string;
}) => { }) => {
const { permission } = await permissionService.getProjectPermission({
actor,
actorId,
projectId,
actorAuthMethod,
actorOrgId,
actionProjectType: ActionProjectType.SecretManager
});
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionCommitsActions.PerformRollback,
ProjectPermissionSub.Commits
);
// Check permissions first // Check permissions first
await checkProjectPermission({ await checkProjectPermission({
actor, actor,
@@ -1516,7 +1539,7 @@ export const folderCommitServiceFactory = ({
}); });
// Get the commit to revert // Get the commit to revert
const commitToRevert = await folderCommitDAL.findById(commitId); const commitToRevert = await folderCommitDAL.findById(commitId, undefined, projectId);
if (!commitToRevert) { if (!commitToRevert) {
throw new NotFoundError({ message: `Commit with ID ${commitId} not found` }); throw new NotFoundError({ message: `Commit with ID ${commitId} not found` });
} }
@@ -0,0 +1,84 @@
import { z } from "zod";
const secretVersionSchema = z.object({
secretKey: z.string(),
secretComment: z.string().optional().nullable(),
skipMultilineEncoding: z.boolean().optional().nullable(),
secretReminderRepeatDays: z.number().optional().nullable(),
secretReminderNote: z.string().optional().nullable(),
metadata: z.unknown().optional().nullable(),
tags: z.array(z.string()).optional().nullable(),
secretReminderRecipients: z.array(z.any()).optional().nullable(),
secretValue: z.string().optional().nullable()
});
const folderVersionSchema = z.object({
name: z.string()
});
const baseChangeSchema = z.object({
id: z.string(),
folderCommitId: z.string(),
changeType: z.string(),
isUpdate: z.boolean().optional(),
createdAt: z.union([z.string(), z.date()]),
updatedAt: z.union([z.string(), z.date()]),
actorMetadata: z
.union([
z.object({
id: z.string().optional(),
name: z.string().optional()
}),
z.unknown()
])
.optional(),
actorType: z.string().optional(),
message: z.string().optional().nullable(),
folderId: z.string().optional()
});
const secretChangeSchema = baseChangeSchema.extend({
objectType: z.literal("secret"),
secretVersionId: z.string(),
folderVersionId: z.null(),
folderName: z.null(),
folderChangeId: z.null(),
secretKey: z.string(),
secretVersion: z.union([z.string(), z.number()]),
secretId: z.string(),
versions: z.array(secretVersionSchema).optional()
});
const folderChangeSchema = baseChangeSchema.extend({
objectType: z.literal("folder"),
secretVersionId: z.null(),
folderVersionId: z.string(),
folderName: z.string(),
folderChangeId: z.string(),
folderVersion: z.union([z.string(), z.number()]),
secretKey: z.null(),
secretId: z.null(),
versions: z.array(folderVersionSchema).optional()
});
const commitChangeSchema = z.discriminatedUnion("objectType", [secretChangeSchema, folderChangeSchema]);
const commitSchema = z.object({
id: z.string(),
commitId: z.string(),
actorMetadata: z.object({
id: z.string(),
name: z.string()
}),
actorType: z.string(),
message: z.string().nullable(),
folderId: z.string(),
envId: z.string(),
createdAt: z.union([z.string(), z.date()]),
updatedAt: z.union([z.string(), z.date()]),
changes: z.array(commitChangeSchema)
});
export const commitChangesResponseSchema = z.object({
changes: commitSchema
});
@@ -8,7 +8,7 @@ import { buildFindFilter, ormify, selectAllTableCols } from "@app/lib/knex";
export type TFolderTreeCheckpointResourcesDALFactory = ReturnType<typeof folderTreeCheckpointResourcesDALFactory>; export type TFolderTreeCheckpointResourcesDALFactory = ReturnType<typeof folderTreeCheckpointResourcesDALFactory>;
type TFolderTreeCheckpointResourcesWithCommitId = TFolderTreeCheckpointResources & { type TFolderTreeCheckpointResourcesWithCommitId = TFolderTreeCheckpointResources & {
commitId: number; commitId: bigint;
}; };
export const folderTreeCheckpointResourcesDALFactory = (db: TDbClient) => { export const folderTreeCheckpointResourcesDALFactory = (db: TDbClient) => {
@@ -3,12 +3,12 @@ import { Knex } from "knex";
import { TDbClient } from "@app/db"; import { TDbClient } from "@app/db";
import { TableName, TFolderCommits, TFolderTreeCheckpoints } from "@app/db/schemas"; import { TableName, TFolderCommits, TFolderTreeCheckpoints } from "@app/db/schemas";
import { DatabaseError } from "@app/lib/errors"; import { DatabaseError } from "@app/lib/errors";
import { ormify, selectAllTableCols } from "@app/lib/knex"; import { buildFindFilter, ormify, selectAllTableCols } from "@app/lib/knex";
export type TFolderTreeCheckpointDALFactory = ReturnType<typeof folderTreeCheckpointDALFactory>; export type TFolderTreeCheckpointDALFactory = ReturnType<typeof folderTreeCheckpointDALFactory>;
type TreeCheckpointWithCommitInfo = TFolderTreeCheckpoints & { type TreeCheckpointWithCommitInfo = TFolderTreeCheckpoints & {
commitId: number; commitId: bigint;
}; };
export const folderTreeCheckpointDALFactory = (db: TDbClient) => { export const folderTreeCheckpointDALFactory = (db: TDbClient) => {
@@ -17,7 +17,8 @@ export const folderTreeCheckpointDALFactory = (db: TDbClient) => {
const findByCommitId = async (folderCommitId: string, tx?: Knex): Promise<TFolderTreeCheckpoints | undefined> => { const findByCommitId = async (folderCommitId: string, tx?: Knex): Promise<TFolderTreeCheckpoints | undefined> => {
try { try {
const doc = await (tx || db.replicaNode())<TFolderTreeCheckpoints>(TableName.FolderTreeCheckpoint) const doc = await (tx || db.replicaNode())<TFolderTreeCheckpoints>(TableName.FolderTreeCheckpoint)
.where({ folderCommitId }) // eslint-disable-next-line @typescript-eslint/no-misused-promises
.where(buildFindFilter({ folderCommitId }, TableName.FolderTreeCheckpoint))
.select(selectAllTableCols(TableName.FolderTreeCheckpoint)) .select(selectAllTableCols(TableName.FolderTreeCheckpoint))
.first(); .first();
return doc; return doc;
@@ -27,27 +28,20 @@ export const folderTreeCheckpointDALFactory = (db: TDbClient) => {
}; };
const findNearestCheckpoint = async ( const findNearestCheckpoint = async (
folderCommitId: string, folderCommitId: bigint,
envId: string, envId: string,
tx?: Knex tx?: Knex
): Promise<TreeCheckpointWithCommitInfo | undefined> => { ): Promise<TreeCheckpointWithCommitInfo | undefined> => {
try { try {
const targetCommit = await (tx || db.replicaNode())(TableName.FolderCommit)
.where({ id: folderCommitId })
.select("id", "commitId", "folderId", "envId")
.first();
if (!targetCommit) {
return undefined;
}
const nearestCheckpoint = await (tx || db.replicaNode())(TableName.FolderTreeCheckpoint) const nearestCheckpoint = await (tx || db.replicaNode())(TableName.FolderTreeCheckpoint)
.leftJoin<TFolderCommits>( .join<TFolderCommits>(
TableName.FolderCommit, TableName.FolderCommit,
`${TableName.FolderTreeCheckpoint}.folderCommitId`, `${TableName.FolderTreeCheckpoint}.folderCommitId`,
`${TableName.FolderCommit}.id` `${TableName.FolderCommit}.id`
) )
.where(`${TableName.FolderCommit}.envId`, "=", targetCommit.envId) // eslint-disable-next-line @typescript-eslint/no-misused-promises
.where(`${TableName.FolderCommit}.envId`, "=", envId)
.where(`${TableName.FolderCommit}.commitId`, "<=", folderCommitId)
.select(selectAllTableCols(TableName.FolderTreeCheckpoint)) .select(selectAllTableCols(TableName.FolderTreeCheckpoint))
.select(db.ref("commitId").withSchema(TableName.FolderCommit)) .select(db.ref("commitId").withSchema(TableName.FolderCommit))
.orderBy(`${TableName.FolderCommit}.commitId`, "desc") .orderBy(`${TableName.FolderCommit}.commitId`, "desc")
@@ -67,12 +61,12 @@ export const folderTreeCheckpointDALFactory = (db: TDbClient) => {
`${TableName.FolderTreeCheckpoint}.folderCommitId`, `${TableName.FolderTreeCheckpoint}.folderCommitId`,
`${TableName.FolderCommit}.id` `${TableName.FolderCommit}.id`
) )
.where(`${TableName.FolderCommit}.envId`, envId) .where(`${TableName.FolderCommit}.envId`, "=", envId)
.orderBy(`${TableName.FolderTreeCheckpoint}.createdAt`, "desc") .orderBy(`${TableName.FolderTreeCheckpoint}.createdAt`, "desc")
.first(); .first();
return doc; return doc;
} catch (error) { } catch (error) {
throw new DatabaseError({ error, name: "FindLatestByFolderId" }); throw new DatabaseError({ error, name: "FindLatestByEnvId" });
} }
}; };
@@ -10,7 +10,7 @@ import { BadRequestError, NotFoundError } from "@app/lib/errors";
import { OrderByDirection, OrgServiceActor } from "@app/lib/types"; import { OrderByDirection, OrgServiceActor } from "@app/lib/types";
import { buildFolderPath } from "@app/services/secret-folder/secret-folder-fns"; import { buildFolderPath } from "@app/services/secret-folder/secret-folder-fns";
import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; import { CommitType, TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service";
import { TProjectDALFactory } from "../project/project-dal"; import { TProjectDALFactory } from "../project/project-dal";
import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal";
import { TSecretFolderDALFactory } from "./secret-folder-dal"; import { TSecretFolderDALFactory } from "./secret-folder-dal";
@@ -134,7 +134,7 @@ export const secretFolderServiceFactory = ({
message: "Folder created", message: "Folder created",
folderId: parentFolderId, folderId: parentFolderId,
changes: folderVersions.map((fv) => ({ changes: folderVersions.map((fv) => ({
type: "add", type: CommitType.ADD,
folderVersionId: fv.id folderVersionId: fv.id
})) }))
}, },
@@ -168,7 +168,7 @@ export const secretFolderServiceFactory = ({
folderId: parentFolderId, folderId: parentFolderId,
changes: [ changes: [
{ {
type: "add", type: CommitType.ADD,
folderVersionId: folderVersion.id folderVersionId: folderVersion.id
} }
] ]
@@ -285,7 +285,7 @@ export const secretFolderServiceFactory = ({
folderId: parentFolder.id, folderId: parentFolder.id,
changes: [ changes: [
{ {
type: "add", type: CommitType.ADD,
isUpdate: true, isUpdate: true,
folderVersionId: folderVersion.id folderVersionId: folderVersion.id
} }
@@ -401,7 +401,7 @@ export const secretFolderServiceFactory = ({
folderId: parentFolder.id, folderId: parentFolder.id,
changes: [ changes: [
{ {
type: "add", type: CommitType.ADD,
isUpdate: true, isUpdate: true,
folderVersionId: folderVersion.id folderVersionId: folderVersion.id
} }
@@ -463,7 +463,7 @@ export const secretFolderServiceFactory = ({
if (!doc) throw new NotFoundError({ message: `Failed to delete folder with ID '${idOrName}', not found` }); if (!doc) throw new NotFoundError({ message: `Failed to delete folder with ID '${idOrName}', not found` });
const folderVersions = await folderVersionDAL.findLatestFolderVersions([doc.id]); const folderVersions = await folderVersionDAL.findLatestFolderVersions([doc.id], tx);
await folderCommitService.createCommit( await folderCommitService.createCommit(
{ {
@@ -477,7 +477,7 @@ export const secretFolderServiceFactory = ({
folderId: parentFolder.id, folderId: parentFolder.id,
changes: [ changes: [
{ {
type: "delete", type: CommitType.DELETE,
folderVersionId: folderVersions[doc.id].id folderVersionId: folderVersions[doc.id].id
} }
] ]
@@ -97,67 +97,78 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => {
logger.info(`${QueueName.DailyResourceCleanUp}: pruning secret folder versions completed`); logger.info(`${QueueName.DailyResourceCleanUp}: pruning secret folder versions completed`);
}; };
// Get latest versions by folderIds
const getLatestFolderVersions = async (folderIds: string[], tx?: Knex): Promise<Array<TSecretFolderVersions>> => {
if (!folderIds.length) return [];
const knexInstance = tx || db.replicaNode();
return knexInstance(TableName.SecretFolderVersion)
.whereIn(`${TableName.SecretFolderVersion}.folderId`, folderIds)
.join(
knexInstance(TableName.SecretFolderVersion)
.groupBy("folderId")
.max("version")
.select("folderId")
.as("latestVersion"),
(bd) => {
bd.on(`${TableName.SecretFolderVersion}.folderId`, "latestVersion.folderId").andOn(
`${TableName.SecretFolderVersion}.version`,
"latestVersion.max"
);
}
);
};
// Get specific versions and update with max version
const getSpecificFolderVersionsWithLatest = async (
versionIds: string[],
tx?: Knex
): Promise<Array<TSecretFolderVersions>> => {
if (!versionIds.length) return [];
const knexInstance = tx || db.replicaNode();
// Get specific versions
const specificVersions = await knexInstance(TableName.SecretFolderVersion).whereIn("id", versionIds);
// Get folderIds from these versions
const specificFolderIds = [...new Set(specificVersions.map((v) => v.folderId).filter(Boolean))];
if (!specificFolderIds.length) return specificVersions;
// Get max versions for these folderIds
const maxVersionsQuery = await knexInstance(TableName.SecretFolderVersion)
.whereIn("folderId", specificFolderIds)
.groupBy("folderId")
.select("folderId")
.max("version", { as: "maxVersion" });
// Create lookup map for max versions
const maxVersionMap = maxVersionsQuery.reduce<Record<string, number>>((acc, item) => {
acc[item.folderId] = item.maxVersion;
return acc;
}, {});
// Replace version with max version
return specificVersions.map((version) => ({
...version,
version: maxVersionMap[version.folderId] || version.version
}));
};
const findByIdsWithLatestVersion = async (folderIds: string[], versionIds?: string[], tx?: Knex) => { const findByIdsWithLatestVersion = async (folderIds: string[], versionIds?: string[], tx?: Knex) => {
try { try {
if (!folderIds.length && (!versionIds || !versionIds.length)) return {}; if (!folderIds.length && (!versionIds || !versionIds.length)) return {};
const knexInstance = tx || db.replicaNode(); const [latestVersions, specificVersionsWithLatest] = await Promise.all([
let allDocs: Array<TSecretFolderVersions & { max?: number }> = []; folderIds.length ? getLatestFolderVersions(folderIds, tx) : [],
versionIds?.length ? getSpecificFolderVersionsWithLatest(versionIds, tx) : []
]);
// Process latest versions for folderIds const allDocs = [...latestVersions, ...specificVersionsWithLatest];
if (folderIds.length) {
const latestVersions: Array<TSecretFolderVersions & { max: number }> = await knexInstance(
TableName.SecretFolderVersion
)
.whereIn(`${TableName.SecretFolderVersion}.folderId`, folderIds)
.join(
knexInstance(TableName.SecretFolderVersion)
.groupBy("folderId")
.max("version")
.select("folderId")
.as("latestVersion"),
(bd) => {
bd.on(`${TableName.SecretFolderVersion}.folderId`, "latestVersion.folderId").andOn(
`${TableName.SecretFolderVersion}.version`,
"latestVersion.max"
);
}
);
allDocs = [...allDocs, ...latestVersions]; // Convert array to record with folderId as key
} return allDocs.reduce<Record<string, TSecretFolderVersions>>(
// Process specific versions by versionIds
if (versionIds && versionIds.length) {
// Get the specific versions
const specificVersions = await knexInstance(TableName.SecretFolderVersion).whereIn("id", versionIds);
// Get the folderIds from these versions
const specificFolderIds = [...new Set(specificVersions.map((v) => v.folderId).filter(Boolean))];
// Get max versions for these folderIds
const maxVersionsQuery = (await knexInstance(TableName.SecretFolderVersion)
.whereIn("folderId", specificFolderIds)
.groupBy("folderId")
.select("folderId")
.max("version as max")) as Array<{ folderId: string; max: number }>;
// Create a lookup map for max versions
const maxVersionMap = maxVersionsQuery.reduce<Record<string, number>>((acc, item) => {
acc[item.folderId] = item.max;
return acc;
}, {});
// Add max version to each specific version
const specificVersionsWithMax = specificVersions.map((version) => ({
...version,
max: maxVersionMap[version.folderId]
}));
allDocs = [...allDocs, ...specificVersionsWithMax];
}
return allDocs.reduce<Record<string, TSecretFolderVersions & { max?: number }>>(
(prev, curr) => ({ ...prev, [curr.folderId || ""]: curr }), (prev, curr) => ({ ...prev, [curr.folderId || ""]: curr }),
{} {}
); );
@@ -8,6 +8,7 @@ import { groupBy } from "@app/lib/fn";
import { logger } from "@app/lib/logger"; import { logger } from "@app/lib/logger";
import { ActorType } from "../auth/auth-type"; import { ActorType } from "../auth/auth-type";
import { CommitType } from "../folder-commit/folder-commit-service";
import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal";
import { ResourceMetadataDTO } from "../resource-metadata/resource-metadata-schema"; import { ResourceMetadataDTO } from "../resource-metadata/resource-metadata-schema";
import { INFISICAL_SECRET_VALUE_HIDDEN_MASK } from "../secret/secret-fns"; import { INFISICAL_SECRET_VALUE_HIDDEN_MASK } from "../secret/secret-fns";
@@ -135,7 +136,7 @@ export const fnSecretBulkInsert = async ({
const commitChanges = secretVersions const commitChanges = secretVersions
.filter(({ type }) => type === SecretType.Shared) .filter(({ type }) => type === SecretType.Shared)
.map((sv) => ({ .map((sv) => ({
type: "add", type: CommitType.ADD,
secretVersionId: sv.id secretVersionId: sv.id
})); }));
@@ -289,7 +290,7 @@ export const fnSecretBulkUpdate = async ({
const commitChanges = secretVersions const commitChanges = secretVersions
.filter(({ type }) => type === SecretType.Shared) .filter(({ type }) => type === SecretType.Shared)
.map((sv) => ({ .map((sv) => ({
type: "add", type: CommitType.ADD,
isUpdate: true, isUpdate: true,
secretVersionId: sv.id secretVersionId: sv.id
})); }));
@@ -421,7 +422,7 @@ export const fnSecretBulkDelete = async ({
const commitChanges = deletedSecrets const commitChanges = deletedSecrets
.filter(({ type }) => type === SecretType.Shared) .filter(({ type }) => type === SecretType.Shared)
.map(({ id }) => ({ .map(({ id }) => ({
type: "delete", type: CommitType.DELETE,
secretVersionId: secretVersions[id].id secretVersionId: secretVersions[id].id
})); }));
if (commitChanges.length > 0) { if (commitChanges.length > 0) {
@@ -16,7 +16,7 @@ import {
} from "@app/ee/services/permission/permission-fns"; } from "@app/ee/services/permission/permission-fns";
import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
import { import {
ProjectPermissionActions, ProjectPermissionCommitsActions,
ProjectPermissionSecretActions, ProjectPermissionSecretActions,
ProjectPermissionSet, ProjectPermissionSet,
ProjectPermissionSub ProjectPermissionSub
@@ -2173,7 +2173,7 @@ export const secretV2BridgeServiceFactory = ({
actorOrgId, actorOrgId,
actionProjectType: ActionProjectType.SecretManager actionProjectType: ActionProjectType.SecretManager
}); });
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretRollback); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionCommitsActions.Read, ProjectPermissionSub.Commits);
const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.SecretManager, type: KmsDataKey.SecretManager,
projectId: folder.projectId projectId: folder.projectId
@@ -2889,7 +2889,7 @@ export const secretV2BridgeServiceFactory = ({
actorOrgId, actorOrgId,
actionProjectType: ActionProjectType.SecretManager actionProjectType: ActionProjectType.SecretManager
}); });
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretRollback); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionCommitsActions.Read, ProjectPermissionSub.Commits);
const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.SecretManager, type: KmsDataKey.SecretManager,
projectId: folder.projectId projectId: folder.projectId
@@ -272,6 +272,79 @@ export const secretVersionV2BridgeDALFactory = (db: TDbClient) => {
} }
}; };
// Function to fetch latest versions by secretIds
const getLatestVersionsBySecretIds = async (
folderId: string,
secretIds: string[],
tx?: Knex
): Promise<Array<TSecretVersionsV2>> => {
if (!secretIds.length) return [];
const knexInstance = tx || db.replicaNode();
return knexInstance(TableName.SecretVersionV2)
.where("folderId", folderId)
.whereIn(`${TableName.SecretVersionV2}.secretId`, secretIds)
.join(
knexInstance(TableName.SecretVersionV2)
.groupBy("secretId")
.max("version")
.select("secretId")
.as("latestVersion"),
(bd) => {
bd.on(`${TableName.SecretVersionV2}.secretId`, "latestVersion.secretId").andOn(
`${TableName.SecretVersionV2}.version`,
"latestVersion.max"
);
}
);
};
// Function to fetch specific versions by versionIds
const getSpecificVersionsWithLatestInfo = async (
folderId: string,
versionIds: string[],
tx?: Knex
): Promise<Array<TSecretVersionsV2>> => {
if (!versionIds.length) return [];
const knexInstance = tx || db.replicaNode();
// Get the specific versions
const specificVersions = await knexInstance(TableName.SecretVersionV2)
.where("folderId", folderId)
.whereIn("id", versionIds);
// Get the secretIds from these versions
const specificSecretIds = [...new Set(specificVersions.map((v) => v.secretId).filter(Boolean))];
if (!specificSecretIds.length) return specificVersions;
// Get max versions for these secretIds
const maxVersionsQuery = await knexInstance(TableName.SecretVersionV2)
.whereIn("secretId", specificSecretIds)
.groupBy("secretId")
.select("secretId")
.max("version as maxVersion");
// Create a lookup map for max versions
const maxVersionMap = maxVersionsQuery.reduce(
(acc, item) => {
acc[item.secretId] = item.maxVersion;
return acc;
},
{} as Record<string, number>
);
// Update the version field with maxVersion when needed
return specificVersions.map((version) => {
// Replace version with maxVersion
return {
...version,
version: maxVersionMap[version.secretId] || version.version
};
});
};
const findByIdsWithLatestVersion = async ( const findByIdsWithLatestVersion = async (
folderId: string, folderId: string,
secretIds: string[], secretIds: string[],
@@ -281,67 +354,15 @@ export const secretVersionV2BridgeDALFactory = (db: TDbClient) => {
try { try {
if (!secretIds.length && (!versionIds || !versionIds.length)) return {}; if (!secretIds.length && (!versionIds || !versionIds.length)) return {};
const knexInstance = tx || db.replicaNode(); const [latestVersions, specificVersionsWithLatest] = await Promise.all([
let allDocs: Array<TSecretVersionsV2 & { max?: number }> = []; secretIds.length ? getLatestVersionsBySecretIds(folderId, secretIds, tx) : [],
versionIds?.length ? getSpecificVersionsWithLatestInfo(folderId, versionIds, tx) : []
]);
// Process latest versions for secretIds const allDocs = [...latestVersions, ...specificVersionsWithLatest];
if (secretIds.length) {
const latestVersions: Array<TSecretVersionsV2 & { max: number }> = await knexInstance(TableName.SecretVersionV2)
.where("folderId", folderId)
.whereIn(`${TableName.SecretVersionV2}.secretId`, secretIds)
.join(
knexInstance(TableName.SecretVersionV2)
.groupBy("secretId")
.max("version")
.select("secretId")
.as("latestVersion"),
(bd) => {
bd.on(`${TableName.SecretVersionV2}.secretId`, "latestVersion.secretId").andOn(
`${TableName.SecretVersionV2}.version`,
"latestVersion.max"
);
}
);
allDocs = [...allDocs, ...latestVersions]; // Convert array to record with secretId as key
} return allDocs.reduce<Record<string, TSecretVersionsV2>>(
// Process specific versions by versionIds
if (versionIds && versionIds.length) {
// Get the specific versions
const specificVersions = await knexInstance(TableName.SecretVersionV2)
.where("folderId", folderId)
.whereIn("id", versionIds);
// Get the secretIds from these versions
const specificSecretIds = [...new Set(specificVersions.map((v) => v.secretId).filter(Boolean))];
// Get max versions for these secretIds
const maxVersionsQuery = (await knexInstance(TableName.SecretVersionV2)
.whereIn("secretId", specificSecretIds)
.groupBy("secretId")
.select("secretId")
.max("version as max")) as Array<{ secretId: string; max: number }>;
// Create a lookup map for max versions
const maxVersionMap = maxVersionsQuery.reduce(
(acc, item) => {
acc[item.secretId] = item.max;
return acc;
},
{} as Record<string, number>
);
// Add max version to each specific version
const specificVersionsWithMax = specificVersions.map((version) => ({
...version,
max: maxVersionMap[version.secretId]
}));
allDocs = [...allDocs, ...specificVersionsWithMax];
}
return allDocs.reduce<Record<string, TSecretVersionsV2 & { max?: number }>>(
(prev, curr) => ({ ...prev, [curr.secretId || ""]: curr }), (prev, curr) => ({ ...prev, [curr.secretId || ""]: curr }),
{} {}
); );
+9 -15
View File
@@ -2533,21 +2533,15 @@ export const secretServiceFactory = ({
secretVersions: string[]; secretVersions: string[];
folderId: string; folderId: string;
}) => { }) => {
const secretVersionV2 = await secretV2BridgeService const secretVersionV2 = await secretV2BridgeService.getSecretVersionsByIds({
.getSecretVersionsByIds({ actorId,
actorId, actor,
actor, actorOrgId,
actorOrgId, actorAuthMethod,
actorAuthMethod, secretId,
secretId, folderId,
folderId, secretVersionNumbers: secretVersions
secretVersionNumbers: secretVersions });
})
.catch((err) => {
if ((err as Error).message === "BadRequest: Failed to find secret") {
return null;
}
});
return secretVersionV2; return secretVersionV2;
}; };
@@ -176,6 +176,13 @@ Supports conditions and permission inversion
| `read` | View secret versions and snapshots | | `read` | View secret versions and snapshots |
| `create` | Roll back secrets to snapshots | | `create` | Roll back secrets to snapshots |
#### Subject: `commits`
| Action | Description |
| -------- | ---------------------------------- |
| `read` | View commits and changes across folders |
| `perform-rollback` | Roll back commits changes and restore folders to previous state|
#### Subject: `secret-approval` #### Subject: `secret-approval`
| Action | Description | | Action | Description |
@@ -53,11 +53,12 @@ const fetchFolderCommitsCount = async ({
directory?: string; directory?: string;
}) => { }) => {
const res = await apiRequest.get<{ count: number; folderId: string }>( const res = await apiRequest.get<{ count: number; folderId: string }>(
`/api/v1/pit/commits/count/${workspaceId}`, "/api/v1/pit/commits/count",
{ {
params: { params: {
environment, environment,
path: directory path: directory,
workspaceId
} }
} }
); );
@@ -69,10 +70,11 @@ const fetchFolderCommitHistory = async (
environment: string, environment: string,
directory: string directory: string
): Promise<CommitHistoryItem[]> => { ): Promise<CommitHistoryItem[]> => {
const res = await apiRequest.get<CommitHistoryItem[]>(`/api/v1/pit/commits/${workspaceId}`, { const res = await apiRequest.get<CommitHistoryItem[]>("/api/v1/pit/commits", {
params: { params: {
environment, environment,
path: directory path: directory,
workspaceId
} }
}); });
return res.data; return res.data;
@@ -80,7 +82,12 @@ const fetchFolderCommitHistory = async (
export const fetchCommitDetails = async (workspaceId: string, commitId: string) => { export const fetchCommitDetails = async (workspaceId: string, commitId: string) => {
const { data } = await apiRequest.get<CommitWithChanges>( const { data } = await apiRequest.get<CommitWithChanges>(
`/api/v1/pit/commits/${workspaceId}/${commitId}/changes` `/api/v1/pit/commits/${commitId}/changes`,
{
params: {
workspaceId
}
}
); );
return data; return data;
}; };
@@ -89,18 +96,19 @@ export const fetchRollbackPreview = async (
folderId: string, folderId: string,
commitId: string, commitId: string,
envId: string, envId: string,
projectId: string, workspaceId: string,
deepRollback: boolean, deepRollback: boolean,
secretPath: string secretPath: string
): Promise<RollbackPreview[]> => { ): Promise<RollbackPreview[]> => {
const { data } = await apiRequest.get<RollbackPreview[]>( const { data } = await apiRequest.get<RollbackPreview[]>(
`/api/v1/pit/commits/${projectId}/${commitId}/compare`, `/api/v1/pit/commits/${commitId}/compare`,
{ {
params: { params: {
folderId, folderId,
envId, envId,
deepRollback, deepRollback,
secretPath secretPath,
workspaceId
} }
} }
); );
@@ -110,26 +118,30 @@ export const fetchRollbackPreview = async (
const fetchRollback = async ( const fetchRollback = async (
folderId: string, folderId: string,
commitId: string, commitId: string,
projectId: string, workspaceId: string,
deepRollback: boolean, deepRollback: boolean,
message?: string, message?: string,
envId?: string envId?: string
) => { ) => {
const { data } = await apiRequest.post<{ success: boolean }>( const { data } = await apiRequest.post<{ success: boolean }>(
`/api/v1/pit/commits/${projectId}/${commitId}/rollback`, `/api/v1/pit/commits/${commitId}/rollback`,
{ {
folderId, folderId,
deepRollback, deepRollback,
message, message,
envId envId,
workspaceId
} }
); );
return data; return data;
}; };
const fetchRevert = async (commitId: string, projectId: string) => { const fetchRevert = async (commitId: string, workspaceId: string) => {
const { data } = await apiRequest.post<{ success: boolean; message: string }>( const { data } = await apiRequest.post<{ success: boolean; message: string }>(
`/api/v1/pit/commits/${projectId}/${commitId}/revert` `/api/v1/pit/commits/${commitId}/revert`,
{
workspaceId
}
); );
return data; return data;
}; };
@@ -1,4 +1,4 @@
import { SecretVersions } from "../types"; import { CommitType, SecretVersions } from "../types";
export type CommitHistoryItem = { export type CommitHistoryItem = {
id: string; id: string;
@@ -18,7 +18,7 @@ export type CommitHistoryItem = {
export type TFolderCommitChanges = { export type TFolderCommitChanges = {
id: string; id: string;
folderCommitId: string; folderCommitId: string;
changeType: "add" | "delete"; changeType: CommitType;
isUpdate: boolean; isUpdate: boolean;
secretVersionId: string | null; secretVersionId: string | null;
folderVersionId: string | null; folderVersionId: string | null;
@@ -1291,7 +1291,7 @@ const SecretsManagerPermissionSubjects = (enabled = false) => ({
[ProjectPermissionSub.Tags]: enabled, [ProjectPermissionSub.Tags]: enabled,
[ProjectPermissionSub.Webhooks]: enabled, [ProjectPermissionSub.Webhooks]: enabled,
[ProjectPermissionSub.IpAllowList]: enabled, [ProjectPermissionSub.IpAllowList]: enabled,
[ProjectPermissionSub.SecretRollback]: enabled, [ProjectPermissionSub.SecretRollback]: false,
[ProjectPermissionSub.SecretRotation]: enabled, [ProjectPermissionSub.SecretRotation]: enabled,
[ProjectPermissionSub.ServiceTokens]: enabled, [ProjectPermissionSub.ServiceTokens]: enabled,
[ProjectPermissionSub.Commits]: enabled [ProjectPermissionSub.Commits]: enabled
@@ -213,38 +213,35 @@ export const RollbackPreviewTab = (): JSX.Element => {
)} )}
</div> </div>
</div> </div>
{deepRollback && nestedFolderChanges.length > 0 && ( {deepRollback && nestedFolderChanges.length > 0 && (
<> <>
<div className="border-b border-mineshaft-600 bg-mineshaft-800 px-4 py-2"> <div className="border-b border-mineshaft-600 bg-mineshaft-800 px-4 py-2">
<span className="text-sm font-semibold text-white"> <span className="text-sm font-semibold text-white">Affected Child Folders</span>
Affected Child Folders
</span>
</div> </div>
{(nestedFolderChanges.map((folder) => ( {nestedFolderChanges.map((folder) => (
<div <div
key={folder.folderId} key={folder.folderId}
className={`cursor-pointer border-b border-mineshaft-600 ${ className={`cursor-pointer border-b border-mineshaft-600 ${
selectedFolderId === folder.folderId ? "bg-mineshaft-700" : "" selectedFolderId === folder.folderId ? "bg-mineshaft-700" : ""
}`} }`}
onClick={() => setSelectedFolderId(folder.folderId)} onClick={() => setSelectedFolderId(folder.folderId)}
> >
<div className="flex items-center justify-between px-4 py-2"> <div className="flex items-center justify-between px-4 py-2">
<div className="flex items-center"> <div className="flex items-center">
<FontAwesomeIcon icon={faFolder} className="mr-2 text-yellow-500" size="sm" /> <FontAwesomeIcon icon={faFolder} className="mr-2 text-yellow-500" size="sm" />
<span className="max-w-[150px] truncate text-sm font-medium text-white"> <span className="max-w-[150px] truncate text-sm font-medium text-white">
{folder.folderPath || folder.folderName} {folder.folderPath || folder.folderName}
</span> </span>
</div>
{folder.changes.length > 0 && (
<span className="rounded-full bg-mineshaft-600 px-2 py-0.5 text-xs text-gray-300">
{folder.changes.length}
</span>
)}
</div> </div>
{folder.changes.length > 0 && (
<span className="rounded-full bg-mineshaft-600 px-2 py-0.5 text-xs text-gray-300">
{folder.changes.length}
</span>
)}
</div> </div>
)) </div>
)} ))}
</> </>
)} )}
</div> </div>
@@ -305,10 +302,10 @@ export const RollbackPreviewTab = (): JSX.Element => {
description="This will restore all changes to how they appeared at the point in time of this commit. Any modifications made after this commit will be undone." description="This will restore all changes to how they appeared at the point in time of this commit. Any modifications made after this commit will be undone."
/> />
<div className="flex w-full border border-mineshaft-600"> <div className="flex w-full border border-mineshaft-600">
{renderSidebar()} {renderSidebar()}
{renderMainContent()} {renderMainContent()}
</div> </div>
<div className="border-x border-mineshaft-600 bg-mineshaft-800 px-6 py-3"> <div className="border-x border-mineshaft-600 bg-mineshaft-800 px-6 py-3">
<div className="flex items-center justify-end"> <div className="flex items-center justify-end">
@@ -353,7 +350,13 @@ export const RollbackPreviewTab = (): JSX.Element => {
}} }}
colorSchema="primary" colorSchema="primary"
className="px-6 py-2" className="px-6 py-2"
isDisabled={message.length === 0 || !rollbackChangesNested || !rollbackChangesNested?.some((folder) => folder.changes.length > 0)} isDisabled={
message.length === 0 ||
!rollbackChangesNested ||
!rollbackChangesNested?.some((folder) => {
return folder.changes.length > 0;
})
}
> >
Restore Restore
</Button> </Button>