awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-10-14 13:22:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

PIT: general improvements and fixes

Browse Source
This commit is contained in:
carlosmonastyrski
2025-05-25 11:00:06 -03:00
parent b7b90aea33
commit 14362dbe6a
18 changed files with 390 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/db/migrations/20250507185010_pit-projects-commits-initialization.ts
+3 -1
View File
@@ -3,13 +3,15 @@ import { Knex } from "knex";
import { inMemoryKeyStore } from "@app/keystore/memory";
import { ProjectType, TableName } from "../schemas";
import { getMigrationEnvConfig } from "./utils/env-config";
import { getMigrationPITServices } from "./utils/services";
export async function up(knex: Knex): Promise<void> {
const hasFolderCommitTable = await knex.schema.hasTable(TableName.FolderCommit);
if (hasFolderCommitTable) {
const keyStore = inMemoryKeyStore();
const { folderCommitService } = await getMigrationPITServices({ db: knex, keyStore });
const envConfig = getMigrationEnvConfig();
const { folderCommitService } = await getMigrationPITServices({ db: knex, keyStore, envConfig });
const projects = await knex(TableName.Project).where({ version: 3, type: ProjectType.SecretManager }).select("id");
for (const project of projects) {
// eslint-disable-next-line no-await-in-loop
backend/src/db/migrations/utils/services.ts
+38 -2
View File
@@ -64,7 +64,15 @@ export const getMigrationEncryptionServices = async ({ envConfig, db, keyStore }
return { kmsService };
};
export const getMigrationPITServices = async ({ db, keyStore }: { db: Knex; keyStore: TKeyStoreFactory }) => {
export const getMigrationPITServices = async ({
db,
keyStore,
envConfig
}: {
db: Knex;
keyStore: TKeyStoreFactory;
envConfig: TMigrationEnvConfig;
}) => {
const projectDAL = projectDALFactory(db);
const folderCommitDAL = folderCommitDALFactory(db);
const folderCommitChangesDAL = folderCommitChangesDALFactory(db);
@@ -79,6 +87,33 @@ export const getMigrationPITServices = async ({ db, keyStore }: { db: Knex; keyS
const secretV2BridgeDAL = secretV2BridgeDALFactory({ db, keyStore });
const folderTreeCheckpointResourcesDAL = folderTreeCheckpointResourcesDALFactory(db);
const orgDAL = orgDALFactory(db);
const kmsRootConfigDAL = kmsRootConfigDALFactory(db);
const kmsDAL = kmskeyDALFactory(db);
const internalKmsDAL = internalKmsDALFactory(db);
const hsmModule = initializeHsmModule(envConfig);
hsmModule.initialize();
const hsmService = hsmServiceFactory({
hsmModule: hsmModule.getModule(),
envConfig
});
const kmsService = kmsServiceFactory({
kmsRootConfigDAL,
keyStore,
kmsDAL,
internalKmsDAL,
orgDAL,
projectDAL,
hsmService,
envConfig
});
await hsmService.startService();
await kmsService.startService();
const folderCommitService = folderCommitServiceFactory({
folderCommitDAL,
folderCommitChangesDAL,
@@ -92,7 +127,8 @@ export const getMigrationPITServices = async ({ db, keyStore }: { db: Knex; keyS
projectDAL,
folderCheckpointResourcesDAL,
secretV2BridgeDAL,
folderTreeCheckpointResourcesDAL
folderTreeCheckpointResourcesDAL,
kmsService
});
return { folderCommitService };
backend/src/ee/routes/v1/pit-router.ts
+1 -1
View File
@@ -267,7 +267,7 @@ export const registerPITRouter = async (server: FastifyZodProvider) => {
body: z.object({
folderId: z.string().trim(),
deepRollback: z.boolean().default(false),
message: z.string().trim().optional(),
message: z.string().max(256).trim().optional(),
envId: z.string().trim(),
projectId: z.string().trim()
}),
backend/src/ee/services/pit/pit-service.ts
+12 -3
View File
@@ -12,6 +12,7 @@ import {
isSecretCommitChange
} from "@app/services/folder-commit-changes/folder-commit-changes-dal";
import { TSecretServiceFactory } from "@app/services/secret/secret-service";
import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";
import { TSecretFolderServiceFactory } from "@app/services/secret-folder/secret-folder-service";
import { TPermissionServiceFactory } from "../permission/permission-service";
@@ -21,6 +22,7 @@ type TPitServiceFactoryDep = {
secretService: Pick<TSecretServiceFactory, "getSecretVersionsV2ByIds" | "getChangeVersions">;
folderService: Pick<TSecretFolderServiceFactory, "getFolderById" | "getFolderVersions">;
permissionService: Pick<TPermissionServiceFactory, "getProjectPermission">;
folderDAL: Pick<TSecretFolderDALFactory, "findSecretPathByFolderIds">;
};
export type TPitServiceFactory = ReturnType<typeof pitServiceFactory>;
@@ -29,7 +31,8 @@ export const pitServiceFactory = ({
folderCommitService,
secretService,
folderService,
permissionService
permissionService,
folderDAL
}: TPitServiceFactoryDep) => {
const getCommitsCount = async ({
actor,
@@ -134,6 +137,8 @@ export const pitServiceFactory = ({
commitId
});
const [folderWithPath] = await folderDAL.findSecretPathByFolderIds(projectId, [changes.folderId]);
for (const change of changes.changes) {
if (isSecretCommitChange(change)) {
change.versions = await secretService.getChangeVersions(
@@ -149,7 +154,9 @@ export const pitServiceFactory = ({
actor,
actorOrgId,
actorAuthMethod,
change.folderId
changes.envId,
projectId,
folderWithPath?.path || ""
);
} else if (isFolderCommitChange(change)) {
change.versions = await folderService.getFolderVersions(
@@ -261,7 +268,9 @@ export const pitServiceFactory = ({
actor,
actorOrgId,
actorAuthMethod,
diff.folderId
envId,
projectId,
diff.folderPath || ""
);
}
} else if (change.type === ResourceType.FOLDER) {
backend/src/server/routes/index.ts
+4 -2
View File
@@ -598,7 +598,8 @@ export const registerRoutes = async (
secretV2BridgeDAL,
folderTreeCheckpointResourcesDAL,
folderCommitQueueService,
permissionService
permissionService,
kmsService
});
const scimService = scimServiceFactory({
licenseService,
@@ -1525,7 +1526,8 @@ export const registerRoutes = async (
folderCommitService,
secretService,
folderService,
permissionService
permissionService,
folderDAL
});
const identityOidcAuthService = identityOidcAuthServiceFactory({
backend/src/services/folder-commit-changes/folder-commit-changes-dal.ts
+1
View File
@@ -52,6 +52,7 @@ export type FolderCommitChange = BaseCommitChangeInfo & {
folderVersion: string;
folderChangeId: string;
versions?: {
version: string;
name?: string;
}[];
};
backend/src/services/folder-commit/folder-commit-schemas.ts
+1
View File
@@ -36,6 +36,7 @@ const secretVersionSchema = z.object({
// Folder-specific versions schema
const folderVersionSchema = z.object({
version: z.string().optional(),
name: z.string().optional()
});
backend/src/services/folder-commit/folder-commit-service.test.ts
+14 -3
View File
@@ -4,7 +4,7 @@
import { Knex } from "knex";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { TSecretFolderVersions, TSecretVersionsV2 } from "@app/db/schemas";
import { ProjectType, TSecretFolderVersions, TSecretVersionsV2 } from "@app/db/schemas";
import { BadRequestError, NotFoundError } from "@app/lib/errors";
import { ActorType } from "../auth/auth-type";
@@ -53,6 +53,10 @@ describe("folderCommitServiceFactory", () => {
findLatestCommitByFolderIds: vi.fn().mockResolvedValue({})
};
const mockKmsService = {
createCipherPairWithDataKey: vi.fn().mockResolvedValue({})
};
const mockFolderCommitChangesDAL = {
create: vi.fn().mockResolvedValue({}),
findByCommitId: vi.fn().mockResolvedValue([]),
@@ -136,7 +140,8 @@ describe("folderCommitServiceFactory", () => {
};
const mockProjectDAL = {
findById: vi.fn().mockResolvedValue({})
findById: vi.fn().mockResolvedValue({}),
findProjectByEnvId: vi.fn().mockResolvedValue({})
};
const mockFolderCommitQueueService = {
@@ -181,7 +186,8 @@ describe("folderCommitServiceFactory", () => {
secretV2BridgeDAL: mockSecretV2BridgeDAL,
folderCommitQueueService: mockFolderCommitQueueService,
// @ts-expect-error - Mock implementation doesn't need all interface methods for testing
permissionService: mockPermissionService
permissionService: mockPermissionService,
kmsService: mockKmsService
});
});
@@ -408,6 +414,11 @@ describe("folderCommitServiceFactory", () => {
{ folderVersionId: "folder-version-1", referencedFolderId: "folder-1" }
]);
mockFolderCommitDAL.findCommitsToRecreate.mockResolvedValue([]);
mockProjectDAL.findProjectByEnvId.mockResolvedValue({
id: "project-id",
name: "test-project",
type: ProjectType.SecretManager
});
// Act
const result = await folderCommitService.compareFolderStates({
backend/src/services/folder-commit/folder-commit-service.ts
+158 -7
View File
@@ -17,6 +17,8 @@ import { TFolderCommitChangesDALFactory } from "../folder-commit-changes/folder-
import { TFolderTreeCheckpointDALFactory } from "../folder-tree-checkpoint/folder-tree-checkpoint-dal";
import { TFolderTreeCheckpointResourcesDALFactory } from "../folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal";
import { TIdentityDALFactory } from "../identity/identity-dal";
import { TKmsServiceFactory } from "../kms/kms-service";
import { KmsDataKey } from "../kms/kms-types";
import { TProjectDALFactory } from "../project/project-dal";
import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
import { TSecretFolderVersionDALFactory } from "../secret-folder/secret-folder-version-dal";
@@ -80,7 +82,7 @@ type BaseChange = {
fromVersion?: string;
};
type SecretChange = {
type SecretChange = BaseChange & {
type: ResourceType.SECRET;
secretKey?: string;
secretVersion?: string;
@@ -98,7 +100,7 @@ type SecretChange = {
}[];
};
type FolderChange = {
type FolderChange = BaseChange & {
type: ResourceType.FOLDER;
folderName?: string;
folderVersion?: string;
@@ -107,7 +109,7 @@ type FolderChange = {
}[];
};
export type ResourceChange = BaseChange & (SecretChange | FolderChange);
export type ResourceChange = SecretChange | FolderChange;
type ActorInfo = {
actorType: string;
@@ -134,12 +136,13 @@ type TFolderCommitServiceFactoryDep = {
folderVersionDAL: TSecretFolderVersionDALFactory;
secretVersionV2BridgeDAL: TSecretVersionV2DALFactory;
secretV2BridgeDAL: secretV2BridgeDal.TSecretV2BridgeDALFactory;
projectDAL: Pick<TProjectDALFactory, "findById">;
projectDAL: Pick<TProjectDALFactory, "findById" | "findProjectByEnvId">;
folderCommitQueueService?: Pick<
TFolderCommitQueueServiceFactory,
"scheduleTreeCheckpoint" | "createFolderTreeCheckpoint"
>;
permissionService?: TPermissionServiceFactory;
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
};
export const folderCommitServiceFactory = ({
@@ -157,7 +160,8 @@ export const folderCommitServiceFactory = ({
secretV2BridgeDAL,
folderTreeCheckpointResourcesDAL,
folderCommitQueueService,
permissionService
permissionService,
kmsService
}: TFolderCommitServiceFactoryDep) => {
const appCfg = getConfig();
@@ -394,6 +398,12 @@ export const folderCommitServiceFactory = ({
throw new NotFoundError({ message: `Commit with ID ${targetCommitId} not found` });
}
const project = await projectDAL.findProjectByEnvId(targetCommit.envId, tx);
if (!project) {
throw new NotFoundError({ message: `No project found for envId ${targetCommit.envId}` });
}
// If currentCommitId is not provided, mark all resources in target as creates
if (!currentCommitId) {
const targetState = await reconstructFolderState(targetCommitId, tx);
@@ -565,7 +575,63 @@ export const folderCommitServiceFactory = ({
}
});
return differences;
const removeNoChangeUpdate: string[] = [];
const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.SecretManager,
projectId: project.id
});
await Promise.all(
differences.map(async (change) => {
if (change.changeType === ChangeType.UPDATE) {
if (change.type === ResourceType.FOLDER && change.folderVersion && change.fromVersion) {
const versions = await folderVersionDAL.find({
folderId: change.id,
$in: {
version: [Number(change.folderVersion), Number(change.fromVersion)]
}
});
const versionsShaped = [...new Set(versions.map((version) => version.name))];
if (versionsShaped.length === 1) {
removeNoChangeUpdate.push(change.id);
}
} else if (change.type === ResourceType.SECRET && change.secretVersion && change.fromVersion) {
const versions = await secretVersionV2BridgeDAL.findVersionsBySecretIdWithActors({
secretId: change.id,
projectId: project.id,
secretVersions: [change.secretVersion, change.fromVersion]
});
const versionsShaped = versions.map((el) => ({
secretKey: el.key,
secretComment: el.encryptedComment
? secretManagerDecryptor({ cipherTextBlob: el.encryptedComment }).toString()
: "",
skipMultilineEncoding: el.skipMultilineEncoding,
secretReminderRepeatDays: el.reminderRepeatDays,
tags: el.tags,
metadata: el.metadata,
secretReminderNote: el.reminderNote,
secretValue: el.encryptedValue
? secretManagerDecryptor({ cipherTextBlob: el.encryptedValue }).toString()
: ""
}));
const uniqueVersions = versionsShaped.filter(
(item, index, arr) =>
arr.findIndex((other) =>
Object.entries(item).every(
([key, value]) => JSON.stringify(value) === JSON.stringify(other[key as keyof typeof other])
)
) === index
);
if (uniqueVersions.length === 1) {
removeNoChangeUpdate.push(change.id);
}
}
}
})
);
return differences.filter((change) => !removeNoChangeUpdate.includes(change.id));
};
/**
@@ -1375,6 +1441,63 @@ export const folderCommitServiceFactory = ({
);
};
const addNestedFolderChanges = async ({
changes,
beforeCommit,
folderId,
folderName,
folderPath,
step = 1,
tx
}: {
changes: {
folderId: string;
folderName: string;
changes: ResourceChange[];
folderPath?: string;
}[];
beforeCommit: bigint;
folderId: string;
folderName?: string;
folderPath?: string;
step?: number;
tx?: Knex;
}) => {
if (step > 20) {
return;
}
const latestFolderCommit = await folderCommitDAL.findCommitBefore(folderId, beforeCommit, tx);
if (!latestFolderCommit) {
return;
}
const diff = await compareFolderStates({
targetCommitId: latestFolderCommit.id,
tx
});
changes.push({
folderId,
folderName: folderName || "",
changes: diff,
folderPath: folderPath || ""
});
await Promise.all(
diff.map(async (change) => {
if (change.type === ResourceType.FOLDER && change.changeType === ChangeType.CREATE) {
await addNestedFolderChanges({
changes,
beforeCommit,
folderId: change.id,
folderName: change.folderName,
folderPath: `${folderPath}/${change.folderName}`,
step: step + 1,
tx
});
}
})
);
};
const deepCompareFolder = async ({
targetCommitId,
envId,
@@ -1463,7 +1586,12 @@ export const folderCommitServiceFactory = ({
const folders = await folderDAL.findFoldersByRootAndIds({ rootId: targetCommit.folderId, folderIds }, tx);
const sortedFolders = sortFoldersByHierarchy(folders);
const response = [];
const response: {
folderId: string;
folderName: string;
changes: ResourceChange[];
folderPath?: string;
}[] = [];
for (const folder of sortedFolders) {
const diff = folderDiffs.get(folder.id);
if (diff) {
@@ -1474,6 +1602,29 @@ export const folderCommitServiceFactory = ({
changes: diff,
folderPath: folderPath?.[0]?.path
});
const recreatedFolders = diff
.filter(
(change): change is FolderChange =>
change.type === ResourceType.FOLDER && change.changeType === ChangeType.CREATE
)
.map((change) => ({
id: change.id,
folderName: change.folderName,
folderPath: folderPath?.[0]?.path
}));
await Promise.all(
recreatedFolders.map(async (change) => {
const nestedFolderPath = folderPath?.[0]?.path;
await addNestedFolderChanges({
changes: response,
beforeCommit: targetCommit.commitId,
folderId: change.id,
folderName: change.folderName,
folderPath: `${nestedFolderPath !== "/" ? nestedFolderPath : ""}/${change.folderName}`,
tx
});
})
);
}
}
return response;
backend/src/services/project/project-dal.ts
+13 -2
View File
@@ -12,7 +12,7 @@ import {
TProjectsUpdate
} from "@app/db/schemas";
import { BadRequestError, DatabaseError, NotFoundError, UnauthorizedError } from "@app/lib/errors";
import { ormify, selectAllTableCols, sqlNestRelationships } from "@app/lib/knex";
import { buildFindFilter, ormify, selectAllTableCols, sqlNestRelationships } from "@app/lib/knex";
import { ActorType } from "../auth/auth-type";
import { Filter, ProjectFilterType, SearchProjectSortBy } from "./project-types";
@@ -425,6 +425,16 @@ export const projectDALFactory = (db: TDbClient) => {
return { docs, totalCount: Number(docs?.[0]?.count ?? 0) };
};
const findProjectByEnvId = async (envId: string, tx?: Knex) => {
const project = await (tx || db.replicaNode())(TableName.Project)
.leftJoin(TableName.Environment, `${TableName.Environment}.projectId`, `${TableName.Project}.id`)
// eslint-disable-next-line @typescript-eslint/no-misused-promises
.where(buildFindFilter({ id: envId }, TableName.Environment))
.select(selectAllTableCols(TableName.Project))
.first();
return project;
};
return {
...projectOrm,
findUserProjects,
@@ -437,6 +447,7 @@ export const projectDALFactory = (db: TDbClient) => {
findProjectWithOrg,
checkProjectUpgradeStatus,
getProjectFromSplitId,
searchProjects
searchProjects,
findProjectByEnvId
};
};
backend/src/services/secret-folder/secret-folder-service.ts
+1
View File
@@ -800,6 +800,7 @@ export const secretFolderServiceFactory = ({
change.isUpdate || change.changeType === ChangeType.UPDATE ? [currentVersion, fromVersion] : [currentVersion]
});
return versions.map((v) => ({
version: v.version?.toString() || "1",
name: v.name
}));
};
backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts
+15 -18
View File
@@ -2875,24 +2875,21 @@ export const secretV2BridgeServiceFactory = ({
actorAuthMethod,
secretId,
secretVersionNumbers,
folderId
secretPath,
envId,
projectId
}: TGetSecretVersionsDTO & {
secretVersionNumbers: string[];
folderId: string;
secretPath: string;
envId: string;
projectId: string;
}) => {
const folder = await folderDAL.findById(folderId);
if (!folder) throw new NotFoundError({ message: `Folder with ID '${folderId}' not found` });
const [folderWithPath] = await folderDAL.findSecretPathByFolderIds(folder.projectId, [folder.id]);
if (!folderWithPath) {
throw new NotFoundError({ message: `Folder with ID '${folder.id}' not found` });
}
const environment = await projectEnvDAL.findOne({ id: envId });
const { permission } = await permissionService.getProjectPermission({
actor,
actorId,
projectId: folder.projectId,
projectId,
actorAuthMethod,
actorOrgId,
actionProjectType: ActionProjectType.SecretManager
@@ -2906,11 +2903,11 @@ export const secretV2BridgeServiceFactory = ({
const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.SecretManager,
projectId: folder.projectId
projectId
});
const secretVersions = await secretVersionDAL.findVersionsBySecretIdWithActors({
secretId,
projectId: folder.projectId,
projectId,
secretVersions: secretVersionNumbers
});
return secretVersions.map((el) => {
@@ -2918,8 +2915,8 @@ export const secretV2BridgeServiceFactory = ({
permission,
ProjectPermissionSecretActions.ReadValue,
{
environment: folder.environment.envSlug,
secretPath: folderWithPath.path,
environment: environment.slug,
secretPath,
secretName: el.key,
...(el.tags?.length && {
secretTags: el.tags.map((tag) => tag.slug)
@@ -2928,9 +2925,9 @@ export const secretV2BridgeServiceFactory = ({
);
return reshapeBridgeSecret(
folder.projectId,
folder.environment.envSlug,
folderWithPath.path,
projectId,
environment.slug,
secretPath,
{
...el,
value: el.encryptedValue ? secretManagerDecryptor({ cipherTextBlob: el.encryptedValue }).toString() : "",
backend/src/services/secret/secret-service.ts
+16 -6
View File
@@ -2529,10 +2529,14 @@ export const secretServiceFactory = ({
actorAuthMethod,
secretId,
secretVersions,
folderId
secretPath,
envId,
projectId
}: TGetSecretVersionsDTO & {
secretVersions: string[];
folderId: string;
secretPath: string;
envId: string;
projectId: string;
}) => {
const secretVersionV2 = await secretV2BridgeService.getSecretVersionsByIds({
actorId,
@@ -2540,8 +2544,10 @@ export const secretServiceFactory = ({
actorOrgId,
actorAuthMethod,
secretId,
folderId,
secretVersionNumbers: secretVersions
secretVersionNumbers: secretVersions,
secretPath,
envId,
projectId
});
return secretVersionV2;
};
@@ -3317,7 +3323,9 @@ export const secretServiceFactory = ({
actor: ActorType,
actorOrgId: string,
actorAuthMethod: ActorAuthMethod,
folderId: string
envId: string,
projectId: string,
secretPath: string
) => {
const currentVersion = change.secretVersion;
const secretId = change.secretId ? change.secretId : change.id;
@@ -3335,7 +3343,9 @@ export const secretServiceFactory = ({
change.isUpdate || change.changeType === ChangeType.UPDATE
? [currentVersion, previousVersion]
: [currentVersion],
folderId
secretPath,
envId,
projectId
});
return versions?.map((v) => ({
secretKey: v.secretKey,