awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-10-15 05:19:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: added cache versioning to sql for geo invalidation

Browse Source
This commit is contained in:
=
2025-09-10 16:12:00 +05:30
parent b59657f64a
commit d6f8189a96
21 changed files with 258 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/services/folder-commit/folder-commit-service.ts
+1 -1
View File
@@ -1386,7 +1386,7 @@ export const folderCommitServiceFactory = ({
);
// Invalidate cache to reflect the changes
await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId);
await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId, tx);
return {
secretChangesCount: secretChanges.length,
backend/src/services/resource-cleanup/resource-cleanup-queue.ts
+9 -1
View File
@@ -1,5 +1,6 @@
import { TAuditLogDALFactory } from "@app/ee/services/audit-log/audit-log-dal";
import { TSnapshotDALFactory } from "@app/ee/services/secret-snapshot/snapshot-dal";
import { TKeyValueStoreDALFactory } from "@app/keystore/key-value-store-dal";
import { getConfig } from "@app/lib/config/env";
import { logger } from "@app/lib/logger";
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
@@ -27,6 +28,7 @@ type TDailyResourceCleanUpQueueServiceFactoryDep = {
queueService: TQueueServiceFactory;
orgService: TOrgServiceFactory;
userNotificationDAL: Pick<TUserNotificationDALFactory, "pruneNotifications">;
keyValueStoreDAL: Pick<TKeyValueStoreDALFactory, "pruneExpiredKeys">;
};
export type TDailyResourceCleanUpQueueServiceFactory = ReturnType<typeof dailyResourceCleanUpQueueServiceFactory>;
@@ -43,7 +45,8 @@ export const dailyResourceCleanUpQueueServiceFactory = ({
identityUniversalAuthClientSecretDAL,
serviceTokenService,
orgService,
userNotificationDAL
userNotificationDAL,
keyValueStoreDAL
}: TDailyResourceCleanUpQueueServiceFactoryDep) => {
const appCfg = getConfig();
@@ -52,6 +55,10 @@ export const dailyResourceCleanUpQueueServiceFactory = ({
}
const init = async () => {
if (appCfg.isSecondaryInstance) {
return;
}
await queueService.stopRepeatableJob(
QueueName.AuditLogPrune,
QueueJobs.AuditLogPrune,
@@ -82,6 +89,7 @@ export const dailyResourceCleanUpQueueServiceFactory = ({
await orgService.notifyInvitedUsers();
await auditLogDAL.pruneAuditLog();
await userNotificationDAL.pruneNotifications();
await keyValueStoreDAL.pruneExpiredKeys();
logger.info(`${QueueName.DailyResourceCleanUp}: queue task completed`);
} catch (error) {
logger.error(error, `${QueueName.DailyResourceCleanUp}: resource cleanup failed`);
backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts
+3 -4
View File
@@ -50,15 +50,14 @@ interface TSecretV2DalArg {
}
export const SECRET_DAL_TTL = () => applyJitter(10 * 60, 2 * 60);
export const SECRET_DAL_VERSION_TTL = 15 * 60;
export const SECRET_DAL_VERSION_TTL = "15m";
export const MAX_SECRET_CACHE_BYTES = 25 * 1024 * 1024;
export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => {
const secretOrm = ormify(db, TableName.SecretV2);
const invalidateSecretCacheByProjectId = async (projectId: string) => {
const invalidateSecretCacheByProjectId = async (projectId: string, tx?: Knex) => {
const secretDalVersionKey = SecretServiceCacheKeys.getSecretDalVersion(projectId);
await keyStore.incrementBy(secretDalVersionKey, 1);
await keyStore.setExpiry(secretDalVersionKey, SECRET_DAL_VERSION_TTL);
await keyStore.pgIncrementBy(secretDalVersionKey, { incr: 1, tx, expiry: SECRET_DAL_VERSION_TTL });
};
const findOne = async (filter: Partial<TSecretsV2>, tx?: Knex) => {
backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts
+23 -18
View File
@@ -118,7 +118,7 @@ type TSecretV2BridgeServiceFactoryDep = {
>;
snapshotService: Pick<TSecretSnapshotServiceFactory, "performSnapshot">;
resourceMetadataDAL: Pick<TResourceMetadataDALFactory, "insertMany" | "delete">;
keyStore: Pick<TKeyStoreFactory, "getItem" | "setExpiry" | "setItemWithExpiry" | "deleteItem">;
keyStore: Pick<TKeyStoreFactory, "getItem" | "setExpiry" | "setItemWithExpiry" | "deleteItem" | "pgGetIntItem">;
reminderService: Pick<TReminderServiceFactory, "createReminder" | "getReminder">;
};
@@ -360,6 +360,7 @@ export const secretV2BridgeServiceFactory = ({
tx
});
await secretDAL.invalidateSecretCacheByProjectId(projectId, tx);
return createdSecret;
});
@@ -377,7 +378,6 @@ export const secretV2BridgeServiceFactory = ({
});
}
await secretDAL.invalidateSecretCacheByProjectId(projectId);
if (inputSecret.type === SecretType.Shared) {
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({
@@ -566,8 +566,8 @@ export const secretV2BridgeServiceFactory = ({
await $validateSecretReferences(projectId, permission, allSecretReferences);
}
const updatedSecret = await secretDAL.transaction(async (tx) =>
fnSecretBulkUpdate({
const updatedSecret = await secretDAL.transaction(async (tx) => {
const modifiedSecretsInDB = await fnSecretBulkUpdate({
folderId,
orgId: actorOrgId,
resourceMetadataDAL,
@@ -598,8 +598,11 @@ export const secretV2BridgeServiceFactory = ({
actorId
},
tx
})
);
});
await secretDAL.invalidateSecretCacheByProjectId(projectId, tx);
return modifiedSecretsInDB;
});
if (inputSecret.secretReminderRepeatDays) {
await reminderService.createReminder({
actor,
@@ -615,7 +618,6 @@ export const secretV2BridgeServiceFactory = ({
});
}
await secretDAL.invalidateSecretCacheByProjectId(projectId);
if (inputSecret.type === SecretType.Shared) {
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({
@@ -715,8 +717,8 @@ export const secretV2BridgeServiceFactory = ({
);
try {
const deletedSecret = await secretDAL.transaction(async (tx) =>
fnSecretBulkDelete({
const deletedSecret = await secretDAL.transaction(async (tx) => {
const modifiedSecretsInDB = await fnSecretBulkDelete({
projectId,
folderId,
actorId,
@@ -732,10 +734,11 @@ export const secretV2BridgeServiceFactory = ({
}
],
tx
})
);
});
await secretDAL.invalidateSecretCacheByProjectId(projectId, tx);
return modifiedSecretsInDB;
});
await secretDAL.invalidateSecretCacheByProjectId(projectId);
if (inputSecret.type === SecretType.Shared) {
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({
@@ -1027,7 +1030,7 @@ export const secretV2BridgeServiceFactory = ({
});
throwIfMissingSecretReadValueOrDescribePermission(permission, ProjectPermissionSecretActions.DescribeSecret);
const cachedSecretDalVersion = await keyStore.getItem(SecretServiceCacheKeys.getSecretDalVersion(projectId));
const cachedSecretDalVersion = await keyStore.pgGetIntItem(SecretServiceCacheKeys.getSecretDalVersion(projectId));
const secretDalVersion = Number(cachedSecretDalVersion || 0);
const cacheKey = SecretServiceCacheKeys.getSecretsOfServiceLayer(projectId, secretDalVersion, {
...dto,
@@ -1692,7 +1695,7 @@ export const secretV2BridgeServiceFactory = ({
await kmsService.createCipherPairWithDataKey({ type: KmsDataKey.SecretManager, projectId });
const executeBulkInsert = async (tx: Knex) => {
return fnSecretBulkInsert({
const modifiedSecretsInDB = await fnSecretBulkInsert({
inputSecrets: inputSecrets.map((el) => {
const references = secretReferencesGroupByInputSecretKey[el.secretKey]?.nestedReferences;
@@ -1728,13 +1731,14 @@ export const secretV2BridgeServiceFactory = ({
},
tx
});
await secretDAL.invalidateSecretCacheByProjectId(projectId, tx);
return modifiedSecretsInDB;
};
const newSecrets = providedTx
? await executeBulkInsert(providedTx)
: await secretDAL.transaction(executeBulkInsert);
await secretDAL.invalidateSecretCacheByProjectId(projectId);
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({
actor,
@@ -2099,6 +2103,7 @@ export const secretV2BridgeServiceFactory = ({
}
}
await secretDAL.invalidateSecretCacheByProjectId(projectId, tx);
return updatedSecrets;
};
@@ -2106,7 +2111,6 @@ export const secretV2BridgeServiceFactory = ({
? await executeBulkUpdate(providedTx)
: await secretDAL.transaction(executeBulkUpdate);
await secretDAL.invalidateSecretCacheByProjectId(projectId);
await Promise.allSettled(folders.map((el) => (el?.id ? snapshotService.performSnapshot(el.id) : undefined)));
await Promise.allSettled(
folders.map((el) =>
@@ -2233,7 +2237,7 @@ export const secretV2BridgeServiceFactory = ({
});
const executeBulkDelete = async (tx: Knex) => {
return fnSecretBulkDelete({
const modifiedSecretsInDB = await fnSecretBulkDelete({
secretDAL,
secretQueueService,
folderCommitService,
@@ -2249,6 +2253,8 @@ export const secretV2BridgeServiceFactory = ({
commitChanges,
tx
});
await secretDAL.invalidateSecretCacheByProjectId(projectId, tx);
return modifiedSecretsInDB;
};
try {
@@ -2256,7 +2262,6 @@ export const secretV2BridgeServiceFactory = ({
? await executeBulkDelete(providedTx)
: await secretDAL.transaction(executeBulkDelete);
await secretDAL.invalidateSecretCacheByProjectId(projectId);
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({
actor,