mirror of
https://github.com/awatertrevi/infisical.git
synced 2026-10-17 16:27:39 +00:00
PIT: address PR suggestions
This commit is contained in:
carlosmonastyrski
@@ -1,6 +1,7 @@
|
||||
import { Knex } from "knex";
|
||||
|
||||
import { TSecretFolders } from "@app/db/schemas";
|
||||
import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore";
|
||||
import { getConfig } from "@app/lib/config/env";
|
||||
import { logger } from "@app/lib/logger";
|
||||
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
|
||||
@@ -10,8 +11,16 @@ import { TFolderTreeCheckpointResourcesDALFactory } from "../folder-tree-checkpo
|
||||
import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
|
||||
import { TFolderCommitDALFactory } from "./folder-commit-dal";
|
||||
|
||||
// Define types for job data
|
||||
type TCreateFolderTreeCheckpointDTO = {
|
||||
envId: string;
|
||||
failedToAcquireLockCount?: number;
|
||||
folderCommitId?: string;
|
||||
};
|
||||
|
||||
type TFolderCommitQueueServiceFactoryDep = {
|
||||
queueService: TQueueServiceFactory;
|
||||
keyStore: Pick<TKeyStoreFactory, "acquireLock" | "getItem" | "deleteItem">;
|
||||
folderTreeCheckpointDAL: Pick<
|
||||
TFolderTreeCheckpointDALFactory,
|
||||
"create" | "findLatestByEnvId" | "findNearestCheckpoint"
|
||||
@@ -31,6 +40,7 @@ export type TFolderCommitQueueServiceFactory = ReturnType<typeof folderCommitQue
|
||||
|
||||
export const folderCommitQueueServiceFactory = ({
|
||||
queueService,
|
||||
keyStore,
|
||||
folderTreeCheckpointDAL,
|
||||
folderTreeCheckpointResourcesDAL,
|
||||
folderCommitDAL,
|
||||
@@ -38,48 +48,38 @@ export const folderCommitQueueServiceFactory = ({
|
||||
}: TFolderCommitQueueServiceFactoryDep) => {
|
||||
const appCfg = getConfig();
|
||||
|
||||
const scheduleTreeCheckpoint = async (envId: string) => {
|
||||
await queueService.queue(
|
||||
QueueName.FolderTreeCheckpoint,
|
||||
QueueJobs.CreateFolderTreeCheckpoint,
|
||||
{ envId },
|
||||
{
|
||||
jobId: `${envId}`,
|
||||
backoff: {
|
||||
type: "exponential",
|
||||
delay: 3000
|
||||
},
|
||||
removeOnFail: {
|
||||
count: 3
|
||||
},
|
||||
removeOnComplete: true
|
||||
}
|
||||
);
|
||||
// Helper function to calculate delay for requeuing
|
||||
const getRequeueDelay = (failureCount?: number) => {
|
||||
if (!failureCount) return 0;
|
||||
|
||||
const baseDelay = 5000;
|
||||
const maxDelay = 30000;
|
||||
|
||||
const delay = Math.min(baseDelay * 2 ** failureCount, maxDelay);
|
||||
const jitter = delay * (0.5 + Math.random() * 0.5);
|
||||
|
||||
return jitter;
|
||||
};
|
||||
|
||||
const schedulePeriodicTreeCheckpoint = async (envId: string, intervalMs: number) => {
|
||||
await queueService.queue(
|
||||
QueueName.FolderTreeCheckpoint,
|
||||
QueueJobs.CreateFolderTreeCheckpoint,
|
||||
{ envId },
|
||||
{
|
||||
jobId: `periodic-${envId}`,
|
||||
repeat: {
|
||||
every: intervalMs
|
||||
},
|
||||
backoff: {
|
||||
type: "exponential",
|
||||
delay: 3000
|
||||
},
|
||||
removeOnFail: false,
|
||||
removeOnComplete: false
|
||||
}
|
||||
);
|
||||
};
|
||||
const scheduleTreeCheckpoint = async (payload: TCreateFolderTreeCheckpointDTO) => {
|
||||
const { envId, failedToAcquireLockCount = 0 } = payload;
|
||||
|
||||
const cancelScheduledTreeCheckpoint = async (envId: string) => {
|
||||
await queueService.stopJobById(QueueName.FolderTreeCheckpoint, envId);
|
||||
await queueService.stopRepeatableJobByJobId(QueueName.FolderTreeCheckpoint, `periodic-${envId}`);
|
||||
// Create a unique jobId for each retry to prevent conflicts
|
||||
const jobId =
|
||||
failedToAcquireLockCount > 0 ? `${envId}-retry-${failedToAcquireLockCount}-${Date.now()}` : `${envId}`;
|
||||
|
||||
await queueService.queue(QueueName.FolderTreeCheckpoint, QueueJobs.CreateFolderTreeCheckpoint, payload, {
|
||||
jobId,
|
||||
delay: getRequeueDelay(failedToAcquireLockCount),
|
||||
backoff: {
|
||||
type: "exponential",
|
||||
delay: 3000
|
||||
},
|
||||
removeOnFail: {
|
||||
count: 3
|
||||
},
|
||||
removeOnComplete: true
|
||||
});
|
||||
};
|
||||
|
||||
// Sort folders by hierarchy (copied from the source code)
|
||||
@@ -121,66 +121,152 @@ export const folderCommitQueueServiceFactory = ({
|
||||
return result;
|
||||
};
|
||||
|
||||
const createFolderTreeCheckpoint = async (envId: string, folderCommitId?: string, tx?: Knex) => {
|
||||
logger.info("Folder tree checkpoint creation started:", envId);
|
||||
const createFolderTreeCheckpoint = async (jobData: TCreateFolderTreeCheckpointDTO, tx?: Knex) => {
|
||||
const { envId, folderCommitId, failedToAcquireLockCount = 0 } = jobData;
|
||||
|
||||
const latestTreeCheckpoint = await folderTreeCheckpointDAL.findLatestByEnvId(envId, tx);
|
||||
logger.info(`Folder tree checkpoint creation started [envId=${envId}] [attempt=${failedToAcquireLockCount + 1}]`);
|
||||
|
||||
let latestCommit;
|
||||
if (folderCommitId) {
|
||||
latestCommit = await folderCommitDAL.findById(folderCommitId, tx);
|
||||
} else {
|
||||
latestCommit = await folderCommitDAL.findLatestEnvCommit(envId, tx);
|
||||
}
|
||||
if (!latestCommit) {
|
||||
logger.info(`Latest commit ID not found for envId ${envId}`);
|
||||
return;
|
||||
}
|
||||
const latestCommitId = latestCommit.id;
|
||||
|
||||
if (latestTreeCheckpoint) {
|
||||
const commitsSinceLastCheckpoint = await folderCommitDAL.getEnvNumberOfCommitsSince(
|
||||
envId,
|
||||
latestTreeCheckpoint.folderCommitId,
|
||||
tx
|
||||
);
|
||||
if (commitsSinceLastCheckpoint < Number(appCfg.PIT_TREE_CHECKPOINT_WINDOW)) {
|
||||
logger.info(
|
||||
`Commits since last checkpoint ${commitsSinceLastCheckpoint} is less than ${appCfg.PIT_TREE_CHECKPOINT_WINDOW}`
|
||||
);
|
||||
return;
|
||||
// First, try to clear any stale locks before attempting to acquire
|
||||
if (failedToAcquireLockCount > 1) {
|
||||
try {
|
||||
await keyStore.deleteItem(KeyStorePrefixes.FolderTreeCheckpoint(envId));
|
||||
logger.info(`Cleared potential stale lock for envId ${envId} before attempt ${failedToAcquireLockCount + 1}`);
|
||||
} catch (error) {
|
||||
// This is fine if it fails, we'll still try to acquire the lock
|
||||
logger.info(`No stale lock found for envId ${envId}`);
|
||||
}
|
||||
}
|
||||
|
||||
const folders = await folderDAL.findByEnvId(envId, tx);
|
||||
const sortedFolders = sortFoldersByHierarchy(folders);
|
||||
const filteredFoldersIds = sortedFolders.filter((folder) => !folder.isReserved).map((folder) => folder.id);
|
||||
let lock: Awaited<ReturnType<typeof keyStore.acquireLock>> | undefined;
|
||||
|
||||
const folderCommits = await folderCommitDAL.findMultipleLatestCommits(filteredFoldersIds, tx);
|
||||
const folderTreeCheckpoint = await folderTreeCheckpointDAL.create(
|
||||
{
|
||||
folderCommitId: latestCommitId
|
||||
},
|
||||
tx
|
||||
);
|
||||
try {
|
||||
// Attempt to acquire the lock with a shorter timeout for first attempts
|
||||
const timeout = failedToAcquireLockCount > 3 ? 60 * 1000 : 15 * 1000;
|
||||
|
||||
await folderTreeCheckpointResourcesDAL.insertMany(
|
||||
folderCommits.map((folderCommit) => ({
|
||||
folderTreeCheckpointId: folderTreeCheckpoint.id,
|
||||
folderId: folderCommit.folderId,
|
||||
folderCommitId: folderCommit.id
|
||||
})),
|
||||
tx
|
||||
);
|
||||
logger.info(`Attempting to acquire lock for envId=${envId} with timeout ${timeout}ms`);
|
||||
|
||||
logger.info("Folder tree checkpoint created successfully:", folderTreeCheckpoint.id);
|
||||
lock = await keyStore.acquireLock([KeyStorePrefixes.FolderTreeCheckpoint(envId)], timeout);
|
||||
|
||||
logger.info(`Successfully acquired lock for envId=${envId}`);
|
||||
} catch (e) {
|
||||
logger.info(
|
||||
`Failed to acquire lock for folder tree checkpoint [envId=${envId}] [attempt=${failedToAcquireLockCount + 1}]`
|
||||
);
|
||||
|
||||
// Requeue with incremented failure count if under max attempts
|
||||
if (failedToAcquireLockCount < 10) {
|
||||
// Force a delay between retries
|
||||
const nextRetryCount = failedToAcquireLockCount + 1;
|
||||
|
||||
logger.info(`Scheduling retry #${nextRetryCount} for folder tree checkpoint [envId=${envId}]`);
|
||||
|
||||
// Create a new job with incremented counter
|
||||
await scheduleTreeCheckpoint({
|
||||
envId,
|
||||
folderCommitId,
|
||||
failedToAcquireLockCount: nextRetryCount
|
||||
});
|
||||
} else {
|
||||
// Max retries reached
|
||||
logger.error(`Maximum lock acquisition attempts (10) reached for envId ${envId}. Giving up.`);
|
||||
// Try to force-clear the lock for next time
|
||||
try {
|
||||
await keyStore.deleteItem(KeyStorePrefixes.FolderTreeCheckpoint(envId));
|
||||
} catch (clearError) {
|
||||
logger.error(clearError, `Failed to clear lock after maximum retries for envId=${envId}`);
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (!lock) {
|
||||
logger.error(`Lock is undefined after acquisition for envId=${envId}. This should never happen.`);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
logger.info(`Processing tree checkpoint data for envId=${envId}`);
|
||||
|
||||
const latestTreeCheckpoint = await folderTreeCheckpointDAL.findLatestByEnvId(envId, tx);
|
||||
|
||||
let latestCommit;
|
||||
if (folderCommitId) {
|
||||
latestCommit = await folderCommitDAL.findById(folderCommitId, tx);
|
||||
} else {
|
||||
latestCommit = await folderCommitDAL.findLatestEnvCommit(envId, tx);
|
||||
}
|
||||
if (!latestCommit) {
|
||||
logger.info(`Latest commit ID not found for envId ${envId}`);
|
||||
return;
|
||||
}
|
||||
const latestCommitId = latestCommit.id;
|
||||
|
||||
if (latestTreeCheckpoint) {
|
||||
const commitsSinceLastCheckpoint = await folderCommitDAL.getEnvNumberOfCommitsSince(
|
||||
envId,
|
||||
latestTreeCheckpoint.folderCommitId,
|
||||
tx
|
||||
);
|
||||
if (commitsSinceLastCheckpoint < Number(appCfg.PIT_TREE_CHECKPOINT_WINDOW)) {
|
||||
logger.info(
|
||||
`Commits since last checkpoint ${commitsSinceLastCheckpoint} is less than ${appCfg.PIT_TREE_CHECKPOINT_WINDOW}`
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
const folders = await folderDAL.findByEnvId(envId, tx);
|
||||
const sortedFolders = sortFoldersByHierarchy(folders);
|
||||
const filteredFoldersIds = sortedFolders.filter((folder) => !folder.isReserved).map((folder) => folder.id);
|
||||
|
||||
const folderCommits = await folderCommitDAL.findMultipleLatestCommits(filteredFoldersIds, tx);
|
||||
const folderTreeCheckpoint = await folderTreeCheckpointDAL.create(
|
||||
{
|
||||
folderCommitId: latestCommitId
|
||||
},
|
||||
tx
|
||||
);
|
||||
|
||||
await folderTreeCheckpointResourcesDAL.insertMany(
|
||||
folderCommits.map((folderCommit) => ({
|
||||
folderTreeCheckpointId: folderTreeCheckpoint.id,
|
||||
folderId: folderCommit.folderId,
|
||||
folderCommitId: folderCommit.id
|
||||
})),
|
||||
tx
|
||||
);
|
||||
|
||||
logger.info(`Folder tree checkpoint created successfully: ${folderTreeCheckpoint.id}`);
|
||||
} catch (error) {
|
||||
logger.error(error, `Error processing folder tree checkpoint [envId=${envId}]`);
|
||||
throw error;
|
||||
} finally {
|
||||
// Always release the lock
|
||||
try {
|
||||
if (lock) {
|
||||
await lock.release();
|
||||
logger.info(`Released lock for folder tree checkpoint [envId=${envId}]`);
|
||||
} else {
|
||||
logger.error(`No lock to release for envId=${envId}. This should never happen.`);
|
||||
}
|
||||
} catch (releaseError) {
|
||||
logger.error(releaseError, `Error releasing lock for folder tree checkpoint [envId=${envId}]`);
|
||||
// Try to force delete the lock if release fails
|
||||
try {
|
||||
await keyStore.deleteItem(KeyStorePrefixes.FolderTreeCheckpoint(envId));
|
||||
logger.info(`Force deleted lock after release failure for envId=${envId}`);
|
||||
} catch (deleteError) {
|
||||
logger.error(deleteError, `Failed to force delete lock after release failure for envId=${envId}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
queueService.start(QueueName.FolderTreeCheckpoint, async (job) => {
|
||||
try {
|
||||
if (job.name === QueueJobs.CreateFolderTreeCheckpoint) {
|
||||
const { envId } = job.data as { envId: string };
|
||||
await createFolderTreeCheckpoint(envId);
|
||||
const jobData = job.data as TCreateFolderTreeCheckpointDTO;
|
||||
await createFolderTreeCheckpoint(jobData);
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(error, "Error creating folder tree checkpoint:");
|
||||
@@ -189,9 +275,8 @@ export const folderCommitQueueServiceFactory = ({
|
||||
});
|
||||
|
||||
return {
|
||||
scheduleTreeCheckpoint,
|
||||
schedulePeriodicTreeCheckpoint,
|
||||
cancelScheduledTreeCheckpoint,
|
||||
createFolderTreeCheckpoint
|
||||
scheduleTreeCheckpoint: (envId: string) => scheduleTreeCheckpoint({ envId }),
|
||||
createFolderTreeCheckpoint: (envId: string, folderCommitId?: string, tx?: Knex) =>
|
||||
createFolderTreeCheckpoint({ envId, folderCommitId }, tx)
|
||||
};
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user