checkpoint

This commit is contained in:
x
2025-05-02 18:43:07 -04:00
parent 877485b45a
commit 85c1a1081e
12 changed files with 220 additions and 47 deletions
@@ -0,0 +1,21 @@
import { Knex } from "knex";
import { TableName } from "../schemas";
export async function up(knex: Knex): Promise<void> {
const hasColumn = await knex.schema.hasColumn(TableName.SuperAdmin, "invalidatingCache");
if (!hasColumn) {
await knex.schema.alterTable(TableName.SuperAdmin, (t) => {
t.boolean("invalidatingCache").notNullable().defaultTo(false);
});
}
}
export async function down(knex: Knex): Promise<void> {
const hasColumn = await knex.schema.hasColumn(TableName.SuperAdmin, "invalidatingCache");
if (hasColumn) {
await knex.schema.alterTable(TableName.SuperAdmin, (t) => {
t.dropColumn("invalidatingCache");
});
}
}
-1
View File
@@ -23,7 +23,6 @@ export const OrganizationsSchema = z.object({
defaultMembershipRole: z.string().default("member"), defaultMembershipRole: z.string().default("member"),
enforceMfa: z.boolean().default(false), enforceMfa: z.boolean().default(false),
selectedMfaMethod: z.string().nullable().optional(), selectedMfaMethod: z.string().nullable().optional(),
secretShareSendToAnyone: z.boolean().default(true).nullable().optional(),
allowSecretSharingOutsideOrganization: z.boolean().default(true).nullable().optional(), allowSecretSharingOutsideOrganization: z.boolean().default(true).nullable().optional(),
shouldUseNewPrivilegeSystem: z.boolean().default(true), shouldUseNewPrivilegeSystem: z.boolean().default(true),
privilegeUpgradeInitiatedByUsername: z.string().nullable().optional(), privilegeUpgradeInitiatedByUsername: z.string().nullable().optional(),
+1 -1
View File
@@ -27,7 +27,7 @@ export const ProjectsSchema = z.object({
description: z.string().nullable().optional(), description: z.string().nullable().optional(),
type: z.string(), type: z.string(),
enforceCapitalization: z.boolean().default(false), enforceCapitalization: z.boolean().default(false),
hasDeleteProtection: z.boolean().default(true).nullable().optional() hasDeleteProtection: z.boolean().default(false).nullable().optional()
}); });
export type TProjects = z.infer<typeof ProjectsSchema>; export type TProjects = z.infer<typeof ProjectsSchema>;
+2 -1
View File
@@ -29,7 +29,8 @@ export const SuperAdminSchema = z.object({
adminIdentityIds: z.string().array().nullable().optional(), adminIdentityIds: z.string().array().nullable().optional(),
encryptedMicrosoftTeamsAppId: zodBuffer.nullable().optional(), encryptedMicrosoftTeamsAppId: zodBuffer.nullable().optional(),
encryptedMicrosoftTeamsClientSecret: zodBuffer.nullable().optional(), encryptedMicrosoftTeamsClientSecret: zodBuffer.nullable().optional(),
encryptedMicrosoftTeamsBotId: zodBuffer.nullable().optional() encryptedMicrosoftTeamsBotId: zodBuffer.nullable().optional(),
invalidatingCache: z.boolean().default(false)
}); });
export type TSuperAdmin = z.infer<typeof SuperAdminSchema>; export type TSuperAdmin = z.infer<typeof SuperAdminSchema>;
+1
View File
@@ -103,6 +103,7 @@ export const keyStoreFactory = (redisUrl: string) => {
// eslint-disable-next-line no-await-in-loop // eslint-disable-next-line no-await-in-loop
await pipeline.exec(); await pipeline.exec();
totalDeleted += batch.length; totalDeleted += batch.length;
console.log("BATCH DONE");
// eslint-disable-next-line no-await-in-loop // eslint-disable-next-line no-await-in-loop
await delayMs(Math.max(0, applyJitter(delay, jitter))); await delayMs(Math.max(0, applyJitter(delay, jitter)));
@@ -587,4 +587,31 @@ export const registerAdminRouter = async (server: FastifyZodProvider) => {
}; };
} }
}); });
server.route({
method: "GET",
url: "/invalidating-cache-status",
config: {
rateLimit: readLimit
},
schema: {
response: {
200: z.object({
invalidating: z.boolean()
})
}
},
onRequest: (req, res, done) => {
verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN])(req, res, () => {
verifySuperAdmin(req, res, done);
});
},
handler: async () => {
const invalidating = await server.services.superAdmin.checkIfInvalidatingCache();
return {
invalidating
};
}
});
}; };
@@ -1,4 +1,5 @@
import { TKeyStoreFactory } from "@app/keystore/keystore"; import { TKeyStoreFactory } from "@app/keystore/keystore";
import { delay } from "@app/lib/delay";
import { logger } from "@app/lib/logger"; import { logger } from "@app/lib/logger";
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
@@ -7,7 +8,7 @@ import { CacheType } from "./super-admin-types";
export type TInvalidateCacheQueueFactoryDep = { export type TInvalidateCacheQueueFactoryDep = {
queueService: TQueueServiceFactory; queueService: TQueueServiceFactory;
keyStore: Pick<TKeyStoreFactory, "deleteItems">; keyStore: Pick<TKeyStoreFactory, "deleteItems" | "setItemWithExpiry" | "deleteItem">;
}; };
export type TInvalidateCacheQueueFactory = ReturnType<typeof invalidateCacheQueueFactory>; export type TInvalidateCacheQueueFactory = ReturnType<typeof invalidateCacheQueueFactory>;
@@ -18,9 +19,18 @@ export const invalidateCacheQueueFactory = ({ queueService, keyStore }: TInvalid
type: CacheType; type: CacheType;
}; };
}) => { }) => {
// Cancel existing jobs if any
try {
console.log("stopping job");
await queueService.clearQueue(QueueName.InvalidateCache);
} catch (err) {
logger.warn(err, "Failed to clear queue");
}
await queueService.queue(QueueName.InvalidateCache, QueueJobs.InvalidateCache, dto, { await queueService.queue(QueueName.InvalidateCache, QueueJobs.InvalidateCache, dto, {
removeOnComplete: true, removeOnComplete: true,
removeOnFail: true removeOnFail: true,
jobId: "invalidate-cache"
}); });
}; };
@@ -30,10 +40,19 @@ export const invalidateCacheQueueFactory = ({ queueService, keyStore }: TInvalid
data: { type } data: { type }
} = job.data; } = job.data;
await keyStore.setItemWithExpiry("invalidating-cache", 3600, "true"); // 1 hour max (in case the job somehow silently fails)
console.log("STARTING JOB");
if (type === CacheType.ALL || type === CacheType.SECRETS) if (type === CacheType.ALL || type === CacheType.SECRETS)
await keyStore.deleteItems({ pattern: "secret-manager:*" }); await keyStore.deleteItems({ pattern: "secret-manager:*" });
// await delay(12000); // TODO(andrey): Remove. It's for debug
await keyStore.deleteItem("invalidating-cache");
} catch (err) { } catch (err) {
logger.error(err, "Failed to invalidate cache"); logger.error(err, "Failed to invalidate cache");
await keyStore.deleteItem("invalidating-cache");
} }
}); });
@@ -67,7 +67,7 @@ export let getServerCfg: () => Promise<
const ADMIN_CONFIG_KEY = "infisical-admin-cfg"; const ADMIN_CONFIG_KEY = "infisical-admin-cfg";
const ADMIN_CONFIG_KEY_EXP = 60; // 60s const ADMIN_CONFIG_KEY_EXP = 60; // 60s
const ADMIN_CONFIG_DB_UUID = "00000000-0000-0000-0000-000000000000"; export const ADMIN_CONFIG_DB_UUID = "00000000-0000-0000-0000-000000000000";
export const superAdminServiceFactory = ({ export const superAdminServiceFactory = ({
serverCfgDAL, serverCfgDAL,
@@ -641,6 +641,10 @@ export const superAdminServiceFactory = ({
}); });
}; };
const checkIfInvalidatingCache = async () => {
return (await keyStore.getItem("invalidating-cache")) !== null;
};
return { return {
initServerCfg, initServerCfg,
updateServerCfg, updateServerCfg,
@@ -655,6 +659,7 @@ export const superAdminServiceFactory = ({
grantServerAdminAccessToUser, grantServerAdminAccessToUser,
deleteIdentitySuperAdminAccess, deleteIdentitySuperAdminAccess,
deleteUserSuperAdminAccess, deleteUserSuperAdminAccess,
invalidateCache invalidateCache,
checkIfInvalidatingCache
}; };
}; };
+14
View File
@@ -8,6 +8,7 @@ import {
AdminGetIdentitiesFilters, AdminGetIdentitiesFilters,
AdminGetUsersFilters, AdminGetUsersFilters,
AdminIntegrationsConfig, AdminIntegrationsConfig,
TGetInvalidatingCacheStatus,
TGetServerRootKmsEncryptionDetails, TGetServerRootKmsEncryptionDetails,
TServerConfig TServerConfig
} from "./types"; } from "./types";
@@ -120,3 +121,16 @@ export const useGetServerRootKmsEncryptionDetails = () => {
} }
}); });
}; };
export const useGetInvalidatingCacheStatus = () => {
return useQuery({
queryKey: adminQueryKeys.getInvalidateCache(),
queryFn: async () => {
const { data } = await apiRequest.get<TGetInvalidatingCacheStatus>(
"/api/v1/admin/invalidating-cache-status"
);
return data.invalidating;
}
});
};
+5
View File
@@ -24,6 +24,7 @@ export type TServerConfig = {
enabledLoginMethods: LoginMethod[]; enabledLoginMethods: LoginMethod[];
authConsentContent?: string; authConsentContent?: string;
pageFrameContent?: string; pageFrameContent?: string;
invalidatingCache: boolean;
}; };
export type TUpdateServerConfigDTO = { export type TUpdateServerConfigDTO = {
@@ -93,3 +94,7 @@ export enum CacheType {
export type TInvalidateCacheDTO = { export type TInvalidateCacheDTO = {
type: CacheType; type: CacheType;
}; };
export type TGetInvalidatingCacheStatus = {
invalidating: boolean;
};
@@ -1,37 +1,66 @@
import { useState } from "react"; import { useEffect, useRef, useState } from "react";
import { createNotification } from "@app/components/notifications"; import { createNotification } from "@app/components/notifications";
import { Button, DeleteActionModal } from "@app/components/v2"; import { Badge, Button, DeleteActionModal } from "@app/components/v2";
import { useOrgPermission } from "@app/context"; import { useOrgPermission } from "@app/context";
import { usePopUp } from "@app/hooks"; import { usePopUp } from "@app/hooks";
import { useInvalidateCache } from "@app/hooks/api"; import { useInvalidateCache } from "@app/hooks/api";
import { CacheType } from "@app/hooks/api/admin/types"; import { CacheType } from "@app/hooks/api/admin/types";
import { useGetInvalidatingCacheStatus } from "@app/hooks/api/admin/queries";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import { faRotate } from "@fortawesome/free-solid-svg-icons";
export const CachingPanel = () => { export const CachingPanel = () => {
const { mutateAsync: invalidateCache } = useInvalidateCache(); const { mutateAsync: invalidateCache } = useInvalidateCache();
const { data: isInvalidating, refetch: refetchInvalidatingStatus } =
useGetInvalidatingCacheStatus();
const { membership } = useOrgPermission(); const { membership } = useOrgPermission();
const wasInvalidating = useRef(false);
const [type, setType] = useState<CacheType | null>(null); const [type, setType] = useState<CacheType | null>(null);
const [isLoading, setIsLoading] = useState(false); const [buttonsDisabled, setButtonsDisabled] = useState(false);
const { popUp, handlePopUpOpen, handlePopUpClose, handlePopUpToggle } = usePopUp([ const { popUp, handlePopUpOpen, handlePopUpClose, handlePopUpToggle } = usePopUp([
"invalidateCache" "invalidateCache"
] as const); ] as const);
const success = () => {
createNotification({
text: `Successfully invalidated cache`,
type: "success"
});
setButtonsDisabled(false);
};
const timeoutRef = useRef<NodeJS.Timeout | null>(null);
const disableButtons = () => {
// Enable buttons after 10 seconds, even if still invalidating
setButtonsDisabled(true);
timeoutRef.current = setTimeout(() => {
setButtonsDisabled(false);
}, 10000);
};
const handleInvalidateCacheSubmit = async () => { const handleInvalidateCacheSubmit = async () => {
if (!type) return; if (!type) return;
setIsLoading(true);
try { try {
await invalidateCache({ type }); await invalidateCache({ type });
wasInvalidating.current = true;
createNotification({ createNotification({
text: `Successfully invalidated ${type} cache`, text: `Began invalidating ${type} cache`,
type: "success" type: "success"
}); });
setType(null); disableButtons();
handlePopUpClose("invalidateCache"); handlePopUpClose("invalidateCache");
if (!(await refetchInvalidatingStatus()).data) {
success();
return;
}
} catch (err) { } catch (err) {
console.error(err); console.error(err);
createNotification({ createNotification({
@@ -40,14 +69,61 @@ export const CachingPanel = () => {
}); });
} }
setIsLoading(false); setType(null);
}; };
const pollingRef = useRef<NodeJS.Timeout | null>(null);
// Update the "invalidating cache" status
useEffect(() => {
if (!isInvalidating) return;
if (pollingRef.current) clearInterval(pollingRef.current);
if (timeoutRef.current) clearTimeout(timeoutRef.current);
// Start polling every 3 seconds
pollingRef.current = setInterval(async () => {
try {
await refetchInvalidatingStatus();
} catch (err) {
console.error("Polling error:", err);
}
}, 3000);
disableButtons();
return () => {
if (pollingRef.current) clearInterval(pollingRef.current);
if (timeoutRef.current) clearTimeout(timeoutRef.current);
};
}, [isInvalidating]);
useEffect(() => {
if (isInvalidating === false && wasInvalidating.current) {
success();
wasInvalidating.current = false;
if (pollingRef.current) clearInterval(pollingRef.current);
if (timeoutRef.current) clearTimeout(timeoutRef.current);
}
}, [isInvalidating]);
return ( return (
<> <>
<div className="mb-6 flex flex-wrap items-end justify-between gap-4 rounded-lg border border-mineshaft-600 bg-mineshaft-900 p-4"> <div className="mb-6 flex flex-wrap items-end justify-between gap-4 rounded-lg border border-mineshaft-600 bg-mineshaft-900 p-4">
<div className="flex flex-col"> <div className="flex flex-col">
<span className="mb-2 text-xl font-semibold text-mineshaft-100">Secrets Cache</span> <div className="mb-2 flex items-center gap-3">
<span className="text-xl font-semibold text-mineshaft-100">Secrets Cache</span>
{isInvalidating && (
<Badge
variant="danger"
className="flex h-5 w-min items-center gap-1.5 whitespace-nowrap"
>
<FontAwesomeIcon icon={faRotate} className="animate-spin" />
Invalidating Cache
</Badge>
)}
</div>
<span className="max-w-xl text-sm text-mineshaft-400"> <span className="max-w-xl text-sm text-mineshaft-400">
The encrypted secrets cache encompasses all secrets stored within the system and The encrypted secrets cache encompasses all secrets stored within the system and
provides a temporary, secure storage location for frequently accessed credentials. provides a temporary, secure storage location for frequently accessed credentials.
@@ -56,12 +132,11 @@ export const CachingPanel = () => {
<Button <Button
colorSchema="danger" colorSchema="danger"
isLoading={isLoading}
onClick={() => { onClick={() => {
setType(CacheType.SECRETS); setType(CacheType.SECRETS);
handlePopUpOpen("invalidateCache"); handlePopUpOpen("invalidateCache");
}} }}
isDisabled={Boolean(membership && membership.role !== "admin") || isLoading} isDisabled={Boolean(membership && membership.role !== "admin") || buttonsDisabled}
> >
Invalidate Secrets Cache Invalidate Secrets Cache
</Button> </Button>
@@ -93,7 +168,7 @@ export const CachingPanel = () => {
<DeleteActionModal <DeleteActionModal
isOpen={popUp.invalidateCache.isOpen} isOpen={popUp.invalidateCache.isOpen}
title={`Are you sure you want to invalidate ${type} cache?`} title={`Are you sure you want to invalidate ${type} cache?`}
subTitle="This action is permanent and irreversible. The cache clearing process may take several minutes to complete." subTitle="This action is permanent and irreversible. The cache invalidation process may take several minutes to complete."
onChange={(isOpen) => handlePopUpToggle("invalidateCache", isOpen)} onChange={(isOpen) => handlePopUpToggle("invalidateCache", isOpen)}
deleteKey="confirm" deleteKey="confirm"
onDeleteApproved={handleInvalidateCacheSubmit} onDeleteApproved={handleInvalidateCacheSubmit}
+8 -2
View File
@@ -1,6 +1,6 @@
import { TanStackRouterVite } from "@tanstack/router-plugin/vite"; import { TanStackRouterVite } from "@tanstack/router-plugin/vite";
import react from "@vitejs/plugin-react-swc"; import react from "@vitejs/plugin-react-swc";
import { defineConfig, PluginOption } from "vite"; import { defineConfig, loadEnv, PluginOption } from "vite";
import { nodePolyfills } from "vite-plugin-node-polyfills"; import { nodePolyfills } from "vite-plugin-node-polyfills";
import topLevelAwait from "vite-plugin-top-level-await"; import topLevelAwait from "vite-plugin-top-level-await";
import wasm from "vite-plugin-wasm"; import wasm from "vite-plugin-wasm";
@@ -20,8 +20,13 @@ const virtualRouteFileChangeReloadPlugin: PluginOption = {
}; };
// https://vite.dev/config/ // https://vite.dev/config/
export default defineConfig({ export default defineConfig(({ mode }) => {
const env = loadEnv(mode, process.cwd());
const allowedHosts = env.VITE_ALLOWED_HOSTS?.split(",") ?? [];
return {
server: { server: {
allowedHosts,
host: true, host: true,
port: 3000 port: 3000
// proxy: { // proxy: {
@@ -48,4 +53,5 @@ export default defineConfig({
react(), react(),
virtualRouteFileChangeReloadPlugin virtualRouteFileChangeReloadPlugin
] ]
};
}); });