diff --git a/backend/src/db/migrations/20240520064127_add-integration-sync-status.ts b/backend/src/db/migrations/20240520064127_add-integration-sync-status.ts new file mode 100644 index 000000000..74b828714 --- /dev/null +++ b/backend/src/db/migrations/20240520064127_add-integration-sync-status.ts @@ -0,0 +1,43 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + const hasIsSyncedColumn = await knex.schema.hasColumn(TableName.Integration, "isSynced"); + const hasSyncMessageColumn = await knex.schema.hasColumn(TableName.Integration, "syncMessage"); + const hasLastSyncJobId = await knex.schema.hasColumn(TableName.Integration, "lastSyncJobId"); + + await knex.schema.alterTable(TableName.Integration, (t) => { + if (!hasIsSyncedColumn) { + t.boolean("isSynced").nullable(); + } + + if (!hasSyncMessageColumn) { + t.text("syncMessage").nullable(); + } + + if (!hasLastSyncJobId) { + t.string("lastSyncJobId").nullable(); + } + }); +} + +export async function down(knex: Knex): Promise { + const hasIsSyncedColumn = await knex.schema.hasColumn(TableName.Integration, "isSynced"); + const hasSyncMessageColumn = await knex.schema.hasColumn(TableName.Integration, "syncMessage"); + const hasLastSyncJobId = await knex.schema.hasColumn(TableName.Integration, "lastSyncJobId"); + + await knex.schema.alterTable(TableName.Integration, (t) => { + if (hasIsSyncedColumn) { + t.dropColumn("isSynced"); + } + + if (hasSyncMessageColumn) { + t.dropColumn("syncMessage"); + } + + if (hasLastSyncJobId) { + t.dropColumn("lastSyncJobId"); + } + }); +} diff --git a/backend/src/db/schemas/integrations.ts b/backend/src/db/schemas/integrations.ts index 203498c85..47cf9e627 100644 --- a/backend/src/db/schemas/integrations.ts +++ b/backend/src/db/schemas/integrations.ts @@ -28,7 +28,10 @@ export const IntegrationsSchema = z.object({ secretPath: z.string().default("/"), createdAt: z.date(), updatedAt: z.date(), - lastUsed: z.date().nullable().optional() + lastUsed: z.date().nullable().optional(), + isSynced: z.boolean().nullable().optional(), + syncMessage: z.string().nullable().optional(), + lastSyncJobId: z.string().nullable().optional() }); export type TIntegrations = z.infer; diff --git a/backend/src/ee/services/audit-log/audit-log-types.ts b/backend/src/ee/services/audit-log/audit-log-types.ts index 87546882e..e512389d7 100644 --- a/backend/src/ee/services/audit-log/audit-log-types.ts +++ b/backend/src/ee/services/audit-log/audit-log-types.ts @@ -51,6 +51,7 @@ export enum EventType { UNAUTHORIZE_INTEGRATION = "unauthorize-integration", CREATE_INTEGRATION = "create-integration", DELETE_INTEGRATION = "delete-integration", + MANUAL_SYNC_INTEGRATION = "manual-sync-integration", ADD_TRUSTED_IP = "add-trusted-ip", UPDATE_TRUSTED_IP = "update-trusted-ip", DELETE_TRUSTED_IP = "delete-trusted-ip", @@ -281,6 +282,25 @@ interface DeleteIntegrationEvent { }; } +interface ManualSyncIntegrationEvent { + type: EventType.MANUAL_SYNC_INTEGRATION; + metadata: { + integrationId: string; + integration: string; + environment: string; + secretPath: string; + url?: string; + app?: string; + appId?: string; + targetEnvironment?: string; + targetEnvironmentId?: string; + targetService?: string; + targetServiceId?: string; + path?: string; + region?: string; + }; +} + interface AddTrustedIPEvent { type: EventType.ADD_TRUSTED_IP; metadata: { @@ -791,6 +811,7 @@ export type Event = | UnauthorizeIntegrationEvent | CreateIntegrationEvent | DeleteIntegrationEvent + | ManualSyncIntegrationEvent | AddTrustedIPEvent | UpdateTrustedIPEvent | DeleteTrustedIPEvent diff --git a/backend/src/lib/api-docs/constants.ts b/backend/src/lib/api-docs/constants.ts index bd78436c3..01f0e5142 100644 --- a/backend/src/lib/api-docs/constants.ts +++ b/backend/src/lib/api-docs/constants.ts @@ -683,6 +683,9 @@ export const INTEGRATION = { }, DELETE: { integrationId: "The ID of the integration object." + }, + SYNC: { + integrationId: "The ID of the integration object to manually sync" } }; diff --git a/backend/src/server/routes/v1/integration-router.ts b/backend/src/server/routes/v1/integration-router.ts index e40e3f799..1fd92df3a 100644 --- a/backend/src/server/routes/v1/integration-router.ts +++ b/backend/src/server/routes/v1/integration-router.ts @@ -262,5 +262,64 @@ export const registerIntegrationRouter = async (server: FastifyZodProvider) => { } }); - // TODO(akhilmhdh-pg): manual sync + server.route({ + method: "POST", + url: "/:integrationId/sync", + config: { + rateLimit: writeLimit + }, + schema: { + description: "Manually trigger sync of an integration by integration id", + security: [ + { + bearerAuth: [] + } + ], + params: z.object({ + integrationId: z.string().trim().describe(INTEGRATION.SYNC.integrationId) + }), + response: { + 200: z.object({ + integration: IntegrationsSchema + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const integration = await server.services.integration.syncIntegration({ + actorId: req.permission.id, + actor: req.permission.type, + actorAuthMethod: req.permission.authMethod, + actorOrgId: req.permission.orgId, + id: req.params.integrationId + }); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: integration.projectId, + event: { + type: EventType.MANUAL_SYNC_INTEGRATION, + // eslint-disable-next-line + metadata: shake({ + integrationId: integration.id, + integration: integration.integration, + environment: integration.environment.slug, + secretPath: integration.secretPath, + url: integration.url, + app: integration.app, + appId: integration.appId, + targetEnvironment: integration.targetEnvironment, + targetEnvironmentId: integration.targetEnvironmentId, + targetService: integration.targetService, + targetServiceId: integration.targetServiceId, + path: integration.path, + region: integration.region + // eslint-disable-next-line + }) as any + } + }); + + return { integration }; + } + }); }; diff --git a/backend/src/services/integration/integration-service.ts b/backend/src/services/integration/integration-service.ts index f4f8cdb44..eff73c1b6 100644 --- a/backend/src/services/integration/integration-service.ts +++ b/backend/src/services/integration/integration-service.ts @@ -9,7 +9,12 @@ import { TIntegrationAuthDALFactory } from "../integration-auth/integration-auth import { TSecretQueueFactory } from "../secret/secret-queue"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; import { TIntegrationDALFactory } from "./integration-dal"; -import { TCreateIntegrationDTO, TDeleteIntegrationDTO, TUpdateIntegrationDTO } from "./integration-types"; +import { + TCreateIntegrationDTO, + TDeleteIntegrationDTO, + TSyncIntegrationDTO, + TUpdateIntegrationDTO +} from "./integration-types"; type TIntegrationServiceFactoryDep = { integrationDAL: TIntegrationDALFactory; @@ -201,10 +206,35 @@ export const integrationServiceFactory = ({ return integrations; }; + const syncIntegration = async ({ id, actorId, actor, actorOrgId, actorAuthMethod }: TSyncIntegrationDTO) => { + const integration = await integrationDAL.findById(id); + if (!integration) { + throw new BadRequestError({ message: "Integration not found" }); + } + + const { permission } = await permissionService.getProjectPermission( + actor, + actorId, + integration.projectId, + actorAuthMethod, + actorOrgId + ); + ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Integrations); + + await secretQueueService.syncIntegrations({ + environment: integration.environment.slug, + secretPath: integration.secretPath, + projectId: integration.projectId + }); + + return { ...integration, envId: integration.environment.id }; + }; + return { createIntegration, updateIntegration, deleteIntegration, - listIntegrationByProject + listIntegrationByProject, + syncIntegration }; }; diff --git a/backend/src/services/integration/integration-types.ts b/backend/src/services/integration/integration-types.ts index 50d0de762..1c8772478 100644 --- a/backend/src/services/integration/integration-types.ts +++ b/backend/src/services/integration/integration-types.ts @@ -59,3 +59,7 @@ export type TUpdateIntegrationDTO = { export type TDeleteIntegrationDTO = { id: string; } & Omit; + +export type TSyncIntegrationDTO = { + id: string; +} & Omit; diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index e1c8d61af..f3e3f1731 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -463,20 +463,37 @@ export const secretQueueFactory = ({ }); } - await syncIntegrationSecrets({ - createManySecretsRawFn, - updateManySecretsRawFn, - integrationDAL, - integration, - integrationAuth, - secrets: Object.keys(suffixedSecrets).length !== 0 ? suffixedSecrets : secrets, - accessId: accessId as string, - accessToken, - appendices: { - prefix: metadata?.secretPrefix || "", - suffix: metadata?.secretSuffix || "" - } - }); + try { + await syncIntegrationSecrets({ + createManySecretsRawFn, + updateManySecretsRawFn, + integrationDAL, + integration, + integrationAuth, + secrets: Object.keys(suffixedSecrets).length !== 0 ? suffixedSecrets : secrets, + accessId: accessId as string, + accessToken, + appendices: { + prefix: metadata?.secretPrefix || "", + suffix: metadata?.secretSuffix || "" + } + }); + + await integrationDAL.updateById(integration.id, { + lastSyncJobId: job.id, + lastUsed: new Date(), + syncMessage: "", + isSynced: true + }); + } catch (err: unknown) { + logger.info("Secret integration sync error:", err); + await integrationDAL.updateById(integration.id, { + lastSyncJobId: job.id, + lastUsed: new Date(), + syncMessage: (err as Error)?.message, + isSynced: false + }); + } } logger.info("Secret integration sync ended: %s", job.id); diff --git a/frontend/src/hooks/api/integrations/queries.tsx b/frontend/src/hooks/api/integrations/queries.tsx index 2d4c8a313..9a1ee6fbf 100644 --- a/frontend/src/hooks/api/integrations/queries.tsx +++ b/frontend/src/hooks/api/integrations/queries.tsx @@ -1,5 +1,6 @@ import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { createNotification } from "@app/components/notifications"; import { apiRequest } from "@app/config/request"; import { workspaceKeys } from "../workspace/queries"; @@ -110,3 +111,15 @@ export const useDeleteIntegration = () => { } }); }; + +export const useSyncIntegration = () => { + return useMutation<{}, {}, { id: string; workspaceId: string; lastUsed: string }>({ + mutationFn: ({ id }) => apiRequest.post(`/api/v1/integration/${id}/sync`), + onSuccess: () => { + createNotification({ + text: "Successfully triggered manual sync", + type: "success" + }); + } + }); +}; diff --git a/frontend/src/hooks/api/integrations/types.ts b/frontend/src/hooks/api/integrations/types.ts index a67ce15a8..345e41b1a 100644 --- a/frontend/src/hooks/api/integrations/types.ts +++ b/frontend/src/hooks/api/integrations/types.ts @@ -29,6 +29,9 @@ export type TIntegration = { secretPath: string; createdAt: string; updatedAt: string; + lastUsed?: string; + isSynced?: boolean; + syncMessage?: string; __v: number; metadata?: { secretSuffix?: string; diff --git a/frontend/src/hooks/api/workspace/queries.tsx b/frontend/src/hooks/api/workspace/queries.tsx index cca544df3..71dbb8e01 100644 --- a/frontend/src/hooks/api/workspace/queries.tsx +++ b/frontend/src/hooks/api/workspace/queries.tsx @@ -198,7 +198,8 @@ export const useGetWorkspaceIntegrations = (workspaceId: string) => useQuery({ queryKey: workspaceKeys.getWorkspaceIntegrations(workspaceId), queryFn: () => fetchWorkspaceIntegrations(workspaceId), - enabled: Boolean(workspaceId) + enabled: Boolean(workspaceId), + refetchInterval: 4000 }); export const createWorkspace = ({ diff --git a/frontend/src/views/IntegrationsPage/components/IntegrationsSection/IntegrationsSection.tsx b/frontend/src/views/IntegrationsPage/components/IntegrationsSection/IntegrationsSection.tsx index 185cc411f..a6e3a45e7 100644 --- a/frontend/src/views/IntegrationsPage/components/IntegrationsSection/IntegrationsSection.tsx +++ b/frontend/src/views/IntegrationsPage/components/IntegrationsSection/IntegrationsSection.tsx @@ -1,21 +1,26 @@ import Link from "next/link"; -import { faArrowRight, faXmark } from "@fortawesome/free-solid-svg-icons"; +import { faCalendarCheck } from "@fortawesome/free-regular-svg-icons"; +import { faArrowRight, faRefresh, faWarning, faXmark } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { format } from "date-fns"; import { integrationSlugNameMapping } from "public/data/frequentConstants"; import { ProjectPermissionCan } from "@app/components/permissions"; import { Alert, AlertDescription, + Button, DeleteActionModal, EmptyState, FormLabel, IconButton, Skeleton, + Tag, Tooltip } from "@app/components/v2"; import { ProjectPermissionActions, ProjectPermissionSub } from "@app/context"; import { usePopUp } from "@app/hooks"; +import { useSyncIntegration } from "@app/hooks/api/integrations/queries"; import { TIntegration } from "@app/hooks/api/types"; type Props = { @@ -39,6 +44,8 @@ export const IntegrationsSection = ({ "deleteConfirmation" ] as const); + const { mutate: syncIntegration } = useSyncIntegration(); + return (
@@ -74,7 +81,7 @@ export const IntegrationsSection = ({
)} {!isLoading && isBotActive && ( -
+
{integrations?.map((integration) => (
-
+
{(integration.integration === "hashicorp-vault" && `${integration.app} - path: ${integration.path}`) || (integration.scope === "github-org" && `${integration.owner}`) || - (integration.integration === "aws-parameter-store" && `${integration.path}`) || + (integration.integration === "aws-parameter-store" && + `${integration.path}`) || (integration.scope?.startsWith("github-") && `${integration.owner}/${integration.app}`) || integration.app} @@ -187,13 +195,70 @@ export const IntegrationsSection = ({
)}
-
+
+ {integration.isSynced != null && integration.lastUsed != null && ( + + +
+ +
Last sync
+
+
+ {format(new Date(integration.lastUsed), "yyyy-MM-dd, hh:mm aaa")} +
+ {!integration.isSynced && ( + <> +
+ +
Fail reason
+
+
+ {integration.syncMessage} +
+ + )} +
+ } + > +
+
Sync Status
+ {!integration.isSynced && } +
+ + + )} +
+ + + +
{(isAllowed: boolean) => ( -
+
handlePopUpOpen("deleteConfirmation", integration)} @@ -217,7 +282,9 @@ export const IntegrationsSection = ({ isOpen={popUp.deleteConfirmation.isOpen} title={`Are you sure want to remove ${ (popUp?.deleteConfirmation.data as TIntegration)?.integration || " " - } integration for ${(popUp?.deleteConfirmation.data as TIntegration)?.app || "this project"}?`} + } integration for ${ + (popUp?.deleteConfirmation.data as TIntegration)?.app || "this project" + }?`} onChange={(isOpen) => handlePopUpToggle("deleteConfirmation", isOpen)} deleteKey={ (popUp?.deleteConfirmation?.data as TIntegration)?.app ||