Merge pull request #1848 from Infisical/feat/add-integration-sync-status

feat: added integration sync status

backend/src/db/migrations/20240520064127_add-integration-sync-status.ts

@@ -0,0 +1,43 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  const hasIsSyncedColumn = await knex.schema.hasColumn(TableName.Integration, "isSynced");
+  const hasSyncMessageColumn = await knex.schema.hasColumn(TableName.Integration, "syncMessage");
+  const hasLastSyncJobId = await knex.schema.hasColumn(TableName.Integration, "lastSyncJobId");
+
+  await knex.schema.alterTable(TableName.Integration, (t) => {
+    if (!hasIsSyncedColumn) {
+      t.boolean("isSynced").nullable();
+    }
+
+    if (!hasSyncMessageColumn) {
+      t.text("syncMessage").nullable();
+    }
+
+    if (!hasLastSyncJobId) {
+      t.string("lastSyncJobId").nullable();
+    }
+  });
+}
+
+export async function down(knex: Knex): Promise<void> {
+  const hasIsSyncedColumn = await knex.schema.hasColumn(TableName.Integration, "isSynced");
+  const hasSyncMessageColumn = await knex.schema.hasColumn(TableName.Integration, "syncMessage");
+  const hasLastSyncJobId = await knex.schema.hasColumn(TableName.Integration, "lastSyncJobId");
+
+  await knex.schema.alterTable(TableName.Integration, (t) => {
+    if (hasIsSyncedColumn) {
+      t.dropColumn("isSynced");
+    }
+
+    if (hasSyncMessageColumn) {
+      t.dropColumn("syncMessage");
+    }
+
+    if (hasLastSyncJobId) {
+      t.dropColumn("lastSyncJobId");
+    }
+  });
+}

backend/src/db/schemas/integrations.ts

@@ -28,7 +28,10 @@ export const IntegrationsSchema = z.object({
   secretPath: z.string().default("/"),
   createdAt: z.date(),
   updatedAt: z.date(),
-  lastUsed: z.date().nullable().optional()
+  lastUsed: z.date().nullable().optional(),
+  isSynced: z.boolean().nullable().optional(),
+  syncMessage: z.string().nullable().optional(),
+  lastSyncJobId: z.string().nullable().optional()
 });
 
 export type TIntegrations = z.infer<typeof IntegrationsSchema>;

backend/src/ee/services/audit-log/audit-log-types.ts

@@ -51,6 +51,7 @@ export enum EventType {
   UNAUTHORIZE_INTEGRATION = "unauthorize-integration",
   CREATE_INTEGRATION = "create-integration",
   DELETE_INTEGRATION = "delete-integration",
+  MANUAL_SYNC_INTEGRATION = "manual-sync-integration",
   ADD_TRUSTED_IP = "add-trusted-ip",
   UPDATE_TRUSTED_IP = "update-trusted-ip",
   DELETE_TRUSTED_IP = "delete-trusted-ip",
@@ -281,6 +282,25 @@ interface DeleteIntegrationEvent {
   };
 }
 
+interface ManualSyncIntegrationEvent {
+  type: EventType.MANUAL_SYNC_INTEGRATION;
+  metadata: {
+    integrationId: string;
+    integration: string;
+    environment: string;
+    secretPath: string;
+    url?: string;
+    app?: string;
+    appId?: string;
+    targetEnvironment?: string;
+    targetEnvironmentId?: string;
+    targetService?: string;
+    targetServiceId?: string;
+    path?: string;
+    region?: string;
+  };
+}
+
 interface AddTrustedIPEvent {
   type: EventType.ADD_TRUSTED_IP;
   metadata: {
@@ -791,6 +811,7 @@ export type Event =
   | UnauthorizeIntegrationEvent
   | CreateIntegrationEvent
   | DeleteIntegrationEvent
+  | ManualSyncIntegrationEvent
   | AddTrustedIPEvent
   | UpdateTrustedIPEvent
   | DeleteTrustedIPEvent

backend/src/lib/api-docs/constants.ts

@@ -683,6 +683,9 @@ export const INTEGRATION = {
   },
   DELETE: {
     integrationId: "The ID of the integration object."
+  },
+  SYNC: {
+    integrationId: "The ID of the integration object to manually sync"
   }
 };
 

backend/src/server/routes/v1/integration-router.ts

@@ -262,5 +262,64 @@ export const registerIntegrationRouter = async (server: FastifyZodProvider) => {
     }
   });
 
-  // TODO(akhilmhdh-pg): manual sync
+  server.route({
+    method: "POST",
+    url: "/:integrationId/sync",
+    config: {
+      rateLimit: writeLimit
+    },
+    schema: {
+      description: "Manually trigger sync of an integration by integration id",
+      security: [
+        {
+          bearerAuth: []
+        }
+      ],
+      params: z.object({
+        integrationId: z.string().trim().describe(INTEGRATION.SYNC.integrationId)
+      }),
+      response: {
+        200: z.object({
+          integration: IntegrationsSchema
+        })
+      }
+    },
+    onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
+    handler: async (req) => {
+      const integration = await server.services.integration.syncIntegration({
+        actorId: req.permission.id,
+        actor: req.permission.type,
+        actorAuthMethod: req.permission.authMethod,
+        actorOrgId: req.permission.orgId,
+        id: req.params.integrationId
+      });
+
+      await server.services.auditLog.createAuditLog({
+        ...req.auditLogInfo,
+        projectId: integration.projectId,
+        event: {
+          type: EventType.MANUAL_SYNC_INTEGRATION,
+          // eslint-disable-next-line
+          metadata: shake({
+            integrationId: integration.id,
+            integration: integration.integration,
+            environment: integration.environment.slug,
+            secretPath: integration.secretPath,
+            url: integration.url,
+            app: integration.app,
+            appId: integration.appId,
+            targetEnvironment: integration.targetEnvironment,
+            targetEnvironmentId: integration.targetEnvironmentId,
+            targetService: integration.targetService,
+            targetServiceId: integration.targetServiceId,
+            path: integration.path,
+            region: integration.region
+            // eslint-disable-next-line
+          }) as any
+        }
+      });
+
+      return { integration };
+    }
+  });
 };

backend/src/services/integration/integration-service.ts

@@ -9,7 +9,12 @@ import { TIntegrationAuthDALFactory } from "../integration-auth/integration-auth
 import { TSecretQueueFactory } from "../secret/secret-queue";
 import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
 import { TIntegrationDALFactory } from "./integration-dal";
-import { TCreateIntegrationDTO, TDeleteIntegrationDTO, TUpdateIntegrationDTO } from "./integration-types";
+import {
+  TCreateIntegrationDTO,
+  TDeleteIntegrationDTO,
+  TSyncIntegrationDTO,
+  TUpdateIntegrationDTO
+} from "./integration-types";
 
 type TIntegrationServiceFactoryDep = {
   integrationDAL: TIntegrationDALFactory;
@@ -201,10 +206,35 @@ export const integrationServiceFactory = ({
     return integrations;
   };
 
+  const syncIntegration = async ({ id, actorId, actor, actorOrgId, actorAuthMethod }: TSyncIntegrationDTO) => {
+    const integration = await integrationDAL.findById(id);
+    if (!integration) {
+      throw new BadRequestError({ message: "Integration not found" });
+    }
+
+    const { permission } = await permissionService.getProjectPermission(
+      actor,
+      actorId,
+      integration.projectId,
+      actorAuthMethod,
+      actorOrgId
+    );
+    ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Integrations);
+
+    await secretQueueService.syncIntegrations({
+      environment: integration.environment.slug,
+      secretPath: integration.secretPath,
+      projectId: integration.projectId
+    });
+
+    return { ...integration, envId: integration.environment.id };
+  };
+
   return {
     createIntegration,
     updateIntegration,
     deleteIntegration,
-    listIntegrationByProject
+    listIntegrationByProject,
+    syncIntegration
   };
 };

backend/src/services/integration/integration-types.ts

@@ -59,3 +59,7 @@ export type TUpdateIntegrationDTO = {
 export type TDeleteIntegrationDTO = {
   id: string;
 } & Omit<TProjectPermission, "projectId">;
+
+export type TSyncIntegrationDTO = {
+  id: string;
+} & Omit<TProjectPermission, "projectId">;

backend/src/services/secret/secret-queue.ts

@@ -463,20 +463,37 @@ export const secretQueueFactory = ({
         });
       }
 
-      await syncIntegrationSecrets({
+      try {
-        createManySecretsRawFn,
+        await syncIntegrationSecrets({
-        updateManySecretsRawFn,
+          createManySecretsRawFn,
-        integrationDAL,
+          updateManySecretsRawFn,
-        integration,
+          integrationDAL,
-        integrationAuth,
+          integration,
-        secrets: Object.keys(suffixedSecrets).length !== 0 ? suffixedSecrets : secrets,
+          integrationAuth,
-        accessId: accessId as string,
+          secrets: Object.keys(suffixedSecrets).length !== 0 ? suffixedSecrets : secrets,
-        accessToken,
+          accessId: accessId as string,
-        appendices: {
+          accessToken,
-          prefix: metadata?.secretPrefix || "",
+          appendices: {
-          suffix: metadata?.secretSuffix || ""
+            prefix: metadata?.secretPrefix || "",
-        }
+            suffix: metadata?.secretSuffix || ""
-      });
+          }
+        });
+
+        await integrationDAL.updateById(integration.id, {
+          lastSyncJobId: job.id,
+          lastUsed: new Date(),
+          syncMessage: "",
+          isSynced: true
+        });
+      } catch (err: unknown) {
+        logger.info("Secret integration sync error:", err);
+        await integrationDAL.updateById(integration.id, {
+          lastSyncJobId: job.id,
+          lastUsed: new Date(),
+          syncMessage: (err as Error)?.message,
+          isSynced: false
+        });
+      }
     }
 
     logger.info("Secret integration sync ended: %s", job.id);

frontend/src/hooks/api/integrations/queries.tsx

@@ -1,5 +1,6 @@
 import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
 
+import { createNotification } from "@app/components/notifications";
 import { apiRequest } from "@app/config/request";
 
 import { workspaceKeys } from "../workspace/queries";
@@ -110,3 +111,15 @@ export const useDeleteIntegration = () => {
     }
   });
 };
+
+export const useSyncIntegration = () => {
+  return useMutation<{}, {}, { id: string; workspaceId: string; lastUsed: string }>({
+    mutationFn: ({ id }) => apiRequest.post(`/api/v1/integration/${id}/sync`),
+    onSuccess: () => {
+      createNotification({
+        text: "Successfully triggered manual sync",
+        type: "success"
+      });
+    }
+  });
+};

frontend/src/hooks/api/integrations/types.ts

@@ -29,6 +29,9 @@ export type TIntegration = {
   secretPath: string;
   createdAt: string;
   updatedAt: string;
+  lastUsed?: string;
+  isSynced?: boolean;
+  syncMessage?: string;
   __v: number;
   metadata?: {
     secretSuffix?: string;

frontend/src/hooks/api/workspace/queries.tsx

@@ -198,7 +198,8 @@ export const useGetWorkspaceIntegrations = (workspaceId: string) =>
   useQuery({
     queryKey: workspaceKeys.getWorkspaceIntegrations(workspaceId),
     queryFn: () => fetchWorkspaceIntegrations(workspaceId),
-    enabled: Boolean(workspaceId)
+    enabled: Boolean(workspaceId),
+    refetchInterval: 4000
   });
 
 export const createWorkspace = ({

frontend/src/views/IntegrationsPage/components/IntegrationsSection/IntegrationsSection.tsx

@@ -1,21 +1,26 @@
 import Link from "next/link";
-import { faArrowRight, faXmark } from "@fortawesome/free-solid-svg-icons";
+import { faCalendarCheck } from "@fortawesome/free-regular-svg-icons";
+import { faArrowRight, faRefresh, faWarning, faXmark } from "@fortawesome/free-solid-svg-icons";
 import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
+import { format } from "date-fns";
 import { integrationSlugNameMapping } from "public/data/frequentConstants";
 
 import { ProjectPermissionCan } from "@app/components/permissions";
 import {
   Alert,
   AlertDescription,
+  Button,
   DeleteActionModal,
   EmptyState,
   FormLabel,
   IconButton,
   Skeleton,
+  Tag,
   Tooltip
 } from "@app/components/v2";
 import { ProjectPermissionActions, ProjectPermissionSub } from "@app/context";
 import { usePopUp } from "@app/hooks";
+import { useSyncIntegration } from "@app/hooks/api/integrations/queries";
 import { TIntegration } from "@app/hooks/api/types";
 
 type Props = {
@@ -39,6 +44,8 @@ export const IntegrationsSection = ({
     "deleteConfirmation"
   ] as const);
 
+  const { mutate: syncIntegration } = useSyncIntegration();
+
   return (
     <div className="mb-8">
       <div className="mx-4 mb-4 mt-6 flex flex-col items-start justify-between px-2 text-xl">
@@ -74,7 +81,7 @@ export const IntegrationsSection = ({
         </div>
       )}
       {!isLoading && isBotActive && (
-        <div className="flex flex-col min-w-max space-y-4 p-6 pt-0">
+        <div className="flex min-w-max flex-col space-y-4 p-6 pt-0">
           {integrations?.map((integration) => (
             <div
               className="max-w-8xl flex justify-between rounded-md border border-mineshaft-600 bg-mineshaft-800 p-3"
@@ -137,11 +144,12 @@ export const IntegrationsSection = ({
                       "App"
                     }
                   />
-                  <div className="min-w-[8rem] max-w-[12rem] overflow-scroll no-scrollbar no-scrollbar::-webkit-scrollbar whitespace-nowrap rounded-md border border-mineshaft-700 bg-mineshaft-900 px-3 py-2 font-inter text-sm text-bunker-200">
+                  <div className="no-scrollbar::-webkit-scrollbar min-w-[8rem] max-w-[12rem] overflow-scroll whitespace-nowrap rounded-md border border-mineshaft-700 bg-mineshaft-900 px-3 py-2 font-inter text-sm text-bunker-200 no-scrollbar">
                     {(integration.integration === "hashicorp-vault" &&
                       `${integration.app} - path: ${integration.path}`) ||
                       (integration.scope === "github-org" && `${integration.owner}`) ||
-                      (integration.integration === "aws-parameter-store" && `${integration.path}`) ||
+                      (integration.integration === "aws-parameter-store" &&
+                        `${integration.path}`) ||
                       (integration.scope?.startsWith("github-") &&
                         `${integration.owner}/${integration.app}`) ||
                       integration.app}
@@ -187,13 +195,70 @@ export const IntegrationsSection = ({
                   </div>
                 )}
               </div>
-              <div className="flex cursor-default items-center">
+              <div className="mt-[1.5rem] flex cursor-default">
+                {integration.isSynced != null && integration.lastUsed != null && (
+                  <Tag
+                    key={integration.id}
+                    className={integration.isSynced ? "bg-green-800" : "bg-red/80"}
+                  >
+                    <Tooltip
+                      center
+                      className="max-w-xs whitespace-normal break-words"
+                      content={
+                        <div className="flex max-h-[10rem] flex-col overflow-auto ">
+                          <div className="flex self-start">
+                            <FontAwesomeIcon
+                              icon={faCalendarCheck}
+                              className="pt-0.5 pr-2 text-sm"
+                            />
+                            <div className="text-sm">Last sync</div>
+                          </div>
+                          <div className="pl-5 text-left text-xs">
+                            {format(new Date(integration.lastUsed), "yyyy-MM-dd, hh:mm aaa")}
+                          </div>
+                          {!integration.isSynced && (
+                            <>
+                              <div className="mt-2 flex self-start">
+                                <FontAwesomeIcon icon={faXmark} className="pt-1 pr-2 text-sm" />
+                                <div className="text-sm">Fail reason</div>
+                              </div>
+                              <div className="pl-5 text-left text-xs">
+                                {integration.syncMessage}
+                              </div>
+                            </>
+                          )}
+                        </div>
+                      }
+                    >
+                      <div className="flex items-center space-x-2 text-white">
+                        <div>Sync Status</div>
+                        {!integration.isSynced && <FontAwesomeIcon icon={faWarning} />}
+                      </div>
+                    </Tooltip>
+                  </Tag>
+                )}
+                <div className="mr-1 flex items-end opacity-80 duration-200 hover:opacity-100">
+                  <Tooltip className="text-center" content="Manually sync integration secrets">
+                    <Button
+                      onClick={() =>
+                        syncIntegration({
+                          workspaceId,
+                          id: integration.id,
+                          lastUsed: integration.lastUsed as string
+                        })
+                      }
+                      className="max-w-[2.5rem] border-none bg-mineshaft-500"
+                    >
+                      <FontAwesomeIcon icon={faRefresh} className="px-1 text-bunker-200" />
+                    </Button>
+                  </Tooltip>
+                </div>
                 <ProjectPermissionCan
                   I={ProjectPermissionActions.Delete}
                   a={ProjectPermissionSub.Integrations}
                 >
                   {(isAllowed: boolean) => (
-                    <div className="ml-2 opacity-80 duration-200 hover:opacity-100">
+                    <div className="flex items-end opacity-80 duration-200 hover:opacity-100">
                       <Tooltip content="Remove Integration">
                         <IconButton
                           onClick={() => handlePopUpOpen("deleteConfirmation", integration)}
@@ -217,7 +282,9 @@ export const IntegrationsSection = ({
         isOpen={popUp.deleteConfirmation.isOpen}
         title={`Are you sure want to remove ${
           (popUp?.deleteConfirmation.data as TIntegration)?.integration || " "
-        } integration for ${(popUp?.deleteConfirmation.data as TIntegration)?.app || "this project"}?`}
+        } integration for ${
+          (popUp?.deleteConfirmation.data as TIntegration)?.app || "this project"
+        }?`}
         onChange={(isOpen) => handlePopUpToggle("deleteConfirmation", isOpen)}
         deleteKey={
           (popUp?.deleteConfirmation?.data as TIntegration)?.app ||