Merge remote-tracking branch 'origin/main' into feat/add-version-limits

2026-10-15 18:15:53 +00:00 · 2024-06-08 01:37:14 +08:00
parent ee152f2d20 faa1572faf
commit 7fe7056af4
126 changed files with 3741 additions and 729 deletions
@@ -44,6 +44,7 @@ import { secretApprovalRequestDALFactory } from "@app/ee/services/secret-approva
 import { secretApprovalRequestReviewerDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-reviewer-dal";
 import { secretApprovalRequestSecretDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-secret-dal";
 import { secretApprovalRequestServiceFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-service";
+import { secretReplicationServiceFactory } from "@app/ee/services/secret-replication/secret-replication-service";
 import { secretRotationDALFactory } from "@app/ee/services/secret-rotation/secret-rotation-dal";
 import { secretRotationQueueFactory } from "@app/ee/services/secret-rotation/secret-rotation-queue";
 import { secretRotationServiceFactory } from "@app/ee/services/secret-rotation/secret-rotation-service";
@@ -96,6 +97,9 @@ import { integrationDALFactory } from "@app/services/integration/integration-dal
 import { integrationServiceFactory } from "@app/services/integration/integration-service";
 import { integrationAuthDALFactory } from "@app/services/integration-auth/integration-auth-dal";
 import { integrationAuthServiceFactory } from "@app/services/integration-auth/integration-auth-service";
+import { kmsDALFactory } from "@app/services/kms/kms-dal";
+import { kmsRootConfigDALFactory } from "@app/services/kms/kms-root-config-dal";
+import { kmsServiceFactory } from "@app/services/kms/kms-service";
 import { incidentContactDALFactory } from "@app/services/org/incident-contacts-dal";
 import { orgBotDALFactory } from "@app/services/org/org-bot-dal";
 import { orgDALFactory } from "@app/services/org/org-dal";
@@ -240,8 +244,8 @@ export const registerRoutes = async (
  const sapApproverDAL = secretApprovalPolicyApproverDALFactory(db);
  const secretApprovalPolicyDAL = secretApprovalPolicyDALFactory(db);
  const secretApprovalRequestDAL = secretApprovalRequestDALFactory(db);
-  const sarReviewerDAL = secretApprovalRequestReviewerDALFactory(db);
-  const sarSecretDAL = secretApprovalRequestSecretDALFactory(db);
+  const secretApprovalRequestReviewerDAL = secretApprovalRequestReviewerDALFactory(db);
+  const secretApprovalRequestSecretDAL = secretApprovalRequestSecretDALFactory(db);

  const secretRotationDAL = secretRotationDALFactory(db);
  const snapshotDAL = snapshotDALFactory(db);
@@ -260,6 +264,9 @@ export const registerRoutes = async (
  const dynamicSecretDAL = dynamicSecretDALFactory(db);
  const dynamicSecretLeaseDAL = dynamicSecretLeaseDALFactory(db);

+  const kmsDAL = kmsDALFactory(db);
+  const kmsRootConfigDAL = kmsRootConfigDALFactory(db);
+
  const permissionService = permissionServiceFactory({
    permissionDAL,
    orgRoleDAL,
@@ -268,6 +275,12 @@ export const registerRoutes = async (
    projectDAL
  });
  const licenseService = licenseServiceFactory({ permissionService, orgDAL, licenseDAL, keyStore });
+  const kmsService = kmsServiceFactory({
+    kmsRootConfigDAL,
+    keyStore,
+    kmsDAL
+  });
+
  const trustedIpService = trustedIpServiceFactory({
    licenseService,
    projectDAL,
@@ -288,7 +301,7 @@ export const registerRoutes = async (
    permissionService,
    auditLogStreamDAL
  });
-  const sapService = secretApprovalPolicyServiceFactory({
+  const secretApprovalPolicyService = secretApprovalPolicyServiceFactory({
    projectMembershipDAL,
    projectEnvDAL,
    secretApprovalPolicyApproverDAL: sapApproverDAL,
@@ -489,7 +502,7 @@ export const registerRoutes = async (
    projectBotDAL,
    projectMembershipDAL,
    secretApprovalRequestDAL,
-    secretApprovalSecretDAL: sarSecretDAL,
+    secretApprovalSecretDAL: secretApprovalRequestSecretDAL,
    projectUserMembershipRoleDAL
  });

@@ -587,6 +600,7 @@ export const registerRoutes = async (
    secretVersionTagDAL
  });
  const secretImportService = secretImportServiceFactory({
+    licenseService,
    projectEnvDAL,
    folderDAL,
    permissionService,
@@ -621,19 +635,18 @@ export const registerRoutes = async (
    secretSharingDAL
  });

-  const sarService = secretApprovalRequestServiceFactory({
+  const secretApprovalRequestService = secretApprovalRequestServiceFactory({
    permissionService,
    projectBotService,
    folderDAL,
    secretDAL,
    secretTagDAL,
-    secretApprovalRequestSecretDAL: sarSecretDAL,
-    secretApprovalRequestReviewerDAL: sarReviewerDAL,
+    secretApprovalRequestSecretDAL,
+    secretApprovalRequestReviewerDAL,
    projectDAL,
    secretVersionDAL,
    secretBlindIndexDAL,
    secretApprovalRequestDAL,
-    secretService,
    snapshotService,
    secretVersionTagDAL,
    secretQueueService
@@ -662,6 +675,23 @@ export const registerRoutes = async (
    accessApprovalPolicyApproverDAL
  });

+  const secretReplicationService = secretReplicationServiceFactory({
+    secretTagDAL,
+    secretVersionTagDAL,
+    secretDAL,
+    secretVersionDAL,
+    secretImportDAL,
+    keyStore,
+    queueService,
+    folderDAL,
+    secretApprovalPolicyService,
+    secretBlindIndexDAL,
+    secretApprovalRequestDAL,
+    secretApprovalRequestSecretDAL,
+    secretQueueService,
+    projectMembershipDAL,
+    projectBotService
+  });
  const secretRotationQueue = secretRotationQueueFactory({
    telemetryService,
    secretRotationDAL,
@@ -808,6 +838,7 @@ export const registerRoutes = async (

  await telemetryQueue.startTelemetryCheck();
  await dailyResourceCleanUp.startCleanUp();
+  await kmsService.startService();

  // inject all services
  server.decorate<FastifyZodProvider["services"]>("services", {
@@ -829,6 +860,7 @@ export const registerRoutes = async (
    projectEnv: projectEnvService,
    projectRole: projectRoleService,
    secret: secretService,
+    secretReplication: secretReplicationService,
    secretTag: secretTagService,
    folder: folderService,
    secretImport: secretImportService,
@@ -845,10 +877,10 @@ export const registerRoutes = async (
    identityGcpAuth: identityGcpAuthService,
    identityAwsAuth: identityAwsAuthService,
    identityAzureAuth: identityAzureAuthService,
-    secretApprovalPolicy: sapService,
    accessApprovalPolicy: accessApprovalPolicyService,
    accessApprovalRequest: accessApprovalRequestService,
-    secretApprovalRequest: sarService,
+    secretApprovalPolicy: secretApprovalPolicyService,
+    secretApprovalRequest: secretApprovalRequestService,
    secretRotation: secretRotationService,
    dynamicSecret: dynamicSecretService,
    dynamicSecretLease: dynamicSecretLeaseService,
@@ -42,6 +42,7 @@ export const registerIntegrationRouter = async (server: FastifyZodProvider) => {
        targetService: z.string().trim().optional().describe(INTEGRATION.CREATE.targetService),
        targetServiceId: z.string().trim().optional().describe(INTEGRATION.CREATE.targetServiceId),
        owner: z.string().trim().optional().describe(INTEGRATION.CREATE.owner),
+        url: z.string().trim().optional().describe(INTEGRATION.CREATE.url),
        path: z.string().trim().optional().describe(INTEGRATION.CREATE.path),
        region: z.string().trim().optional().describe(INTEGRATION.CREATE.region),
        scope: z.string().trim().optional().describe(INTEGRATION.CREATE.scope),
@@ -29,7 +29,8 @@ export const registerSecretImportRouter = async (server: FastifyZodProvider) =>
        import: z.object({
          environment: z.string().trim().describe(SECRET_IMPORTS.CREATE.import.environment),
          path: z.string().trim().transform(removeTrailingSlash).describe(SECRET_IMPORTS.CREATE.import.path)
-        })
+        }),
+        isReplication: z.boolean().default(false)
      }),
      response: {
        200: z.object({
@@ -210,6 +211,49 @@ export const registerSecretImportRouter = async (server: FastifyZodProvider) =>
    }
  });

+  server.route({
+    method: "POST",
+    url: "/:secretImportId/replication-resync",
+    config: {
+      rateLimit: secretsLimit
+    },
+    schema: {
+      description: "Resync secret replication of secret imports",
+      security: [
+        {
+          bearerAuth: []
+        }
+      ],
+      params: z.object({
+        secretImportId: z.string().trim().describe(SECRET_IMPORTS.UPDATE.secretImportId)
+      }),
+      body: z.object({
+        workspaceId: z.string().trim().describe(SECRET_IMPORTS.UPDATE.workspaceId),
+        environment: z.string().trim().describe(SECRET_IMPORTS.UPDATE.environment),
+        path: z.string().trim().default("/").transform(removeTrailingSlash).describe(SECRET_IMPORTS.UPDATE.path)
+      }),
+      response: {
+        200: z.object({
+          message: z.string()
+        })
+      }
+    },
+    onRequest: verifyAuth([AuthMode.JWT]),
+    handler: async (req) => {
+      const { message } = await server.services.secretImport.resyncSecretImportReplication({
+        actorId: req.permission.id,
+        actor: req.permission.type,
+        actorAuthMethod: req.permission.authMethod,
+        actorOrgId: req.permission.orgId,
+        id: req.params.secretImportId,
+        ...req.body,
+        projectId: req.body.workspaceId
+      });
+
+      return { message };
+    }
+  });
+
  server.route({
    method: "GET",
    url: "/",
@@ -232,11 +276,9 @@ export const registerSecretImportRouter = async (server: FastifyZodProvider) =>
        200: z.object({
          message: z.string(),
          secretImports: SecretImportsSchema.omit({ importEnv: true })
-            .merge(
-              z.object({
-                importEnv: z.object({ name: z.string(), slug: z.string(), id: z.string() })
-              })
-            )
+            .extend({
+              importEnv: z.object({ name: z.string(), slug: z.string(), id: z.string() })
+            })
            .array()
        })
      }
@@ -45,7 +45,13 @@ export const registerSecretSharingRouter = async (server: FastifyZodProvider) =>
        hashedHex: z.string()
      }),
      response: {
-        200: SecretSharingSchema.pick({ name: true, encryptedValue: true, iv: true, tag: true, expiresAt: true })
+        200: SecretSharingSchema.pick({
+          encryptedValue: true,
+          iv: true,
+          tag: true,
+          expiresAt: true,
+          expiresAfterViews: true
+        })
      }
    },
    handler: async (req) => {
@@ -55,11 +61,11 @@ export const registerSecretSharingRouter = async (server: FastifyZodProvider) =>
      );
      if (!sharedSecret) return undefined;
      return {
-        name: sharedSecret.name,
        encryptedValue: sharedSecret.encryptedValue,
        iv: sharedSecret.iv,
        tag: sharedSecret.tag,
-        expiresAt: sharedSecret.expiresAt
+        expiresAt: sharedSecret.expiresAt,
+        expiresAfterViews: sharedSecret.expiresAfterViews
      };
    }
  });
@@ -72,14 +78,14 @@ export const registerSecretSharingRouter = async (server: FastifyZodProvider) =>
    },
    schema: {
      body: z.object({
-        name: z.string(),
        encryptedValue: z.string(),
        iv: z.string(),
        tag: z.string(),
        hashedHex: z.string(),
-        expiresAt: z.string().refine((date) => new Date(date) > new Date(), {
-          message: "Expires at should be a future date"
-        })
+        expiresAt: z
+          .string()
+          .refine((date) => date === undefined || new Date(date) > new Date(), "Expires at should be a future date"),
+        expiresAfterViews: z.number()
      }),
      response: {
        200: z.object({
@@ -89,19 +95,19 @@ export const registerSecretSharingRouter = async (server: FastifyZodProvider) =>
    },
    onRequest: verifyAuth([AuthMode.JWT]),
    handler: async (req) => {
-      const { name, encryptedValue, iv, tag, hashedHex, expiresAt } = req.body;
+      const { encryptedValue, iv, tag, hashedHex, expiresAt, expiresAfterViews } = req.body;
      const sharedSecret = await req.server.services.secretSharing.createSharedSecret({
        actor: req.permission.type,
        actorId: req.permission.id,
        orgId: req.permission.orgId,
        actorAuthMethod: req.permission.authMethod,
        actorOrgId: req.permission.orgId,
-        name,
        encryptedValue,
        iv,
        tag,
        hashedHex,
-        expiresAt: new Date(expiresAt)
+        expiresAt: new Date(expiresAt),
+        expiresAfterViews
      });
      return { id: sharedSecret.id };
    }
@@ -9,7 +9,6 @@ import {
  ServiceTokenScopes
 } from "@app/db/schemas";
 import { EventType } from "@app/ee/services/audit-log/audit-log-types";
-import { CommitType } from "@app/ee/services/secret-approval-request/secret-approval-request-types";
 import { RAW_SECRETS, SECRETS } from "@app/lib/api-docs";
 import { BadRequestError } from "@app/lib/errors";
 import { removeTrailingSlash } from "@app/lib/fn";
@@ -19,6 +18,7 @@ import { getUserAgentType } from "@app/server/plugins/audit-log";
 import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
 import { ActorType, AuthMode } from "@app/services/auth/auth-type";
 import { ProjectFilterType } from "@app/services/project/project-types";
+import { SecretOperations } from "@app/services/secret/secret-types";
 import { PostHogEventTypes } from "@app/services/telemetry/telemetry-types";

 import { secretRawSchema } from "../sanitizedSchemas";
@@ -902,7 +902,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
            projectId,
            policy,
            data: {
-              [CommitType.Create]: [
+              [SecretOperations.Create]: [
                {
                  secretName: req.params.secretName,
                  secretValueCiphertext,
@@ -1084,7 +1084,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
            projectId,
            policy,
            data: {
-              [CommitType.Update]: [
+              [SecretOperations.Update]: [
                {
                  secretName: req.params.secretName,
                  newSecretName,
@@ -1234,7 +1234,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
            projectId,
            policy,
            data: {
-              [CommitType.Delete]: [
+              [SecretOperations.Delete]: [
                {
                  secretName: req.params.secretName
                }
@@ -1364,7 +1364,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
            projectId,
            policy,
            data: {
-              [CommitType.Create]: inputSecrets
+              [SecretOperations.Create]: inputSecrets
            }
          });

@@ -1491,7 +1491,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
            projectId,
            policy,
            data: {
-              [CommitType.Update]: inputSecrets.filter(({ type }) => type === "shared")
+              [SecretOperations.Update]: inputSecrets.filter(({ type }) => type === "shared")
            }
          });

@@ -1606,7 +1606,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
            projectId,
            policy,
            data: {
-              [CommitType.Delete]: inputSecrets.filter(({ type }) => type === "shared")
+              [SecretOperations.Delete]: inputSecrets.filter(({ type }) => type === "shared")
            }
          });
          await server.services.auditLog.createAuditLog({