feat: added secret raw endpoint and include imports

backend-pg/src/server/routes/index.ts

@@ -338,14 +338,14 @@ export const registerRoutes = async (
   });
   const secretQueueService = secretQueueFactory({
     queueService,
-    webhookService,
     secretDal,
     folderDal,
-    secretImportService,
     integrationAuthService,
     projectBotService,
     integrationDal,
-    secretImportDal
+    secretImportDal,
+    projectEnvDal,
+    webhookDal
   });
   const secretService = secretServiceFactory({
     folderDal,
@@ -355,7 +355,9 @@ export const registerRoutes = async (
     secretDal,
     secretTagDal,
     snapshotService,
-    secretQueueService
+    secretQueueService,
+    secretImportDal,
+    projectBotService
   });
   const sarService = secretApprovalRequestServiceFactory({
     permissionService,

backend-pg/src/server/routes/sanitizedSchemas.ts

@@ -27,3 +27,13 @@ export const sapPubSchema = SecretApprovalPoliciesSchema.merge(
     projectId: z.string()
   })
 );
+
+export const secretRawSchema = z.object({
+  id: z.string(),
+  _id: z.string(),
+  version: z.number(),
+  type: z.string(),
+  secretKey: z.string(),
+  secretValue: z.string(),
+  secretComment: z.string().optional()
+});

backend-pg/src/server/routes/v3/secret-router.ts

@@ -11,7 +11,308 @@ import { CommitType } from "@app/ee/services/secret-approval-request/secret-appr
 import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
 import { ActorType, AuthMode } from "@app/services/auth/auth-type";
 
+import { secretRawSchema } from "../sanitizedSchemas";
+
 export const registerSecretRouter = async (server: FastifyZodProvider) => {
+  server.route({
+    url: "/raw",
+    method: "GET",
+    schema: {
+      querystring: z.object({
+        workspaceId: z.string().trim(),
+        environment: z.string().trim(),
+        secretPath: z.string().trim().default("/"),
+        include_imports: z
+          .enum(["true", "false"])
+          .default("false")
+          .transform((value) => value === "true")
+      }),
+      response: {
+        200: z.object({
+          secrets: secretRawSchema.array(),
+          imports: z
+            .object({
+              secretPath: z.string(),
+              environment: z.object({
+                id: z.string(),
+                name: z.string(),
+                slug: z.string()
+              }),
+              folderId: z.string().optional(),
+              secrets: secretRawSchema.array()
+            })
+            .array()
+            .optional()
+        })
+      }
+    },
+    onRequest: verifyAuth([
+      AuthMode.JWT,
+      AuthMode.API_KEY,
+      AuthMode.SERVICE_TOKEN,
+      AuthMode.IDENTITY_ACCESS_TOKEN
+    ]),
+    handler: async (req) => {
+      const { secrets, imports } = await server.services.secret.getSecretsRaw({
+        actorId: req.permission.id,
+        actor: req.permission.type,
+        environment: req.query.environment,
+        projectId: req.query.workspaceId,
+        path: req.query.secretPath,
+        includeImports: req.query.include_imports
+      });
+
+      await server.services.auditLog.createAuditLog({
+        projectId: req.query.workspaceId,
+        ...req.auditLogInfo,
+        event: {
+          type: EventType.GET_SECRETS,
+          metadata: {
+            environment: req.query.environment,
+            secretPath: req.query.secretPath,
+            numberOfSecrets: secrets.length
+          }
+        }
+      });
+      return { secrets, imports };
+    }
+  });
+
+  server.route({
+    url: "/raw/:secretName",
+    method: "GET",
+    schema: {
+      params: z.object({
+        secretName: z.string().trim()
+      }),
+      querystring: z.object({
+        workspaceId: z.string().trim(),
+        environment: z.string().trim(),
+        secretPath: z.string().trim().default("/"),
+        type: z.nativeEnum(SecretType).default(SecretType.Shared),
+        include_imports: z
+          .enum(["true", "false"])
+          .default("false")
+          .transform((value) => value === "true")
+      }),
+      response: {
+        200: z.object({
+          secret: secretRawSchema
+        })
+      }
+    },
+    onRequest: verifyAuth([
+      AuthMode.JWT,
+      AuthMode.API_KEY,
+      AuthMode.SERVICE_TOKEN,
+      AuthMode.IDENTITY_ACCESS_TOKEN
+    ]),
+    handler: async (req) => {
+      const secret = await server.services.secret.getASecretRaw({
+        actorId: req.permission.id,
+        actor: req.permission.type,
+        environment: req.query.environment,
+        projectId: req.query.workspaceId,
+        path: req.query.secretPath,
+        secretName: req.params.secretName,
+        type: req.query.type,
+        includeImports: req.query.include_imports
+      });
+
+      await server.services.auditLog.createAuditLog({
+        projectId: req.query.workspaceId,
+        ...req.auditLogInfo,
+        event: {
+          type: EventType.GET_SECRET,
+          metadata: {
+            environment: req.query.environment,
+            secretPath: req.query.secretPath,
+            secretId: secret.id,
+            secretKey: req.params.secretName,
+            secretVersion: secret.version
+          }
+        }
+      });
+      return { secret };
+    }
+  });
+
+  server.route({
+    url: "/raw/:secretName",
+    method: "POST",
+    schema: {
+      params: z.object({
+        secretName: z.string().trim()
+      }),
+      body: z.object({
+        workspaceId: z.string().trim(),
+        environment: z.string().trim(),
+        secretPath: z.string().trim().default("/"),
+        secretValue: z
+          .string()
+          .transform((val) => (val.at(-1) === "\n" ? `${val.trim()}\n` : val.trim())),
+        secretComment: z.string().trim().optional().default(""),
+        skipMultilineEncoding: z.boolean().optional(),
+        type: z.nativeEnum(SecretType).default(SecretType.Shared)
+      }),
+      response: {
+        200: z.object({
+          secret: secretRawSchema
+        })
+      }
+    },
+    onRequest: verifyAuth([
+      AuthMode.JWT,
+      AuthMode.API_KEY,
+      AuthMode.SERVICE_TOKEN,
+      AuthMode.IDENTITY_ACCESS_TOKEN
+    ]),
+    handler: async (req) => {
+      const secret = await server.services.secret.createSecretRaw({
+        actorId: req.permission.id,
+        actor: req.permission.type,
+        environment: req.body.environment,
+        projectId: req.body.workspaceId,
+        secretPath: req.body.secretPath,
+        secretName: req.params.secretName,
+        type: req.body.type,
+        secretValue: req.body.secretValue,
+        skipMultilineEncoding: req.body.skipMultilineEncoding,
+        secretComment: req.body.secretComment
+      });
+
+      await server.services.auditLog.createAuditLog({
+        projectId: req.body.workspaceId,
+        ...req.auditLogInfo,
+        event: {
+          type: EventType.CREATE_SECRET,
+          metadata: {
+            environment: req.body.environment,
+            secretPath: req.body.secretPath,
+            secretId: secret.id,
+            secretKey: req.params.secretName,
+            secretVersion: secret.version
+          }
+        }
+      });
+      return { secret };
+    }
+  });
+
+  server.route({
+    url: "/raw/:secretName",
+    method: "PATCH",
+    schema: {
+      params: z.object({
+        secretName: z.string().trim()
+      }),
+      body: z.object({
+        workspaceId: z.string().trim(),
+        environment: z.string().trim(),
+        secretValue: z
+          .string()
+          .transform((val) => (val.at(-1) === "\n" ? `${val.trim()}\n` : val.trim())),
+        secretPath: z.string().trim().default("/"),
+        skipMultilineEncoding: z.boolean().optional(),
+        type: z.nativeEnum(SecretType).default(SecretType.Shared)
+      }),
+      response: {
+        200: z.object({
+          secret: secretRawSchema
+        })
+      }
+    },
+    onRequest: verifyAuth([
+      AuthMode.JWT,
+      AuthMode.API_KEY,
+      AuthMode.SERVICE_TOKEN,
+      AuthMode.IDENTITY_ACCESS_TOKEN
+    ]),
+    handler: async (req) => {
+      const secret = await server.services.secret.updateSecretRaw({
+        actorId: req.permission.id,
+        actor: req.permission.type,
+        environment: req.body.environment,
+        projectId: req.body.workspaceId,
+        secretPath: req.body.secretPath,
+        secretName: req.params.secretName,
+        type: req.body.type,
+        secretValue: req.body.secretValue,
+        skipMultilineEncoding: req.body.skipMultilineEncoding
+      });
+
+      await server.services.auditLog.createAuditLog({
+        projectId: req.body.workspaceId,
+        ...req.auditLogInfo,
+        event: {
+          type: EventType.UPDATE_SECRET,
+          metadata: {
+            environment: req.body.environment,
+            secretPath: req.body.secretPath,
+            secretId: secret.id,
+            secretKey: req.params.secretName,
+            secretVersion: secret.version
+          }
+        }
+      });
+      return { secret };
+    }
+  });
+
+  server.route({
+    url: "/raw/:secretName",
+    method: "DELETE",
+    schema: {
+      params: z.object({
+        secretName: z.string().trim()
+      }),
+      body: z.object({
+        workspaceId: z.string().trim(),
+        environment: z.string().trim(),
+        secretPath: z.string().trim().default("/"),
+        type: z.nativeEnum(SecretType).default(SecretType.Shared)
+      }),
+      response: {
+        200: z.object({
+          secret: secretRawSchema
+        })
+      }
+    },
+    onRequest: verifyAuth([
+      AuthMode.JWT,
+      AuthMode.API_KEY,
+      AuthMode.SERVICE_TOKEN,
+      AuthMode.IDENTITY_ACCESS_TOKEN
+    ]),
+    handler: async (req) => {
+      const secret = await server.services.secret.deleteSecretRaw({
+        actorId: req.permission.id,
+        actor: req.permission.type,
+        environment: req.body.environment,
+        projectId: req.body.workspaceId,
+        secretPath: req.body.secretPath,
+        secretName: req.params.secretName,
+        type: req.body.type
+      });
+
+      await server.services.auditLog.createAuditLog({
+        projectId: req.body.workspaceId,
+        ...req.auditLogInfo,
+        event: {
+          type: EventType.DELETE_SECRET,
+          metadata: {
+            environment: req.body.environment,
+            secretPath: req.body.secretPath,
+            secretId: secret.id,
+            secretKey: req.params.secretName,
+            secretVersion: secret.version
+          }
+        }
+      });
+      return { secret };
+    }
+  });
+
   server.route({
     url: "/",
     method: "GET",
@@ -38,7 +339,20 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
                 }).array()
               })
             )
+            .array(),
+          imports: z
+            .object({
+              secretPath: z.string(),
+              environment: z.object({
+                id: z.string(),
+                name: z.string(),
+                slug: z.string()
+              }),
+              folderId: z.string().optional(),
+              secrets: SecretsSchema.omit({ secretBlindIndex: true }).array()
+            })
             .array()
+            .optional()
         })
       }
     },
@@ -49,12 +363,13 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
       AuthMode.IDENTITY_ACCESS_TOKEN
     ]),
     handler: async (req) => {
-      const secrets = await server.services.secret.getSecrets({
+      const { secrets, imports } = await server.services.secret.getSecrets({
         actorId: req.permission.id,
         actor: req.permission.type,
         environment: req.query.environment,
         projectId: req.query.workspaceId,
-        path: req.query.secretPath
+        path: req.query.secretPath,
+        includeImports: req.query.include_imports
       });
 
       await server.services.auditLog.createAuditLog({
@@ -70,7 +385,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
         }
       });
 
-      return { secrets };
+      return { secrets, imports };
     }
   });
 
@@ -111,7 +426,8 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
         projectId: req.query.workspaceId,
         path: req.query.secretPath,
         secretName: req.params.secretName,
-        type: req.query.type
+        type: req.query.type,
+        includeImports: req.query.include_imports
       });
 
       await server.services.auditLog.createAuditLog({

backend-pg/src/services/secret-import/secret-import-fns.ts

@@ -0,0 +1,42 @@
+import { SecretType, TSecretImports } from "@app/db/schemas";
+import { groupBy } from "@app/lib/fn";
+
+import { TSecretDalFactory } from "../secret/secret-dal";
+import { TSecretFolderDalFactory } from "../secret-folder/secret-folder-dal";
+
+export const fnSecretsFromImports = async ({
+  allowedImports,
+  folderDal,
+  secretDal
+}: {
+  allowedImports: (Omit<TSecretImports, "importEnv"> & {
+    importEnv: { id: string; slug: string; name: string };
+  })[];
+  folderDal: Pick<TSecretFolderDalFactory, "findByManySecretPath">;
+  secretDal: Pick<TSecretDalFactory, "find">;
+}) => {
+  const importedFolders = await folderDal.findByManySecretPath(
+    allowedImports.map(({ importEnv, importPath }) => ({
+      envId: importEnv.id,
+      secretPath: importPath
+    }))
+  );
+  const folderIds = importedFolders.map((el) => el?.id).filter(Boolean) as string[];
+  if (!folderIds.length) {
+    return [];
+  }
+  const importedSecrets = await secretDal.find({
+    $in: { folderId: folderIds },
+    type: SecretType.Shared
+  });
+
+  const importedSecsGroupByFolderId = groupBy(importedSecrets, (i) => i.folderId);
+  return allowedImports.map(({ importPath, importEnv }, i) => ({
+    secretPath: importPath,
+    environment: importEnv,
+    folderId: importedFolders?.[i]?.id,
+    secrets: importedFolders?.[i]?.id
+      ? importedSecsGroupByFolderId[importedFolders?.[i]?.id as string]
+      : []
+  }));
+};

backend-pg/src/services/secret-import/secret-import-service.ts

@@ -1,18 +1,17 @@
 import { ForbiddenError, subject } from "@casl/ability";
 
-import { SecretType, TSecretImports } from "@app/db/schemas";
 import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
 import {
   ProjectPermissionActions,
   ProjectPermissionSub
 } from "@app/ee/services/permission/project-permission";
 import { BadRequestError } from "@app/lib/errors";
-import { groupBy } from "@app/lib/fn";
 
 import { TProjectEnvDalFactory } from "../project-env/project-env-dal";
 import { TSecretDalFactory } from "../secret/secret-dal";
 import { TSecretFolderDalFactory } from "../secret-folder/secret-folder-dal";
 import { TSecretImportDalFactory } from "./secret-import-dal";
+import { fnSecretsFromImports } from "./secret-import-fns";
 import {
   TCreateSecretImportDTO,
   TDeleteSecretImportDTO,
@@ -176,37 +175,6 @@ export const secretImportServiceFactory = ({
     return secImports;
   };
 
-  const fnSecretsFromImports = async (
-    allowedImports: (Omit<TSecretImports, "importEnv"> & {
-      importEnv: { id: string; slug: string; name: string };
-    })[]
-  ) => {
-    const importedFolders = await folderDal.findByManySecretPath(
-      allowedImports.map(({ importEnv, importPath }) => ({
-        envId: importEnv.id,
-        secretPath: importPath
-      }))
-    );
-    const folderIds = importedFolders.map((el) => el?.id).filter(Boolean) as string[];
-    if (!folderIds.length) {
-      return [];
-    }
-    const importedSecrets = await secretDal.find({
-      $in: { folderId: folderIds },
-      type: SecretType.Shared
-    });
-
-    const importedSecsGroupByFolderId = groupBy(importedSecrets, (i) => i.folderId);
-    return allowedImports.map(({ importPath, importEnv }, i) => ({
-      secretPath: importPath,
-      environment: importEnv,
-      folderId: importedFolders?.[i]?.id,
-      secrets: importedFolders?.[i]?.id
-        ? importedSecsGroupByFolderId[importedFolders?.[i]?.id as string]
-        : []
-    }));
-  };
-
   const getSecretsFromImports = async ({
     path,
     environment,
@@ -234,7 +202,7 @@ export const secretImportServiceFactory = ({
         })
       )
     );
-    return fnSecretsFromImports(allowedImports);
+    return fnSecretsFromImports({ allowedImports, folderDal, secretDal });
   };
 
   return {

backend-pg/src/services/secret/secret-fns.ts

@@ -1,11 +1,30 @@
 /* eslint-disable no-await-in-loop */
 import path from "path";
 
-import { decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
+import { SecretKeyEncoding, TSecretBlindIndexes, TSecrets } from "@app/db/schemas";
+import { getConfig } from "@app/lib/config/env";
+import { buildSecretBlindIndexFromName, decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
 
 import { TSecretFolderDalFactory } from "../secret-folder/secret-folder-dal";
 import { TSecretDalFactory } from "./secret-dal";
 
+export const generateSecretBlindIndexBySalt = async (
+  secretName: string,
+  secretBlindIndexDoc: TSecretBlindIndexes
+) => {
+  const appCfg = getConfig();
+  const secretBlindIndex = await buildSecretBlindIndexFromName({
+    secretName,
+    keyEncoding: secretBlindIndexDoc.keyEncoding as SecretKeyEncoding,
+    rootEncryptionKey: appCfg.ROOT_ENCRYPTION_KEY,
+    encryptionKey: appCfg.ENCRYPTION_KEY,
+    tag: secretBlindIndexDoc.saltTag,
+    ciphertext: secretBlindIndexDoc.encryptedSaltCipherText,
+    iv: secretBlindIndexDoc.saltIV
+  });
+  return secretBlindIndex;
+};
+
 type TInterpolateSecretArg = {
   projectId: string;
   secretEncKey: string;
@@ -177,3 +196,41 @@ export const interpolateSecrets = ({
   };
   return expandSecrets;
 };
+
+export const decryptSecretRaw = (secret: TSecrets, key: string) => {
+  const secretKey = decryptSymmetric128BitHexKeyUTF8({
+    ciphertext: secret.secretKeyCiphertext,
+    iv: secret.secretKeyIV,
+    tag: secret.secretKeyTag,
+    key
+  });
+
+  const secretValue = decryptSymmetric128BitHexKeyUTF8({
+    ciphertext: secret.secretValueCiphertext,
+    iv: secret.secretValueIV,
+    tag: secret.secretValueTag,
+    key
+  });
+
+  let secretComment = "";
+
+  if (secret.secretCommentCiphertext && secret.secretCommentIV && secret.secretCommentTag) {
+    secretComment = decryptSymmetric128BitHexKeyUTF8({
+      ciphertext: secret.secretCommentCiphertext,
+      iv: secret.secretCommentIV,
+      tag: secret.secretCommentTag,
+      key
+    });
+  }
+
+  return {
+    secretKey,
+    secretValue,
+    secretComment,
+    version: secret.version,
+    type: secret.type,
+    _id: secret.id,
+    id: secret.id,
+    user: secret.userId
+  };
+};

backend-pg/src/services/secret/secret-queue.ts

@@ -8,10 +8,12 @@ import { TIntegrationDalFactory } from "../integration/integration-dal";
 import { TIntegrationAuthServiceFactory } from "../integration-auth/integration-auth-service";
 import { syncIntegrationSecrets } from "../integration-auth/integration-sync-secret";
 import { TProjectBotServiceFactory } from "../project-bot/project-bot-service";
+import { TProjectEnvDalFactory } from "../project-env/project-env-dal";
 import { TSecretFolderDalFactory } from "../secret-folder/secret-folder-dal";
 import { TSecretImportDalFactory } from "../secret-import/secret-import-dal";
-import { TSecretImportServiceFactory } from "../secret-import/secret-import-service";
+import { fnSecretsFromImports } from "../secret-import/secret-import-fns";
-import { TWebhookServiceFactory } from "../webhook/webhook-service";
+import { TWebhookDalFactory } from "../webhook/webhook-dal";
+import { fnTriggerWebhook } from "../webhook/webhook-fns";
 import { TSecretDalFactory } from "./secret-dal";
 import { interpolateSecrets } from "./secret-fns";
 
@@ -19,14 +21,14 @@ export type TSecretQueueFactory = ReturnType<typeof secretQueueFactory>;
 
 type TSecretQueueFactoryDep = {
   queueService: TQueueServiceFactory;
-  webhookService: Pick<TWebhookServiceFactory, "fnTriggerWebhook">;
   integrationDal: Pick<TIntegrationDalFactory, "findByProjectIdV2">;
   projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
   integrationAuthService: Pick<TIntegrationAuthServiceFactory, "getIntegrationAccessToken">;
-  folderDal: Pick<TSecretFolderDalFactory, "findBySecretPath">;
+  folderDal: Pick<TSecretFolderDalFactory, "findBySecretPath" | "findByManySecretPath">;
-  secretDal: Pick<TSecretDalFactory, "findByFolderId">;
+  secretDal: Pick<TSecretDalFactory, "findByFolderId" | "find">;
   secretImportDal: Pick<TSecretImportDalFactory, "find">;
-  secretImportService: Pick<TSecretImportServiceFactory, "fnSecretsFromImports">;
+  webhookDal: Pick<TWebhookDalFactory, "findAllWebhooks" | "transaction" | "update" | "bulkUpdate">;
+  projectEnvDal: Pick<TProjectEnvDalFactory, "findOne">;
 };
 
 export type TGetSecrets = {
@@ -37,14 +39,14 @@ export type TGetSecrets = {
 
 export const secretQueueFactory = ({
   queueService,
-  webhookService,
   integrationDal,
   projectBotService,
   integrationAuthService,
   secretDal,
   secretImportDal,
-  secretImportService,
+  folderDal,
-  folderDal
+  webhookDal,
+  projectEnvDal
 }: TSecretQueueFactoryDep) => {
   const syncSecrets = async (dto: TGetSecrets) => {
     queueService.queue(QueueName.SecretWebhook, QueueJobs.SecWebhook, dto, {
@@ -79,7 +81,11 @@ export const secretQueueFactory = ({
 
     // get imported secrets
     const secretImport = await secretImportDal.find({ folderId: dto.folderId });
-    const importedSecrets = await secretImportService.fnSecretsFromImports(secretImport);
+    const importedSecrets = await fnSecretsFromImports({
+      allowedImports: secretImport,
+      secretDal,
+      folderDal
+    });
     const content: Record<
       string,
       { value: string; comment?: string; skipMultilineEncoding?: boolean }
@@ -217,7 +223,7 @@ export const secretQueueFactory = ({
 
   queueService.start(QueueName.SecretWebhook, async (job) => {
     logger.info("Secret webhook job started", job.data, job.id);
-    await webhookService.fnTriggerWebhook(job.data);
+    await fnTriggerWebhook({ ...job.data, projectEnvDal, webhookDal });
     logger.info("Secret webhook job ended", job.id);
   });
 

backend-pg/src/services/secret/secret-service.ts

@@ -1,12 +1,6 @@
 import { ForbiddenError, subject } from "@casl/ability";
 
-import {
+import { SecretEncryptionAlgo, SecretKeyEncoding, SecretType, TableName } from "@app/db/schemas";
-  SecretEncryptionAlgo,
-  SecretKeyEncoding,
-  SecretType,
-  TableName,
-  TSecretBlindIndexes
-} from "@app/db/schemas";
 import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
 import {
   ProjectPermissionActions,
@@ -14,31 +8,40 @@ import {
 } from "@app/ee/services/permission/project-permission";
 import { TSecretSnapshotServiceFactory } from "@app/ee/services/secret-snapshot/secret-snapshot-service";
 import { getConfig } from "@app/lib/config/env";
-import { buildSecretBlindIndexFromName } from "@app/lib/crypto";
+import { buildSecretBlindIndexFromName, encryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
 import { BadRequestError } from "@app/lib/errors";
 import { groupBy, pick } from "@app/lib/fn";
 
 import { ActorType } from "../auth/auth-type";
+import { TProjectBotServiceFactory } from "../project-bot/project-bot-service";
 import { TSecretFolderDalFactory } from "../secret-folder/secret-folder-dal";
+import { TSecretImportDalFactory } from "../secret-import/secret-import-dal";
+import { fnSecretsFromImports } from "../secret-import/secret-import-fns";
 import { TSecretTagDalFactory } from "../secret-tag/secret-tag-dal";
 import { TSecretBlindIndexDalFactory } from "./secret-blind-index-dal";
 import { TSecretDalFactory } from "./secret-dal";
+import { decryptSecretRaw, generateSecretBlindIndexBySalt } from "./secret-fns";
 import { TSecretQueueFactory } from "./secret-queue";
 import {
   TCreateBulkSecretDTO,
   TCreateSecretDTO,
+  TCreateSecretRawDTO,
   TDeleteBulkSecretDTO,
   TDeleteSecretDTO,
+  TDeleteSecretRawDTO,
   TFnSecretBlindIndexCheck,
   TFnSecretBlindIndexCheckV2,
   TFnSecretBulkDelete,
   TFnSecretBulkInsert,
   TFnSecretBulkUpdate,
   TGetASecretDTO,
+  TGetASecretRawDTO,
   TGetSecretsDTO,
+  TGetSecretsRawDTO,
   TListSecretVersionDTO,
   TUpdateBulkSecretDTO,
-  TUpdateSecretDTO
+  TUpdateSecretDTO,
+  TUpdateSecretRawDTO
 } from "./secret-types";
 import { TSecretVersionDalFactory } from "./secret-version-dal";
 
@@ -46,32 +49,19 @@ type TSecretServiceFactoryDep = {
   secretDal: TSecretDalFactory;
   secretTagDal: TSecretTagDalFactory;
   secretVersionDal: TSecretVersionDalFactory;
-  folderDal: Pick<TSecretFolderDalFactory, "findBySecretPath" | "updateById" | "findById">;
+  folderDal: Pick<
+    TSecretFolderDalFactory,
+    "findBySecretPath" | "updateById" | "findById" | "findByManySecretPath"
+  >;
   secretBlindIndexDal: TSecretBlindIndexDalFactory;
   permissionService: Pick<TPermissionServiceFactory, "getProjectPermission">;
   snapshotService: Pick<TSecretSnapshotServiceFactory, "performSnapshot">;
   secretQueueService: Pick<TSecretQueueFactory, "syncSecrets">;
+  projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
+  secretImportDal: Pick<TSecretImportDalFactory, "find">;
 };
 
 export type TSecretServiceFactory = ReturnType<typeof secretServiceFactory>;
-
-export const generateSecretBlindIndexBySalt = async (
-  secretName: string,
-  secretBlindIndexDoc: TSecretBlindIndexes
-) => {
-  const appCfg = getConfig();
-  const secretBlindIndex = await buildSecretBlindIndexFromName({
-    secretName,
-    keyEncoding: secretBlindIndexDoc.keyEncoding as SecretKeyEncoding,
-    rootEncryptionKey: appCfg.ROOT_ENCRYPTION_KEY,
-    encryptionKey: appCfg.ENCRYPTION_KEY,
-    tag: secretBlindIndexDoc.saltTag,
-    ciphertext: secretBlindIndexDoc.encryptedSaltCipherText,
-    iv: secretBlindIndexDoc.saltIV
-  });
-  return secretBlindIndex;
-};
-
 export const secretServiceFactory = ({
   secretDal,
   secretTagDal,
@@ -80,7 +70,9 @@ export const secretServiceFactory = ({
   secretBlindIndexDal,
   permissionService,
   snapshotService,
-  secretQueueService
+  secretQueueService,
+  projectBotService,
+  secretImportDal
 }: TSecretServiceFactoryDep) => {
   // utility function to get secret blind index data
   const interalGenSecBlindIndexByName = async (projectId: string, secretName: string) => {
@@ -485,7 +477,14 @@ export const secretServiceFactory = ({
     return deletedSecret[0];
   };
 
-  const getSecrets = async ({ actorId, path, environment, projectId, actor }: TGetSecretsDTO) => {
+  const getSecrets = async ({
+    actorId,
+    path,
+    environment,
+    projectId,
+    actor,
+    includeImports
+  }: TGetSecretsDTO) => {
     const { permission } = await permissionService.getProjectPermission(actor, actorId, projectId);
     ForbiddenError.from(permission).throwUnlessCan(
       ProjectPermissionActions.Read,
@@ -497,7 +496,28 @@ export const secretServiceFactory = ({
     const folderId = folder.id;
 
     const secrets = await secretDal.findByFolderId(folderId, actorId);
-    return secrets;
+    if (includeImports) {
+      const secretImports = await secretImportDal.find({ folderId });
+      const allowedImports = secretImports.filter(({ importEnv, importPath }) =>
+        // if its service token allow full access over imported one
+        actor === ActorType.SERVICE
+          ? true
+          : permission.can(
+              ProjectPermissionActions.Read,
+              subject(ProjectPermissionSub.Secrets, {
+                environment: importEnv.slug,
+                secretPath: importPath
+              })
+            )
+      );
+      const importedSecrets = await fnSecretsFromImports({
+        allowedImports,
+        secretDal,
+        folderDal
+      });
+      return { secrets, imports: importedSecrets };
+    }
+    return { secrets };
   };
 
   const getASecret = async ({
@@ -507,7 +527,8 @@ export const secretServiceFactory = ({
     environment,
     path,
     type,
-    secretName
+    secretName,
+    includeImports
   }: TGetASecretDTO) => {
     const { permission } = await permissionService.getProjectPermission(actor, actorId, projectId);
     ForbiddenError.from(permission).throwUnlessCan(
@@ -526,6 +547,36 @@ export const secretServiceFactory = ({
       userId: type === SecretType.Personal ? actorId : null,
       secretBlindIndex
     });
+    // now if secret is not found
+    // then search for imported secrets
+    // here we consider the import order also thus starting from bottom
+    if (!secret && includeImports) {
+      const secretImports = await secretImportDal.find({ folderId });
+      const allowedImports = secretImports.filter(({ importEnv, importPath }) =>
+        // if its service token allow full access over imported one
+        actor === ActorType.SERVICE
+          ? true
+          : permission.can(
+              ProjectPermissionActions.Read,
+              subject(ProjectPermissionSub.Secrets, {
+                environment: importEnv.slug,
+                secretPath: importPath
+              })
+            )
+      );
+      const importedSecrets = await fnSecretsFromImports({
+        allowedImports,
+        secretDal,
+        folderDal
+      });
+      for (let i = importedSecrets.length; i >= 0; i -= 1) {
+        for (let j = 0; j < importedSecrets[i].secrets.length; j += 1) {
+          if (secretBlindIndex === importedSecrets[i].secrets[j].secretBlindIndex) {
+            return importedSecrets[i].secrets[j];
+          }
+        }
+      }
+    }
     if (!secret) throw new BadRequestError({ message: "Secret not found" });
 
     return secret;
@@ -738,6 +789,170 @@ export const secretServiceFactory = ({
     return secretVersions;
   };
 
+  const getSecretsRaw = async ({
+    projectId,
+    path,
+    actor,
+    actorId,
+    environment,
+    includeImports
+  }: TGetSecretsRawDTO) => {
+    const botKey = await projectBotService.getBotKey(projectId);
+    if (!botKey) throw new BadRequestError({ message: "Project bot not found" });
+
+    const { secrets, imports } = await getSecrets({
+      actorId,
+      projectId,
+      environment,
+      actor,
+      path,
+      includeImports
+    });
+
+    return {
+      secrets: secrets.map((el) => decryptSecretRaw(el, botKey)),
+      imports: (imports || [])?.map(({ secrets: importedSecrets, ...el }) => ({
+        ...el,
+        secrets: importedSecrets.map((sec) => decryptSecretRaw(sec, botKey))
+      }))
+    };
+  };
+
+  const getASecretRaw = async ({
+    type,
+    path,
+    actor,
+    environment,
+    projectId,
+    actorId,
+    secretName,
+    includeImports
+  }: TGetASecretRawDTO) => {
+    const botKey = await projectBotService.getBotKey(projectId);
+    if (!botKey) throw new BadRequestError({ message: "Project bot not found" });
+
+    const secret = await getASecret({
+      actorId,
+      projectId,
+      environment,
+      actor,
+      path,
+      secretName,
+      type,
+      includeImports
+    });
+    return decryptSecretRaw(secret, botKey);
+  };
+
+  const createSecretRaw = async ({
+    secretName,
+    actorId,
+    projectId,
+    environment,
+    actor,
+    type,
+    secretPath,
+    secretValue,
+    secretComment,
+    skipMultilineEncoding
+  }: TCreateSecretRawDTO) => {
+    const botKey = await projectBotService.getBotKey(projectId);
+    if (!botKey) throw new BadRequestError({ message: "Project bot not found" });
+
+    const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secretName, botKey);
+    const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secretValue || "", botKey);
+    const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secretComment || "", botKey);
+
+    const secret = await createSecret({
+      secretName,
+      projectId,
+      environment,
+      type,
+      path: secretPath,
+      actor,
+      actorId,
+      secretKeyCiphertext: secretKeyEncrypted.ciphertext,
+      secretKeyIV: secretKeyEncrypted.iv,
+      secretKeyTag: secretKeyEncrypted.tag,
+      secretValueCiphertext: secretValueEncrypted.ciphertext,
+      secretValueIV: secretValueEncrypted.iv,
+      secretValueTag: secretValueEncrypted.tag,
+      secretCommentCiphertext: secretCommentEncrypted.ciphertext,
+      secretCommentIV: secretCommentEncrypted.iv,
+      secretCommentTag: secretCommentEncrypted.tag,
+      skipMultilineEncoding
+    });
+
+    await snapshotService.performSnapshot(secret.folderId);
+    await secretQueueService.syncSecrets({ secretPath, projectId, environment });
+
+    return decryptSecretRaw(secret, botKey);
+  };
+
+  const updateSecretRaw = async ({
+    secretName,
+    actorId,
+    projectId,
+    environment,
+    actor,
+    type,
+    secretPath,
+    secretValue,
+    skipMultilineEncoding
+  }: TUpdateSecretRawDTO) => {
+    const botKey = await projectBotService.getBotKey(projectId);
+    if (!botKey) throw new BadRequestError({ message: "Project bot not found" });
+
+    const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secretValue || "", botKey);
+
+    const secret = await updateSecret({
+      secretName,
+      projectId,
+      environment,
+      type,
+      path: secretPath,
+      actor,
+      actorId,
+      secretValueCiphertext: secretValueEncrypted.ciphertext,
+      secretValueIV: secretValueEncrypted.iv,
+      secretValueTag: secretValueEncrypted.tag,
+      skipMultilineEncoding
+    });
+
+    await snapshotService.performSnapshot(secret.folderId);
+    await secretQueueService.syncSecrets({ secretPath, projectId, environment });
+
+    return decryptSecretRaw(secret, botKey);
+  };
+
+  const deleteSecretRaw = async ({
+    secretName,
+    actorId,
+    projectId,
+    environment,
+    actor,
+    type,
+    secretPath
+  }: TDeleteSecretRawDTO) => {
+    const botKey = await projectBotService.getBotKey(projectId);
+    if (!botKey) throw new BadRequestError({ message: "Project bot not found" });
+
+    const secret = await deleteSecret({
+      secretName,
+      projectId,
+      environment,
+      type,
+      path: secretPath,
+      actor,
+      actorId
+    });
+
+    await snapshotService.performSnapshot(secret.folderId);
+    await secretQueueService.syncSecrets({ secretPath, projectId, environment });
+
+    return decryptSecretRaw(secret, botKey);
+  };
+
   return {
     createSecret,
     deleteSecret,
@@ -747,6 +962,11 @@ export const secretServiceFactory = ({
     deleteManySecret,
     getASecret,
     getSecrets,
+    getSecretsRaw,
+    getASecretRaw,
+    createSecretRaw,
+    updateSecretRaw,
+    deleteSecretRaw,
     listSecretVersionsBySecretId,
     // external services function
     fnSecretBulkDelete,

backend-pg/src/services/secret/secret-types.ts

@@ -61,6 +61,7 @@ export type TDeleteSecretDTO = {
 export type TGetSecretsDTO = {
   path: string;
   environment: string;
+  includeImports?: boolean;
 } & TProjectPermission;
 
 export type TGetASecretDTO = {
@@ -68,6 +69,7 @@ export type TGetASecretDTO = {
   path: string;
   environment: string;
   type: "shared" | "personal";
+  includeImports?: boolean;
 } & TProjectPermission;
 
 export type TCreateBulkSecretDTO = {
@@ -125,6 +127,46 @@ export type TListSecretVersionDTO = {
   limit?: number;
 } & Omit<TProjectPermission, "projectId">;
 
+export type TGetSecretsRawDTO = {
+  path: string;
+  environment: string;
+  includeImports?: boolean;
+} & TProjectPermission;
+
+export type TGetASecretRawDTO = {
+  secretName: string;
+  path: string;
+  environment: string;
+  type: "shared" | "personal";
+  includeImports?: boolean;
+} & TProjectPermission;
+
+export type TCreateSecretRawDTO = TProjectPermission & {
+  secretPath: string;
+  environment: string;
+  secretName: string;
+  secretValue: string;
+  type: SecretType;
+  secretComment?: string;
+  skipMultilineEncoding?: boolean;
+};
+
+export type TUpdateSecretRawDTO = TProjectPermission & {
+  secretPath: string;
+  environment: string;
+  secretName: string;
+  secretValue?: string;
+  type: SecretType;
+  skipMultilineEncoding?: boolean;
+};
+
+export type TDeleteSecretRawDTO = TProjectPermission & {
+  secretPath: string;
+  environment: string;
+  secretName: string;
+  type: SecretType;
+};
+
 export type TFnSecretBulkInsert = {
   folderId: string;
   tx?: Knex;

backend-pg/src/services/webhook/webhook-fns.ts

@@ -1,9 +1,15 @@
 import crypto from "node:crypto";
 
+import picomatch from "picomatch";
+
 import { SecretKeyEncoding, TWebhooks } from "@app/db/schemas";
 import { getConfig } from "@app/lib/config/env";
 import { request } from "@app/lib/config/request";
 import { decryptSymmetric, decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
+import { BadRequestError } from "@app/lib/errors";
+
+import { TProjectEnvDalFactory } from "../project-env/project-env-dal";
+import { TWebhookDalFactory } from "./webhook-dal";
 
 const WEBHOOK_TRIGGER_TIMEOUT = 15 * 1000;
 export const triggerWebhookRequest = async (
@@ -64,3 +70,67 @@ export const getWebhookPayload = (
     secretPath
   }
 });
+
+export type TFnTriggerWebhookDTO = {
+  projectId: string;
+  secretPath: string;
+  environment: string;
+  webhookDal: Pick<TWebhookDalFactory, "findAllWebhooks" | "transaction" | "update" | "bulkUpdate">;
+  projectEnvDal: Pick<TProjectEnvDalFactory, "findOne">;
+};
+// this is reusable function
+// used in secret queue to trigger webhook and update status when secrets changes
+export const fnTriggerWebhook = async ({
+  environment,
+  secretPath,
+  projectId,
+  webhookDal,
+  projectEnvDal
+}: TFnTriggerWebhookDTO) => {
+  const webhooks = await webhookDal.findAllWebhooks(projectId, environment);
+  const toBeTriggeredHooks = webhooks.filter(
+    ({ secretPath: hookSecretPath, isDisabled }) =>
+      !isDisabled && picomatch.isMatch(secretPath, hookSecretPath, { strictSlashes: false })
+  );
+  if (!toBeTriggeredHooks.length) return;
+  const webhooksTriggered = await Promise.allSettled(
+    toBeTriggeredHooks.map((hook) =>
+      triggerWebhookRequest(
+        hook,
+        getWebhookPayload("secrets.modified", projectId, environment, secretPath)
+      )
+    )
+  );
+  // filter hooks by status
+  const successWebhooks = webhooksTriggered
+    .filter(({ status }) => status === "fulfilled")
+    .map((_, i) => toBeTriggeredHooks[i].id);
+  const failedWebhooks = webhooksTriggered
+    .filter(({ status }) => status === "rejected")
+    .map((data, i) => ({
+      id: toBeTriggeredHooks[i].id,
+      error: data.status === "rejected" && data.reason.message
+    }));
+
+  await webhookDal.transaction(async (tx) => {
+    const env = await projectEnvDal.findOne({ projectId, slug: environment }, tx);
+    if (!env) throw new BadRequestError({ message: "Env not found" });
+    if (successWebhooks.length) {
+      await webhookDal.update(
+        { envId: env.id, $in: { id: successWebhooks } },
+        { lastStatus: "success", lastRunErrorMessage: null },
+        tx
+      );
+    }
+    if (failedWebhooks.length) {
+      await webhookDal.bulkUpdate(
+        failedWebhooks.map(({ id, error }) => ({
+          id,
+          lastRunErrorMessage: error,
+          lastStatus: "failed"
+        })),
+        tx
+      );
+    }
+  });
+};

backend-pg/src/services/webhook/webhook-service.ts

@@ -1,5 +1,4 @@
 import { ForbiddenError } from "@casl/ability";
-import picomatch from "picomatch";
 
 import { SecretEncryptionAlgo, SecretKeyEncoding, TWebhooksInsert } from "@app/db/schemas";
 import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
@@ -17,7 +16,6 @@ import { getWebhookPayload, triggerWebhookRequest } from "./webhook-fns";
 import {
   TCreateWebhookDTO,
   TDeleteWebhookDTO,
-  TFnTriggerWebhookDTO,
   TListWebhookDTO,
   TTestWebhookDTO,
   TUpdateWebhookDTO
@@ -170,63 +168,11 @@ export const webhookServiceFactory = ({
     return webhookDal.findAllWebhooks(projectId, environment, secretPath);
   };
 
-  // this is reusable function
-  // used in secret queue to trigger webhook and update status when secrets changes
-  const fnTriggerWebhook = async ({ environment, secretPath, projectId }: TFnTriggerWebhookDTO) => {
-    const webhooks = await webhookDal.findAllWebhooks(projectId, environment);
-    const toBeTriggeredHooks = webhooks.filter(
-      ({ secretPath: hookSecretPath, isDisabled }) =>
-        !isDisabled && picomatch.isMatch(secretPath, hookSecretPath, { strictSlashes: false })
-    );
-    if (!toBeTriggeredHooks.length) return;
-    const webhooksTriggered = await Promise.allSettled(
-      toBeTriggeredHooks.map((hook) =>
-        triggerWebhookRequest(
-          hook,
-          getWebhookPayload("secrets.modified", projectId, environment, secretPath)
-        )
-      )
-    );
-    // filter hooks by status
-    const successWebhooks = webhooksTriggered
-      .filter(({ status }) => status === "fulfilled")
-      .map((_, i) => toBeTriggeredHooks[i].id);
-    const failedWebhooks = webhooksTriggered
-      .filter(({ status }) => status === "rejected")
-      .map((data, i) => ({
-        id: toBeTriggeredHooks[i].id,
-        error: data.status === "rejected" && data.reason.message
-      }));
-
-    await webhookDal.transaction(async (tx) => {
-      const env = await projectEnvDal.findOne({ projectId, slug: environment }, tx);
-      if (!env) throw new BadRequestError({ message: "Env not found" });
-      if (successWebhooks.length) {
-        await webhookDal.update(
-          { envId: env.id, $in: { id: successWebhooks } },
-          { lastStatus: "success", lastRunErrorMessage: null },
-          tx
-        );
-      }
-      if (failedWebhooks.length) {
-        await webhookDal.bulkUpdate(
-          failedWebhooks.map(({ id, error }) => ({
-            id,
-            lastRunErrorMessage: error,
-            lastStatus: "failed"
-          })),
-          tx
-        );
-      }
-    });
-  };
-
   return {
     createWebhook,
     deleteWebhook,
     listWebhooks,
     updateWebhook,
-    testWebhook,
+    testWebhook
-    fnTriggerWebhook
   };
 };

backend-pg/src/services/webhook/webhook-types.ts

@@ -24,9 +24,3 @@ export type TListWebhookDTO = {
   environment?: string;
   secretPath?: string;
 } & TProjectPermission;
-
-export type TFnTriggerWebhookDTO = {
-  projectId: string;
-  secretPath: string;
-  environment: string;
-};