awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-09-22 13:39:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: added secret raw endpoint and include imports

Browse Source
This commit is contained in:
Akhil Mohan
2024-01-14 19:57:28 +05:30
parent 0506389ada
commit 3519412639
12 changed files with 820 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
backend-pg/src/server/routes/index.ts
View File

@@ -338,14 +338,14 @@ export const registerRoutes = async (
});
const secretQueueService = secretQueueFactory({
queueService,
webhookService,
secretDal,
folderDal,
secretImportService,
integrationAuthService,
projectBotService,
integrationDal,
secretImportDal
secretImportDal,
projectEnvDal,
webhookDal
});
const secretService = secretServiceFactory({
folderDal,
@@ -355,7 +355,9 @@ export const registerRoutes = async (
secretDal,
secretTagDal,
snapshotService,
secretQueueService
secretQueueService,
secretImportDal,
projectBotService
});
const sarService = secretApprovalRequestServiceFactory({
permissionService,

10
backend-pg/src/server/routes/sanitizedSchemas.ts
View File

@@ -27,3 +27,13 @@ export const sapPubSchema = SecretApprovalPoliciesSchema.merge(
projectId: z.string()
})
);
export const secretRawSchema = z.object({
id: z.string(),
_id: z.string(),
version: z.number(),
type: z.string(),
secretKey: z.string(),
secretValue: z.string(),
secretComment: z.string().optional()
});

324
backend-pg/src/server/routes/v3/secret-router.ts
View File

@@ -11,7 +11,308 @@ import { CommitType } from "@app/ee/services/secret-approval-request/secret-appr
import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
import { ActorType, AuthMode } from "@app/services/auth/auth-type";
import { secretRawSchema } from "../sanitizedSchemas";
export const registerSecretRouter = async (server: FastifyZodProvider) => {
server.route({
url: "/raw",
method: "GET",
schema: {
querystring: z.object({
workspaceId: z.string().trim(),
environment: z.string().trim(),
secretPath: z.string().trim().default("/"),
include_imports: z
.enum(["true", "false"])
.default("false")
.transform((value) => value === "true")
}),
response: {
200: z.object({
secrets: secretRawSchema.array(),
imports: z
.object({
secretPath: z.string(),
environment: z.object({
id: z.string(),
name: z.string(),
slug: z.string()
}),
folderId: z.string().optional(),
secrets: secretRawSchema.array()
})
.array()
.optional()
})
}
},
onRequest: verifyAuth([
AuthMode.JWT,
AuthMode.API_KEY,
AuthMode.SERVICE_TOKEN,
AuthMode.IDENTITY_ACCESS_TOKEN
]),
handler: async (req) => {
const { secrets, imports } = await server.services.secret.getSecretsRaw({
actorId: req.permission.id,
actor: req.permission.type,
environment: req.query.environment,
projectId: req.query.workspaceId,
path: req.query.secretPath,
includeImports: req.query.include_imports
});
await server.services.auditLog.createAuditLog({
projectId: req.query.workspaceId,
...req.auditLogInfo,
event: {
type: EventType.GET_SECRETS,
metadata: {
environment: req.query.environment,
secretPath: req.query.secretPath,
numberOfSecrets: secrets.length
}
}
});
return { secrets, imports };
}
});
server.route({
url: "/raw/:secretName",
method: "GET",
schema: {
params: z.object({
secretName: z.string().trim()
}),
querystring: z.object({
workspaceId: z.string().trim(),
environment: z.string().trim(),
secretPath: z.string().trim().default("/"),
type: z.nativeEnum(SecretType).default(SecretType.Shared),
include_imports: z
.enum(["true", "false"])
.default("false")
.transform((value) => value === "true")
}),
response: {
200: z.object({
secret: secretRawSchema
})
}
},
onRequest: verifyAuth([
AuthMode.JWT,
AuthMode.API_KEY,
AuthMode.SERVICE_TOKEN,
AuthMode.IDENTITY_ACCESS_TOKEN
]),
handler: async (req) => {
const secret = await server.services.secret.getASecretRaw({
actorId: req.permission.id,
actor: req.permission.type,
environment: req.query.environment,
projectId: req.query.workspaceId,
path: req.query.secretPath,
secretName: req.params.secretName,
type: req.query.type,
includeImports: req.query.include_imports
});
await server.services.auditLog.createAuditLog({
projectId: req.query.workspaceId,
...req.auditLogInfo,
event: {
type: EventType.GET_SECRET,
metadata: {
environment: req.query.environment,
secretPath: req.query.secretPath,
secretId: secret.id,
secretKey: req.params.secretName,
secretVersion: secret.version
}
}
});
return { secret };
}
});
server.route({
url: "/raw/:secretName",
method: "POST",
schema: {
params: z.object({
secretName: z.string().trim()
}),
body: z.object({
workspaceId: z.string().trim(),
environment: z.string().trim(),
secretPath: z.string().trim().default("/"),
secretValue: z
.string()
.transform((val) => (val.at(-1) === "\n" ? `${val.trim()}\n` : val.trim())),
secretComment: z.string().trim().optional().default(""),
skipMultilineEncoding: z.boolean().optional(),
type: z.nativeEnum(SecretType).default(SecretType.Shared)
}),
response: {
200: z.object({
secret: secretRawSchema
})
}
},
onRequest: verifyAuth([
AuthMode.JWT,
AuthMode.API_KEY,
AuthMode.SERVICE_TOKEN,
AuthMode.IDENTITY_ACCESS_TOKEN
]),
handler: async (req) => {
const secret = await server.services.secret.createSecretRaw({
actorId: req.permission.id,
actor: req.permission.type,
environment: req.body.environment,
projectId: req.body.workspaceId,
secretPath: req.body.secretPath,
secretName: req.params.secretName,
type: req.body.type,
secretValue: req.body.secretValue,
skipMultilineEncoding: req.body.skipMultilineEncoding,
secretComment: req.body.secretComment
});
await server.services.auditLog.createAuditLog({
projectId: req.body.workspaceId,
...req.auditLogInfo,
event: {
type: EventType.CREATE_SECRET,
metadata: {
environment: req.body.environment,
secretPath: req.body.secretPath,
secretId: secret.id,
secretKey: req.params.secretName,
secretVersion: secret.version
}
}
});
return { secret };
}
});
server.route({
url: "/raw/:secretName",
method: "PATCH",
schema: {
params: z.object({
secretName: z.string().trim()
}),
body: z.object({
workspaceId: z.string().trim(),
environment: z.string().trim(),
secretValue: z
.string()
.transform((val) => (val.at(-1) === "\n" ? `${val.trim()}\n` : val.trim())),
secretPath: z.string().trim().default("/"),
skipMultilineEncoding: z.boolean().optional(),
type: z.nativeEnum(SecretType).default(SecretType.Shared)
}),
response: {
200: z.object({
secret: secretRawSchema
})
}
},
onRequest: verifyAuth([
AuthMode.JWT,
AuthMode.API_KEY,
AuthMode.SERVICE_TOKEN,
AuthMode.IDENTITY_ACCESS_TOKEN
]),
handler: async (req) => {
const secret = await server.services.secret.updateSecretRaw({
actorId: req.permission.id,
actor: req.permission.type,
environment: req.body.environment,
projectId: req.body.workspaceId,
secretPath: req.body.secretPath,
secretName: req.params.secretName,
type: req.body.type,
secretValue: req.body.secretValue,
skipMultilineEncoding: req.body.skipMultilineEncoding
});
await server.services.auditLog.createAuditLog({
projectId: req.body.workspaceId,
...req.auditLogInfo,
event: {
type: EventType.UPDATE_SECRET,
metadata: {
environment: req.body.environment,
secretPath: req.body.secretPath,
secretId: secret.id,
secretKey: req.params.secretName,
secretVersion: secret.version
}
}
});
return { secret };
}
});
server.route({
url: "/raw/:secretName",
method: "DELETE",
schema: {
params: z.object({
secretName: z.string().trim()
}),
body: z.object({
workspaceId: z.string().trim(),
environment: z.string().trim(),
secretPath: z.string().trim().default("/"),
type: z.nativeEnum(SecretType).default(SecretType.Shared)
}),
response: {
200: z.object({
secret: secretRawSchema
})
}
},
onRequest: verifyAuth([
AuthMode.JWT,
AuthMode.API_KEY,
AuthMode.SERVICE_TOKEN,
AuthMode.IDENTITY_ACCESS_TOKEN
]),
handler: async (req) => {
const secret = await server.services.secret.deleteSecretRaw({
actorId: req.permission.id,
actor: req.permission.type,
environment: req.body.environment,
projectId: req.body.workspaceId,
secretPath: req.body.secretPath,
secretName: req.params.secretName,
type: req.body.type
});
await server.services.auditLog.createAuditLog({
projectId: req.body.workspaceId,
...req.auditLogInfo,
event: {
type: EventType.DELETE_SECRET,
metadata: {
environment: req.body.environment,
secretPath: req.body.secretPath,
secretId: secret.id,
secretKey: req.params.secretName,
secretVersion: secret.version
}
}
});
return { secret };
}
});
server.route({
url: "/",
method: "GET",
@@ -38,7 +339,20 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
}).array()
})
)
.array(),
imports: z
.object({
secretPath: z.string(),
environment: z.object({
id: z.string(),
name: z.string(),
slug: z.string()
}),
folderId: z.string().optional(),
secrets: SecretsSchema.omit({ secretBlindIndex: true }).array()
})
.array()
.optional()
})
}
},
@@ -49,12 +363,13 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
AuthMode.IDENTITY_ACCESS_TOKEN
]),
handler: async (req) => {
const secrets = await server.services.secret.getSecrets({
const { secrets, imports } = await server.services.secret.getSecrets({
actorId: req.permission.id,
actor: req.permission.type,
environment: req.query.environment,
projectId: req.query.workspaceId,
path: req.query.secretPath
path: req.query.secretPath,
includeImports: req.query.include_imports
});
await server.services.auditLog.createAuditLog({
@@ -70,7 +385,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
}
});
return { secrets };
return { secrets, imports };
}
});
@@ -111,7 +426,8 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
projectId: req.query.workspaceId,
path: req.query.secretPath,
secretName: req.params.secretName,
type: req.query.type
type: req.query.type,
includeImports: req.query.include_imports
});
await server.services.auditLog.createAuditLog({

42
backend-pg/src/services/secret-import/secret-import-fns.ts Normal file
View File

@@ -0,0 +1,42 @@
import { SecretType, TSecretImports } from "@app/db/schemas";
import { groupBy } from "@app/lib/fn";
import { TSecretDalFactory } from "../secret/secret-dal";
import { TSecretFolderDalFactory } from "../secret-folder/secret-folder-dal";
export const fnSecretsFromImports = async ({
allowedImports,
folderDal,
secretDal
}: {
allowedImports: (Omit<TSecretImports, "importEnv"> & {
importEnv: { id: string; slug: string; name: string };
})[];
folderDal: Pick<TSecretFolderDalFactory, "findByManySecretPath">;
secretDal: Pick<TSecretDalFactory, "find">;
}) => {
const importedFolders = await folderDal.findByManySecretPath(
allowedImports.map(({ importEnv, importPath }) => ({
envId: importEnv.id,
secretPath: importPath
}))
);
const folderIds = importedFolders.map((el) => el?.id).filter(Boolean) as string[];
if (!folderIds.length) {
return [];
}
const importedSecrets = await secretDal.find({
$in: { folderId: folderIds },
type: SecretType.Shared
});
const importedSecsGroupByFolderId = groupBy(importedSecrets, (i) => i.folderId);
return allowedImports.map(({ importPath, importEnv }, i) => ({
secretPath: importPath,
environment: importEnv,
folderId: importedFolders?.[i]?.id,
secrets: importedFolders?.[i]?.id
? importedSecsGroupByFolderId[importedFolders?.[i]?.id as string]
: []
}));
};

36
backend-pg/src/services/secret-import/secret-import-service.ts
View File

@@ -1,18 +1,17 @@
import { ForbiddenError, subject } from "@casl/ability";
import { SecretType, TSecretImports } from "@app/db/schemas";
import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
import {
ProjectPermissionActions,
ProjectPermissionSub
} from "@app/ee/services/permission/project-permission";
import { BadRequestError } from "@app/lib/errors";
import { groupBy } from "@app/lib/fn";
import { TProjectEnvDalFactory } from "../project-env/project-env-dal";
import { TSecretDalFactory } from "../secret/secret-dal";
import { TSecretFolderDalFactory } from "../secret-folder/secret-folder-dal";
import { TSecretImportDalFactory } from "./secret-import-dal";
import { fnSecretsFromImports } from "./secret-import-fns";
import {
TCreateSecretImportDTO,
TDeleteSecretImportDTO,
@@ -176,37 +175,6 @@ export const secretImportServiceFactory = ({
return secImports;
};
const fnSecretsFromImports = async (
allowedImports: (Omit<TSecretImports, "importEnv"> & {
importEnv: { id: string; slug: string; name: string };
})[]
) => {
const importedFolders = await folderDal.findByManySecretPath(
allowedImports.map(({ importEnv, importPath }) => ({
envId: importEnv.id,
secretPath: importPath
}))
);
const folderIds = importedFolders.map((el) => el?.id).filter(Boolean) as string[];
if (!folderIds.length) {
return [];
}
const importedSecrets = await secretDal.find({
$in: { folderId: folderIds },
type: SecretType.Shared
});
const importedSecsGroupByFolderId = groupBy(importedSecrets, (i) => i.folderId);
return allowedImports.map(({ importPath, importEnv }, i) => ({
secretPath: importPath,
environment: importEnv,
folderId: importedFolders?.[i]?.id,
secrets: importedFolders?.[i]?.id
? importedSecsGroupByFolderId[importedFolders?.[i]?.id as string]
: []
}));
};
const getSecretsFromImports = async ({
path,
environment,
@@ -234,7 +202,7 @@ export const secretImportServiceFactory = ({
})
)
);
return fnSecretsFromImports(allowedImports);
return fnSecretsFromImports({ allowedImports, folderDal, secretDal });
};
return {

59
backend-pg/src/services/secret/secret-fns.ts
View File

@@ -1,11 +1,30 @@
/* eslint-disable no-await-in-loop */
import path from "path";
import { decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
import { SecretKeyEncoding, TSecretBlindIndexes, TSecrets } from "@app/db/schemas";
import { getConfig } from "@app/lib/config/env";
import { buildSecretBlindIndexFromName, decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
import { TSecretFolderDalFactory } from "../secret-folder/secret-folder-dal";
import { TSecretDalFactory } from "./secret-dal";
export const generateSecretBlindIndexBySalt = async (
secretName: string,
secretBlindIndexDoc: TSecretBlindIndexes
) => {
const appCfg = getConfig();
const secretBlindIndex = await buildSecretBlindIndexFromName({
secretName,
keyEncoding: secretBlindIndexDoc.keyEncoding as SecretKeyEncoding,
rootEncryptionKey: appCfg.ROOT_ENCRYPTION_KEY,
encryptionKey: appCfg.ENCRYPTION_KEY,
tag: secretBlindIndexDoc.saltTag,
ciphertext: secretBlindIndexDoc.encryptedSaltCipherText,
iv: secretBlindIndexDoc.saltIV
});
return secretBlindIndex;
};
type TInterpolateSecretArg = {
projectId: string;
secretEncKey: string;
@@ -177,3 +196,41 @@ export const interpolateSecrets = ({
};
return expandSecrets;
};
export const decryptSecretRaw = (secret: TSecrets, key: string) => {
const secretKey = decryptSymmetric128BitHexKeyUTF8({
ciphertext: secret.secretKeyCiphertext,
iv: secret.secretKeyIV,
tag: secret.secretKeyTag,
key
});
const secretValue = decryptSymmetric128BitHexKeyUTF8({
ciphertext: secret.secretValueCiphertext,
iv: secret.secretValueIV,
tag: secret.secretValueTag,
key
});
let secretComment = "";
if (secret.secretCommentCiphertext && secret.secretCommentIV && secret.secretCommentTag) {
secretComment = decryptSymmetric128BitHexKeyUTF8({
ciphertext: secret.secretCommentCiphertext,
iv: secret.secretCommentIV,
tag: secret.secretCommentTag,
key
});
}
return {
secretKey,
secretValue,
secretComment,
version: secret.version,
type: secret.type,
_id: secret.id,
id: secret.id,
user: secret.userId
};
};

28
backend-pg/src/services/secret/secret-queue.ts
View File

@@ -8,10 +8,12 @@ import { TIntegrationDalFactory } from "../integration/integration-dal";
import { TIntegrationAuthServiceFactory } from "../integration-auth/integration-auth-service";
import { syncIntegrationSecrets } from "../integration-auth/integration-sync-secret";
import { TProjectBotServiceFactory } from "../project-bot/project-bot-service";
import { TProjectEnvDalFactory } from "../project-env/project-env-dal";
import { TSecretFolderDalFactory } from "../secret-folder/secret-folder-dal";
import { TSecretImportDalFactory } from "../secret-import/secret-import-dal";
import { TSecretImportServiceFactory } from "../secret-import/secret-import-service";
import { TWebhookServiceFactory } from "../webhook/webhook-service";
import { fnSecretsFromImports } from "../secret-import/secret-import-fns";
import { TWebhookDalFactory } from "../webhook/webhook-dal";
import { fnTriggerWebhook } from "../webhook/webhook-fns";
import { TSecretDalFactory } from "./secret-dal";
import { interpolateSecrets } from "./secret-fns";
@@ -19,14 +21,14 @@ export type TSecretQueueFactory = ReturnType<typeof secretQueueFactory>;
type TSecretQueueFactoryDep = {
queueService: TQueueServiceFactory;
webhookService: Pick<TWebhookServiceFactory, "fnTriggerWebhook">;
integrationDal: Pick<TIntegrationDalFactory, "findByProjectIdV2">;
projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
integrationAuthService: Pick<TIntegrationAuthServiceFactory, "getIntegrationAccessToken">;
folderDal: Pick<TSecretFolderDalFactory, "findBySecretPath">;
secretDal: Pick<TSecretDalFactory, "findByFolderId">;
folderDal: Pick<TSecretFolderDalFactory, "findBySecretPath" | "findByManySecretPath">;
secretDal: Pick<TSecretDalFactory, "findByFolderId" | "find">;
secretImportDal: Pick<TSecretImportDalFactory, "find">;
secretImportService: Pick<TSecretImportServiceFactory, "fnSecretsFromImports">;
webhookDal: Pick<TWebhookDalFactory, "findAllWebhooks" | "transaction" | "update" | "bulkUpdate">;
projectEnvDal: Pick<TProjectEnvDalFactory, "findOne">;
};
export type TGetSecrets = {
@@ -37,14 +39,14 @@ export type TGetSecrets = {
export const secretQueueFactory = ({
queueService,
webhookService,
integrationDal,
projectBotService,
integrationAuthService,
secretDal,
secretImportDal,
secretImportService,
folderDal
folderDal,
webhookDal,
projectEnvDal
}: TSecretQueueFactoryDep) => {
const syncSecrets = async (dto: TGetSecrets) => {
queueService.queue(QueueName.SecretWebhook, QueueJobs.SecWebhook, dto, {
@@ -79,7 +81,11 @@ export const secretQueueFactory = ({
// get imported secrets
const secretImport = await secretImportDal.find({ folderId: dto.folderId });
const importedSecrets = await secretImportService.fnSecretsFromImports(secretImport);
const importedSecrets = await fnSecretsFromImports({
allowedImports: secretImport,
secretDal,
folderDal
});
const content: Record<
string,
{ value: string; comment?: string; skipMultilineEncoding?: boolean }
@@ -217,7 +223,7 @@ export const secretQueueFactory = ({
queueService.start(QueueName.SecretWebhook, async (job) => {
logger.info("Secret webhook job started", job.data, job.id);
await webhookService.fnTriggerWebhook(job.data);
await fnTriggerWebhook({ ...job.data, projectEnvDal, webhookDal });
logger.info("Secret webhook job ended", job.id);
});

284
backend-pg/src/services/secret/secret-service.ts
View File

@@ -1,12 +1,6 @@
import { ForbiddenError, subject } from "@casl/ability";
import {
SecretEncryptionAlgo,
SecretKeyEncoding,
SecretType,
TableName,
TSecretBlindIndexes
} from "@app/db/schemas";
import { SecretEncryptionAlgo, SecretKeyEncoding, SecretType, TableName } from "@app/db/schemas";
import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
import {
ProjectPermissionActions,
@@ -14,31 +8,40 @@ import {
} from "@app/ee/services/permission/project-permission";
import { TSecretSnapshotServiceFactory } from "@app/ee/services/secret-snapshot/secret-snapshot-service";
import { getConfig } from "@app/lib/config/env";
import { buildSecretBlindIndexFromName } from "@app/lib/crypto";
import { buildSecretBlindIndexFromName, encryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
import { BadRequestError } from "@app/lib/errors";
import { groupBy, pick } from "@app/lib/fn";
import { ActorType } from "../auth/auth-type";
import { TProjectBotServiceFactory } from "../project-bot/project-bot-service";
import { TSecretFolderDalFactory } from "../secret-folder/secret-folder-dal";
import { TSecretImportDalFactory } from "../secret-import/secret-import-dal";
import { fnSecretsFromImports } from "../secret-import/secret-import-fns";
import { TSecretTagDalFactory } from "../secret-tag/secret-tag-dal";
import { TSecretBlindIndexDalFactory } from "./secret-blind-index-dal";
import { TSecretDalFactory } from "./secret-dal";
import { decryptSecretRaw, generateSecretBlindIndexBySalt } from "./secret-fns";
import { TSecretQueueFactory } from "./secret-queue";
import {
TCreateBulkSecretDTO,
TCreateSecretDTO,
TCreateSecretRawDTO,
TDeleteBulkSecretDTO,
TDeleteSecretDTO,
TDeleteSecretRawDTO,
TFnSecretBlindIndexCheck,
TFnSecretBlindIndexCheckV2,
TFnSecretBulkDelete,
TFnSecretBulkInsert,
TFnSecretBulkUpdate,
TGetASecretDTO,
TGetASecretRawDTO,
TGetSecretsDTO,
TGetSecretsRawDTO,
TListSecretVersionDTO,
TUpdateBulkSecretDTO,
TUpdateSecretDTO
TUpdateSecretDTO,
TUpdateSecretRawDTO
} from "./secret-types";
import { TSecretVersionDalFactory } from "./secret-version-dal";
@@ -46,32 +49,19 @@ type TSecretServiceFactoryDep = {
secretDal: TSecretDalFactory;
secretTagDal: TSecretTagDalFactory;
secretVersionDal: TSecretVersionDalFactory;
folderDal: Pick<TSecretFolderDalFactory, "findBySecretPath" | "updateById" | "findById">;
folderDal: Pick<
TSecretFolderDalFactory,
"findBySecretPath" | "updateById" | "findById" | "findByManySecretPath"
>;
secretBlindIndexDal: TSecretBlindIndexDalFactory;
permissionService: Pick<TPermissionServiceFactory, "getProjectPermission">;
snapshotService: Pick<TSecretSnapshotServiceFactory, "performSnapshot">;
secretQueueService: Pick<TSecretQueueFactory, "syncSecrets">;
projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
secretImportDal: Pick<TSecretImportDalFactory, "find">;
};
export type TSecretServiceFactory = ReturnType<typeof secretServiceFactory>;
export const generateSecretBlindIndexBySalt = async (
secretName: string,
secretBlindIndexDoc: TSecretBlindIndexes
) => {
const appCfg = getConfig();
const secretBlindIndex = await buildSecretBlindIndexFromName({
secretName,
keyEncoding: secretBlindIndexDoc.keyEncoding as SecretKeyEncoding,
rootEncryptionKey: appCfg.ROOT_ENCRYPTION_KEY,
encryptionKey: appCfg.ENCRYPTION_KEY,
tag: secretBlindIndexDoc.saltTag,
ciphertext: secretBlindIndexDoc.encryptedSaltCipherText,
iv: secretBlindIndexDoc.saltIV
});
return secretBlindIndex;
};
export const secretServiceFactory = ({
secretDal,
secretTagDal,
@@ -80,7 +70,9 @@ export const secretServiceFactory = ({
secretBlindIndexDal,
permissionService,
snapshotService,
secretQueueService
secretQueueService,
projectBotService,
secretImportDal
}: TSecretServiceFactoryDep) => {
// utility function to get secret blind index data
const interalGenSecBlindIndexByName = async (projectId: string, secretName: string) => {
@@ -485,7 +477,14 @@ export const secretServiceFactory = ({
return deletedSecret[0];
};
const getSecrets = async ({ actorId, path, environment, projectId, actor }: TGetSecretsDTO) => {
const getSecrets = async ({
actorId,
path,
environment,
projectId,
actor,
includeImports
}: TGetSecretsDTO) => {
const { permission } = await permissionService.getProjectPermission(actor, actorId, projectId);
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionActions.Read,
@@ -497,7 +496,28 @@ export const secretServiceFactory = ({
const folderId = folder.id;
const secrets = await secretDal.findByFolderId(folderId, actorId);
return secrets;
if (includeImports) {
const secretImports = await secretImportDal.find({ folderId });
const allowedImports = secretImports.filter(({ importEnv, importPath }) =>
// if its service token allow full access over imported one
actor === ActorType.SERVICE
? true
: permission.can(
ProjectPermissionActions.Read,
subject(ProjectPermissionSub.Secrets, {
environment: importEnv.slug,
secretPath: importPath
})
)
);
const importedSecrets = await fnSecretsFromImports({
allowedImports,
secretDal,
folderDal
});
return { secrets, imports: importedSecrets };
}
return { secrets };
};
const getASecret = async ({
@@ -507,7 +527,8 @@ export const secretServiceFactory = ({
environment,
path,
type,
secretName
secretName,
includeImports
}: TGetASecretDTO) => {
const { permission } = await permissionService.getProjectPermission(actor, actorId, projectId);
ForbiddenError.from(permission).throwUnlessCan(
@@ -526,6 +547,36 @@ export const secretServiceFactory = ({
userId: type === SecretType.Personal ? actorId : null,
secretBlindIndex
});
// now if secret is not found
// then search for imported secrets
// here we consider the import order also thus starting from bottom
if (!secret && includeImports) {
const secretImports = await secretImportDal.find({ folderId });
const allowedImports = secretImports.filter(({ importEnv, importPath }) =>
// if its service token allow full access over imported one
actor === ActorType.SERVICE
? true
: permission.can(
ProjectPermissionActions.Read,
subject(ProjectPermissionSub.Secrets, {
environment: importEnv.slug,
secretPath: importPath
})
)
);
const importedSecrets = await fnSecretsFromImports({
allowedImports,
secretDal,
folderDal
});
for (let i = importedSecrets.length; i >= 0; i -= 1) {
for (let j = 0; j < importedSecrets[i].secrets.length; j += 1) {
if (secretBlindIndex === importedSecrets[i].secrets[j].secretBlindIndex) {
return importedSecrets[i].secrets[j];
}
}
}
}
if (!secret) throw new BadRequestError({ message: "Secret not found" });
return secret;
@@ -738,6 +789,170 @@ export const secretServiceFactory = ({
return secretVersions;
};
const getSecretsRaw = async ({
projectId,
path,
actor,
actorId,
environment,
includeImports
}: TGetSecretsRawDTO) => {
const botKey = await projectBotService.getBotKey(projectId);
if (!botKey) throw new BadRequestError({ message: "Project bot not found" });
const { secrets, imports } = await getSecrets({
actorId,
projectId,
environment,
actor,
path,
includeImports
});
return {
secrets: secrets.map((el) => decryptSecretRaw(el, botKey)),
imports: (imports || [])?.map(({ secrets: importedSecrets, ...el }) => ({
...el,
secrets: importedSecrets.map((sec) => decryptSecretRaw(sec, botKey))
}))
};
};
const getASecretRaw = async ({
type,
path,
actor,
environment,
projectId,
actorId,
secretName,
includeImports
}: TGetASecretRawDTO) => {
const botKey = await projectBotService.getBotKey(projectId);
if (!botKey) throw new BadRequestError({ message: "Project bot not found" });
const secret = await getASecret({
actorId,
projectId,
environment,
actor,
path,
secretName,
type,
includeImports
});
return decryptSecretRaw(secret, botKey);
};
const createSecretRaw = async ({
secretName,
actorId,
projectId,
environment,
actor,
type,
secretPath,
secretValue,
secretComment,
skipMultilineEncoding
}: TCreateSecretRawDTO) => {
const botKey = await projectBotService.getBotKey(projectId);
if (!botKey) throw new BadRequestError({ message: "Project bot not found" });
const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secretName, botKey);
const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secretValue || "", botKey);
const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secretComment || "", botKey);
const secret = await createSecret({
secretName,
projectId,
environment,
type,
path: secretPath,
actor,
actorId,
secretKeyCiphertext: secretKeyEncrypted.ciphertext,
secretKeyIV: secretKeyEncrypted.iv,
secretKeyTag: secretKeyEncrypted.tag,
secretValueCiphertext: secretValueEncrypted.ciphertext,
secretValueIV: secretValueEncrypted.iv,
secretValueTag: secretValueEncrypted.tag,
secretCommentCiphertext: secretCommentEncrypted.ciphertext,
secretCommentIV: secretCommentEncrypted.iv,
secretCommentTag: secretCommentEncrypted.tag,
skipMultilineEncoding
});
await snapshotService.performSnapshot(secret.folderId);
await secretQueueService.syncSecrets({ secretPath, projectId, environment });
return decryptSecretRaw(secret, botKey);
};
const updateSecretRaw = async ({
secretName,
actorId,
projectId,
environment,
actor,
type,
secretPath,
secretValue,
skipMultilineEncoding
}: TUpdateSecretRawDTO) => {
const botKey = await projectBotService.getBotKey(projectId);
if (!botKey) throw new BadRequestError({ message: "Project bot not found" });
const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secretValue || "", botKey);
const secret = await updateSecret({
secretName,
projectId,
environment,
type,
path: secretPath,
actor,
actorId,
secretValueCiphertext: secretValueEncrypted.ciphertext,
secretValueIV: secretValueEncrypted.iv,
secretValueTag: secretValueEncrypted.tag,
skipMultilineEncoding
});
await snapshotService.performSnapshot(secret.folderId);
await secretQueueService.syncSecrets({ secretPath, projectId, environment });
return decryptSecretRaw(secret, botKey);
};
const deleteSecretRaw = async ({
secretName,
actorId,
projectId,
environment,
actor,
type,
secretPath
}: TDeleteSecretRawDTO) => {
const botKey = await projectBotService.getBotKey(projectId);
if (!botKey) throw new BadRequestError({ message: "Project bot not found" });
const secret = await deleteSecret({
secretName,
projectId,
environment,
type,
path: secretPath,
actor,
actorId
});
await snapshotService.performSnapshot(secret.folderId);
await secretQueueService.syncSecrets({ secretPath, projectId, environment });
return decryptSecretRaw(secret, botKey);
};
return {
createSecret,
deleteSecret,
@@ -747,6 +962,11 @@ export const secretServiceFactory = ({
deleteManySecret,
getASecret,
getSecrets,
getSecretsRaw,
getASecretRaw,
createSecretRaw,
updateSecretRaw,
deleteSecretRaw,
listSecretVersionsBySecretId,
// external services function
fnSecretBulkDelete,

42
backend-pg/src/services/secret/secret-types.ts
View File

@@ -61,6 +61,7 @@ export type TDeleteSecretDTO = {
export type TGetSecretsDTO = {
path: string;
environment: string;
includeImports?: boolean;
} & TProjectPermission;
export type TGetASecretDTO = {
@@ -68,6 +69,7 @@ export type TGetASecretDTO = {
path: string;
environment: string;
type: "shared" | "personal";
includeImports?: boolean;
} & TProjectPermission;
export type TCreateBulkSecretDTO = {
@@ -125,6 +127,46 @@ export type TListSecretVersionDTO = {
limit?: number;
} & Omit<TProjectPermission, "projectId">;
export type TGetSecretsRawDTO = {
path: string;
environment: string;
includeImports?: boolean;
} & TProjectPermission;
export type TGetASecretRawDTO = {
secretName: string;
path: string;
environment: string;
type: "shared" | "personal";
includeImports?: boolean;
} & TProjectPermission;
export type TCreateSecretRawDTO = TProjectPermission & {
secretPath: string;
environment: string;
secretName: string;
secretValue: string;
type: SecretType;
secretComment?: string;
skipMultilineEncoding?: boolean;
};
export type TUpdateSecretRawDTO = TProjectPermission & {
secretPath: string;
environment: string;
secretName: string;
secretValue?: string;
type: SecretType;
skipMultilineEncoding?: boolean;
};
export type TDeleteSecretRawDTO = TProjectPermission & {
secretPath: string;
environment: string;
secretName: string;
type: SecretType;
};
export type TFnSecretBulkInsert = {
folderId: string;
tx?: Knex;

70
backend-pg/src/services/webhook/webhook-fns.ts
View File

@@ -1,9 +1,15 @@
import crypto from "node:crypto";
import picomatch from "picomatch";
import { SecretKeyEncoding, TWebhooks } from "@app/db/schemas";
import { getConfig } from "@app/lib/config/env";
import { request } from "@app/lib/config/request";
import { decryptSymmetric, decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
import { BadRequestError } from "@app/lib/errors";
import { TProjectEnvDalFactory } from "../project-env/project-env-dal";
import { TWebhookDalFactory } from "./webhook-dal";
const WEBHOOK_TRIGGER_TIMEOUT = 15 * 1000;
export const triggerWebhookRequest = async (
@@ -64,3 +70,67 @@ export const getWebhookPayload = (
secretPath
}
});
export type TFnTriggerWebhookDTO = {
projectId: string;
secretPath: string;
environment: string;
webhookDal: Pick<TWebhookDalFactory, "findAllWebhooks" | "transaction" | "update" | "bulkUpdate">;
projectEnvDal: Pick<TProjectEnvDalFactory, "findOne">;
};
// this is reusable function
// used in secret queue to trigger webhook and update status when secrets changes
export const fnTriggerWebhook = async ({
environment,
secretPath,
projectId,
webhookDal,
projectEnvDal
}: TFnTriggerWebhookDTO) => {
const webhooks = await webhookDal.findAllWebhooks(projectId, environment);
const toBeTriggeredHooks = webhooks.filter(
({ secretPath: hookSecretPath, isDisabled }) =>
!isDisabled && picomatch.isMatch(secretPath, hookSecretPath, { strictSlashes: false })
);
if (!toBeTriggeredHooks.length) return;
const webhooksTriggered = await Promise.allSettled(
toBeTriggeredHooks.map((hook) =>
triggerWebhookRequest(
hook,
getWebhookPayload("secrets.modified", projectId, environment, secretPath)
)
)
);
// filter hooks by status
const successWebhooks = webhooksTriggered
.filter(({ status }) => status === "fulfilled")
.map((_, i) => toBeTriggeredHooks[i].id);
const failedWebhooks = webhooksTriggered
.filter(({ status }) => status === "rejected")
.map((data, i) => ({
id: toBeTriggeredHooks[i].id,
error: data.status === "rejected" && data.reason.message
}));
await webhookDal.transaction(async (tx) => {
const env = await projectEnvDal.findOne({ projectId, slug: environment }, tx);
if (!env) throw new BadRequestError({ message: "Env not found" });
if (successWebhooks.length) {
await webhookDal.update(
{ envId: env.id, $in: { id: successWebhooks } },
{ lastStatus: "success", lastRunErrorMessage: null },
tx
);
}
if (failedWebhooks.length) {
await webhookDal.bulkUpdate(
failedWebhooks.map(({ id, error }) => ({
id,
lastRunErrorMessage: error,
lastStatus: "failed"
})),
tx
);
}
});
};

56
backend-pg/src/services/webhook/webhook-service.ts
View File

@@ -1,5 +1,4 @@
import { ForbiddenError } from "@casl/ability";
import picomatch from "picomatch";
import { SecretEncryptionAlgo, SecretKeyEncoding, TWebhooksInsert } from "@app/db/schemas";
import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
@@ -17,7 +16,6 @@ import { getWebhookPayload, triggerWebhookRequest } from "./webhook-fns";
import {
TCreateWebhookDTO,
TDeleteWebhookDTO,
TFnTriggerWebhookDTO,
TListWebhookDTO,
TTestWebhookDTO,
TUpdateWebhookDTO
@@ -170,63 +168,11 @@ export const webhookServiceFactory = ({
return webhookDal.findAllWebhooks(projectId, environment, secretPath);
};
// this is reusable function
// used in secret queue to trigger webhook and update status when secrets changes
const fnTriggerWebhook = async ({ environment, secretPath, projectId }: TFnTriggerWebhookDTO) => {
const webhooks = await webhookDal.findAllWebhooks(projectId, environment);
const toBeTriggeredHooks = webhooks.filter(
({ secretPath: hookSecretPath, isDisabled }) =>
!isDisabled && picomatch.isMatch(secretPath, hookSecretPath, { strictSlashes: false })
);
if (!toBeTriggeredHooks.length) return;
const webhooksTriggered = await Promise.allSettled(
toBeTriggeredHooks.map((hook) =>
triggerWebhookRequest(
hook,
getWebhookPayload("secrets.modified", projectId, environment, secretPath)
)
)
);
// filter hooks by status
const successWebhooks = webhooksTriggered
.filter(({ status }) => status === "fulfilled")
.map((_, i) => toBeTriggeredHooks[i].id);
const failedWebhooks = webhooksTriggered
.filter(({ status }) => status === "rejected")
.map((data, i) => ({
id: toBeTriggeredHooks[i].id,
error: data.status === "rejected" && data.reason.message
}));
await webhookDal.transaction(async (tx) => {
const env = await projectEnvDal.findOne({ projectId, slug: environment }, tx);
if (!env) throw new BadRequestError({ message: "Env not found" });
if (successWebhooks.length) {
await webhookDal.update(
{ envId: env.id, $in: { id: successWebhooks } },
{ lastStatus: "success", lastRunErrorMessage: null },
tx
);
}
if (failedWebhooks.length) {
await webhookDal.bulkUpdate(
failedWebhooks.map(({ id, error }) => ({
id,
lastRunErrorMessage: error,
lastStatus: "failed"
})),
tx
);
}
});
};
return {
createWebhook,
deleteWebhook,
listWebhooks,
updateWebhook,
testWebhook,
fnTriggerWebhook
testWebhook
};
};

6
backend-pg/src/services/webhook/webhook-types.ts
View File

@@ -24,9 +24,3 @@ export type TListWebhookDTO = {
environment?: string;
secretPath?: string;
} & TProjectPermission;
export type TFnTriggerWebhookDTO = {
projectId: string;
secretPath: string;
environment: string;
};