awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-09-22 13:39:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1712 from akhilmhdh/feat/batch-raw-secrets-api

Browse Source 
Batch raw secrets api
This commit is contained in:
Maidul Islam
2024-04-22 18:48:14 -04:00
committed by GitHub
parent 62aebe2fd4 aa5cd0fd0f
commit 08cb105fe4
8 changed files with 570 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

107
backend/e2e-test/routes/v3/secrets.spec.ts
View File

@@ -942,6 +942,113 @@ describe.each([{ auth: AuthMode.JWT }, { auth: AuthMode.IDENTITY_ACCESS_TOKEN }]
const secrets = await getSecrets(seedData1.environment.slug, path);
expect(secrets).toEqual([]);
});
test.each(testRawSecrets)("Bulk create secret raw in path $path", async ({ path, secret }) => {
const createSecretReqBody = {
projectSlug: seedData1.project.slug,
environment: seedData1.environment.slug,
secretPath: path,
secrets: [
{
secretKey: secret.key,
secretValue: secret.value,
secretComment: secret.comment
}
]
};
const createSecRes = await testServer.inject({
method: "POST",
url: `/api/v3/secrets/batch/raw`,
headers: {
authorization: `Bearer ${authToken}`
},
body: createSecretReqBody
});
expect(createSecRes.statusCode).toBe(200);
const createdSecretPayload = JSON.parse(createSecRes.payload);
expect(createdSecretPayload).toHaveProperty("secrets");
// fetch secrets
const secrets = await getSecrets(seedData1.environment.slug, path);
expect(secrets).toEqual(
expect.arrayContaining([
expect.objectContaining({
key: secret.key,
value: secret.value,
type: SecretType.Shared
})
])
);
await deleteRawSecret({ path, key: secret.key });
});
test.each(testRawSecrets)("Bulk update secret raw in path $path", async ({ secret, path }) => {
await createRawSecret({ path, ...secret });
const updateSecretReqBody = {
projectSlug: seedData1.project.slug,
environment: seedData1.environment.slug,
secretPath: path,
secrets: [
{
secretValue: "new-value",
secretKey: secret.key
}
]
};
const updateSecRes = await testServer.inject({
method: "PATCH",
url: `/api/v3/secrets/batch/raw`,
headers: {
authorization: `Bearer ${authToken}`
},
body: updateSecretReqBody
});
expect(updateSecRes.statusCode).toBe(200);
const updatedSecretPayload = JSON.parse(updateSecRes.payload);
expect(updatedSecretPayload).toHaveProperty("secrets");
// fetch secrets
const secrets = await getSecrets(seedData1.environment.slug, path);
expect(secrets).toEqual(
expect.arrayContaining([
expect.objectContaining({
key: secret.key,
value: "new-value",
version: 2,
type: SecretType.Shared
})
])
);
await deleteRawSecret({ path, key: secret.key });
});
test.each(testRawSecrets)("Bulk delete secret raw in path $path", async ({ path, secret }) => {
await createRawSecret({ path, ...secret });
const deletedSecretReqBody = {
projectSlug: seedData1.project.slug,
environment: seedData1.environment.slug,
secretPath: path,
secrets: [{ secretKey: secret.key }]
};
const deletedSecRes = await testServer.inject({
method: "DELETE",
url: `/api/v3/secrets/batch/raw`,
headers: {
authorization: `Bearer ${authToken}`
},
body: deletedSecretReqBody
});
expect(deletedSecRes.statusCode).toBe(200);
const deletedSecretPayload = JSON.parse(deletedSecRes.payload);
expect(deletedSecretPayload).toHaveProperty("secrets");
// fetch secrets
const secrets = await getSecrets(seedData1.environment.slug, path);
expect(secrets).toEqual([]);
});
}
);

4
backend/src/lib/api-docs/constants.ts
View File

@@ -282,6 +282,7 @@ export const RAW_SECRETS = {
},
CREATE: {
secretName: "The name of the secret to create.",
projectSlug: "The slug of the project to create the secret in.",
environment: "The slug of the environment to create the secret in.",
secretComment: "Attach a comment to the secret.",
secretPath: "The path to create the secret in.",
@@ -301,11 +302,13 @@ export const RAW_SECRETS = {
},
UPDATE: {
secretName: "The name of the secret to update.",
secretComment: "Update comment to the secret.",
environment: "The slug of the environment where the secret is located.",
secretPath: "The path of the secret to update",
secretValue: "The new value of the secret.",
skipMultilineEncoding: "Skip multiline encoding for the secret value.",
type: "The type of the secret to update.",
projectSlug: "The slug of the project to update the secret in.",
workspaceId: "The ID of the project to update the secret in."
},
DELETE: {
@@ -313,6 +316,7 @@ export const RAW_SECRETS = {
environment: "The slug of the environment where the secret is located.",
secretPath: "The path of the secret.",
type: "The type of the secret to delete.",
projectSlug: "The slug of the project to delete the secret in.",
workspaceId: "The ID of the project where the secret is located."
}
} as const;

259
backend/src/server/routes/v3/secret-router.ts
View File

@@ -1656,4 +1656,263 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
return { secrets };
}
});
server.route({
method: "POST",
url: "/batch/raw",
config: {
rateLimit: secretsLimit
},
schema: {
description: "Create many secrets",
security: [
{
bearerAuth: []
}
],
body: z.object({
projectSlug: z.string().trim().describe(RAW_SECRETS.CREATE.projectSlug),
environment: z.string().trim().describe(RAW_SECRETS.CREATE.environment),
secretPath: z
.string()
.trim()
.default("/")
.transform(removeTrailingSlash)
.describe(RAW_SECRETS.CREATE.secretPath),
secrets: z
.object({
secretKey: z.string().trim().describe(RAW_SECRETS.CREATE.secretName),
secretValue: z
.string()
.transform((val) => (val.at(-1) === "\n" ? `${val.trim()}\n` : val.trim()))
.describe(RAW_SECRETS.CREATE.secretValue),
secretComment: z.string().trim().optional().default("").describe(RAW_SECRETS.CREATE.secretComment),
skipMultilineEncoding: z.boolean().optional().describe(RAW_SECRETS.CREATE.skipMultilineEncoding)
})
.array()
.min(1)
}),
response: {
200: z.object({
secrets: secretRawSchema.array()
})
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY, AuthMode.SERVICE_TOKEN, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const { environment, projectSlug, secretPath, secrets: inputSecrets } = req.body;
const secrets = await server.services.secret.createManySecretsRaw({
actorId: req.permission.id,
actor: req.permission.type,
actorAuthMethod: req.permission.authMethod,
actorOrgId: req.permission.orgId,
secretPath,
environment,
projectSlug,
secrets: inputSecrets
});
await server.services.auditLog.createAuditLog({
projectId: secrets[0].workspace,
...req.auditLogInfo,
event: {
type: EventType.CREATE_SECRETS,
metadata: {
environment: req.body.environment,
secretPath: req.body.secretPath,
secrets: secrets.map((secret, i) => ({
secretId: secret.id,
secretKey: inputSecrets[i].secretKey,
secretVersion: secret.version
}))
}
}
});
await server.services.telemetry.sendPostHogEvents({
event: PostHogEventTypes.SecretCreated,
distinctId: getTelemetryDistinctId(req),
properties: {
numberOfSecrets: secrets.length,
workspaceId: secrets[0].workspace,
environment: req.body.environment,
secretPath: req.body.secretPath,
channel: getUserAgentType(req.headers["user-agent"]),
...req.auditLogInfo
}
});
return { secrets };
}
});
server.route({
method: "PATCH",
url: "/batch/raw",
config: {
rateLimit: secretsLimit
},
schema: {
description: "Update many secrets",
security: [
{
bearerAuth: []
}
],
body: z.object({
projectSlug: z.string().trim().describe(RAW_SECRETS.UPDATE.projectSlug),
environment: z.string().trim().describe(RAW_SECRETS.UPDATE.environment),
secretPath: z
.string()
.trim()
.default("/")
.transform(removeTrailingSlash)
.describe(RAW_SECRETS.UPDATE.secretPath),
secrets: z
.object({
secretKey: z.string().trim().describe(RAW_SECRETS.UPDATE.secretName),
secretValue: z
.string()
.transform((val) => (val.at(-1) === "\n" ? `${val.trim()}\n` : val.trim()))
.describe(RAW_SECRETS.UPDATE.secretValue),
secretComment: z.string().trim().optional().describe(RAW_SECRETS.UPDATE.secretComment),
skipMultilineEncoding: z.boolean().optional().describe(RAW_SECRETS.UPDATE.skipMultilineEncoding)
})
.array()
.min(1)
}),
response: {
200: z.object({
secrets: secretRawSchema.array()
})
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY, AuthMode.SERVICE_TOKEN, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const { environment, projectSlug, secretPath, secrets: inputSecrets } = req.body;
const secrets = await server.services.secret.updateManySecretsRaw({
actorId: req.permission.id,
actor: req.permission.type,
actorAuthMethod: req.permission.authMethod,
actorOrgId: req.permission.orgId,
secretPath,
environment,
projectSlug,
secrets: inputSecrets
});
await server.services.auditLog.createAuditLog({
projectId: secrets[0].workspace,
...req.auditLogInfo,
event: {
type: EventType.UPDATE_SECRETS,
metadata: {
environment: req.body.environment,
secretPath: req.body.secretPath,
secrets: secrets.map((secret, i) => ({
secretId: secret.id,
secretKey: inputSecrets[i].secretKey,
secretVersion: secret.version
}))
}
}
});
await server.services.telemetry.sendPostHogEvents({
event: PostHogEventTypes.SecretUpdated,
distinctId: getTelemetryDistinctId(req),
properties: {
numberOfSecrets: secrets.length,
workspaceId: secrets[0].workspace,
environment: req.body.environment,
secretPath: req.body.secretPath,
channel: getUserAgentType(req.headers["user-agent"]),
...req.auditLogInfo
}
});
return { secrets };
}
});
server.route({
method: "DELETE",
url: "/batch/raw",
config: {
rateLimit: secretsLimit
},
schema: {
description: "Delete many secrets",
security: [
{
bearerAuth: []
}
],
body: z.object({
projectSlug: z.string().trim().describe(RAW_SECRETS.DELETE.projectSlug),
environment: z.string().trim().describe(RAW_SECRETS.DELETE.environment),
secretPath: z
.string()
.trim()
.default("/")
.transform(removeTrailingSlash)
.describe(RAW_SECRETS.DELETE.secretPath),
secrets: z
.object({
secretKey: z.string().trim().describe(RAW_SECRETS.DELETE.secretName)
})
.array()
.min(1)
}),
response: {
200: z.object({
secrets: secretRawSchema.array()
})
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY, AuthMode.SERVICE_TOKEN, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const { environment, projectSlug, secretPath, secrets: inputSecrets } = req.body;
const secrets = await server.services.secret.deleteManySecretsRaw({
actorId: req.permission.id,
actor: req.permission.type,
actorAuthMethod: req.permission.authMethod,
actorOrgId: req.permission.orgId,
environment,
projectSlug,
secretPath,
secrets: inputSecrets
});
await server.services.auditLog.createAuditLog({
projectId: secrets[0].workspace,
...req.auditLogInfo,
event: {
type: EventType.DELETE_SECRETS,
metadata: {
environment: req.body.environment,
secretPath: req.body.secretPath,
secrets: secrets.map((secret, i) => ({
secretId: secret.id,
secretKey: inputSecrets[i].secretKey,
secretVersion: secret.version
}))
}
}
});
await server.services.telemetry.sendPostHogEvents({
event: PostHogEventTypes.SecretDeleted,
distinctId: getTelemetryDistinctId(req),
properties: {
numberOfSecrets: secrets.length,
workspaceId: secrets[0].workspace,
environment: req.body.environment,
secretPath: req.body.secretPath,
channel: getUserAgentType(req.headers["user-agent"]),
...req.auditLogInfo
}
});
return { secrets };
}
});
};

143
backend/src/services/secret/secret-service.ts
View File

@@ -33,9 +33,11 @@ import { TSecretQueueFactory } from "./secret-queue";
import {
TAttachSecretTagsDTO,
TCreateBulkSecretDTO,
TCreateManySecretRawDTO,
TCreateSecretDTO,
TCreateSecretRawDTO,
TDeleteBulkSecretDTO,
TDeleteManySecretRawDTO,
TDeleteSecretDTO,
TDeleteSecretRawDTO,
TFnSecretBlindIndexCheckV2,
@@ -46,6 +48,7 @@ import {
TGetSecretsRawDTO,
TGetSecretVersionsDTO,
TUpdateBulkSecretDTO,
TUpdateManySecretRawDTO,
TUpdateSecretDTO,
TUpdateSecretRawDTO
} from "./secret-types";
@@ -1036,6 +1039,143 @@ export const secretServiceFactory = ({
return decryptSecretRaw(secret, botKey);
};
const createManySecretsRaw = async ({
actorId,
projectSlug,
environment,
actor,
actorOrgId,
actorAuthMethod,
secretPath,
secrets: inputSecrets = []
}: TCreateManySecretRawDTO) => {
const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId);
if (!project) throw new BadRequestError({ message: "Project not found" });
const projectId = project.id;
const botKey = await projectBotService.getBotKey(projectId);
if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" });
const secrets = await createManySecret({
projectId,
environment,
path: secretPath,
actor,
actorId,
actorOrgId,
actorAuthMethod,
secrets: inputSecrets.map(({ secretComment, secretKey, secretValue, skipMultilineEncoding }) => {
const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secretKey, botKey);
const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secretValue || "", botKey);
const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secretComment || "", botKey);
return {
secretName: secretKey,
skipMultilineEncoding,
secretKeyCiphertext: secretKeyEncrypted.ciphertext,
secretKeyIV: secretKeyEncrypted.iv,
secretKeyTag: secretKeyEncrypted.tag,
secretValueCiphertext: secretValueEncrypted.ciphertext,
secretValueIV: secretValueEncrypted.iv,
secretValueTag: secretValueEncrypted.tag,
secretCommentCiphertext: secretCommentEncrypted.ciphertext,
secretCommentIV: secretCommentEncrypted.iv,
secretCommentTag: secretCommentEncrypted.tag
};
})
});
await snapshotService.performSnapshot(secrets[0].folderId);
await secretQueueService.syncSecrets({ secretPath, projectId, environment });
return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment }, botKey));
};
const updateManySecretsRaw = async ({
actorId,
projectSlug,
environment,
actor,
actorOrgId,
actorAuthMethod,
secretPath,
secrets: inputSecrets = []
}: TUpdateManySecretRawDTO) => {
const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId);
if (!project) throw new BadRequestError({ message: "Project not found" });
const projectId = project.id;
const botKey = await projectBotService.getBotKey(projectId);
if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" });
const secrets = await updateManySecret({
projectId,
environment,
path: secretPath,
actor,
actorId,
actorOrgId,
actorAuthMethod,
secrets: inputSecrets.map(({ secretComment, secretKey, secretValue, skipMultilineEncoding }) => {
const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secretKey, botKey);
const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secretValue || "", botKey);
const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secretComment || "", botKey);
return {
secretName: secretKey,
type: SecretType.Shared,
skipMultilineEncoding,
secretKeyCiphertext: secretKeyEncrypted.ciphertext,
secretKeyIV: secretKeyEncrypted.iv,
secretKeyTag: secretKeyEncrypted.tag,
secretValueCiphertext: secretValueEncrypted.ciphertext,
secretValueIV: secretValueEncrypted.iv,
secretValueTag: secretValueEncrypted.tag,
secretCommentCiphertext: secretCommentEncrypted.ciphertext,
secretCommentIV: secretCommentEncrypted.iv,
secretCommentTag: secretCommentEncrypted.tag
};
})
});
await snapshotService.performSnapshot(secrets[0].folderId);
await secretQueueService.syncSecrets({ secretPath, projectId, environment });
return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment }, botKey));
};
const deleteManySecretsRaw = async ({
actorId,
projectSlug,
environment,
actor,
actorOrgId,
actorAuthMethod,
secretPath,
secrets: inputSecrets = []
}: TDeleteManySecretRawDTO) => {
const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId);
if (!project) throw new BadRequestError({ message: "Project not found" });
const projectId = project.id;
const botKey = await projectBotService.getBotKey(projectId);
if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" });
const secrets = await deleteManySecret({
projectId,
environment,
path: secretPath,
actor,
actorId,
actorOrgId,
actorAuthMethod,
secrets: inputSecrets.map(({ secretKey }) => ({ secretName: secretKey, type: SecretType.Shared }))
});
await snapshotService.performSnapshot(secrets[0].folderId);
await secretQueueService.syncSecrets({ secretPath, projectId, environment });
return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment }, botKey));
};
const getSecretVersions = async ({
actorId,
actor,
@@ -1280,6 +1420,9 @@ export const secretServiceFactory = ({
createSecretRaw,
updateSecretRaw,
deleteSecretRaw,
createManySecretsRaw,
updateManySecretsRaw,
deleteManySecretsRaw,
getSecretVersions,
// external services function
fnSecretBulkDelete,

33
backend/src/services/secret/secret-types.ts
View File

@@ -181,6 +181,39 @@ export type TDeleteSecretRawDTO = TProjectPermission & {
type: SecretType;
};
export type TCreateManySecretRawDTO = Omit<TProjectPermission, "projectId"> & {
secretPath: string;
projectSlug: string;
environment: string;
secrets: {
secretKey: string;
secretValue: string;
secretComment?: string;
skipMultilineEncoding?: boolean;
}[];
};
export type TUpdateManySecretRawDTO = Omit<TProjectPermission, "projectId"> & {
secretPath: string;
projectSlug: string;
environment: string;
secrets: {
secretKey: string;
secretValue: string;
secretComment?: string;
skipMultilineEncoding?: boolean;
}[];
};
export type TDeleteManySecretRawDTO = Omit<TProjectPermission, "projectId"> & {
secretPath: string;
projectSlug: string;
environment: string;
secrets: {
secretKey: string;
}[];
};
export type TGetSecretVersionsDTO = Omit<TProjectPermission, "projectId"> & {
limit?: number;
offset?: number;

8
docs/api-reference/endpoints/secrets/create-many.mdx Normal file
View File

@@ -0,0 +1,8 @@
---
title: "Bulk Create"
openapi: "POST /api/v3/secrets/batch/raw"
---
<Tip>
This endpoint requires you to disable end-to-end encryption. For more information, you should consult this [note](https://infisical.com/docs/api-reference/overview/examples/note).
</Tip>

8
docs/api-reference/endpoints/secrets/delete-many.mdx Normal file
View File

@@ -0,0 +1,8 @@
---
title: "Bulk Delete"
openapi: "DELETE /api/v3/secrets/batch/raw"
---
<Tip>
This endpoint requires you to disable end-to-end encryption. For more information, you should consult this [note](https://infisical.com/docs/api-reference/overview/examples/note).
</Tip>

8
docs/api-reference/endpoints/secrets/update-many.mdx Normal file
View File

@@ -0,0 +1,8 @@
---
title: "Bulk Update"
openapi: "PATCH /api/v3/secrets/batch/raw"
---
<Tip>
This endpoint requires you to disable end-to-end encryption. For more information, you should consult this [note](https://infisical.com/docs/api-reference/overview/examples/note).
</Tip>