From 718cabe49b83bf04942dea8a64513ccae4ed7169 Mon Sep 17 00:00:00 2001 From: Akhil Mohan Date: Fri, 19 Apr 2024 20:53:54 +0530 Subject: [PATCH 01/18] feat(server): added batch raw bulk secret ops api --- backend/e2e-test/routes/v3/secrets.spec.ts | 107 ++++++++ backend/src/lib/api-docs/constants.ts | 1 + backend/src/server/routes/v3/secret-router.ts | 259 ++++++++++++++++++ backend/src/services/secret/secret-service.ts | 131 +++++++++ backend/src/services/secret/secret-types.ts | 30 ++ 5 files changed, 528 insertions(+) diff --git a/backend/e2e-test/routes/v3/secrets.spec.ts b/backend/e2e-test/routes/v3/secrets.spec.ts index 03e1c2f50..ab73a7f1f 100644 --- a/backend/e2e-test/routes/v3/secrets.spec.ts +++ b/backend/e2e-test/routes/v3/secrets.spec.ts @@ -942,6 +942,113 @@ describe.each([{ auth: AuthMode.JWT }, { auth: AuthMode.IDENTITY_ACCESS_TOKEN }] const secrets = await getSecrets(seedData1.environment.slug, path); expect(secrets).toEqual([]); }); + + test.each(testRawSecrets)("Bulk create secret raw in path $path", async ({ path, secret }) => { + const createSecretReqBody = { + workspaceId: seedData1.project.id, + environment: seedData1.environment.slug, + secretPath: path, + secrets: [ + { + secretKey: secret.key, + secretValue: secret.value, + secretComment: secret.comment + } + ] + }; + const createSecRes = await testServer.inject({ + method: "POST", + url: `/api/v3/secrets/batch/raw`, + headers: { + authorization: `Bearer ${authToken}` + }, + body: createSecretReqBody + }); + expect(createSecRes.statusCode).toBe(200); + const createdSecretPayload = JSON.parse(createSecRes.payload); + expect(createdSecretPayload).toHaveProperty("secrets"); + + // fetch secrets + const secrets = await getSecrets(seedData1.environment.slug, path); + expect(secrets).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + key: secret.key, + value: secret.value, + type: SecretType.Shared + }) + ]) + ); + + await deleteRawSecret({ path, key: secret.key }); + }); + + test.each(testRawSecrets)("Bulk update secret raw in path $path", async ({ secret, path }) => { + await createRawSecret({ path, ...secret }); + const updateSecretReqBody = { + workspaceId: seedData1.project.id, + environment: seedData1.environment.slug, + secretPath: path, + secrets: [ + { + secretValue: "new-value", + secretKey: secret.key + } + ] + }; + const updateSecRes = await testServer.inject({ + method: "PATCH", + url: `/api/v3/secrets/batch/raw`, + headers: { + authorization: `Bearer ${authToken}` + }, + body: updateSecretReqBody + }); + expect(updateSecRes.statusCode).toBe(200); + const updatedSecretPayload = JSON.parse(updateSecRes.payload); + expect(updatedSecretPayload).toHaveProperty("secrets"); + + // fetch secrets + const secrets = await getSecrets(seedData1.environment.slug, path); + expect(secrets).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + key: secret.key, + value: "new-value", + version: 2, + type: SecretType.Shared + }) + ]) + ); + + await deleteRawSecret({ path, key: secret.key }); + }); + + test.each(testRawSecrets)("Bulk delete secret raw in path $path", async ({ path, secret }) => { + await createRawSecret({ path, ...secret }); + + const deletedSecretReqBody = { + workspaceId: seedData1.project.id, + environment: seedData1.environment.slug, + secretPath: path, + secrets: [{ secretKey: secret.key }] + }; + const deletedSecRes = await testServer.inject({ + method: "DELETE", + url: `/api/v3/secrets/batch/raw`, + headers: { + authorization: `Bearer ${authToken}` + }, + body: deletedSecretReqBody + }); + expect(deletedSecRes.statusCode).toBe(200); + const deletedSecretPayload = JSON.parse(deletedSecRes.payload); + expect(deletedSecretPayload).toHaveProperty("secrets"); + + // fetch secrets + const secrets = await getSecrets(seedData1.environment.slug, path); + expect(secrets).toEqual([]); + }); } ); diff --git a/backend/src/lib/api-docs/constants.ts b/backend/src/lib/api-docs/constants.ts index c83234c1f..4a98f73c8 100644 --- a/backend/src/lib/api-docs/constants.ts +++ b/backend/src/lib/api-docs/constants.ts @@ -301,6 +301,7 @@ export const RAW_SECRETS = { }, UPDATE: { secretName: "The name of the secret to update.", + secretComment: "Update comment to the secret.", environment: "The slug of the environment where the secret is located.", secretPath: "The path of the secret to update", secretValue: "The new value of the secret.", diff --git a/backend/src/server/routes/v3/secret-router.ts b/backend/src/server/routes/v3/secret-router.ts index b1d852a88..ceecc0805 100644 --- a/backend/src/server/routes/v3/secret-router.ts +++ b/backend/src/server/routes/v3/secret-router.ts @@ -1656,4 +1656,263 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { return { secrets }; } }); + + server.route({ + method: "POST", + url: "/batch/raw", + config: { + rateLimit: secretsLimit + }, + schema: { + description: "Create many secrets", + security: [ + { + bearerAuth: [] + } + ], + body: z.object({ + workspaceId: z.string().trim().describe(RAW_SECRETS.CREATE.workspaceId), + environment: z.string().trim().describe(RAW_SECRETS.CREATE.environment), + secretPath: z + .string() + .trim() + .default("/") + .transform(removeTrailingSlash) + .describe(RAW_SECRETS.CREATE.secretPath), + secrets: z + .object({ + secretKey: z.string().trim().describe(RAW_SECRETS.CREATE.secretName), + secretValue: z + .string() + .transform((val) => (val.at(-1) === "\n" ? `${val.trim()}\n` : val.trim())) + .describe(RAW_SECRETS.CREATE.secretValue), + secretComment: z.string().trim().optional().default("").describe(RAW_SECRETS.CREATE.secretComment), + skipMultilineEncoding: z.boolean().optional().describe(RAW_SECRETS.CREATE.skipMultilineEncoding) + }) + .array() + .min(1) + }), + response: { + 200: z.object({ + secrets: secretRawSchema.array() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY, AuthMode.SERVICE_TOKEN, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const { environment, workspaceId: projectId, secretPath, secrets: inputSecrets } = req.body; + + const secrets = await server.services.secret.createManySecretsRaw({ + actorId: req.permission.id, + actor: req.permission.type, + actorAuthMethod: req.permission.authMethod, + actorOrgId: req.permission.orgId, + secretPath, + environment, + projectId, + secrets: inputSecrets + }); + + await server.services.auditLog.createAuditLog({ + projectId: req.body.workspaceId, + ...req.auditLogInfo, + event: { + type: EventType.CREATE_SECRETS, + metadata: { + environment: req.body.environment, + secretPath: req.body.secretPath, + secrets: secrets.map((secret, i) => ({ + secretId: secret.id, + secretKey: inputSecrets[i].secretKey, + secretVersion: secret.version + })) + } + } + }); + + await server.services.telemetry.sendPostHogEvents({ + event: PostHogEventTypes.SecretCreated, + distinctId: getTelemetryDistinctId(req), + properties: { + numberOfSecrets: secrets.length, + workspaceId: req.body.workspaceId, + environment: req.body.environment, + secretPath: req.body.secretPath, + channel: getUserAgentType(req.headers["user-agent"]), + ...req.auditLogInfo + } + }); + return { secrets }; + } + }); + + server.route({ + method: "PATCH", + url: "/batch/raw", + config: { + rateLimit: secretsLimit + }, + schema: { + description: "Update many secrets", + security: [ + { + bearerAuth: [] + } + ], + body: z.object({ + workspaceId: z.string().trim().describe(RAW_SECRETS.UPDATE.workspaceId), + environment: z.string().trim().describe(RAW_SECRETS.UPDATE.environment), + secretPath: z + .string() + .trim() + .default("/") + .transform(removeTrailingSlash) + .describe(RAW_SECRETS.UPDATE.secretPath), + secrets: z + .object({ + secretKey: z.string().trim().describe(RAW_SECRETS.UPDATE.secretName), + secretValue: z + .string() + .transform((val) => (val.at(-1) === "\n" ? `${val.trim()}\n` : val.trim())) + .describe(RAW_SECRETS.UPDATE.secretValue), + secretComment: z.string().trim().optional().describe(RAW_SECRETS.UPDATE.secretComment), + skipMultilineEncoding: z.boolean().optional().describe(RAW_SECRETS.UPDATE.skipMultilineEncoding) + }) + .array() + .min(1) + }), + response: { + 200: z.object({ + secrets: secretRawSchema.array() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY, AuthMode.SERVICE_TOKEN, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const { environment, workspaceId: projectId, secretPath, secrets: inputSecrets } = req.body; + const secrets = await server.services.secret.updateManySecretsRaw({ + actorId: req.permission.id, + actor: req.permission.type, + actorAuthMethod: req.permission.authMethod, + actorOrgId: req.permission.orgId, + secretPath, + environment, + projectId, + secrets: inputSecrets + }); + + await server.services.auditLog.createAuditLog({ + projectId: req.body.workspaceId, + ...req.auditLogInfo, + event: { + type: EventType.UPDATE_SECRETS, + metadata: { + environment: req.body.environment, + secretPath: req.body.secretPath, + secrets: secrets.map((secret, i) => ({ + secretId: secret.id, + secretKey: inputSecrets[i].secretKey, + secretVersion: secret.version + })) + } + } + }); + + await server.services.telemetry.sendPostHogEvents({ + event: PostHogEventTypes.SecretUpdated, + distinctId: getTelemetryDistinctId(req), + properties: { + numberOfSecrets: secrets.length, + workspaceId: req.body.workspaceId, + environment: req.body.environment, + secretPath: req.body.secretPath, + channel: getUserAgentType(req.headers["user-agent"]), + ...req.auditLogInfo + } + }); + return { secrets }; + } + }); + + server.route({ + method: "DELETE", + url: "/batch/raw", + config: { + rateLimit: secretsLimit + }, + schema: { + description: "Delete many secrets", + security: [ + { + bearerAuth: [] + } + ], + body: z.object({ + workspaceId: z.string().trim().describe(RAW_SECRETS.DELETE.workspaceId), + environment: z.string().trim().describe(RAW_SECRETS.DELETE.environment), + secretPath: z + .string() + .trim() + .default("/") + .transform(removeTrailingSlash) + .describe(RAW_SECRETS.DELETE.secretPath), + secrets: z + .object({ + secretKey: z.string().trim().describe(RAW_SECRETS.DELETE.secretName) + }) + .array() + .min(1) + }), + response: { + 200: z.object({ + secrets: secretRawSchema.array() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY, AuthMode.SERVICE_TOKEN, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const { environment, workspaceId: projectId, secretPath, secrets: inputSecrets } = req.body; + const secrets = await server.services.secret.deleteManySecretsRaw({ + actorId: req.permission.id, + actor: req.permission.type, + actorAuthMethod: req.permission.authMethod, + actorOrgId: req.permission.orgId, + environment, + projectId, + secretPath, + secrets: inputSecrets + }); + + await server.services.auditLog.createAuditLog({ + projectId: req.body.workspaceId, + ...req.auditLogInfo, + event: { + type: EventType.DELETE_SECRETS, + metadata: { + environment: req.body.environment, + secretPath: req.body.secretPath, + secrets: secrets.map((secret, i) => ({ + secretId: secret.id, + secretKey: inputSecrets[i].secretKey, + secretVersion: secret.version + })) + } + } + }); + + await server.services.telemetry.sendPostHogEvents({ + event: PostHogEventTypes.SecretDeleted, + distinctId: getTelemetryDistinctId(req), + properties: { + numberOfSecrets: secrets.length, + workspaceId: req.body.workspaceId, + environment: req.body.environment, + secretPath: req.body.secretPath, + channel: getUserAgentType(req.headers["user-agent"]), + ...req.auditLogInfo + } + }); + return { secrets }; + } + }); }; diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index 3b504fbf4..9c7c0cbd6 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -33,9 +33,11 @@ import { TSecretQueueFactory } from "./secret-queue"; import { TAttachSecretTagsDTO, TCreateBulkSecretDTO, + TCreateManySecretRawDTO, TCreateSecretDTO, TCreateSecretRawDTO, TDeleteBulkSecretDTO, + TDeleteManySecretRawDTO, TDeleteSecretDTO, TDeleteSecretRawDTO, TFnSecretBlindIndexCheckV2, @@ -46,6 +48,7 @@ import { TGetSecretsRawDTO, TGetSecretVersionsDTO, TUpdateBulkSecretDTO, + TUpdateManySecretRawDTO, TUpdateSecretDTO, TUpdateSecretRawDTO } from "./secret-types"; @@ -1036,6 +1039,131 @@ export const secretServiceFactory = ({ return decryptSecretRaw(secret, botKey); }; + const createManySecretsRaw = async ({ + actorId, + projectId, + environment, + actor, + actorOrgId, + actorAuthMethod, + secretPath, + secrets: inputSecrets = [] + }: TCreateManySecretRawDTO) => { + const botKey = await projectBotService.getBotKey(projectId); + if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" }); + + const secrets = await createManySecret({ + projectId, + environment, + path: secretPath, + actor, + actorId, + actorOrgId, + actorAuthMethod, + secrets: inputSecrets.map(({ secretComment, secretKey, secretValue, skipMultilineEncoding }) => { + const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secretKey, botKey); + const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secretValue || "", botKey); + const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secretComment || "", botKey); + return { + secretName: secretKey, + skipMultilineEncoding, + secretKeyCiphertext: secretKeyEncrypted.ciphertext, + secretKeyIV: secretKeyEncrypted.iv, + secretKeyTag: secretKeyEncrypted.tag, + secretValueCiphertext: secretValueEncrypted.ciphertext, + secretValueIV: secretValueEncrypted.iv, + secretValueTag: secretValueEncrypted.tag, + secretCommentCiphertext: secretCommentEncrypted.ciphertext, + secretCommentIV: secretCommentEncrypted.iv, + secretCommentTag: secretCommentEncrypted.tag + }; + }) + }); + + await snapshotService.performSnapshot(secrets[0].folderId); + await secretQueueService.syncSecrets({ secretPath, projectId, environment }); + + return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment }, botKey)); + }; + + const updateManySecretsRaw = async ({ + actorId, + projectId, + environment, + actor, + actorOrgId, + actorAuthMethod, + secretPath, + secrets: inputSecrets = [] + }: TUpdateManySecretRawDTO) => { + const botKey = await projectBotService.getBotKey(projectId); + if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" }); + + const secrets = await updateManySecret({ + projectId, + environment, + path: secretPath, + actor, + actorId, + actorOrgId, + actorAuthMethod, + secrets: inputSecrets.map(({ secretComment, secretKey, secretValue, skipMultilineEncoding }) => { + const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secretKey, botKey); + const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secretValue || "", botKey); + const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secretComment || "", botKey); + return { + secretName: secretKey, + type: SecretType.Shared, + skipMultilineEncoding, + secretKeyCiphertext: secretKeyEncrypted.ciphertext, + secretKeyIV: secretKeyEncrypted.iv, + secretKeyTag: secretKeyEncrypted.tag, + secretValueCiphertext: secretValueEncrypted.ciphertext, + secretValueIV: secretValueEncrypted.iv, + secretValueTag: secretValueEncrypted.tag, + secretCommentCiphertext: secretCommentEncrypted.ciphertext, + secretCommentIV: secretCommentEncrypted.iv, + secretCommentTag: secretCommentEncrypted.tag + }; + }) + }); + + await snapshotService.performSnapshot(secrets[0].folderId); + await secretQueueService.syncSecrets({ secretPath, projectId, environment }); + + return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment }, botKey)); + }; + + const deleteManySecretsRaw = async ({ + actorId, + projectId, + environment, + actor, + actorOrgId, + actorAuthMethod, + secretPath, + secrets: inputSecrets = [] + }: TDeleteManySecretRawDTO) => { + const botKey = await projectBotService.getBotKey(projectId); + if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" }); + + const secrets = await deleteManySecret({ + projectId, + environment, + path: secretPath, + actor, + actorId, + actorOrgId, + actorAuthMethod, + secrets: inputSecrets.map(({ secretKey }) => ({ secretName: secretKey, type: SecretType.Shared })) + }); + + await snapshotService.performSnapshot(secrets[0].folderId); + await secretQueueService.syncSecrets({ secretPath, projectId, environment }); + + return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment }, botKey)); + }; + const getSecretVersions = async ({ actorId, actor, @@ -1280,6 +1408,9 @@ export const secretServiceFactory = ({ createSecretRaw, updateSecretRaw, deleteSecretRaw, + createManySecretsRaw, + updateManySecretsRaw, + deleteManySecretsRaw, getSecretVersions, // external services function fnSecretBulkDelete, diff --git a/backend/src/services/secret/secret-types.ts b/backend/src/services/secret/secret-types.ts index 22347de4e..34b8bc822 100644 --- a/backend/src/services/secret/secret-types.ts +++ b/backend/src/services/secret/secret-types.ts @@ -181,6 +181,36 @@ export type TDeleteSecretRawDTO = TProjectPermission & { type: SecretType; }; +export type TCreateManySecretRawDTO = TProjectPermission & { + secretPath: string; + environment: string; + secrets: { + secretKey: string; + secretValue: string; + secretComment?: string; + skipMultilineEncoding?: boolean; + }[]; +}; + +export type TUpdateManySecretRawDTO = TProjectPermission & { + secretPath: string; + environment: string; + secrets: { + secretKey: string; + secretValue: string; + secretComment?: string; + skipMultilineEncoding?: boolean; + }[]; +}; + +export type TDeleteManySecretRawDTO = TProjectPermission & { + secretPath: string; + environment: string; + secrets: { + secretKey: string; + }[]; +}; + export type TGetSecretVersionsDTO = Omit & { limit?: number; offset?: number; From a339c473d5dd1730fa1a2343ac6b0922f7d20488 Mon Sep 17 00:00:00 2001 From: Akhil Mohan Date: Fri, 19 Apr 2024 20:54:41 +0530 Subject: [PATCH 02/18] docs: updated api doc with bulk raw secret ops --- docs/api-reference/endpoints/secrets/create-many.mdx | 8 ++++++++ docs/api-reference/endpoints/secrets/delete-many.mdx | 8 ++++++++ docs/api-reference/endpoints/secrets/update-many.mdx | 8 ++++++++ 3 files changed, 24 insertions(+) create mode 100644 docs/api-reference/endpoints/secrets/create-many.mdx create mode 100644 docs/api-reference/endpoints/secrets/delete-many.mdx create mode 100644 docs/api-reference/endpoints/secrets/update-many.mdx diff --git a/docs/api-reference/endpoints/secrets/create-many.mdx b/docs/api-reference/endpoints/secrets/create-many.mdx new file mode 100644 index 000000000..9b0609c0a --- /dev/null +++ b/docs/api-reference/endpoints/secrets/create-many.mdx @@ -0,0 +1,8 @@ +--- +title: "Bulk Create" +openapi: "POST /api/v3/secrets/batch/raw" +--- + + + This endpoint requires you to disable end-to-end encryption. For more information, you should consult this [note](https://infisical.com/docs/api-reference/overview/examples/note). + diff --git a/docs/api-reference/endpoints/secrets/delete-many.mdx b/docs/api-reference/endpoints/secrets/delete-many.mdx new file mode 100644 index 000000000..6477b2a98 --- /dev/null +++ b/docs/api-reference/endpoints/secrets/delete-many.mdx @@ -0,0 +1,8 @@ +--- +title: "Bulk Delete" +openapi: "DELETE /api/v3/secrets/batch/raw" +--- + + + This endpoint requires you to disable end-to-end encryption. For more information, you should consult this [note](https://infisical.com/docs/api-reference/overview/examples/note). + diff --git a/docs/api-reference/endpoints/secrets/update-many.mdx b/docs/api-reference/endpoints/secrets/update-many.mdx new file mode 100644 index 000000000..9feaf2ca2 --- /dev/null +++ b/docs/api-reference/endpoints/secrets/update-many.mdx @@ -0,0 +1,8 @@ +--- +title: "Bulk Update" +openapi: "PATCH /api/v3/secrets/batch/raw" +--- + + + This endpoint requires you to disable end-to-end encryption. For more information, you should consult this [note](https://infisical.com/docs/api-reference/overview/examples/note). + From aa5cd0fd0fc186cd491620ce543954500405bf0a Mon Sep 17 00:00:00 2001 From: Akhil Mohan Date: Mon, 22 Apr 2024 21:19:06 +0530 Subject: [PATCH 03/18] feat(server): switched from workspace id to project slug --- backend/e2e-test/routes/v3/secrets.spec.ts | 6 ++-- backend/src/lib/api-docs/constants.ts | 3 ++ backend/src/server/routes/v3/secret-router.ts | 30 +++++++++---------- backend/src/services/secret/secret-service.ts | 18 +++++++++-- backend/src/services/secret/secret-types.ts | 9 ++++-- 5 files changed, 42 insertions(+), 24 deletions(-) diff --git a/backend/e2e-test/routes/v3/secrets.spec.ts b/backend/e2e-test/routes/v3/secrets.spec.ts index ab73a7f1f..e7e271279 100644 --- a/backend/e2e-test/routes/v3/secrets.spec.ts +++ b/backend/e2e-test/routes/v3/secrets.spec.ts @@ -945,7 +945,7 @@ describe.each([{ auth: AuthMode.JWT }, { auth: AuthMode.IDENTITY_ACCESS_TOKEN }] test.each(testRawSecrets)("Bulk create secret raw in path $path", async ({ path, secret }) => { const createSecretReqBody = { - workspaceId: seedData1.project.id, + projectSlug: seedData1.project.slug, environment: seedData1.environment.slug, secretPath: path, secrets: [ @@ -986,7 +986,7 @@ describe.each([{ auth: AuthMode.JWT }, { auth: AuthMode.IDENTITY_ACCESS_TOKEN }] test.each(testRawSecrets)("Bulk update secret raw in path $path", async ({ secret, path }) => { await createRawSecret({ path, ...secret }); const updateSecretReqBody = { - workspaceId: seedData1.project.id, + projectSlug: seedData1.project.slug, environment: seedData1.environment.slug, secretPath: path, secrets: [ @@ -1028,7 +1028,7 @@ describe.each([{ auth: AuthMode.JWT }, { auth: AuthMode.IDENTITY_ACCESS_TOKEN }] await createRawSecret({ path, ...secret }); const deletedSecretReqBody = { - workspaceId: seedData1.project.id, + projectSlug: seedData1.project.slug, environment: seedData1.environment.slug, secretPath: path, secrets: [{ secretKey: secret.key }] diff --git a/backend/src/lib/api-docs/constants.ts b/backend/src/lib/api-docs/constants.ts index 4a98f73c8..9dbcc4fb4 100644 --- a/backend/src/lib/api-docs/constants.ts +++ b/backend/src/lib/api-docs/constants.ts @@ -282,6 +282,7 @@ export const RAW_SECRETS = { }, CREATE: { secretName: "The name of the secret to create.", + projectSlug: "The slug of the project to create the secret in.", environment: "The slug of the environment to create the secret in.", secretComment: "Attach a comment to the secret.", secretPath: "The path to create the secret in.", @@ -307,6 +308,7 @@ export const RAW_SECRETS = { secretValue: "The new value of the secret.", skipMultilineEncoding: "Skip multiline encoding for the secret value.", type: "The type of the secret to update.", + projectSlug: "The slug of the project to update the secret in.", workspaceId: "The ID of the project to update the secret in." }, DELETE: { @@ -314,6 +316,7 @@ export const RAW_SECRETS = { environment: "The slug of the environment where the secret is located.", secretPath: "The path of the secret.", type: "The type of the secret to delete.", + projectSlug: "The slug of the project to delete the secret in.", workspaceId: "The ID of the project where the secret is located." } } as const; diff --git a/backend/src/server/routes/v3/secret-router.ts b/backend/src/server/routes/v3/secret-router.ts index ceecc0805..955aa01be 100644 --- a/backend/src/server/routes/v3/secret-router.ts +++ b/backend/src/server/routes/v3/secret-router.ts @@ -1671,7 +1671,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { } ], body: z.object({ - workspaceId: z.string().trim().describe(RAW_SECRETS.CREATE.workspaceId), + projectSlug: z.string().trim().describe(RAW_SECRETS.CREATE.projectSlug), environment: z.string().trim().describe(RAW_SECRETS.CREATE.environment), secretPath: z .string() @@ -1700,7 +1700,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { }, onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY, AuthMode.SERVICE_TOKEN, AuthMode.IDENTITY_ACCESS_TOKEN]), handler: async (req) => { - const { environment, workspaceId: projectId, secretPath, secrets: inputSecrets } = req.body; + const { environment, projectSlug, secretPath, secrets: inputSecrets } = req.body; const secrets = await server.services.secret.createManySecretsRaw({ actorId: req.permission.id, @@ -1709,12 +1709,12 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { actorOrgId: req.permission.orgId, secretPath, environment, - projectId, + projectSlug, secrets: inputSecrets }); await server.services.auditLog.createAuditLog({ - projectId: req.body.workspaceId, + projectId: secrets[0].workspace, ...req.auditLogInfo, event: { type: EventType.CREATE_SECRETS, @@ -1735,7 +1735,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { distinctId: getTelemetryDistinctId(req), properties: { numberOfSecrets: secrets.length, - workspaceId: req.body.workspaceId, + workspaceId: secrets[0].workspace, environment: req.body.environment, secretPath: req.body.secretPath, channel: getUserAgentType(req.headers["user-agent"]), @@ -1760,7 +1760,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { } ], body: z.object({ - workspaceId: z.string().trim().describe(RAW_SECRETS.UPDATE.workspaceId), + projectSlug: z.string().trim().describe(RAW_SECRETS.UPDATE.projectSlug), environment: z.string().trim().describe(RAW_SECRETS.UPDATE.environment), secretPath: z .string() @@ -1789,7 +1789,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { }, onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY, AuthMode.SERVICE_TOKEN, AuthMode.IDENTITY_ACCESS_TOKEN]), handler: async (req) => { - const { environment, workspaceId: projectId, secretPath, secrets: inputSecrets } = req.body; + const { environment, projectSlug, secretPath, secrets: inputSecrets } = req.body; const secrets = await server.services.secret.updateManySecretsRaw({ actorId: req.permission.id, actor: req.permission.type, @@ -1797,12 +1797,12 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { actorOrgId: req.permission.orgId, secretPath, environment, - projectId, + projectSlug, secrets: inputSecrets }); await server.services.auditLog.createAuditLog({ - projectId: req.body.workspaceId, + projectId: secrets[0].workspace, ...req.auditLogInfo, event: { type: EventType.UPDATE_SECRETS, @@ -1823,7 +1823,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { distinctId: getTelemetryDistinctId(req), properties: { numberOfSecrets: secrets.length, - workspaceId: req.body.workspaceId, + workspaceId: secrets[0].workspace, environment: req.body.environment, secretPath: req.body.secretPath, channel: getUserAgentType(req.headers["user-agent"]), @@ -1848,7 +1848,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { } ], body: z.object({ - workspaceId: z.string().trim().describe(RAW_SECRETS.DELETE.workspaceId), + projectSlug: z.string().trim().describe(RAW_SECRETS.DELETE.projectSlug), environment: z.string().trim().describe(RAW_SECRETS.DELETE.environment), secretPath: z .string() @@ -1871,20 +1871,20 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { }, onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY, AuthMode.SERVICE_TOKEN, AuthMode.IDENTITY_ACCESS_TOKEN]), handler: async (req) => { - const { environment, workspaceId: projectId, secretPath, secrets: inputSecrets } = req.body; + const { environment, projectSlug, secretPath, secrets: inputSecrets } = req.body; const secrets = await server.services.secret.deleteManySecretsRaw({ actorId: req.permission.id, actor: req.permission.type, actorAuthMethod: req.permission.authMethod, actorOrgId: req.permission.orgId, environment, - projectId, + projectSlug, secretPath, secrets: inputSecrets }); await server.services.auditLog.createAuditLog({ - projectId: req.body.workspaceId, + projectId: secrets[0].workspace, ...req.auditLogInfo, event: { type: EventType.DELETE_SECRETS, @@ -1905,7 +1905,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { distinctId: getTelemetryDistinctId(req), properties: { numberOfSecrets: secrets.length, - workspaceId: req.body.workspaceId, + workspaceId: secrets[0].workspace, environment: req.body.environment, secretPath: req.body.secretPath, channel: getUserAgentType(req.headers["user-agent"]), diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index 9c7c0cbd6..01557f992 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -1041,7 +1041,7 @@ export const secretServiceFactory = ({ const createManySecretsRaw = async ({ actorId, - projectId, + projectSlug, environment, actor, actorOrgId, @@ -1049,6 +1049,10 @@ export const secretServiceFactory = ({ secretPath, secrets: inputSecrets = [] }: TCreateManySecretRawDTO) => { + const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId); + if (!project) throw new BadRequestError({ message: "Project not found" }); + const projectId = project.id; + const botKey = await projectBotService.getBotKey(projectId); if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" }); @@ -1088,7 +1092,7 @@ export const secretServiceFactory = ({ const updateManySecretsRaw = async ({ actorId, - projectId, + projectSlug, environment, actor, actorOrgId, @@ -1096,6 +1100,10 @@ export const secretServiceFactory = ({ secretPath, secrets: inputSecrets = [] }: TUpdateManySecretRawDTO) => { + const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId); + if (!project) throw new BadRequestError({ message: "Project not found" }); + const projectId = project.id; + const botKey = await projectBotService.getBotKey(projectId); if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" }); @@ -1136,7 +1144,7 @@ export const secretServiceFactory = ({ const deleteManySecretsRaw = async ({ actorId, - projectId, + projectSlug, environment, actor, actorOrgId, @@ -1144,6 +1152,10 @@ export const secretServiceFactory = ({ secretPath, secrets: inputSecrets = [] }: TDeleteManySecretRawDTO) => { + const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId); + if (!project) throw new BadRequestError({ message: "Project not found" }); + const projectId = project.id; + const botKey = await projectBotService.getBotKey(projectId); if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" }); diff --git a/backend/src/services/secret/secret-types.ts b/backend/src/services/secret/secret-types.ts index 34b8bc822..c2a0d5cf6 100644 --- a/backend/src/services/secret/secret-types.ts +++ b/backend/src/services/secret/secret-types.ts @@ -181,8 +181,9 @@ export type TDeleteSecretRawDTO = TProjectPermission & { type: SecretType; }; -export type TCreateManySecretRawDTO = TProjectPermission & { +export type TCreateManySecretRawDTO = Omit & { secretPath: string; + projectSlug: string; environment: string; secrets: { secretKey: string; @@ -192,8 +193,9 @@ export type TCreateManySecretRawDTO = TProjectPermission & { }[]; }; -export type TUpdateManySecretRawDTO = TProjectPermission & { +export type TUpdateManySecretRawDTO = Omit & { secretPath: string; + projectSlug: string; environment: string; secrets: { secretKey: string; @@ -203,8 +205,9 @@ export type TUpdateManySecretRawDTO = TProjectPermission & { }[]; }; -export type TDeleteManySecretRawDTO = TProjectPermission & { +export type TDeleteManySecretRawDTO = Omit & { secretPath: string; + projectSlug: string; environment: string; secrets: { secretKey: string; From 44ff1abd747068861f95a18cef0adf75ebf5b66a Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 23 Apr 2024 01:49:26 +0200 Subject: [PATCH 04/18] Update 20240405000045_org-memberships-unique-constraint.ts --- .../20240405000045_org-memberships-unique-constraint.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backend/src/db/migrations/20240405000045_org-memberships-unique-constraint.ts b/backend/src/db/migrations/20240405000045_org-memberships-unique-constraint.ts index 9a342d542..1743c7524 100644 --- a/backend/src/db/migrations/20240405000045_org-memberships-unique-constraint.ts +++ b/backend/src/db/migrations/20240405000045_org-memberships-unique-constraint.ts @@ -42,6 +42,7 @@ export async function up(knex: Knex): Promise { await knex.transaction(async (tx) => { const duplicateRows = await tx(TableName.OrgMembership) .select("userId", "orgId") // Select the userId and orgId so we can group by them + .whereNotNull("userId") // Ensure that the userId is not null .count("* as cnt") // Count the number of rows for each userId and orgId, so we can make sure there are more than 1 row (a duplicate) .groupBy("userId", "orgId") .havingRaw("count(*) > ?", [1]); // Using havingRaw for direct SQL expressions @@ -97,6 +98,8 @@ export async function up(knex: Knex): Promise { `Deleted ${numberOfRowsDeleted} duplicate organization memberships for ${row.userId} and ${row.orgId}` ); } + + throw new Error("This is a test error"); }); await knex.schema.alterTable(TableName.OrgMembership, (table) => { From 233a4f7d77f9cd55072aa010e4e618d6b1f74fa7 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 23 Apr 2024 01:49:44 +0200 Subject: [PATCH 05/18] Update 20240405000045_org-memberships-unique-constraint.ts --- .../20240405000045_org-memberships-unique-constraint.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/backend/src/db/migrations/20240405000045_org-memberships-unique-constraint.ts b/backend/src/db/migrations/20240405000045_org-memberships-unique-constraint.ts index 1743c7524..b97c024de 100644 --- a/backend/src/db/migrations/20240405000045_org-memberships-unique-constraint.ts +++ b/backend/src/db/migrations/20240405000045_org-memberships-unique-constraint.ts @@ -98,8 +98,6 @@ export async function up(knex: Knex): Promise { `Deleted ${numberOfRowsDeleted} duplicate organization memberships for ${row.userId} and ${row.orgId}` ); } - - throw new Error("This is a test error"); }); await knex.schema.alterTable(TableName.OrgMembership, (table) => { From f2b3b7b72609c2d87fc90c7b1b8e6be7961a7e55 Mon Sep 17 00:00:00 2001 From: Akhil Mohan Date: Tue, 23 Apr 2024 11:23:03 +0530 Subject: [PATCH 06/18] docs: added -y flag in infisical cli installation in amplify doc to skip confirmation prompt --- docs/integrations/cloud/aws-amplify.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/cloud/aws-amplify.mdx b/docs/integrations/cloud/aws-amplify.mdx index 9edb5cf33..825c6c356 100644 --- a/docs/integrations/cloud/aws-amplify.mdx +++ b/docs/integrations/cloud/aws-amplify.mdx @@ -33,7 +33,7 @@ This approach enables you to fetch secrets from Infisical during Amplify build t preBuild: commands: - sudo curl -1sLf 'https://dl.cloudsmith.io/public/infisical/infisical-cli/setup.rpm.sh' | sudo -E bash - - sudo yum install infisical + - sudo yum -y install infisical ``` From ad354c106e213e9e974299dc8e4d50bc33082ba3 Mon Sep 17 00:00:00 2001 From: Maidul Islam Date: Tue, 23 Apr 2024 13:56:12 -0400 Subject: [PATCH 07/18] update folder not found error message --- .../secret-approval-request-service.ts | 6 ++- backend/src/services/secret/secret-fns.ts | 12 +++++- backend/src/services/secret/secret-service.ts | 42 +++++++++++++++---- 3 files changed, 50 insertions(+), 10 deletions(-) diff --git a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts index b8ad89e45..2e66ab2ce 100644 --- a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts +++ b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts @@ -495,7 +495,11 @@ export const secretApprovalRequestServiceFactory = ({ await projectDAL.checkProjectUpgradeStatus(projectId); const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); - if (!folder) throw new BadRequestError({ message: "Folder not found", name: "GenSecretApproval" }); + if (!folder) + throw new BadRequestError({ + message: "Folder not found for the given environment slug & secret path", + name: "GenSecretApproval" + }); const folderId = folder.id; const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId }); diff --git a/backend/src/services/secret/secret-fns.ts b/backend/src/services/secret/secret-fns.ts index 2bfda2cbe..fb2b90ba9 100644 --- a/backend/src/services/secret/secret-fns.ts +++ b/backend/src/services/secret/secret-fns.ts @@ -575,7 +575,11 @@ export const createManySecretsRawFnFactory = ({ await projectDAL.checkProjectUpgradeStatus(projectId); const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); - if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Create secret" }); + if (!folder) + throw new BadRequestError({ + message: "Folder not found for the given environment slug & secret path", + name: "Create secret" + }); const folderId = folder.id; const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId }); @@ -680,7 +684,11 @@ export const updateManySecretsRawFnFactory = ({ await projectDAL.checkProjectUpgradeStatus(projectId); const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); - if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Update secret" }); + if (!folder) + throw new BadRequestError({ + message: "Folder not found for the given environment slug & secret path", + name: "Update secret" + }); const folderId = folder.id; const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId }); diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index 3b504fbf4..7c105f2f0 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -179,7 +179,11 @@ export const secretServiceFactory = ({ await projectDAL.checkProjectUpgradeStatus(projectId); const folder = await folderDAL.findBySecretPath(projectId, environment, path); - if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Create secret" }); + if (!folder) + throw new BadRequestError({ + message: "Folder not found for the given environment slug & secret path", + name: "Create secret" + }); const folderId = folder.id; const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId }); @@ -275,7 +279,11 @@ export const secretServiceFactory = ({ } const folder = await folderDAL.findBySecretPath(projectId, environment, path); - if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Create secret" }); + if (!folder) + throw new BadRequestError({ + message: "Folder not found for the given environment slug & secret path", + name: "Create secret" + }); const folderId = folder.id; const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId }); @@ -391,7 +399,11 @@ export const secretServiceFactory = ({ await projectDAL.checkProjectUpgradeStatus(projectId); const folder = await folderDAL.findBySecretPath(projectId, environment, path); - if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Create secret" }); + if (!folder) + throw new BadRequestError({ + message: "Folder not found for the given environment slug & secret path", + name: "Create secret" + }); const folderId = folder.id; const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId }); @@ -559,7 +571,11 @@ export const secretServiceFactory = ({ subject(ProjectPermissionSub.Secrets, { environment, secretPath: path }) ); const folder = await folderDAL.findBySecretPath(projectId, environment, path); - if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Create secret" }); + if (!folder) + throw new BadRequestError({ + message: "Folder not found for the given environment slug & secret path", + name: "Create secret" + }); const folderId = folder.id; const secretBlindIndex = await interalGenSecBlindIndexByName(projectId, secretName); @@ -655,7 +671,11 @@ export const secretServiceFactory = ({ await projectDAL.checkProjectUpgradeStatus(projectId); const folder = await folderDAL.findBySecretPath(projectId, environment, path); - if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Create secret" }); + if (!folder) + throw new BadRequestError({ + message: "Folder not found for the given environment slug & secret path", + name: "Create secret" + }); const folderId = folder.id; const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId }); @@ -724,7 +744,11 @@ export const secretServiceFactory = ({ await projectDAL.checkProjectUpgradeStatus(projectId); const folder = await folderDAL.findBySecretPath(projectId, environment, path); - if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Update secret" }); + if (!folder) + throw new BadRequestError({ + message: "Folder not found for the given environment slug & secret path", + name: "Update secret" + }); const folderId = folder.id; const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId }); @@ -810,7 +834,11 @@ export const secretServiceFactory = ({ await projectDAL.checkProjectUpgradeStatus(projectId); const folder = await folderDAL.findBySecretPath(projectId, environment, path); - if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Create secret" }); + if (!folder) + throw new BadRequestError({ + message: "Folder not found for the given environment slug & secret path", + name: "Create secret" + }); const folderId = folder.id; const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId }); From 246b8728a418db17c54eb867182163ea98778aba Mon Sep 17 00:00:00 2001 From: Maidul Islam Date: Tue, 23 Apr 2024 14:49:01 -0400 Subject: [PATCH 08/18] add patroni gha --- .../workflows/build-patroni-docker-img.yml | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 .github/workflows/build-patroni-docker-img.yml diff --git a/.github/workflows/build-patroni-docker-img.yml b/.github/workflows/build-patroni-docker-img.yml new file mode 100644 index 000000000..4ee99f27a --- /dev/null +++ b/.github/workflows/build-patroni-docker-img.yml @@ -0,0 +1,38 @@ +name: Build patroni +on: [workflow_dispatch] + +jobs: + patroni-image: + name: Build patroni + runs-on: ubuntu-latest + steps: + - name: ☁️ Checkout source + uses: actions/checkout@v3 + with: + repository: 'zalando/patroni' + - name: Save commit hashes for tag + id: commit + uses: pr-mpt/actions-commit-hash@v2 + - name: 🔧 Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: 🐋 Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Set up Depot CLI + uses: depot/setup-action@v1 + - name: 🏗️ Build backend and push to docker hub + uses: depot/build-push-action@v1 + with: + project: 64mmf0n610 + token: ${{ secrets.DEPOT_PROJECT_TOKEN }} + push: true + context: . + file: Dockerfile + tags: | + infisical/patroni:${{ steps.commit.outputs.short }} + infisical/patroni:latest + platforms: linux/amd64,linux/arm64 + + \ No newline at end of file From b94ffb8a8212c1ebbd6b4f21f326bab867cf4ee6 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 23 Apr 2024 22:00:32 +0200 Subject: [PATCH 09/18] Fix: UA Token being overwritten by INFISICAL_TOKEN env variable --- cli/packages/cmd/secrets.go | 5 ++++- cli/packages/util/secrets.go | 4 ---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/cli/packages/cmd/secrets.go b/cli/packages/cmd/secrets.go index f2f694075..305a1f0fd 100644 --- a/cli/packages/cmd/secrets.go +++ b/cli/packages/cmd/secrets.go @@ -7,6 +7,7 @@ import ( "crypto/sha256" "encoding/base64" "fmt" + "os" "regexp" "sort" "strings" @@ -204,8 +205,10 @@ var secretsSetCmd = &cobra.Command{ // decrypt workspace key plainTextEncryptionKey := crypto.DecryptAsymmetric(encryptedWorkspaceKey, encryptedWorkspaceKeyNonce, encryptedWorkspaceKeySenderPublicKey, currentUsersPrivateKey) + infisicalTokenEnv := os.Getenv(util.INFISICAL_TOKEN_NAME) + // pull current secrets - secrets, err := util.GetAllEnvironmentVariables(models.GetAllSecretsParameters{Environment: environmentName, SecretsPath: secretsPath}, "") + secrets, err := util.GetAllEnvironmentVariables(models.GetAllSecretsParameters{Environment: environmentName, SecretsPath: secretsPath, InfisicalToken: infisicalTokenEnv}, "") if err != nil { util.HandleError(err, "unable to retrieve secrets") } diff --git a/cli/packages/util/secrets.go b/cli/packages/util/secrets.go index bedd77572..27f0636a9 100644 --- a/cli/packages/util/secrets.go +++ b/cli/packages/util/secrets.go @@ -307,10 +307,6 @@ func FilterSecretsByTag(plainTextSecrets []models.SingleEnvironmentVariable, tag } func GetAllEnvironmentVariables(params models.GetAllSecretsParameters, projectConfigFilePath string) ([]models.SingleEnvironmentVariable, error) { - if params.InfisicalToken == "" { - params.InfisicalToken = os.Getenv(INFISICAL_TOKEN_NAME) - } - isConnected := CheckIsConnectedToInternet() var secretsToReturn []models.SingleEnvironmentVariable // var serviceTokenDetails api.GetServiceTokenDetailsResponse From 7a3d425b0e228d4fde221bd74bc674bf78a49356 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 23 Apr 2024 22:20:43 +0200 Subject: [PATCH 10/18] Fix: Undefined env variables --- .github/workflows/run-cli-tests.yml | 19 +++++++++++++++---- .github/workflows/test-workflow.yml | 25 +++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/test-workflow.yml diff --git a/.github/workflows/run-cli-tests.yml b/.github/workflows/run-cli-tests.yml index 846998528..840a0c69f 100644 --- a/.github/workflows/run-cli-tests.yml +++ b/.github/workflows/run-cli-tests.yml @@ -1,12 +1,23 @@ name: Go CLI Tests on: - pull_request: - types: [opened, synchronize] - paths: - - "cli/**" + #pull_request: + # types: [opened, synchronize] + # paths: + # - "cli/**" workflow_call: + secrets: + CLI_TESTS_UA_CLIENT_ID: + required: true + CLI_TESTS_UA_CLIENT_SECRET: + required: true + CLI_TESTS_SERVICE_TOKEN: + required: true + CLI_TESTS_PROJECT_ID: + required: true + CLI_TESTS_ENV_SLUG: + required: true jobs: test: diff --git a/.github/workflows/test-workflow.yml b/.github/workflows/test-workflow.yml new file mode 100644 index 000000000..e745684fe --- /dev/null +++ b/.github/workflows/test-workflow.yml @@ -0,0 +1,25 @@ +name: Go CLI Tests + +on: + pull_request: + types: [opened, synchronize] + paths: + - "cli/**" + +jobs: + call-workflow: + uses: ./.github/workflows/run-cli-tests.yml + secrets: + CLI_TESTS_UA_CLIENT_ID: ${{ secrets.CLI_TESTS_UA_CLIENT_ID }} + CLI_TESTS_UA_CLIENT_SECRET: ${{ secrets.CLI_TESTS_UA_CLIENT_SECRET }} + CLI_TESTS_SERVICE_TOKEN: ${{ secrets.CLI_TESTS_SERVICE_TOKEN }} + CLI_TESTS_PROJECT_ID: ${{ secrets.CLI_TESTS_PROJECT_ID }} + CLI_TESTS_ENV_SLUG: ${{ secrets.CLI_TESTS_ENV_SLUG }} + + test: + name: Go CLI Tests + runs-on: ubuntu-latest + needs: [call-workflow] + steps: + - name: Hello world + run: echo "Hello world" From b7893a6a72bb830cfb25489badcb790c9e569d9e Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 23 Apr 2024 22:21:32 +0200 Subject: [PATCH 11/18] Update test-workflow.yml --- .github/workflows/test-workflow.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-workflow.yml b/.github/workflows/test-workflow.yml index e745684fe..39f1fdce1 100644 --- a/.github/workflows/test-workflow.yml +++ b/.github/workflows/test-workflow.yml @@ -3,8 +3,8 @@ name: Go CLI Tests on: pull_request: types: [opened, synchronize] - paths: - - "cli/**" + # paths: + # - "cli/**" jobs: call-workflow: From c44c7810cefd269cafbdb65fb963db41c6b468b0 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 23 Apr 2024 22:24:17 +0200 Subject: [PATCH 12/18] Fix: CLI Tests failing when called as a dependency workflow --- .../workflows/release_build_infisical_cli.yml | 6 +++++ .github/workflows/run-cli-tests.yml | 8 +++--- .github/workflows/test-workflow.yml | 25 ------------------- 3 files changed, 10 insertions(+), 29 deletions(-) delete mode 100644 .github/workflows/test-workflow.yml diff --git a/.github/workflows/release_build_infisical_cli.yml b/.github/workflows/release_build_infisical_cli.yml index 40137a455..4a0da7f69 100644 --- a/.github/workflows/release_build_infisical_cli.yml +++ b/.github/workflows/release_build_infisical_cli.yml @@ -14,6 +14,12 @@ jobs: cli-integration-tests: name: Run tests before deployment uses: ./.github/workflows/run-cli-tests.yml + secrets: + CLI_TESTS_UA_CLIENT_ID: ${{ secrets.CLI_TESTS_UA_CLIENT_ID }} + CLI_TESTS_UA_CLIENT_SECRET: ${{ secrets.CLI_TESTS_UA_CLIENT_SECRET }} + CLI_TESTS_SERVICE_TOKEN: ${{ secrets.CLI_TESTS_SERVICE_TOKEN }} + CLI_TESTS_PROJECT_ID: ${{ secrets.CLI_TESTS_PROJECT_ID }} + CLI_TESTS_ENV_SLUG: ${{ secrets.CLI_TESTS_ENV_SLUG }} goreleaser: runs-on: ubuntu-20.04 diff --git a/.github/workflows/run-cli-tests.yml b/.github/workflows/run-cli-tests.yml index 840a0c69f..bca989e37 100644 --- a/.github/workflows/run-cli-tests.yml +++ b/.github/workflows/run-cli-tests.yml @@ -1,10 +1,10 @@ name: Go CLI Tests on: - #pull_request: - # types: [opened, synchronize] - # paths: - # - "cli/**" + pull_request: + types: [opened, synchronize] + paths: + - "cli/**" workflow_call: secrets: diff --git a/.github/workflows/test-workflow.yml b/.github/workflows/test-workflow.yml deleted file mode 100644 index 39f1fdce1..000000000 --- a/.github/workflows/test-workflow.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Go CLI Tests - -on: - pull_request: - types: [opened, synchronize] - # paths: - # - "cli/**" - -jobs: - call-workflow: - uses: ./.github/workflows/run-cli-tests.yml - secrets: - CLI_TESTS_UA_CLIENT_ID: ${{ secrets.CLI_TESTS_UA_CLIENT_ID }} - CLI_TESTS_UA_CLIENT_SECRET: ${{ secrets.CLI_TESTS_UA_CLIENT_SECRET }} - CLI_TESTS_SERVICE_TOKEN: ${{ secrets.CLI_TESTS_SERVICE_TOKEN }} - CLI_TESTS_PROJECT_ID: ${{ secrets.CLI_TESTS_PROJECT_ID }} - CLI_TESTS_ENV_SLUG: ${{ secrets.CLI_TESTS_ENV_SLUG }} - - test: - name: Go CLI Tests - runs-on: ubuntu-latest - needs: [call-workflow] - steps: - - name: Hello world - run: echo "Hello world" From b330c5570d9b78871ab2e2e64c08d1fe5f69ce0a Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Wed, 24 Apr 2024 00:06:35 +0200 Subject: [PATCH 13/18] Allow trigger through Github UI --- .github/workflows/run-cli-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/run-cli-tests.yml b/.github/workflows/run-cli-tests.yml index bca989e37..e814f9143 100644 --- a/.github/workflows/run-cli-tests.yml +++ b/.github/workflows/run-cli-tests.yml @@ -6,6 +6,8 @@ on: paths: - "cli/**" + workflow_dispatch: + workflow_call: secrets: CLI_TESTS_UA_CLIENT_ID: From cbd568b71469ed2d45c81c3b384cce12236bf906 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Wed, 24 Apr 2024 00:18:25 +0200 Subject: [PATCH 14/18] Update release_build_infisical_cli.yml --- .../workflows/release_build_infisical_cli.yml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release_build_infisical_cli.yml b/.github/workflows/release_build_infisical_cli.yml index 4a0da7f69..23a3fcb24 100644 --- a/.github/workflows/release_build_infisical_cli.yml +++ b/.github/workflows/release_build_infisical_cli.yml @@ -1,6 +1,8 @@ name: Build and release CLI on: + workflow_dispatch: + push: # run only against tags tags: @@ -12,7 +14,7 @@ permissions: # issues: write jobs: cli-integration-tests: - name: Run tests before deployment + name: CLI Integration Tests uses: ./.github/workflows/run-cli-tests.yml secrets: CLI_TESTS_UA_CLIENT_ID: ${{ secrets.CLI_TESTS_UA_CLIENT_ID }} @@ -21,9 +23,17 @@ jobs: CLI_TESTS_PROJECT_ID: ${{ secrets.CLI_TESTS_PROJECT_ID }} CLI_TESTS_ENV_SLUG: ${{ secrets.CLI_TESTS_ENV_SLUG }} - goreleaser: - runs-on: ubuntu-20.04 + run-tests: + name: Run tests before deployment needs: [cli-integration-tests] + runs-on: ubuntu-latest + + # only run if the event is not a workflow_dispatch + goreleaser: + if: github.event_name != 'workflow_dispatch' + name: Build and release CLI + runs-on: ubuntu-20.04 + needs: [run-tests] steps: - uses: actions/checkout@v3 with: From f27d9f8ceeac185db2f6602238ccc5a03277abbf Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Wed, 24 Apr 2024 00:21:46 +0200 Subject: [PATCH 15/18] Update release_build_infisical_cli.yml --- .github/workflows/release_build_infisical_cli.yml | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/.github/workflows/release_build_infisical_cli.yml b/.github/workflows/release_build_infisical_cli.yml index 23a3fcb24..e4a5945e0 100644 --- a/.github/workflows/release_build_infisical_cli.yml +++ b/.github/workflows/release_build_infisical_cli.yml @@ -14,7 +14,7 @@ permissions: # issues: write jobs: cli-integration-tests: - name: CLI Integration Tests + name: Run tests before deployment uses: ./.github/workflows/run-cli-tests.yml secrets: CLI_TESTS_UA_CLIENT_ID: ${{ secrets.CLI_TESTS_UA_CLIENT_ID }} @@ -23,17 +23,9 @@ jobs: CLI_TESTS_PROJECT_ID: ${{ secrets.CLI_TESTS_PROJECT_ID }} CLI_TESTS_ENV_SLUG: ${{ secrets.CLI_TESTS_ENV_SLUG }} - run-tests: - name: Run tests before deployment - needs: [cli-integration-tests] - runs-on: ubuntu-latest - - # only run if the event is not a workflow_dispatch goreleaser: - if: github.event_name != 'workflow_dispatch' - name: Build and release CLI runs-on: ubuntu-20.04 - needs: [run-tests] + needs: [cli-integration-tests] steps: - uses: actions/checkout@v3 with: From e5333e271828235ac1ed706352ae72ab822990c4 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Wed, 24 Apr 2024 02:07:45 +0200 Subject: [PATCH 16/18] Fix: UA token being overwritten by service token --- cli/packages/util/folders.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/cli/packages/util/folders.go b/cli/packages/util/folders.go index 165b97534..82e5451a5 100644 --- a/cli/packages/util/folders.go +++ b/cli/packages/util/folders.go @@ -2,7 +2,6 @@ package util import ( "fmt" - "os" "strings" "github.com/Infisical/infisical-merge/packages/api" @@ -13,10 +12,6 @@ import ( func GetAllFolders(params models.GetAllFoldersParameters) ([]models.SingleFolder, error) { - if params.InfisicalToken == "" { - params.InfisicalToken = os.Getenv(INFISICAL_TOKEN_NAME) - } - var foldersToReturn []models.SingleFolder var folderErr error if params.InfisicalToken == "" && params.UniversalAuthAccessToken == "" { From f04f3aee2555edb3f4b45e35479a26ce531d3ce7 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Wed, 24 Apr 2024 02:36:29 +0200 Subject: [PATCH 17/18] Fix: Allow service token & UA access token to be used as authentication --- cli/packages/cmd/folder.go | 4 ---- cli/packages/util/folders.go | 2 ++ 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/cli/packages/cmd/folder.go b/cli/packages/cmd/folder.go index b306960f4..9cb76a312 100644 --- a/cli/packages/cmd/folder.go +++ b/cli/packages/cmd/folder.go @@ -22,10 +22,6 @@ var folderCmd = &cobra.Command{ var getCmd = &cobra.Command{ Use: "get", Short: "Get folders in a directory", - PersistentPreRun: func(cmd *cobra.Command, args []string) { - util.RequireLocalWorkspaceFile() - util.RequireLogin() - }, Run: func(cmd *cobra.Command, args []string) { environmentName, _ := cmd.Flags().GetString("env") diff --git a/cli/packages/util/folders.go b/cli/packages/util/folders.go index 82e5451a5..18fe5c888 100644 --- a/cli/packages/util/folders.go +++ b/cli/packages/util/folders.go @@ -15,6 +15,8 @@ func GetAllFolders(params models.GetAllFoldersParameters) ([]models.SingleFolder var foldersToReturn []models.SingleFolder var folderErr error if params.InfisicalToken == "" && params.UniversalAuthAccessToken == "" { + RequireLogin() + RequireLocalWorkspaceFile() log.Debug().Msg("GetAllFolders: Trying to fetch folders using logged in details") From c61602370eba8956d39ab61c8d0c3b2b6ace9350 Mon Sep 17 00:00:00 2001 From: vmatsiiako <78047717+vmatsiiako@users.noreply.github.com> Date: Tue, 23 Apr 2024 19:32:26 -0700 Subject: [PATCH 18/18] Update kubernetes-helm.mdx --- docs/self-hosting/deployment-options/kubernetes-helm.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/self-hosting/deployment-options/kubernetes-helm.mdx b/docs/self-hosting/deployment-options/kubernetes-helm.mdx index b95a3fd4d..d127dbfb1 100644 --- a/docs/self-hosting/deployment-options/kubernetes-helm.mdx +++ b/docs/self-hosting/deployment-options/kubernetes-helm.mdx @@ -22,7 +22,7 @@ description: "Learn how to use Helm chart to install Infisical on your Kubernete By default, the Infisical version set in your helm chart will likely be outdated. - Choose the latest Infisical docker image tag from here [here](https://hub.docker.com/r/infisical/infisical/tags). + Choose the latest Infisical docker image tag from [here](https://hub.docker.com/r/infisical/infisical/tags). ```yaml values.yaml