diff --git a/backend/e2e-test/routes/v3/secrets-v2.spec.ts b/backend/e2e-test/routes/v3/secrets-v2.spec.ts new file mode 100644 index 000000000..70066c061 --- /dev/null +++ b/backend/e2e-test/routes/v3/secrets-v2.spec.ts @@ -0,0 +1,576 @@ +import { SecretType } from "@app/db/schemas"; +import { seedData1 } from "@app/db/seed-data"; +import { AuthMode } from "@app/services/auth/auth-type"; + +type TRawSecret = { + secretKey: string; + secretValue: string; + secretComment?: string; + version: number; +}; +const createSecret = async (dto: { path: string; key: string; value: string; comment: string; type?: SecretType }) => { + const createSecretReqBody = { + workspaceId: seedData1.projectV3.id, + environment: seedData1.environment.slug, + type: dto.type || SecretType.Shared, + secretPath: dto.path, + secretKey: dto.key, + secretValue: dto.value, + secretComment: dto.comment + }; + const createSecRes = await testServer.inject({ + method: "POST", + url: `/api/v3/secrets/raw/${dto.key}`, + headers: { + authorization: `Bearer ${jwtAuthToken}` + }, + body: createSecretReqBody + }); + expect(createSecRes.statusCode).toBe(200); + const createdSecretPayload = JSON.parse(createSecRes.payload); + expect(createdSecretPayload).toHaveProperty("secret"); + return createdSecretPayload.secret as TRawSecret; +}; + +const deleteSecret = async (dto: { path: string; key: string }) => { + const deleteSecRes = await testServer.inject({ + method: "DELETE", + url: `/api/v3/secrets/raw/${dto.key}`, + headers: { + authorization: `Bearer ${jwtAuthToken}` + }, + body: { + workspaceId: seedData1.projectV3.id, + environment: seedData1.environment.slug, + secretPath: dto.path + } + }); + expect(deleteSecRes.statusCode).toBe(200); + const updatedSecretPayload = JSON.parse(deleteSecRes.payload); + expect(updatedSecretPayload).toHaveProperty("secret"); + return updatedSecretPayload.secret as TRawSecret; +}; + +describe.each([{ auth: AuthMode.JWT }, { auth: AuthMode.IDENTITY_ACCESS_TOKEN }])( + "Secret V2 Architecture - $auth mode", + async ({ auth }) => { + let folderId = ""; + let authToken = ""; + const secretTestCases = [ + { + path: "/", + secret: { + key: "SEC1", + value: "something-secret", + comment: "some comment" + } + }, + { + path: "/nested1/nested2/folder", + secret: { + key: "NESTED-SEC1", + value: "something-secret", + comment: "some comment" + } + }, + { + path: "/", + secret: { + key: "secret-key-2", + value: `-----BEGIN PRIVATE KEY----- + MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCa6eeFk+cMVqFn + hoVQDYgn2Ptp5Azysr2UPq6P73pCL9BzUtOXKZROqDyGehzzfg3wE2KdYU1Jk5Uq + fP0ZOWDIlM2SaVCSI3FW32o5+ZiggjpqcVdLFc/PS0S/ZdSmpPd8h11iO2brtIAI + ugTW8fcKlGSNUwx9aFmE7A6JnTRliTxB1l6QaC+YAwTK39VgeVH2gDSWC407aS15 + QobAkaBKKmFkzB5D7i2ZJwt+uXJV/rbLmyDmtnw0lubciGn7NX9wbYef180fisqT + aPNAz0nPKk0fFH2Wd5MZixNGbrrpDA+FCYvI5doThZyT2hpj08qWP07oXXCAqw46 + IEupNSILAgMBAAECggEBAIJb5KzeaiZS3B3O8G4OBQ5rJB3WfyLYUHnoSWLsBbie + nc392/ovThLmtZAAQE6SO85Tsb93+t64Z2TKqv1H8G658UeMgfWIB78v4CcLJ2mi + TN/3opqXrzjkQOTDHzBgT7al/mpETHZ6fOdbCemK0fVALGFUioUZg4M8VXtuI4Jw + q28jAyoRKrCrzda4BeQ553NZ4G5RvwhX3O2I8B8upTbt5hLcisBKy8MPLYY5LUFj + YKAP+raf6QLliP6KYHuVxUlgzxjLTxVG41etcyqqZF+foyiKBO3PU3n8oh++tgQP + ExOxiR0JSkBG5b+oOBD0zxcvo3/SjBHn0dJOZCSU2SkCgYEAyCe676XnNyBZMRD7 + 6trsaoiCWBpA6M8H44+x3w4cQFtqV38RyLy60D+iMKjIaLqeBbnay61VMzo24Bz3 + EuF2n4+9k/MetLJ0NCw8HmN5k0WSMD2BFsJWG8glVbzaqzehP4tIclwDTYc1jQVt + IoV2/iL7HGT+x2daUwbU5kN5hK0CgYEAxiLB+fmjxJW7VY4SHDLqPdpIW0q/kv4K + d/yZBrCX799vjmFb9vLh7PkQUfJhMJ/ttJOd7EtT3xh4mfkBeLfHwVU0d/ahbmSH + UJu/E9ZGxAW3PP0kxHZtPrLKQwBnfq8AxBauIhR3rPSorQTIOKtwz1jMlHFSUpuL + 3KeK2YfDYJcCgYEAkQnJOlNcAuRb/WQzSHIvktssqK8NjiZHryy3Vc0hx7j2jES2 + HGI2dSVHYD9OSiXA0KFm3OTTsnViwm/60iGzFdjRJV6tR39xGUVcoyCuPnvRfUd0 + PYvBXgxgkYpyYlPDcwp5CvWGJy3tLi1acgOIwIuUr3S38sL//t4adGk8q1kCgYB8 + Jbs1Tl53BvrimKpwUNbE+sjrquJu0A7vL68SqgQJoQ7dP9PH4Ff/i+/V6PFM7mib + BQOm02wyFbs7fvKVGVJoqWK+6CIucX732x7W5yRgHtS5ukQXdbzt1Ek3wkEW98Cb + HTruz7RNAt/NyXlLSODeit1lBbx3Vk9EaxZtRsv88QKBgGn7JwXgez9NOyobsNIo + QVO80rpUeenSjuFi+R0VmbLKe/wgAQbYJ0xTAsQ0btqViMzB27D6mJyC+KUIwWNX + MN8a+m46v4kqvZkKL2c4gmDibyURNe/vCtCHFuanJS/1mo2tr4XDyEeiuK52eTd9 + omQDpP86RX/hIIQ+JyLSaWYa + -----END PRIVATE KEY-----`, + comment: + "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation" + } + }, + { + path: "/nested1/nested2/folder", + secret: { + key: "secret-key-3", + value: `-----BEGIN PRIVATE KEY----- + MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCa6eeFk+cMVqFn + hoVQDYgn2Ptp5Azysr2UPq6P73pCL9BzUtOXKZROqDyGehzzfg3wE2KdYU1Jk5Uq + fP0ZOWDIlM2SaVCSI3FW32o5+ZiggjpqcVdLFc/PS0S/ZdSmpPd8h11iO2brtIAI + ugTW8fcKlGSNUwx9aFmE7A6JnTRliTxB1l6QaC+YAwTK39VgeVH2gDSWC407aS15 + QobAkaBKKmFkzB5D7i2ZJwt+uXJV/rbLmyDmtnw0lubciGn7NX9wbYef180fisqT + aPNAz0nPKk0fFH2Wd5MZixNGbrrpDA+FCYvI5doThZyT2hpj08qWP07oXXCAqw46 + IEupNSILAgMBAAECggEBAIJb5KzeaiZS3B3O8G4OBQ5rJB3WfyLYUHnoSWLsBbie + nc392/ovThLmtZAAQE6SO85Tsb93+t64Z2TKqv1H8G658UeMgfWIB78v4CcLJ2mi + TN/3opqXrzjkQOTDHzBgT7al/mpETHZ6fOdbCemK0fVALGFUioUZg4M8VXtuI4Jw + q28jAyoRKrCrzda4BeQ553NZ4G5RvwhX3O2I8B8upTbt5hLcisBKy8MPLYY5LUFj + YKAP+raf6QLliP6KYHuVxUlgzxjLTxVG41etcyqqZF+foyiKBO3PU3n8oh++tgQP + ExOxiR0JSkBG5b+oOBD0zxcvo3/SjBHn0dJOZCSU2SkCgYEAyCe676XnNyBZMRD7 + 6trsaoiCWBpA6M8H44+x3w4cQFtqV38RyLy60D+iMKjIaLqeBbnay61VMzo24Bz3 + EuF2n4+9k/MetLJ0NCw8HmN5k0WSMD2BFsJWG8glVbzaqzehP4tIclwDTYc1jQVt + IoV2/iL7HGT+x2daUwbU5kN5hK0CgYEAxiLB+fmjxJW7VY4SHDLqPdpIW0q/kv4K + d/yZBrCX799vjmFb9vLh7PkQUfJhMJ/ttJOd7EtT3xh4mfkBeLfHwVU0d/ahbmSH + UJu/E9ZGxAW3PP0kxHZtPrLKQwBnfq8AxBauIhR3rPSorQTIOKtwz1jMlHFSUpuL + 3KeK2YfDYJcCgYEAkQnJOlNcAuRb/WQzSHIvktssqK8NjiZHryy3Vc0hx7j2jES2 + HGI2dSVHYD9OSiXA0KFm3OTTsnViwm/60iGzFdjRJV6tR39xGUVcoyCuPnvRfUd0 + PYvBXgxgkYpyYlPDcwp5CvWGJy3tLi1acgOIwIuUr3S38sL//t4adGk8q1kCgYB8 + Jbs1Tl53BvrimKpwUNbE+sjrquJu0A7vL68SqgQJoQ7dP9PH4Ff/i+/V6PFM7mib + BQOm02wyFbs7fvKVGVJoqWK+6CIucX732x7W5yRgHtS5ukQXdbzt1Ek3wkEW98Cb + HTruz7RNAt/NyXlLSODeit1lBbx3Vk9EaxZtRsv88QKBgGn7JwXgez9NOyobsNIo + QVO80rpUeenSjuFi+R0VmbLKe/wgAQbYJ0xTAsQ0btqViMzB27D6mJyC+KUIwWNX + MN8a+m46v4kqvZkKL2c4gmDibyURNe/vCtCHFuanJS/1mo2tr4XDyEeiuK52eTd9 + omQDpP86RX/hIIQ+JyLSaWYa + -----END PRIVATE KEY-----`, + comment: + "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation" + } + }, + { + path: "/nested1/nested2/folder", + secret: { + key: "secret-key-3", + value: + "TG9yZW0gaXBzdW0gZG9sb3Igc2l0IGFtZXQsIGNvbnNlY3RldHVyIGFkaXBpc2NpbmcgZWxpdC4gU2VkIGRvIGVpdXNtb2QgdGVtcG9yIGluY2lkaWR1bnQgdXQgbGFib3JlIGV0IGRvbG9yZSBtYWduYSBhbGlxdWEuIFV0IGVuaW0gYWQgbWluaW0gdmVuaWFtLCBxdWlzIG5vc3RydWQgZXhlcmNpdGF0aW9uCg==", + comment: "" + } + } + ]; + + beforeAll(async () => { + if (auth === AuthMode.JWT) { + authToken = jwtAuthToken; + } else if (auth === AuthMode.IDENTITY_ACCESS_TOKEN) { + const identityLogin = await testServer.inject({ + method: "POST", + url: "/api/v1/auth/universal-auth/login", + body: { + clientSecret: seedData1.machineIdentity.clientCredentials.secret, + clientId: seedData1.machineIdentity.clientCredentials.id + } + }); + expect(identityLogin.statusCode).toBe(200); + authToken = identityLogin.json().accessToken; + } + // create a deep folder + const folderCreate = await testServer.inject({ + method: "POST", + url: `/api/v1/folders`, + headers: { + authorization: `Bearer ${jwtAuthToken}` + }, + body: { + workspaceId: seedData1.projectV3.id, + environment: seedData1.environment.slug, + name: "folder", + path: "/nested1/nested2" + } + }); + expect(folderCreate.statusCode).toBe(200); + folderId = folderCreate.json().folder.id; + }); + + afterAll(async () => { + const deleteFolder = await testServer.inject({ + method: "DELETE", + url: `/api/v1/folders/${folderId}`, + headers: { + authorization: `Bearer ${authToken}` + }, + body: { + workspaceId: seedData1.projectV3.id, + environment: seedData1.environment.slug, + path: "/nested1/nested2" + } + }); + expect(deleteFolder.statusCode).toBe(200); + }); + + const getSecrets = async (environment: string, secretPath = "/") => { + const res = await testServer.inject({ + method: "GET", + url: `/api/v3/secrets/raw`, + headers: { + authorization: `Bearer ${authToken}` + }, + query: { + secretPath, + environment, + workspaceId: seedData1.projectV3.id + } + }); + const secrets: TRawSecret[] = JSON.parse(res.payload).secrets || []; + return secrets; + }; + + test.each(secretTestCases)("Create secret in path $path", async ({ secret, path }) => { + const createdSecret = await createSecret({ path, ...secret }); + expect(createdSecret.secretKey).toEqual(secret.key); + expect(createdSecret.secretValue).toEqual(secret.value); + expect(createdSecret.secretComment || "").toEqual(secret.comment); + expect(createdSecret.version).toEqual(1); + + const secrets = await getSecrets(seedData1.environment.slug, path); + expect(secrets).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + secretKey: secret.key, + secretValue: secret.value, + type: SecretType.Shared + }) + ]) + ); + await deleteSecret({ path, key: secret.key }); + }); + + test.each(secretTestCases)("Get secret by name in path $path", async ({ secret, path }) => { + await createSecret({ path, ...secret }); + + const getSecByNameRes = await testServer.inject({ + method: "GET", + url: `/api/v3/secrets/raw/${secret.key}`, + headers: { + authorization: `Bearer ${authToken}` + }, + query: { + secretPath: path, + workspaceId: seedData1.projectV3.id, + environment: seedData1.environment.slug + } + }); + expect(getSecByNameRes.statusCode).toBe(200); + const getSecretByNamePayload = JSON.parse(getSecByNameRes.payload); + expect(getSecretByNamePayload).toHaveProperty("secret"); + const decryptedSecret = getSecretByNamePayload.secret as TRawSecret; + expect(decryptedSecret.secretKey).toEqual(secret.key); + expect(decryptedSecret.secretValue).toEqual(secret.value); + expect(decryptedSecret.secretComment || "").toEqual(secret.comment); + + await deleteSecret({ path, key: secret.key }); + }); + + if (auth === AuthMode.JWT) { + test.each(secretTestCases)( + "Creating personal secret without shared throw error in path $path", + async ({ secret }) => { + const createSecretReqBody = { + workspaceId: seedData1.projectV3.id, + environment: seedData1.environment.slug, + type: SecretType.Personal, + secretKey: secret.key, + secretValue: secret.value, + secretComment: secret.comment + }; + const createSecRes = await testServer.inject({ + method: "POST", + url: `/api/v3/secrets/raw/SEC2`, + headers: { + authorization: `Bearer ${authToken}` + }, + body: createSecretReqBody + }); + const payload = JSON.parse(createSecRes.payload); + expect(createSecRes.statusCode).toBe(400); + expect(payload.error).toEqual("BadRequest"); + } + ); + + test.each(secretTestCases)("Creating personal secret in path $path", async ({ secret, path }) => { + await createSecret({ path, ...secret }); + + const createSecretReqBody = { + workspaceId: seedData1.projectV3.id, + environment: seedData1.environment.slug, + type: SecretType.Personal, + secretPath: path, + secretKey: secret.key, + secretValue: "personal-value", + secretComment: secret.comment + }; + const createSecRes = await testServer.inject({ + method: "POST", + url: `/api/v3/secrets/raw/${secret.key}`, + headers: { + authorization: `Bearer ${authToken}` + }, + body: createSecretReqBody + }); + expect(createSecRes.statusCode).toBe(200); + + // list secrets should contain personal one and shared one + const secrets = await getSecrets(seedData1.environment.slug, path); + expect(secrets).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + secretKey: secret.key, + secretValue: secret.value, + type: SecretType.Shared + }), + expect.objectContaining({ + secretKey: secret.key, + secretValue: "personal-value", + type: SecretType.Personal + }) + ]) + ); + + await deleteSecret({ path, key: secret.key }); + }); + + test.each(secretTestCases)( + "Deleting personal one should not delete shared secret in path $path", + async ({ secret, path }) => { + await createSecret({ path, ...secret }); // shared one + await createSecret({ path, ...secret, type: SecretType.Personal }); + + // shared secret deletion should delete personal ones also + const secrets = await getSecrets(seedData1.environment.slug, path); + expect(secrets).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + secretKey: secret.key, + type: SecretType.Shared + }), + expect.not.objectContaining({ + secretKey: secret.key, + type: SecretType.Personal + }) + ]) + ); + await deleteSecret({ path, key: secret.key }); + } + ); + } + + test.each(secretTestCases)("Update secret in path $path", async ({ path, secret }) => { + await createSecret({ path, ...secret }); + const updateSecretReqBody = { + workspaceId: seedData1.projectV3.id, + environment: seedData1.environment.slug, + type: SecretType.Shared, + secretPath: path, + secretKey: secret.key, + secretValue: "new-value", + secretComment: secret.comment + }; + const updateSecRes = await testServer.inject({ + method: "PATCH", + url: `/api/v3/secrets/raw/${secret.key}`, + headers: { + authorization: `Bearer ${authToken}` + }, + body: updateSecretReqBody + }); + expect(updateSecRes.statusCode).toBe(200); + const updatedSecretPayload = JSON.parse(updateSecRes.payload); + expect(updatedSecretPayload).toHaveProperty("secret"); + const decryptedSecret = updatedSecretPayload.secret; + expect(decryptedSecret.secretKey).toEqual(secret.key); + expect(decryptedSecret.secretValue).toEqual("new-value"); + expect(decryptedSecret.secretComment || "").toEqual(secret.comment); + + // list secret should have updated value + const secrets = await getSecrets(seedData1.environment.slug, path); + expect(secrets).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + secretKey: secret.key, + secretValue: "new-value", + type: SecretType.Shared + }) + ]) + ); + + await deleteSecret({ path, key: secret.key }); + }); + + test.each(secretTestCases)("Delete secret in path $path", async ({ secret, path }) => { + await createSecret({ path, ...secret }); + const deletedSecret = await deleteSecret({ path, key: secret.key }); + expect(deletedSecret.secretKey).toEqual(secret.key); + + // shared secret deletion should delete personal ones also + const secrets = await getSecrets(seedData1.environment.slug, path); + expect(secrets).toEqual( + expect.not.arrayContaining([ + expect.objectContaining({ + secretKey: secret.key, + type: SecretType.Shared + }), + expect.objectContaining({ + secretKey: secret.key, + type: SecretType.Personal + }) + ]) + ); + }); + + test.each(secretTestCases)("Bulk create secrets in path $path", async ({ secret, path }) => { + const createSharedSecRes = await testServer.inject({ + method: "POST", + url: `/api/v3/secrets/batch/raw`, + headers: { + authorization: `Bearer ${authToken}` + }, + body: { + workspaceId: seedData1.projectV3.id, + environment: seedData1.environment.slug, + secretPath: path, + secrets: Array.from(Array(5)).map((_e, i) => ({ + secretKey: `BULK-${secret.key}-${i + 1}`, + secretValue: secret.value, + secretComment: secret.comment + })) + } + }); + expect(createSharedSecRes.statusCode).toBe(200); + const createSharedSecPayload = JSON.parse(createSharedSecRes.payload); + expect(createSharedSecPayload).toHaveProperty("secrets"); + + // bulk ones should exist + const secrets = await getSecrets(seedData1.environment.slug, path); + expect(secrets).toEqual( + expect.arrayContaining( + Array.from(Array(5)).map((_e, i) => + expect.objectContaining({ + secretKey: `BULK-${secret.key}-${i + 1}`, + secretValue: secret.value, + type: SecretType.Shared + }) + ) + ) + ); + + await Promise.all( + Array.from(Array(5)).map((_e, i) => deleteSecret({ path, key: `BULK-${secret.key}-${i + 1}` })) + ); + }); + + test.each(secretTestCases)("Bulk create fail on existing secret in path $path", async ({ secret, path }) => { + await createSecret({ ...secret, key: `BULK-${secret.key}-1`, path }); + + const createSharedSecRes = await testServer.inject({ + method: "POST", + url: `/api/v3/secrets/batch/raw`, + headers: { + authorization: `Bearer ${authToken}` + }, + body: { + workspaceId: seedData1.projectV3.id, + environment: seedData1.environment.slug, + secretPath: path, + secrets: Array.from(Array(5)).map((_e, i) => ({ + secretKey: `BULK-${secret.key}-${i + 1}`, + secretValue: secret.value, + secretComment: secret.comment + })) + } + }); + expect(createSharedSecRes.statusCode).toBe(400); + + await deleteSecret({ path, key: `BULK-${secret.key}-1` }); + }); + + test.each(secretTestCases)("Bulk update secrets in path $path", async ({ secret, path }) => { + await Promise.all( + Array.from(Array(5)).map((_e, i) => createSecret({ ...secret, key: `BULK-${secret.key}-${i + 1}`, path })) + ); + + const updateSharedSecRes = await testServer.inject({ + method: "PATCH", + url: `/api/v3/secrets/batch/raw`, + headers: { + authorization: `Bearer ${authToken}` + }, + body: { + workspaceId: seedData1.projectV3.id, + environment: seedData1.environment.slug, + secretPath: path, + secrets: Array.from(Array(5)).map((_e, i) => ({ + secretKey: `BULK-${secret.key}-${i + 1}`, + secretValue: "update-value", + secretComment: secret.comment + })) + } + }); + expect(updateSharedSecRes.statusCode).toBe(200); + const updateSharedSecPayload = JSON.parse(updateSharedSecRes.payload); + expect(updateSharedSecPayload).toHaveProperty("secrets"); + + // bulk ones should exist + const secrets = await getSecrets(seedData1.environment.slug, path); + expect(secrets).toEqual( + expect.arrayContaining( + Array.from(Array(5)).map((_e, i) => + expect.objectContaining({ + secretKey: `BULK-${secret.key}-${i + 1}`, + secretValue: "update-value", + type: SecretType.Shared + }) + ) + ) + ); + await Promise.all( + Array.from(Array(5)).map((_e, i) => deleteSecret({ path, key: `BULK-${secret.key}-${i + 1}` })) + ); + }); + + test.each(secretTestCases)("Bulk delete secrets in path $path", async ({ secret, path }) => { + await Promise.all( + Array.from(Array(5)).map((_e, i) => createSecret({ ...secret, key: `BULK-${secret.key}-${i + 1}`, path })) + ); + + const deletedSharedSecRes = await testServer.inject({ + method: "DELETE", + url: `/api/v3/secrets/batch/raw`, + headers: { + authorization: `Bearer ${authToken}` + }, + body: { + workspaceId: seedData1.projectV3.id, + environment: seedData1.environment.slug, + secretPath: path, + secrets: Array.from(Array(5)).map((_e, i) => ({ + secretKey: `BULK-${secret.key}-${i + 1}` + })) + } + }); + + expect(deletedSharedSecRes.statusCode).toBe(200); + const deletedSecretPayload = JSON.parse(deletedSharedSecRes.payload); + expect(deletedSecretPayload).toHaveProperty("secrets"); + + // bulk ones should exist + const secrets = await getSecrets(seedData1.environment.slug, path); + expect(secrets).toEqual( + expect.not.arrayContaining( + Array.from(Array(5)).map((_e, i) => + expect.objectContaining({ + secretKey: `BULK-${secret.value}-${i + 1}`, + type: SecretType.Shared + }) + ) + ) + ); + }); + } +); diff --git a/backend/package.json b/backend/package.json index dd66d4cd6..1328db4dd 100644 --- a/backend/package.json +++ b/backend/package.json @@ -40,8 +40,8 @@ "type:check": "tsc --noEmit", "lint:fix": "eslint --fix --ext js,ts ./src", "lint": "eslint 'src/**/*.ts'", - "test:e2e": "vitest run -c vitest.e2e.config.ts", - "test:e2e-watch": "vitest -c vitest.e2e.config.ts", + "test:e2e": "vitest run -c vitest.e2e.config.ts --bail=1", + "test:e2e-watch": "vitest -c vitest.e2e.config.ts --bail=1", "test:e2e-coverage": "vitest run --coverage -c vitest.e2e.config.ts", "generate:component": "tsx ./scripts/create-backend-file.ts", "generate:schema": "tsx ./scripts/generate-schema-types.ts", diff --git a/backend/src/@types/knex.d.ts b/backend/src/@types/knex.d.ts index 8aa00ec79..5b2f1f2c8 100644 --- a/backend/src/@types/knex.d.ts +++ b/backend/src/@types/knex.d.ts @@ -204,6 +204,9 @@ import { TSecretApprovalRequestSecretTags, TSecretApprovalRequestSecretTagsInsert, TSecretApprovalRequestSecretTagsUpdate, + TSecretApprovalRequestSecretTagsV2, + TSecretApprovalRequestSecretTagsV2Insert, + TSecretApprovalRequestSecretTagsV2Update, TSecretApprovalRequestsInsert, TSecretApprovalRequestsReviewers, TSecretApprovalRequestsReviewersInsert, @@ -211,6 +214,9 @@ import { TSecretApprovalRequestsSecrets, TSecretApprovalRequestsSecretsInsert, TSecretApprovalRequestsSecretsUpdate, + TSecretApprovalRequestsSecretsV2, + TSecretApprovalRequestsSecretsV2Insert, + TSecretApprovalRequestsSecretsV2Update, TSecretApprovalRequestsUpdate, TSecretBlindIndexes, TSecretBlindIndexesInsert, @@ -227,9 +233,15 @@ import { TSecretReferences, TSecretReferencesInsert, TSecretReferencesUpdate, + TSecretReferencesV2, + TSecretReferencesV2Insert, + TSecretReferencesV2Update, TSecretRotationOutputs, TSecretRotationOutputsInsert, TSecretRotationOutputsUpdate, + TSecretRotationOutputV2, + TSecretRotationOutputV2Insert, + TSecretRotationOutputV2Update, TSecretRotations, TSecretRotationsInsert, TSecretRotationsUpdate, @@ -248,6 +260,9 @@ import { TSecretSnapshotSecrets, TSecretSnapshotSecretsInsert, TSecretSnapshotSecretsUpdate, + TSecretSnapshotSecretsV2, + TSecretSnapshotSecretsV2Insert, + TSecretSnapshotSecretsV2Update, TSecretSnapshotsInsert, TSecretSnapshotsUpdate, TSecretsUpdate, @@ -263,6 +278,9 @@ import { TSecretVersionTagJunction, TSecretVersionTagJunctionInsert, TSecretVersionTagJunctionUpdate, + TSecretVersionV2TagJunction, + TSecretVersionV2TagJunctionInsert, + TSecretVersionV2TagJunctionUpdate, TServiceTokens, TServiceTokensInsert, TServiceTokensUpdate, @@ -291,6 +309,17 @@ import { TWebhooksInsert, TWebhooksUpdate } from "@app/db/schemas"; +import { + TSecretV2TagJunction, + TSecretV2TagJunctionInsert, + TSecretV2TagJunctionUpdate +} from "@app/db/schemas/secret-v2-tag-junction"; +import { + TSecretVersionsV2, + TSecretVersionsV2Insert, + TSecretVersionsV2Update +} from "@app/db/schemas/secret-versions-v2"; +import { TSecretsV2, TSecretsV2Insert, TSecretsV2Update } from "@app/db/schemas/secrets-v2"; declare module "knex" { namespace Knex { @@ -645,7 +674,23 @@ declare module "knex/types/tables" { TSecretScanningGitRisksUpdate >; [TableName.TrustedIps]: KnexOriginal.CompositeTableType; + [TableName.SecretV2]: KnexOriginal.CompositeTableType; + [TableName.SecretVersionV2]: KnexOriginal.CompositeTableType< + TSecretVersionsV2, + TSecretVersionsV2Insert, + TSecretVersionsV2Update + >; + [TableName.SecretReferenceV2]: KnexOriginal.CompositeTableType< + TSecretReferencesV2, + TSecretReferencesV2Insert, + TSecretReferencesV2Update + >; // Junction tables + [TableName.SecretV2JnTag]: KnexOriginal.CompositeTableType< + TSecretV2TagJunction, + TSecretV2TagJunctionInsert, + TSecretV2TagJunctionUpdate + >; [TableName.JnSecretTag]: KnexOriginal.CompositeTableType< TSecretTagJunction, TSecretTagJunctionInsert, @@ -656,6 +701,31 @@ declare module "knex/types/tables" { TSecretVersionTagJunctionInsert, TSecretVersionTagJunctionUpdate >; + [TableName.SecretVersionV2Tag]: KnexOriginal.CompositeTableType< + TSecretVersionV2TagJunction, + TSecretVersionV2TagJunctionInsert, + TSecretVersionV2TagJunctionUpdate + >; + [TableName.SnapshotSecretV2]: KnexOriginal.CompositeTableType< + TSecretSnapshotSecretsV2, + TSecretSnapshotSecretsV2Insert, + TSecretSnapshotSecretsV2Update + >; + [TableName.SecretApprovalRequestSecretV2]: KnexOriginal.CompositeTableType< + TSecretApprovalRequestsSecretsV2, + TSecretApprovalRequestsSecretsV2Insert, + TSecretApprovalRequestsSecretsV2Update + >; + [TableName.SecretApprovalRequestSecretTagV2]: KnexOriginal.CompositeTableType< + TSecretApprovalRequestSecretTagsV2, + TSecretApprovalRequestSecretTagsV2Insert, + TSecretApprovalRequestSecretTagsV2Update + >; + [TableName.SecretRotationOutputV2]: KnexOriginal.CompositeTableType< + TSecretRotationOutputV2, + TSecretRotationOutputV2Insert, + TSecretRotationOutputV2Update + >; // KMS service [TableName.KmsServerRootConfig]: KnexOriginal.CompositeTableType< TKmsRootConfig, diff --git a/backend/src/db/migrations/20240730181830_add-org-kms-data-key.ts b/backend/src/db/migrations/20240730181830_add-org-kms-data-key.ts new file mode 100644 index 000000000..13e6a1ef6 --- /dev/null +++ b/backend/src/db/migrations/20240730181830_add-org-kms-data-key.ts @@ -0,0 +1,21 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + const hasKmsDataKeyCol = await knex.schema.hasColumn(TableName.Organization, "kmsEncryptedDataKey"); + await knex.schema.alterTable(TableName.Organization, (tb) => { + if (!hasKmsDataKeyCol) { + tb.binary("kmsEncryptedDataKey"); + } + }); +} + +export async function down(knex: Knex): Promise { + const hasKmsDataKeyCol = await knex.schema.hasColumn(TableName.Organization, "kmsEncryptedDataKey"); + await knex.schema.alterTable(TableName.Organization, (t) => { + if (hasKmsDataKeyCol) { + t.dropColumn("kmsEncryptedDataKey"); + } + }); +} diff --git a/backend/src/db/migrations/20240730181840_add-project-data-key.ts b/backend/src/db/migrations/20240730181840_add-project-data-key.ts new file mode 100644 index 000000000..e8ef083a3 --- /dev/null +++ b/backend/src/db/migrations/20240730181840_add-project-data-key.ts @@ -0,0 +1,29 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + const hasKmsSecretManagerEncryptedDataKey = await knex.schema.hasColumn( + TableName.Project, + "kmsSecretManagerEncryptedDataKey" + ); + + await knex.schema.alterTable(TableName.Project, (tb) => { + if (!hasKmsSecretManagerEncryptedDataKey) { + tb.binary("kmsSecretManagerEncryptedDataKey"); + } + }); +} + +export async function down(knex: Knex): Promise { + const hasKmsSecretManagerEncryptedDataKey = await knex.schema.hasColumn( + TableName.Project, + "kmsSecretManagerEncryptedDataKey" + ); + + await knex.schema.alterTable(TableName.Project, (t) => { + if (hasKmsSecretManagerEncryptedDataKey) { + t.dropColumn("kmsSecretManagerEncryptedDataKey"); + } + }); +} diff --git a/backend/src/db/migrations/20240730181850_secret-v2.ts b/backend/src/db/migrations/20240730181850_secret-v2.ts new file mode 100644 index 000000000..a9188c608 --- /dev/null +++ b/backend/src/db/migrations/20240730181850_secret-v2.ts @@ -0,0 +1,404 @@ +/* eslint-disable @typescript-eslint/ban-ts-comment */ +import { Knex } from "knex"; + +import { infisicalSymmetricDecrypt } from "@app/lib/crypto/encryption"; +import { selectAllTableCols } from "@app/lib/knex/select"; + +import { SecretKeyEncoding, SecretType, TableName } from "../schemas"; +import { createJunctionTable, createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils"; +import { getSecretManagerDataKey } from "./utils/kms"; + +const backfillWebhooks = async (knex: Knex) => { + const hasEncryptedSecretKeyWithKms = await knex.schema.hasColumn(TableName.Webhook, "encryptedSecretKeyWithKms"); + const hasEncryptedWebhookUrl = await knex.schema.hasColumn(TableName.Webhook, "encryptedUrl"); + const hasUrlCipherText = await knex.schema.hasColumn(TableName.Webhook, "urlCipherText"); + const hasUrlIV = await knex.schema.hasColumn(TableName.Webhook, "urlIV"); + const hasUrlTag = await knex.schema.hasColumn(TableName.Webhook, "urlTag"); + const hasEncryptedSecretKey = await knex.schema.hasColumn(TableName.Webhook, "encryptedSecretKey"); + const hasIV = await knex.schema.hasColumn(TableName.Webhook, "iv"); + const hasTag = await knex.schema.hasColumn(TableName.Webhook, "tag"); + const hasKeyEncoding = await knex.schema.hasColumn(TableName.Webhook, "keyEncoding"); + const hasAlgorithm = await knex.schema.hasColumn(TableName.Webhook, "algorithm"); + const hasUrl = await knex.schema.hasColumn(TableName.Webhook, "url"); + + await knex.schema.alterTable(TableName.Webhook, (t) => { + if (!hasEncryptedSecretKeyWithKms) t.binary("encryptedSecretKeyWithKms"); + if (!hasEncryptedWebhookUrl) t.binary("encryptedUrl"); + if (hasUrl) t.string("url").nullable().alter(); + }); + + const kmsEncryptorGroupByProjectId: Record>["encryptor"]> = + {}; + if (hasUrlCipherText && hasUrlIV && hasUrlTag && hasEncryptedSecretKey && hasIV && hasTag) { + // eslint-disable-next-line + const webhooksToFill = await knex(TableName.Webhook) + .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.Webhook}.envId`) + .whereNull("encryptedUrl") + // eslint-disable-next-line + // @ts-ignore knex migration fails + .select(selectAllTableCols(TableName.Webhook)) + .select("projectId"); + + const updatedWebhooks = []; + for (const webhook of webhooksToFill) { + if (!kmsEncryptorGroupByProjectId[webhook.projectId]) { + // eslint-disable-next-line + const { encryptor } = await getSecretManagerDataKey(knex, webhook.projectId); + kmsEncryptorGroupByProjectId[webhook.projectId] = encryptor; + } + + const kmsEncryptor = kmsEncryptorGroupByProjectId[webhook.projectId]; + + // @ts-ignore post migration fails + let webhookUrl = webhook.url; + let webhookSecretKey; + + // @ts-ignore post migration fails + if (webhook.urlTag && webhook.urlCipherText && webhook.urlIV) { + webhookUrl = infisicalSymmetricDecrypt({ + // @ts-ignore post migration fails + keyEncoding: webhook.keyEncoding as SecretKeyEncoding, + // @ts-ignore post migration fails + ciphertext: webhook.urlCipherText, + // @ts-ignore post migration fails + iv: webhook.urlIV, + // @ts-ignore post migration fails + tag: webhook.urlTag + }); + } + // @ts-ignore post migration fails + if (webhook.encryptedSecretKey && webhook.iv && webhook.tag) { + webhookSecretKey = infisicalSymmetricDecrypt({ + // @ts-ignore post migration fails + keyEncoding: webhook.keyEncoding as SecretKeyEncoding, + // @ts-ignore post migration fails + ciphertext: webhook.encryptedSecretKey, + // @ts-ignore post migration fails + iv: webhook.iv, + // @ts-ignore post migration fails + tag: webhook.tag + }); + } + const { projectId, ...el } = webhook; + updatedWebhooks.push({ + ...el, + encryptedSecretKeyWithKms: webhookSecretKey + ? kmsEncryptor({ plainText: Buffer.from(webhookSecretKey) }).cipherTextBlob + : null, + encryptedUrl: kmsEncryptor({ plainText: Buffer.from(webhookUrl) }).cipherTextBlob + }); + } + if (updatedWebhooks.length) { + // eslint-disable-next-line + await knex(TableName.Webhook).insert(updatedWebhooks).onConflict("id").merge(); + } + } + await knex.schema.alterTable(TableName.Webhook, (t) => { + t.binary("encryptedUrl").notNullable().alter(); + + if (hasUrlIV) t.dropColumn("urlIV"); + if (hasUrlCipherText) t.dropColumn("urlCipherText"); + if (hasUrlTag) t.dropColumn("urlTag"); + if (hasIV) t.dropColumn("iv"); + if (hasTag) t.dropColumn("tag"); + if (hasEncryptedSecretKey) t.dropColumn("encryptedSecretKey"); + if (hasKeyEncoding) t.dropColumn("keyEncoding"); + if (hasAlgorithm) t.dropColumn("algorithm"); + if (hasUrl) t.dropColumn("url"); + }); +}; + +const backfillDynamicSecretConfigs = async (knex: Knex) => { + const hasEncryptedConfig = await knex.schema.hasColumn(TableName.DynamicSecret, "encryptedConfig"); + + const hasInputCipherText = await knex.schema.hasColumn(TableName.DynamicSecret, "inputCiphertext"); + const hasInputIV = await knex.schema.hasColumn(TableName.DynamicSecret, "inputIV"); + const hasInputTag = await knex.schema.hasColumn(TableName.DynamicSecret, "inputTag"); + const hasKeyEncoding = await knex.schema.hasColumn(TableName.DynamicSecret, "keyEncoding"); + const hasAlgorithm = await knex.schema.hasColumn(TableName.DynamicSecret, "algorithm"); + + await knex.schema.alterTable(TableName.DynamicSecret, (t) => { + if (!hasEncryptedConfig) t.binary("encryptedConfig"); + }); + const kmsEncryptorGroupByProjectId: Record>["encryptor"]> = + {}; + if (hasInputCipherText && hasInputIV && hasInputTag) { + // eslint-disable-next-line + const dynamicSecretConfigs = await knex(TableName.DynamicSecret) + .join(TableName.SecretFolder, `${TableName.SecretFolder}.id`, `${TableName.DynamicSecret}.folderId`) + .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) + .whereNull("encryptedConfig") + // @ts-ignore post migration fails + .select(selectAllTableCols(TableName.DynamicSecret)) + .select("projectId"); + + const updatedConfigs = []; + for (const dynamicSecretConfig of dynamicSecretConfigs) { + if (!kmsEncryptorGroupByProjectId[dynamicSecretConfig.projectId]) { + // eslint-disable-next-line + const { encryptor } = await getSecretManagerDataKey(knex, dynamicSecretConfig.projectId); + kmsEncryptorGroupByProjectId[dynamicSecretConfig.projectId] = encryptor; + } + + const kmsEncryptor = kmsEncryptorGroupByProjectId[dynamicSecretConfig.projectId]; + const inputConfig = infisicalSymmetricDecrypt({ + // @ts-ignore post migration fails + keyEncoding: dynamicSecretConfig.keyEncoding as SecretKeyEncoding, + // @ts-ignore post migration fails + ciphertext: dynamicSecretConfig.inputCiphertext as string, + // @ts-ignore post migration fails + iv: dynamicSecretConfig.inputIV as string, + // @ts-ignore post migration fails + tag: dynamicSecretConfig.inputTag as string + }); + + const { projectId, ...el } = dynamicSecretConfig; + updatedConfigs.push({ + ...el, + encryptedConfig: kmsEncryptor({ plainText: Buffer.from(inputConfig) }).cipherTextBlob + }); + } + if (updatedConfigs.length) { + // eslint-disable-next-line + await knex(TableName.DynamicSecret).insert(updatedConfigs).onConflict("id").merge(); + } + } + await knex.schema.alterTable(TableName.DynamicSecret, (t) => { + t.binary("encryptedConfig").notNullable().alter(); + + if (hasInputTag) t.dropColumn("inputTag"); + if (hasInputIV) t.dropColumn("inputIV"); + if (hasInputCipherText) t.dropColumn("inputCiphertext"); + if (hasKeyEncoding) t.dropColumn("keyEncoding"); + if (hasAlgorithm) t.dropColumn("algorithm"); + }); +}; + +export async function up(knex: Knex): Promise { + const doesSecretV2TableExist = await knex.schema.hasTable(TableName.SecretV2); + if (!doesSecretV2TableExist) { + await knex.schema.createTable(TableName.SecretV2, (t) => { + t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid()); + t.integer("version").defaultTo(1).notNullable(); + t.string("type").notNullable().defaultTo(SecretType.Shared); + t.string("key", 500).notNullable(); + t.binary("encryptedValue"); + t.binary("encryptedComment"); + t.string("reminderNote"); + t.integer("reminderRepeatDays"); + t.boolean("skipMultilineEncoding").defaultTo(false); + t.jsonb("metadata"); + t.uuid("userId"); + t.foreign("userId").references("id").inTable(TableName.Users).onDelete("CASCADE"); + t.uuid("folderId").notNullable(); + t.foreign("folderId").references("id").inTable(TableName.SecretFolder).onDelete("CASCADE"); + t.timestamps(true, true, true); + t.index(["folderId", "userId"]); + }); + } + await createOnUpdateTrigger(knex, TableName.SecretV2); + + // many to many relation between tags + await createJunctionTable(knex, TableName.SecretV2JnTag, TableName.SecretV2, TableName.SecretTag); + + const doesSecretV2VersionTableExist = await knex.schema.hasTable(TableName.SecretVersionV2); + if (!doesSecretV2VersionTableExist) { + await knex.schema.createTable(TableName.SecretVersionV2, (t) => { + t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid()); + t.integer("version").defaultTo(1).notNullable(); + t.string("type").notNullable().defaultTo(SecretType.Shared); + t.string("key", 500).notNullable(); + t.binary("encryptedValue"); + t.binary("encryptedComment"); + t.string("reminderNote"); + t.integer("reminderRepeatDays"); + t.boolean("skipMultilineEncoding").defaultTo(false); + t.jsonb("metadata"); + // to avoid orphan rows + t.uuid("envId"); + t.foreign("envId").references("id").inTable(TableName.Environment).onDelete("CASCADE"); + t.uuid("secretId").notNullable(); + t.uuid("folderId").notNullable(); + t.uuid("userId"); + t.foreign("userId").references("id").inTable(TableName.Users).onDelete("CASCADE"); + t.timestamps(true, true, true); + }); + } + await createOnUpdateTrigger(knex, TableName.SecretVersionV2); + + if (!(await knex.schema.hasTable(TableName.SecretReferenceV2))) { + await knex.schema.createTable(TableName.SecretReferenceV2, (t) => { + t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid()); + t.string("environment").notNullable(); + t.string("secretPath").notNullable(); + t.string("secretKey", 500).notNullable(); + t.uuid("secretId").notNullable(); + t.foreign("secretId").references("id").inTable(TableName.SecretV2).onDelete("CASCADE"); + }); + } + + await createJunctionTable(knex, TableName.SecretVersionV2Tag, TableName.SecretVersionV2, TableName.SecretTag); + + if (!(await knex.schema.hasTable(TableName.SecretApprovalRequestSecretV2))) { + await knex.schema.createTable(TableName.SecretApprovalRequestSecretV2, (t) => { + // everything related to secret + t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid()); + t.integer("version").defaultTo(1); + t.string("key", 500).notNullable(); + t.binary("encryptedValue"); + t.binary("encryptedComment"); + t.string("reminderNote"); + t.integer("reminderRepeatDays"); + t.boolean("skipMultilineEncoding").defaultTo(false); + t.jsonb("metadata"); + t.timestamps(true, true, true); + // commit details + t.uuid("requestId").notNullable(); + t.foreign("requestId").references("id").inTable(TableName.SecretApprovalRequest).onDelete("CASCADE"); + t.string("op").notNullable(); + t.uuid("secretId"); + t.foreign("secretId").references("id").inTable(TableName.SecretV2).onDelete("SET NULL"); + t.uuid("secretVersion"); + t.foreign("secretVersion").references("id").inTable(TableName.SecretVersionV2).onDelete("SET NULL"); + }); + } + + if (!(await knex.schema.hasTable(TableName.SecretApprovalRequestSecretTagV2))) { + await knex.schema.createTable(TableName.SecretApprovalRequestSecretTagV2, (t) => { + t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid()); + t.uuid("secretId").notNullable(); + t.foreign("secretId").references("id").inTable(TableName.SecretApprovalRequestSecretV2).onDelete("CASCADE"); + t.uuid("tagId").notNullable(); + t.foreign("tagId").references("id").inTable(TableName.SecretTag).onDelete("CASCADE"); + t.timestamps(true, true, true); + }); + } + + if (!(await knex.schema.hasTable(TableName.SnapshotSecretV2))) { + await knex.schema.createTable(TableName.SnapshotSecretV2, (t) => { + t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid()); + t.uuid("envId").index().notNullable(); + t.foreign("envId").references("id").inTable(TableName.Environment).onDelete("CASCADE"); + // not a relation kept like that to keep it when rolled back + t.uuid("secretVersionId").index().notNullable(); + t.foreign("secretVersionId").references("id").inTable(TableName.SecretVersionV2).onDelete("CASCADE"); + t.uuid("snapshotId").index().notNullable(); + t.foreign("snapshotId").references("id").inTable(TableName.Snapshot).onDelete("CASCADE"); + t.timestamps(true, true, true); + }); + } + + if (await knex.schema.hasTable(TableName.IntegrationAuth)) { + const hasEncryptedAccess = await knex.schema.hasColumn(TableName.IntegrationAuth, "encryptedAccess"); + const hasEncryptedAccessId = await knex.schema.hasColumn(TableName.IntegrationAuth, "encryptedAccessId"); + const hasEncryptedRefresh = await knex.schema.hasColumn(TableName.IntegrationAuth, "encryptedRefresh"); + const hasEncryptedAwsIamAssumRole = await knex.schema.hasColumn( + TableName.IntegrationAuth, + "encryptedAwsAssumeIamRoleArn" + ); + await knex.schema.alterTable(TableName.IntegrationAuth, (t) => { + if (!hasEncryptedAccess) t.binary("encryptedAccess"); + if (!hasEncryptedAccessId) t.binary("encryptedAccessId"); + if (!hasEncryptedRefresh) t.binary("encryptedRefresh"); + if (!hasEncryptedAwsIamAssumRole) t.binary("encryptedAwsAssumeIamRoleArn"); + }); + } + + if (!(await knex.schema.hasTable(TableName.SecretRotationOutputV2))) { + await knex.schema.createTable(TableName.SecretRotationOutputV2, (t) => { + t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid()); + t.string("key").notNullable(); + t.uuid("secretId").notNullable(); + t.foreign("secretId").references("id").inTable(TableName.SecretV2).onDelete("CASCADE"); + t.uuid("rotationId").notNullable(); + t.foreign("rotationId").references("id").inTable(TableName.SecretRotation).onDelete("CASCADE"); + }); + } + + if (await knex.schema.hasTable(TableName.Webhook)) { + await backfillWebhooks(knex); + } + + if (await knex.schema.hasTable(TableName.DynamicSecret)) { + await backfillDynamicSecretConfigs(knex); + } +} + +export async function down(knex: Knex): Promise { + await knex.schema.dropTableIfExists(TableName.SnapshotSecretV2); + await knex.schema.dropTableIfExists(TableName.SecretApprovalRequestSecretTagV2); + await knex.schema.dropTableIfExists(TableName.SecretApprovalRequestSecretV2); + + await knex.schema.dropTableIfExists(TableName.SecretV2JnTag); + await knex.schema.dropTableIfExists(TableName.SecretReferenceV2); + + await knex.schema.dropTableIfExists(TableName.SecretRotationOutputV2); + + await dropOnUpdateTrigger(knex, TableName.SecretVersionV2); + await knex.schema.dropTableIfExists(TableName.SecretVersionV2Tag); + await knex.schema.dropTableIfExists(TableName.SecretVersionV2); + + await dropOnUpdateTrigger(knex, TableName.SecretV2); + await knex.schema.dropTableIfExists(TableName.SecretV2); + + if (await knex.schema.hasTable(TableName.IntegrationAuth)) { + const hasEncryptedAccess = await knex.schema.hasColumn(TableName.IntegrationAuth, "encryptedAccess"); + const hasEncryptedAccessId = await knex.schema.hasColumn(TableName.IntegrationAuth, "encryptedAccessId"); + const hasEncryptedRefresh = await knex.schema.hasColumn(TableName.IntegrationAuth, "encryptedRefresh"); + const hasEncryptedAwsIamAssumRole = await knex.schema.hasColumn( + TableName.IntegrationAuth, + "encryptedAwsAssumeIamRoleArn" + ); + await knex.schema.alterTable(TableName.IntegrationAuth, (t) => { + if (hasEncryptedAccess) t.dropColumn("encryptedAccess"); + if (hasEncryptedAccessId) t.dropColumn("encryptedAccessId"); + if (hasEncryptedRefresh) t.dropColumn("encryptedRefresh"); + if (hasEncryptedAwsIamAssumRole) t.dropColumn("encryptedAwsAssumeIamRoleArn"); + }); + } + if (await knex.schema.hasTable(TableName.Webhook)) { + const hasEncryptedWebhookSecretKey = await knex.schema.hasColumn(TableName.Webhook, "encryptedSecretKeyWithKms"); + const hasEncryptedWebhookUrl = await knex.schema.hasColumn(TableName.Webhook, "encryptedUrl"); + const hasUrlCipherText = await knex.schema.hasColumn(TableName.Webhook, "urlCipherText"); + const hasUrlIV = await knex.schema.hasColumn(TableName.Webhook, "urlIV"); + const hasUrlTag = await knex.schema.hasColumn(TableName.Webhook, "urlTag"); + const hasEncryptedSecretKey = await knex.schema.hasColumn(TableName.Webhook, "encryptedSecretKey"); + const hasIV = await knex.schema.hasColumn(TableName.Webhook, "iv"); + const hasTag = await knex.schema.hasColumn(TableName.Webhook, "tag"); + const hasKeyEncoding = await knex.schema.hasColumn(TableName.Webhook, "keyEncoding"); + const hasAlgorithm = await knex.schema.hasColumn(TableName.Webhook, "algorithm"); + const hasUrl = await knex.schema.hasColumn(TableName.Webhook, "url"); + + await knex.schema.alterTable(TableName.Webhook, (t) => { + if (hasEncryptedWebhookSecretKey) t.dropColumn("encryptedSecretKeyWithKms"); + if (hasEncryptedWebhookUrl) t.dropColumn("encryptedUrl"); + if (!hasUrl) t.string("url"); + if (!hasEncryptedSecretKey) t.string("encryptedSecretKey"); + if (!hasIV) t.string("iv"); + if (!hasTag) t.string("tag"); + if (!hasAlgorithm) t.string("algorithm"); + if (!hasKeyEncoding) t.string("keyEncoding"); + if (!hasUrlCipherText) t.string("urlCipherText"); + if (!hasUrlIV) t.string("urlIV"); + if (!hasUrlTag) t.string("urlTag"); + }); + } + + if (await knex.schema.hasTable(TableName.DynamicSecret)) { + const hasEncryptedConfig = await knex.schema.hasColumn(TableName.DynamicSecret, "encryptedConfig"); + + const hasInputIV = await knex.schema.hasColumn(TableName.DynamicSecret, "inputIV"); + const hasInputCipherText = await knex.schema.hasColumn(TableName.DynamicSecret, "inputCiphertext"); + const hasInputTag = await knex.schema.hasColumn(TableName.DynamicSecret, "inputTag"); + const hasAlgorithm = await knex.schema.hasColumn(TableName.DynamicSecret, "algorithm"); + const hasKeyEncoding = await knex.schema.hasColumn(TableName.DynamicSecret, "keyEncoding"); + await knex.schema.alterTable(TableName.DynamicSecret, (t) => { + if (hasEncryptedConfig) t.dropColumn("encryptedConfig"); + if (!hasInputIV) t.string("inputIV"); + if (!hasInputCipherText) t.text("inputCiphertext"); + if (!hasInputTag) t.string("inputTag"); + if (!hasAlgorithm) t.string("algorithm"); + if (!hasKeyEncoding) t.string("keyEncoding"); + }); + } +} diff --git a/backend/src/db/migrations/utils/kms.ts b/backend/src/db/migrations/utils/kms.ts new file mode 100644 index 000000000..b4698a42d --- /dev/null +++ b/backend/src/db/migrations/utils/kms.ts @@ -0,0 +1,105 @@ +import slugify from "@sindresorhus/slugify"; +import { Knex } from "knex"; + +import { TableName } from "@app/db/schemas"; +import { randomSecureBytes } from "@app/lib/crypto"; +import { symmetricCipherService, SymmetricEncryption } from "@app/lib/crypto/cipher"; +import { alphaNumericNanoId } from "@app/lib/nanoid"; + +const getInstanceRootKey = async (knex: Knex) => { + const encryptionKey = process.env.ENCRYPTION_KEY || process.env.ROOT_ENCRYPTION_KEY; + // if root key its base64 encoded + const isBase64 = !process.env.ENCRYPTION_KEY; + if (!encryptionKey) throw new Error("ENCRYPTION_KEY variable needed for migration"); + const encryptionKeyBuffer = Buffer.from(encryptionKey, isBase64 ? "base64" : "utf8"); + + const KMS_ROOT_CONFIG_UUID = "00000000-0000-0000-0000-000000000000"; + const kmsRootConfig = await knex(TableName.KmsServerRootConfig).where({ id: KMS_ROOT_CONFIG_UUID }).first(); + const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256); + if (kmsRootConfig) { + const decryptedRootKey = cipher.decrypt(kmsRootConfig.encryptedRootKey, encryptionKeyBuffer); + // set the flag so that other instancen nodes can start + return decryptedRootKey; + } + + const newRootKey = randomSecureBytes(32); + const encryptedRootKey = cipher.encrypt(newRootKey, encryptionKeyBuffer); + await knex(TableName.KmsServerRootConfig).insert({ + encryptedRootKey, + // eslint-disable-next-line + // @ts-ignore id is kept as fixed for idempotence and to avoid race condition + id: KMS_ROOT_CONFIG_UUID + }); + return encryptedRootKey; +}; + +export const getSecretManagerDataKey = async (knex: Knex, projectId: string) => { + const KMS_VERSION = "v01"; + const KMS_VERSION_BLOB_LENGTH = 3; + const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256); + const project = await knex(TableName.Project).where({ id: projectId }).first(); + if (!project) throw new Error("Missing project id"); + + const ROOT_ENCRYPTION_KEY = await getInstanceRootKey(knex); + + let secretManagerKmsKey; + const projectSecretManagerKmsId = project?.kmsSecretManagerKeyId; + if (projectSecretManagerKmsId) { + const kmsDoc = await knex(TableName.KmsKey) + .leftJoin(TableName.InternalKms, `${TableName.KmsKey}.id`, `${TableName.InternalKms}.kmsKeyId`) + .where({ [`${TableName.KmsKey}.id` as "id"]: projectSecretManagerKmsId }) + .first(); + if (!kmsDoc) throw new Error("missing kms"); + secretManagerKmsKey = cipher.decrypt(kmsDoc.encryptedKey, ROOT_ENCRYPTION_KEY); + } else { + const [kmsDoc] = await knex(TableName.KmsKey) + .insert({ + slug: slugify(alphaNumericNanoId(8).toLowerCase()), + orgId: project.orgId, + isReserved: false + }) + .returning("*"); + + secretManagerKmsKey = randomSecureBytes(32); + const encryptedKeyMaterial = cipher.encrypt(secretManagerKmsKey, ROOT_ENCRYPTION_KEY); + await knex(TableName.InternalKms).insert({ + version: 1, + encryptedKey: encryptedKeyMaterial, + encryptionAlgorithm: SymmetricEncryption.AES_GCM_256, + kmsKeyId: kmsDoc.id + }); + } + + const encryptedSecretManagerDataKey = project?.kmsSecretManagerEncryptedDataKey; + let dataKey: Buffer; + if (!encryptedSecretManagerDataKey) { + dataKey = randomSecureBytes(); + // the below versioning we do it automatically in kms service + const unversionedDataKey = cipher.encrypt(dataKey, secretManagerKmsKey); + const versionBlob = Buffer.from(KMS_VERSION, "utf8"); // length is 3 + await knex(TableName.Project) + .where({ id: projectId }) + .update({ + kmsSecretManagerEncryptedDataKey: Buffer.concat([unversionedDataKey, versionBlob]) + }); + } else { + const cipherTextBlob = encryptedSecretManagerDataKey.subarray(0, -KMS_VERSION_BLOB_LENGTH); + dataKey = cipher.decrypt(cipherTextBlob, secretManagerKmsKey); + } + + return { + encryptor: ({ plainText }: { plainText: Buffer }) => { + const encryptedPlainTextBlob = cipher.encrypt(plainText, dataKey); + + // Buffer#1 encrypted text + Buffer#2 version number + const versionBlob = Buffer.from(KMS_VERSION, "utf8"); // length is 3 + const cipherTextBlob = Buffer.concat([encryptedPlainTextBlob, versionBlob]); + return { cipherTextBlob }; + }, + decryptor: ({ cipherTextBlob: versionedCipherTextBlob }: { cipherTextBlob: Buffer }) => { + const cipherTextBlob = versionedCipherTextBlob.subarray(0, -KMS_VERSION_BLOB_LENGTH); + const decryptedBlob = cipher.decrypt(cipherTextBlob, dataKey); + return decryptedBlob; + } + }; +}; diff --git a/backend/src/db/schemas/dynamic-secrets.ts b/backend/src/db/schemas/dynamic-secrets.ts index b27da396c..d90f1f7d2 100644 --- a/backend/src/db/schemas/dynamic-secrets.ts +++ b/backend/src/db/schemas/dynamic-secrets.ts @@ -5,6 +5,8 @@ import { z } from "zod"; +import { zodBuffer } from "@app/lib/zod"; + import { TImmutableDBKeys } from "./models"; export const DynamicSecretsSchema = z.object({ @@ -14,16 +16,12 @@ export const DynamicSecretsSchema = z.object({ type: z.string(), defaultTTL: z.string(), maxTTL: z.string().nullable().optional(), - inputIV: z.string(), - inputCiphertext: z.string(), - inputTag: z.string(), - algorithm: z.string().default("aes-256-gcm"), - keyEncoding: z.string().default("utf8"), folderId: z.string().uuid(), status: z.string().nullable().optional(), statusDetails: z.string().nullable().optional(), createdAt: z.date(), - updatedAt: z.date() + updatedAt: z.date(), + encryptedConfig: zodBuffer }); export type TDynamicSecrets = z.infer; diff --git a/backend/src/db/schemas/index.ts b/backend/src/db/schemas/index.ts index 06c735cee..61031f910 100644 --- a/backend/src/db/schemas/index.ts +++ b/backend/src/db/schemas/index.ts @@ -66,26 +66,35 @@ export * from "./scim-tokens"; export * from "./secret-approval-policies"; export * from "./secret-approval-policies-approvers"; export * from "./secret-approval-request-secret-tags"; +export * from "./secret-approval-request-secret-tags-v2"; export * from "./secret-approval-requests"; export * from "./secret-approval-requests-reviewers"; export * from "./secret-approval-requests-secrets"; +export * from "./secret-approval-requests-secrets-v2"; export * from "./secret-blind-indexes"; export * from "./secret-folder-versions"; export * from "./secret-folders"; export * from "./secret-imports"; export * from "./secret-references"; +export * from "./secret-references-v2"; +export * from "./secret-rotation-output-v2"; export * from "./secret-rotation-outputs"; export * from "./secret-rotations"; export * from "./secret-scanning-git-risks"; export * from "./secret-sharing"; export * from "./secret-snapshot-folders"; export * from "./secret-snapshot-secrets"; +export * from "./secret-snapshot-secrets-v2"; export * from "./secret-snapshots"; export * from "./secret-tag-junction"; export * from "./secret-tags"; +export * from "./secret-v2-tag-junction"; export * from "./secret-version-tag-junction"; +export * from "./secret-version-v2-tag-junction"; export * from "./secret-versions"; +export * from "./secret-versions-v2"; export * from "./secrets"; +export * from "./secrets-v2"; export * from "./service-tokens"; export * from "./super-admin"; export * from "./trusted-ips"; diff --git a/backend/src/db/schemas/integration-auths.ts b/backend/src/db/schemas/integration-auths.ts index 0a980edc8..85368eff7 100644 --- a/backend/src/db/schemas/integration-auths.ts +++ b/backend/src/db/schemas/integration-auths.ts @@ -5,6 +5,8 @@ import { z } from "zod"; +import { zodBuffer } from "@app/lib/zod"; + import { TImmutableDBKeys } from "./models"; export const IntegrationAuthsSchema = z.object({ @@ -32,7 +34,11 @@ export const IntegrationAuthsSchema = z.object({ updatedAt: z.date(), awsAssumeIamRoleArnCipherText: z.string().nullable().optional(), awsAssumeIamRoleArnIV: z.string().nullable().optional(), - awsAssumeIamRoleArnTag: z.string().nullable().optional() + awsAssumeIamRoleArnTag: z.string().nullable().optional(), + encryptedAccess: zodBuffer.nullable().optional(), + encryptedAccessId: zodBuffer.nullable().optional(), + encryptedRefresh: zodBuffer.nullable().optional(), + encryptedAwsAssumeIamRoleArn: zodBuffer.nullable().optional() }); export type TIntegrationAuths = z.infer; diff --git a/backend/src/db/schemas/models.ts b/backend/src/db/schemas/models.ts index 5747fb381..2e91608e3 100644 --- a/backend/src/db/schemas/models.ts +++ b/backend/src/db/schemas/models.ts @@ -90,9 +90,18 @@ export enum TableName { TrustedIps = "trusted_ips", DynamicSecret = "dynamic_secrets", DynamicSecretLease = "dynamic_secret_leases", + SecretV2 = "secrets_v2", + SecretReferenceV2 = "secret_references_v2", + SecretVersionV2 = "secret_versions_v2", + SecretApprovalRequestSecretV2 = "secret_approval_requests_secrets_v2", + SecretApprovalRequestSecretTagV2 = "secret_approval_request_secret_tags_v2", + SnapshotSecretV2 = "secret_snapshot_secrets_v2", // junction tables with tags + SecretV2JnTag = "secret_v2_tag_junction", JnSecretTag = "secret_tag_junction", SecretVersionTag = "secret_version_tag_junction", + SecretVersionV2Tag = "secret_version_v2_tag_junction", + SecretRotationOutputV2 = "secret_rotation_output_v2", // KMS Service KmsServerRootConfig = "kms_root_config", KmsKey = "kms_keys", @@ -157,7 +166,8 @@ export enum SecretType { export enum ProjectVersion { V1 = 1, - V2 = 2 + V2 = 2, + V3 = 3 } export enum ProjectUpgradeStatus { diff --git a/backend/src/db/schemas/organizations.ts b/backend/src/db/schemas/organizations.ts index 7b7a004fc..aa63423c9 100644 --- a/backend/src/db/schemas/organizations.ts +++ b/backend/src/db/schemas/organizations.ts @@ -5,6 +5,8 @@ import { z } from "zod"; +import { zodBuffer } from "@app/lib/zod"; + import { TImmutableDBKeys } from "./models"; export const OrganizationsSchema = z.object({ @@ -16,7 +18,8 @@ export const OrganizationsSchema = z.object({ updatedAt: z.date(), authEnforced: z.boolean().default(false).nullable().optional(), scimEnabled: z.boolean().default(false).nullable().optional(), - kmsDefaultKeyId: z.string().uuid().nullable().optional() + kmsDefaultKeyId: z.string().uuid().nullable().optional(), + kmsEncryptedDataKey: zodBuffer.nullable().optional() }); export type TOrganizations = z.infer; diff --git a/backend/src/db/schemas/projects.ts b/backend/src/db/schemas/projects.ts index 19597c9df..deba51b9a 100644 --- a/backend/src/db/schemas/projects.ts +++ b/backend/src/db/schemas/projects.ts @@ -5,6 +5,8 @@ import { z } from "zod"; +import { zodBuffer } from "@app/lib/zod"; + import { TImmutableDBKeys } from "./models"; export const ProjectsSchema = z.object({ @@ -20,7 +22,8 @@ export const ProjectsSchema = z.object({ pitVersionLimit: z.number().default(10), kmsCertificateKeyId: z.string().uuid().nullable().optional(), auditLogsRetentionDays: z.number().nullable().optional(), - kmsSecretManagerKeyId: z.string().uuid().nullable().optional() + kmsSecretManagerKeyId: z.string().uuid().nullable().optional(), + kmsSecretManagerEncryptedDataKey: zodBuffer.nullable().optional() }); export type TProjects = z.infer; diff --git a/backend/src/db/schemas/secret-approval-request-secret-tags-v2.ts b/backend/src/db/schemas/secret-approval-request-secret-tags-v2.ts new file mode 100644 index 000000000..6ded05a5d --- /dev/null +++ b/backend/src/db/schemas/secret-approval-request-secret-tags-v2.ts @@ -0,0 +1,25 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const SecretApprovalRequestSecretTagsV2Schema = z.object({ + id: z.string().uuid(), + secretId: z.string().uuid(), + tagId: z.string().uuid(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TSecretApprovalRequestSecretTagsV2 = z.infer; +export type TSecretApprovalRequestSecretTagsV2Insert = Omit< + z.input, + TImmutableDBKeys +>; +export type TSecretApprovalRequestSecretTagsV2Update = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/secret-approval-requests-secrets-v2.ts b/backend/src/db/schemas/secret-approval-requests-secrets-v2.ts new file mode 100644 index 000000000..ee25ed6ef --- /dev/null +++ b/backend/src/db/schemas/secret-approval-requests-secrets-v2.ts @@ -0,0 +1,37 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { zodBuffer } from "@app/lib/zod"; + +import { TImmutableDBKeys } from "./models"; + +export const SecretApprovalRequestsSecretsV2Schema = z.object({ + id: z.string().uuid(), + version: z.number().default(1).nullable().optional(), + key: z.string(), + encryptedValue: zodBuffer.nullable().optional(), + encryptedComment: zodBuffer.nullable().optional(), + reminderNote: z.string().nullable().optional(), + reminderRepeatDays: z.number().nullable().optional(), + skipMultilineEncoding: z.boolean().default(false).nullable().optional(), + metadata: z.unknown().nullable().optional(), + createdAt: z.date(), + updatedAt: z.date(), + requestId: z.string().uuid(), + op: z.string(), + secretId: z.string().uuid().nullable().optional(), + secretVersion: z.string().uuid().nullable().optional() +}); + +export type TSecretApprovalRequestsSecretsV2 = z.infer; +export type TSecretApprovalRequestsSecretsV2Insert = Omit< + z.input, + TImmutableDBKeys +>; +export type TSecretApprovalRequestsSecretsV2Update = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/secret-references-v2.ts b/backend/src/db/schemas/secret-references-v2.ts new file mode 100644 index 000000000..ed339e65c --- /dev/null +++ b/backend/src/db/schemas/secret-references-v2.ts @@ -0,0 +1,20 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const SecretReferencesV2Schema = z.object({ + id: z.string().uuid(), + environment: z.string(), + secretPath: z.string(), + secretKey: z.string(), + secretId: z.string().uuid() +}); + +export type TSecretReferencesV2 = z.infer; +export type TSecretReferencesV2Insert = Omit, TImmutableDBKeys>; +export type TSecretReferencesV2Update = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/secret-rotation-output-v2.ts b/backend/src/db/schemas/secret-rotation-output-v2.ts new file mode 100644 index 000000000..28d45413a --- /dev/null +++ b/backend/src/db/schemas/secret-rotation-output-v2.ts @@ -0,0 +1,21 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const SecretRotationOutputV2Schema = z.object({ + id: z.string().uuid(), + key: z.string(), + secretId: z.string().uuid(), + rotationId: z.string().uuid() +}); + +export type TSecretRotationOutputV2 = z.infer; +export type TSecretRotationOutputV2Insert = Omit, TImmutableDBKeys>; +export type TSecretRotationOutputV2Update = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/secret-snapshot-secrets-v2.ts b/backend/src/db/schemas/secret-snapshot-secrets-v2.ts new file mode 100644 index 000000000..d643f7f8c --- /dev/null +++ b/backend/src/db/schemas/secret-snapshot-secrets-v2.ts @@ -0,0 +1,23 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const SecretSnapshotSecretsV2Schema = z.object({ + id: z.string().uuid(), + envId: z.string().uuid(), + secretVersionId: z.string().uuid(), + snapshotId: z.string().uuid(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TSecretSnapshotSecretsV2 = z.infer; +export type TSecretSnapshotSecretsV2Insert = Omit, TImmutableDBKeys>; +export type TSecretSnapshotSecretsV2Update = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/secret-v2-tag-junction.ts b/backend/src/db/schemas/secret-v2-tag-junction.ts new file mode 100644 index 000000000..982e11f50 --- /dev/null +++ b/backend/src/db/schemas/secret-v2-tag-junction.ts @@ -0,0 +1,18 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const SecretV2TagJunctionSchema = z.object({ + id: z.string().uuid(), + secrets_v2Id: z.string().uuid(), + secret_tagsId: z.string().uuid() +}); + +export type TSecretV2TagJunction = z.infer; +export type TSecretV2TagJunctionInsert = Omit, TImmutableDBKeys>; +export type TSecretV2TagJunctionUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/secret-version-v2-tag-junction.ts b/backend/src/db/schemas/secret-version-v2-tag-junction.ts new file mode 100644 index 000000000..4679116a0 --- /dev/null +++ b/backend/src/db/schemas/secret-version-v2-tag-junction.ts @@ -0,0 +1,23 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const SecretVersionV2TagJunctionSchema = z.object({ + id: z.string().uuid(), + secret_versions_v2Id: z.string().uuid(), + secret_tagsId: z.string().uuid() +}); + +export type TSecretVersionV2TagJunction = z.infer; +export type TSecretVersionV2TagJunctionInsert = Omit< + z.input, + TImmutableDBKeys +>; +export type TSecretVersionV2TagJunctionUpdate = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/secret-versions-v2.ts b/backend/src/db/schemas/secret-versions-v2.ts new file mode 100644 index 000000000..160ed1c14 --- /dev/null +++ b/backend/src/db/schemas/secret-versions-v2.ts @@ -0,0 +1,33 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { zodBuffer } from "@app/lib/zod"; + +import { TImmutableDBKeys } from "./models"; + +export const SecretVersionsV2Schema = z.object({ + id: z.string().uuid(), + version: z.number().default(1), + type: z.string().default("shared"), + key: z.string(), + encryptedValue: zodBuffer.nullable().optional(), + encryptedComment: zodBuffer.nullable().optional(), + reminderNote: z.string().nullable().optional(), + reminderRepeatDays: z.number().nullable().optional(), + skipMultilineEncoding: z.boolean().default(false).nullable().optional(), + metadata: z.unknown().nullable().optional(), + envId: z.string().uuid().nullable().optional(), + secretId: z.string().uuid(), + folderId: z.string().uuid(), + userId: z.string().uuid().nullable().optional(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TSecretVersionsV2 = z.infer; +export type TSecretVersionsV2Insert = Omit, TImmutableDBKeys>; +export type TSecretVersionsV2Update = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/secrets-v2.ts b/backend/src/db/schemas/secrets-v2.ts new file mode 100644 index 000000000..18fa3b476 --- /dev/null +++ b/backend/src/db/schemas/secrets-v2.ts @@ -0,0 +1,31 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { zodBuffer } from "@app/lib/zod"; + +import { TImmutableDBKeys } from "./models"; + +export const SecretsV2Schema = z.object({ + id: z.string().uuid(), + version: z.number().default(1), + type: z.string().default("shared"), + key: z.string(), + encryptedValue: zodBuffer.nullable().optional(), + encryptedComment: zodBuffer.nullable().optional(), + reminderNote: z.string().nullable().optional(), + reminderRepeatDays: z.number().nullable().optional(), + skipMultilineEncoding: z.boolean().default(false).nullable().optional(), + metadata: z.unknown().nullable().optional(), + userId: z.string().uuid().nullable().optional(), + folderId: z.string().uuid(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TSecretsV2 = z.infer; +export type TSecretsV2Insert = Omit, TImmutableDBKeys>; +export type TSecretsV2Update = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/webhooks.ts b/backend/src/db/schemas/webhooks.ts index a7aac2933..3f670497f 100644 --- a/backend/src/db/schemas/webhooks.ts +++ b/backend/src/db/schemas/webhooks.ts @@ -5,27 +5,22 @@ import { z } from "zod"; +import { zodBuffer } from "@app/lib/zod"; + import { TImmutableDBKeys } from "./models"; export const WebhooksSchema = z.object({ id: z.string().uuid(), secretPath: z.string().default("/"), - url: z.string(), lastStatus: z.string().nullable().optional(), lastRunErrorMessage: z.string().nullable().optional(), isDisabled: z.boolean().default(false), - encryptedSecretKey: z.string().nullable().optional(), - iv: z.string().nullable().optional(), - tag: z.string().nullable().optional(), - algorithm: z.string().nullable().optional(), - keyEncoding: z.string().nullable().optional(), createdAt: z.date(), updatedAt: z.date(), envId: z.string().uuid(), - urlCipherText: z.string().nullable().optional(), - urlIV: z.string().nullable().optional(), - urlTag: z.string().nullable().optional(), - type: z.string().default("general").nullable().optional() + type: z.string().default("general").nullable().optional(), + encryptedSecretKeyWithKms: zodBuffer.nullable().optional(), + encryptedUrl: zodBuffer }); export type TWebhooks = z.infer; diff --git a/backend/src/db/seed-data.ts b/backend/src/db/seed-data.ts index 5f4ea1b4f..47ef15d90 100644 --- a/backend/src/db/seed-data.ts +++ b/backend/src/db/seed-data.ts @@ -33,6 +33,11 @@ export const seedData1 = { name: "first project", slug: "first-project" }, + projectV3: { + id: "77fa7aed-9288-401e-a4c9-3a9430be62a4", + name: "first project v2", + slug: "first-project-v2" + }, environment: { name: "Development", slug: "dev" diff --git a/backend/src/db/seeds/4-project-v3.ts b/backend/src/db/seeds/4-project-v3.ts new file mode 100644 index 000000000..60431919d --- /dev/null +++ b/backend/src/db/seeds/4-project-v3.ts @@ -0,0 +1,50 @@ +import { Knex } from "knex"; + +import { ProjectMembershipRole, ProjectVersion, TableName } from "../schemas"; +import { seedData1 } from "../seed-data"; + +export const DEFAULT_PROJECT_ENVS = [ + { name: "Development", slug: "dev" }, + { name: "Staging", slug: "staging" }, + { name: "Production", slug: "prod" } +]; + +export async function seed(knex: Knex): Promise { + const [projectV2] = await knex(TableName.Project) + .insert({ + name: seedData1.projectV3.name, + orgId: seedData1.organization.id, + slug: seedData1.projectV3.slug, + version: ProjectVersion.V3, + // eslint-disable-next-line + // @ts-ignore + id: seedData1.projectV3.id + }) + .returning("*"); + + const projectMembershipV3 = await knex(TableName.ProjectMembership) + .insert({ + projectId: projectV2.id, + userId: seedData1.id + }) + .returning("*"); + await knex(TableName.ProjectUserMembershipRole).insert({ + role: ProjectMembershipRole.Admin, + projectMembershipId: projectMembershipV3[0].id + }); + + // create default environments and default folders + const projectV3Envs = await knex(TableName.Environment) + .insert( + DEFAULT_PROJECT_ENVS.map(({ name, slug }, index) => ({ + name, + slug, + projectId: seedData1.projectV3.id, + position: index + 1 + })) + ) + .returning("*"); + await knex(TableName.SecretFolder).insert( + projectV3Envs.map(({ id }) => ({ name: "root", envId: id, parentId: null })) + ); +} diff --git a/backend/src/db/seeds/4-machine-identity.ts b/backend/src/db/seeds/5-machine-identity.ts similarity index 87% rename from backend/src/db/seeds/4-machine-identity.ts rename to backend/src/db/seeds/5-machine-identity.ts index 662232e02..544739100 100644 --- a/backend/src/db/seeds/4-machine-identity.ts +++ b/backend/src/db/seeds/5-machine-identity.ts @@ -86,4 +86,15 @@ export async function seed(knex: Knex): Promise { role: ProjectMembershipRole.Admin, projectMembershipId: identityProjectMembership[0].id }); + const identityProjectMembershipV3 = await knex(TableName.IdentityProjectMembership) + .insert({ + identityId: seedData1.machineIdentity.id, + projectId: seedData1.projectV3.id + }) + .returning("*"); + + await knex(TableName.IdentityProjectMembershipRole).insert({ + role: ProjectMembershipRole.Admin, + projectMembershipId: identityProjectMembershipV3[0].id + }); } diff --git a/backend/src/ee/routes/v1/external-kms-router.ts b/backend/src/ee/routes/v1/external-kms-router.ts index 712b8b456..a029e1cb8 100644 --- a/backend/src/ee/routes/v1/external-kms-router.ts +++ b/backend/src/ee/routes/v1/external-kms-router.ts @@ -1,6 +1,7 @@ import { z } from "zod"; import { ExternalKmsSchema, KmsKeysSchema } from "@app/db/schemas"; +import { EventType } from "@app/ee/services/audit-log/audit-log-types"; import { ExternalKmsAwsSchema, ExternalKmsInputSchema, @@ -19,6 +20,23 @@ const sanitizedExternalSchema = KmsKeysSchema.extend({ }) }); +const sanitizedExternalSchemaForGetAll = KmsKeysSchema.pick({ + id: true, + description: true, + isDisabled: true, + createdAt: true, + updatedAt: true, + slug: true +}) + .extend({ + externalKms: ExternalKmsSchema.pick({ + provider: true, + status: true, + statusDetails: true + }) + }) + .array(); + const sanitizedExternalSchemaForGetById = KmsKeysSchema.extend({ external: ExternalKmsSchema.pick({ id: true, @@ -39,8 +57,8 @@ export const registerExternalKmsRouter = async (server: FastifyZodProvider) => { }, schema: { body: z.object({ - slug: z.string().min(1).trim().toLowerCase().optional(), - description: z.string().min(1).trim().optional(), + slug: z.string().min(1).trim().toLowerCase(), + description: z.string().trim().optional(), provider: ExternalKmsInputSchema }), response: { @@ -60,6 +78,21 @@ export const registerExternalKmsRouter = async (server: FastifyZodProvider) => { provider: req.body.provider, description: req.body.description }); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + orgId: req.permission.orgId, + event: { + type: EventType.CREATE_KMS, + metadata: { + kmsId: externalKms.id, + provider: req.body.provider.type, + slug: req.body.slug, + description: req.body.description + } + } + }); + return { externalKms }; } }); @@ -76,7 +109,7 @@ export const registerExternalKmsRouter = async (server: FastifyZodProvider) => { }), body: z.object({ slug: z.string().min(1).trim().toLowerCase().optional(), - description: z.string().min(1).trim().optional(), + description: z.string().trim().optional(), provider: ExternalKmsInputUpdateSchema }), response: { @@ -97,6 +130,21 @@ export const registerExternalKmsRouter = async (server: FastifyZodProvider) => { description: req.body.description, id: req.params.id }); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + orgId: req.permission.orgId, + event: { + type: EventType.UPDATE_KMS, + metadata: { + kmsId: externalKms.id, + provider: req.body.provider.type, + slug: req.body.slug, + description: req.body.description + } + } + }); + return { externalKms }; } }); @@ -126,6 +174,19 @@ export const registerExternalKmsRouter = async (server: FastifyZodProvider) => { actorOrgId: req.permission.orgId, id: req.params.id }); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + orgId: req.permission.orgId, + event: { + type: EventType.DELETE_KMS, + metadata: { + kmsId: externalKms.id, + slug: externalKms.slug + } + } + }); + return { externalKms }; } }); @@ -155,10 +216,48 @@ export const registerExternalKmsRouter = async (server: FastifyZodProvider) => { actorOrgId: req.permission.orgId, id: req.params.id }); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + orgId: req.permission.orgId, + event: { + type: EventType.GET_KMS, + metadata: { + kmsId: externalKms.id, + slug: externalKms.slug + } + } + }); + return { externalKms }; } }); + server.route({ + method: "GET", + url: "/", + config: { + rateLimit: readLimit + }, + schema: { + response: { + 200: z.object({ + externalKmsList: sanitizedExternalSchemaForGetAll + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const externalKmsList = await server.services.externalKms.list({ + actor: req.permission.type, + actorId: req.permission.id, + actorAuthMethod: req.permission.authMethod, + actorOrgId: req.permission.orgId + }); + return { externalKmsList }; + } + }); + server.route({ method: "GET", url: "/slug/:slug", diff --git a/backend/src/ee/routes/v1/index.ts b/backend/src/ee/routes/v1/index.ts index 6bd9176d8..1892bd5e8 100644 --- a/backend/src/ee/routes/v1/index.ts +++ b/backend/src/ee/routes/v1/index.ts @@ -4,6 +4,7 @@ import { registerAuditLogStreamRouter } from "./audit-log-stream-router"; import { registerCaCrlRouter } from "./certificate-authority-crl-router"; import { registerDynamicSecretLeaseRouter } from "./dynamic-secret-lease-router"; import { registerDynamicSecretRouter } from "./dynamic-secret-router"; +import { registerExternalKmsRouter } from "./external-kms-router"; import { registerGroupRouter } from "./group-router"; import { registerIdentityProjectAdditionalPrivilegeRouter } from "./identity-project-additional-privilege-router"; import { registerLdapRouter } from "./ldap-router"; @@ -87,4 +88,8 @@ export const registerV1EERoutes = async (server: FastifyZodProvider) => { }, { prefix: "/additional-privilege" } ); + + await server.register(registerExternalKmsRouter, { + prefix: "/external-kms" + }); }; diff --git a/backend/src/ee/routes/v1/project-router.ts b/backend/src/ee/routes/v1/project-router.ts index 8639ef1a1..aa32fcb5f 100644 --- a/backend/src/ee/routes/v1/project-router.ts +++ b/backend/src/ee/routes/v1/project-router.ts @@ -4,9 +4,10 @@ import { AuditLogsSchema, SecretSnapshotsSchema } from "@app/db/schemas"; import { EventType, UserAgentType } from "@app/ee/services/audit-log/audit-log-types"; import { AUDIT_LOGS, PROJECTS } from "@app/lib/api-docs"; import { getLastMidnightDateISO, removeTrailingSlash } from "@app/lib/fn"; -import { readLimit } from "@app/server/config/rateLimiter"; +import { readLimit, writeLimit } from "@app/server/config/rateLimiter"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; import { AuthMode } from "@app/services/auth/auth-type"; +import { KmsType } from "@app/services/kms/kms-types"; export const registerProjectRouter = async (server: FastifyZodProvider) => { server.route({ @@ -171,4 +172,212 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { onRequest: verifyAuth([AuthMode.JWT]), handler: async () => ({ actors: [] }) }); + + server.route({ + method: "GET", + url: "/:workspaceId/kms", + config: { + rateLimit: readLimit + }, + schema: { + params: z.object({ + workspaceId: z.string().trim() + }), + response: { + 200: z.object({ + secretManagerKmsKey: z.object({ + id: z.string(), + slug: z.string(), + isExternal: z.boolean() + }) + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const kmsKey = await server.services.project.getProjectKmsKeys({ + actor: req.permission.type, + actorId: req.permission.id, + actorAuthMethod: req.permission.authMethod, + actorOrgId: req.permission.orgId, + projectId: req.params.workspaceId + }); + + return kmsKey; + } + }); + + server.route({ + method: "PATCH", + url: "/:workspaceId/kms", + config: { + rateLimit: writeLimit + }, + schema: { + params: z.object({ + workspaceId: z.string().trim() + }), + body: z.object({ + kms: z.discriminatedUnion("type", [ + z.object({ type: z.literal(KmsType.Internal) }), + z.object({ type: z.literal(KmsType.External), kmsId: z.string() }) + ]) + }), + response: { + 200: z.object({ + secretManagerKmsKey: z.object({ + id: z.string(), + slug: z.string(), + isExternal: z.boolean() + }) + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const { secretManagerKmsKey } = await server.services.project.updateProjectKmsKey({ + actor: req.permission.type, + actorId: req.permission.id, + actorAuthMethod: req.permission.authMethod, + actorOrgId: req.permission.orgId, + projectId: req.params.workspaceId, + ...req.body + }); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: req.params.workspaceId, + event: { + type: EventType.UPDATE_PROJECT_KMS, + metadata: { + secretManagerKmsKey: { + id: secretManagerKmsKey.id, + slug: secretManagerKmsKey.slug + } + } + } + }); + + return { + secretManagerKmsKey + }; + } + }); + + server.route({ + method: "GET", + url: "/:workspaceId/kms/backup", + config: { + rateLimit: readLimit + }, + schema: { + params: z.object({ + workspaceId: z.string().trim() + }), + response: { + 200: z.object({ + secretManager: z.string() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const backup = await server.services.project.getProjectKmsBackup({ + actor: req.permission.type, + actorId: req.permission.id, + actorAuthMethod: req.permission.authMethod, + actorOrgId: req.permission.orgId, + projectId: req.params.workspaceId + }); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: req.params.workspaceId, + event: { + type: EventType.GET_PROJECT_KMS_BACKUP, + metadata: {} + } + }); + + return backup; + } + }); + + server.route({ + method: "POST", + url: "/:workspaceId/kms/backup", + config: { + rateLimit: writeLimit + }, + schema: { + params: z.object({ + workspaceId: z.string().trim() + }), + body: z.object({ + backup: z.string().min(1) + }), + response: { + 200: z.object({ + secretManagerKmsKey: z.object({ + id: z.string(), + slug: z.string(), + isExternal: z.boolean() + }) + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const backup = await server.services.project.loadProjectKmsBackup({ + actor: req.permission.type, + actorId: req.permission.id, + actorAuthMethod: req.permission.authMethod, + actorOrgId: req.permission.orgId, + projectId: req.params.workspaceId, + backup: req.body.backup + }); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: req.params.workspaceId, + event: { + type: EventType.LOAD_PROJECT_KMS_BACKUP, + metadata: {} + } + }); + + return backup; + } + }); + + server.route({ + method: "POST", + url: "/:workspaceId/migrate-v3", + config: { + rateLimit: writeLimit + }, + schema: { + params: z.object({ + workspaceId: z.string().trim() + }), + + response: { + 200: z.object({ + message: z.string() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const migration = await server.services.secret.startSecretV2Migration({ + actor: req.permission.type, + actorId: req.permission.id, + actorAuthMethod: req.permission.authMethod, + actorOrgId: req.permission.orgId, + projectId: req.params.workspaceId + }); + + return migration; + } + }); }; diff --git a/backend/src/ee/routes/v1/secret-approval-request-router.ts b/backend/src/ee/routes/v1/secret-approval-request-router.ts index 0ba7099a3..bcc4a36c5 100644 --- a/backend/src/ee/routes/v1/secret-approval-request-router.ts +++ b/backend/src/ee/routes/v1/secret-approval-request-router.ts @@ -3,16 +3,14 @@ import { z } from "zod"; import { SecretApprovalRequestsReviewersSchema, SecretApprovalRequestsSchema, - SecretApprovalRequestsSecretsSchema, - SecretsSchema, SecretTagsSchema, - SecretVersionsSchema, UsersSchema } from "@app/db/schemas"; import { EventType } from "@app/ee/services/audit-log/audit-log-types"; import { ApprovalStatus, RequestState } from "@app/ee/services/secret-approval-request/secret-approval-request-types"; import { readLimit, writeLimit } from "@app/server/config/rateLimiter"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; +import { secretRawSchema } from "@app/server/routes/sanitizedSchemas"; import { AuthMode } from "@app/services/auth/auth-type"; const approvalRequestUser = z.object({ userId: z.string() }).merge( @@ -261,46 +259,32 @@ export const registerSecretApprovalRequestRouter = async (server: FastifyZodProv committerUser: approvalRequestUser, reviewers: approvalRequestUser.extend({ status: z.string() }).array(), secretPath: z.string(), - commits: SecretApprovalRequestsSecretsSchema.omit({ secretBlindIndex: true }) - .merge( - z.object({ - tags: tagSchema, - secret: SecretsSchema.pick({ - id: true, - version: true, - secretKeyIV: true, - secretKeyTag: true, - secretKeyCiphertext: true, - secretValueIV: true, - secretValueTag: true, - secretValueCiphertext: true, - secretCommentIV: true, - secretCommentTag: true, - secretCommentCiphertext: true + commits: secretRawSchema + .omit({ _id: true, environment: true, workspace: true, type: true, version: true }) + .extend({ + op: z.string(), + tags: tagSchema, + secret: z + .object({ + id: z.string(), + version: z.number(), + secretKey: z.string(), + secretValue: z.string().optional(), + secretComment: z.string().optional() }) - .optional() - .nullable(), - secretVersion: SecretVersionsSchema.pick({ - id: true, - version: true, - secretKeyIV: true, - secretKeyTag: true, - secretKeyCiphertext: true, - secretValueIV: true, - secretValueTag: true, - secretValueCiphertext: true, - secretCommentIV: true, - secretCommentTag: true, - secretCommentCiphertext: true + .optional() + .nullable(), + secretVersion: z + .object({ + id: z.string(), + version: z.number(), + secretKey: z.string(), + secretValue: z.string().optional(), + secretComment: z.string().optional(), + tags: tagSchema }) - .merge( - z.object({ - tags: tagSchema - }) - ) - .optional() - }) - ) + .optional() + }) .array() }) ) diff --git a/backend/src/ee/routes/v1/secret-rotation-router.ts b/backend/src/ee/routes/v1/secret-rotation-router.ts index d951eb744..936459fa1 100644 --- a/backend/src/ee/routes/v1/secret-rotation-router.ts +++ b/backend/src/ee/routes/v1/secret-rotation-router.ts @@ -1,6 +1,6 @@ import { z } from "zod"; -import { SecretRotationOutputsSchema, SecretRotationsSchema, SecretsSchema } from "@app/db/schemas"; +import { SecretRotationOutputsSchema, SecretRotationsSchema } from "@app/db/schemas"; import { removeTrailingSlash } from "@app/lib/fn"; import { readLimit, writeLimit } from "@app/server/config/rateLimiter"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; @@ -112,18 +112,10 @@ export const registerSecretRotationRouter = async (server: FastifyZodProvider) = outputs: z .object({ key: z.string(), - secret: SecretsSchema.pick({ - id: true, - version: true, - secretKeyIV: true, - secretKeyTag: true, - secretKeyCiphertext: true, - secretValueIV: true, - secretValueTag: true, - secretValueCiphertext: true, - secretCommentIV: true, - secretCommentTag: true, - secretCommentCiphertext: true + secret: z.object({ + secretKey: z.string(), + id: z.string(), + version: z.number() }) }) .array() diff --git a/backend/src/ee/routes/v1/secret-version-router.ts b/backend/src/ee/routes/v1/secret-version-router.ts index 0604135ba..11443ebfe 100644 --- a/backend/src/ee/routes/v1/secret-version-router.ts +++ b/backend/src/ee/routes/v1/secret-version-router.ts @@ -1,8 +1,8 @@ import { z } from "zod"; -import { SecretVersionsSchema } from "@app/db/schemas"; import { readLimit } from "@app/server/config/rateLimiter"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; +import { secretRawSchema } from "@app/server/routes/sanitizedSchemas"; import { AuthMode } from "@app/services/auth/auth-type"; export const registerSecretVersionRouter = async (server: FastifyZodProvider) => { @@ -22,7 +22,7 @@ export const registerSecretVersionRouter = async (server: FastifyZodProvider) => }), response: { 200: z.object({ - secretVersions: SecretVersionsSchema.omit({ secretBlindIndex: true }).array() + secretVersions: secretRawSchema.array() }) } }, diff --git a/backend/src/ee/routes/v1/snapshot-router.ts b/backend/src/ee/routes/v1/snapshot-router.ts index 6767f8383..a716aabd7 100644 --- a/backend/src/ee/routes/v1/snapshot-router.ts +++ b/backend/src/ee/routes/v1/snapshot-router.ts @@ -1,9 +1,10 @@ import { z } from "zod"; -import { SecretSnapshotsSchema, SecretTagsSchema, SecretVersionsSchema } from "@app/db/schemas"; +import { SecretSnapshotsSchema, SecretTagsSchema } from "@app/db/schemas"; import { PROJECTS } from "@app/lib/api-docs"; import { readLimit, writeLimit } from "@app/server/config/rateLimiter"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; +import { secretRawSchema } from "@app/server/routes/sanitizedSchemas"; import { AuthMode } from "@app/services/auth/auth-type"; export const registerSnapshotRouter = async (server: FastifyZodProvider) => { @@ -27,17 +28,17 @@ export const registerSnapshotRouter = async (server: FastifyZodProvider) => { slug: z.string(), name: z.string() }), - secretVersions: SecretVersionsSchema.omit({ secretBlindIndex: true }) - .merge( - z.object({ - tags: SecretTagsSchema.pick({ - id: true, - slug: true, - name: true, - color: true - }).array() - }) - ) + secretVersions: secretRawSchema + .omit({ _id: true, environment: true, workspace: true, type: true }) + .extend({ + secretId: z.string(), + tags: SecretTagsSchema.pick({ + id: true, + slug: true, + name: true, + color: true + }).array() + }) .array(), folderVersion: z.object({ id: z.string(), name: z.string() }).array(), createdAt: z.date(), diff --git a/backend/src/ee/services/audit-log/audit-log-types.ts b/backend/src/ee/services/audit-log/audit-log-types.ts index 5dedead70..76bf73d8c 100644 --- a/backend/src/ee/services/audit-log/audit-log-types.ts +++ b/backend/src/ee/services/audit-log/audit-log-types.ts @@ -139,7 +139,14 @@ export enum EventType { GET_CERT = "get-cert", DELETE_CERT = "delete-cert", REVOKE_CERT = "revoke-cert", - GET_CERT_BODY = "get-cert-body" + GET_CERT_BODY = "get-cert-body", + CREATE_KMS = "create-kms", + UPDATE_KMS = "update-kms", + DELETE_KMS = "delete-kms", + GET_KMS = "get-kms", + UPDATE_PROJECT_KMS = "update-project-kms", + GET_PROJECT_KMS_BACKUP = "get-project-kms-backup", + LOAD_PROJECT_KMS_BACKUP = "load-project-kms-backup" } interface UserActorMetadata { @@ -1172,6 +1179,62 @@ interface GetCertBody { }; } +interface CreateKmsEvent { + type: EventType.CREATE_KMS; + metadata: { + kmsId: string; + provider: string; + slug: string; + description?: string; + }; +} + +interface DeleteKmsEvent { + type: EventType.DELETE_KMS; + metadata: { + kmsId: string; + slug: string; + }; +} + +interface UpdateKmsEvent { + type: EventType.UPDATE_KMS; + metadata: { + kmsId: string; + provider: string; + slug?: string; + description?: string; + }; +} + +interface GetKmsEvent { + type: EventType.GET_KMS; + metadata: { + kmsId: string; + slug: string; + }; +} + +interface UpdateProjectKmsEvent { + type: EventType.UPDATE_PROJECT_KMS; + metadata: { + secretManagerKmsKey: { + id: string; + slug: string; + }; + }; +} + +interface GetProjectKmsBackupEvent { + type: EventType.GET_PROJECT_KMS_BACKUP; + metadata: Record; // no metadata yet +} + +interface LoadProjectKmsBackupEvent { + type: EventType.LOAD_PROJECT_KMS_BACKUP; + metadata: Record; // no metadata yet +} + export type Event = | GetSecretsEvent | GetSecretEvent @@ -1273,4 +1336,11 @@ export type Event = | GetCert | DeleteCert | RevokeCert - | GetCertBody; + | GetCertBody + | CreateKmsEvent + | UpdateKmsEvent + | DeleteKmsEvent + | GetKmsEvent + | UpdateProjectKmsEvent + | GetProjectKmsBackupEvent + | LoadProjectKmsBackupEvent; diff --git a/backend/src/ee/services/certificate-authority-crl/certificate-authority-crl-service.ts b/backend/src/ee/services/certificate-authority-crl/certificate-authority-crl-service.ts index 917c55a0f..2ef924ffb 100644 --- a/backend/src/ee/services/certificate-authority-crl/certificate-authority-crl-service.ts +++ b/backend/src/ee/services/certificate-authority-crl/certificate-authority-crl-service.ts @@ -72,7 +72,7 @@ export const certificateAuthorityCrlServiceFactory = ({ kmsId: keyId }); - const decryptedCrl = kmsDecryptor({ cipherTextBlob: caCrl.encryptedCrl }); + const decryptedCrl = await kmsDecryptor({ cipherTextBlob: caCrl.encryptedCrl }); const crl = new x509.X509Crl(decryptedCrl); const base64crl = crl.toString("base64"); diff --git a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-dal.ts b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-dal.ts index 339b2d626..97a682933 100644 --- a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-dal.ts +++ b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-dal.ts @@ -40,14 +40,10 @@ export const dynamicSecretLeaseDALFactory = (db: TDbClient) => { db.ref("type").withSchema(TableName.DynamicSecret).as("dynType"), db.ref("defaultTTL").withSchema(TableName.DynamicSecret).as("dynDefaultTTL"), db.ref("maxTTL").withSchema(TableName.DynamicSecret).as("dynMaxTTL"), - db.ref("inputIV").withSchema(TableName.DynamicSecret).as("dynInputIV"), - db.ref("inputTag").withSchema(TableName.DynamicSecret).as("dynInputTag"), - db.ref("inputCiphertext").withSchema(TableName.DynamicSecret).as("dynInputCiphertext"), - db.ref("algorithm").withSchema(TableName.DynamicSecret).as("dynAlgorithm"), - db.ref("keyEncoding").withSchema(TableName.DynamicSecret).as("dynKeyEncoding"), db.ref("folderId").withSchema(TableName.DynamicSecret).as("dynFolderId"), db.ref("status").withSchema(TableName.DynamicSecret).as("dynStatus"), db.ref("statusDetails").withSchema(TableName.DynamicSecret).as("dynStatusDetails"), + db.ref("encryptedConfig").withSchema(TableName.DynamicSecret).as("dynEncryptedConfig"), db.ref("createdAt").withSchema(TableName.DynamicSecret).as("dynCreatedAt"), db.ref("updatedAt").withSchema(TableName.DynamicSecret).as("dynUpdatedAt") ); @@ -62,16 +58,12 @@ export const dynamicSecretLeaseDALFactory = (db: TDbClient) => { type: doc.dynType, defaultTTL: doc.dynDefaultTTL, maxTTL: doc.dynMaxTTL, - inputIV: doc.dynInputIV, - inputTag: doc.dynInputTag, - inputCiphertext: doc.dynInputCiphertext, - algorithm: doc.dynAlgorithm, - keyEncoding: doc.dynKeyEncoding, folderId: doc.dynFolderId, status: doc.dynStatus, statusDetails: doc.dynStatusDetails, createdAt: doc.dynCreatedAt, - updatedAt: doc.dynUpdatedAt + updatedAt: doc.dynUpdatedAt, + encryptedConfig: doc.dynEncryptedConfig } }; } catch (error) { diff --git a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-queue.ts b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-queue.ts index 9bdb1c24e..82248bd8c 100644 --- a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-queue.ts +++ b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-queue.ts @@ -1,8 +1,9 @@ -import { SecretKeyEncoding } from "@app/db/schemas"; import { DisableRotationErrors } from "@app/ee/services/secret-rotation/secret-rotation-queue"; -import { infisicalSymmetricDecrypt } from "@app/lib/crypto/encryption"; import { logger } from "@app/lib/logger"; import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; +import { TKmsServiceFactory } from "@app/services/kms/kms-service"; +import { KmsDataKey } from "@app/services/kms/kms-types"; +import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal"; import { TDynamicSecretDALFactory } from "../dynamic-secret/dynamic-secret-dal"; import { DynamicSecretStatus } from "../dynamic-secret/dynamic-secret-types"; @@ -14,6 +15,8 @@ type TDynamicSecretLeaseQueueServiceFactoryDep = { dynamicSecretLeaseDAL: Pick; dynamicSecretDAL: Pick; dynamicSecretProviders: Record; + kmsService: Pick; + folderDAL: Pick; }; export type TDynamicSecretLeaseQueueServiceFactory = ReturnType; @@ -22,7 +25,9 @@ export const dynamicSecretLeaseQueueServiceFactory = ({ queueService, dynamicSecretDAL, dynamicSecretProviders, - dynamicSecretLeaseDAL + dynamicSecretLeaseDAL, + kmsService, + folderDAL }: TDynamicSecretLeaseQueueServiceFactoryDep) => { const pruneDynamicSecret = async (dynamicSecretCfgId: string) => { await queueService.queue( @@ -77,15 +82,20 @@ export const dynamicSecretLeaseQueueServiceFactory = ({ if (!dynamicSecretLease) throw new DisableRotationErrors({ message: "Dynamic secret lease not found" }); const dynamicSecretCfg = dynamicSecretLease.dynamicSecret; + const folder = await folderDAL.findById(dynamicSecretCfg.folderId); + if (!folder) throw new DisableRotationErrors({ message: "Folder not found" }); + const { projectId } = folder; + + const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId + }); + + const dynamicSecretInputConfig = secretManagerDecryptor({ + cipherTextBlob: dynamicSecretCfg.encryptedConfig + }).toString(); const selectedProvider = dynamicSecretProviders[dynamicSecretCfg.type as DynamicSecretProviders]; - const decryptedStoredInput = JSON.parse( - infisicalSymmetricDecrypt({ - keyEncoding: dynamicSecretCfg.keyEncoding as SecretKeyEncoding, - ciphertext: dynamicSecretCfg.inputCiphertext, - tag: dynamicSecretCfg.inputTag, - iv: dynamicSecretCfg.inputIV - }) - ) as object; + const decryptedStoredInput = JSON.parse(dynamicSecretInputConfig) as object; await selectedProvider.revoke(decryptedStoredInput, dynamicSecretLease.externalEntityId); await dynamicSecretLeaseDAL.deleteById(dynamicSecretLease.id); @@ -100,17 +110,22 @@ export const dynamicSecretLeaseQueueServiceFactory = ({ if ((dynamicSecretCfg.status as DynamicSecretStatus) !== DynamicSecretStatus.Deleting) throw new DisableRotationErrors({ message: "Document not deleted" }); + const folder = await folderDAL.findById(dynamicSecretCfg.folderId); + if (!folder) throw new DisableRotationErrors({ message: "Folder not found" }); + const { projectId } = folder; + const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId + }); + const dynamicSecretLeases = await dynamicSecretLeaseDAL.find({ dynamicSecretId: dynamicSecretCfgId }); if (dynamicSecretLeases.length) { const selectedProvider = dynamicSecretProviders[dynamicSecretCfg.type as DynamicSecretProviders]; - const decryptedStoredInput = JSON.parse( - infisicalSymmetricDecrypt({ - keyEncoding: dynamicSecretCfg.keyEncoding as SecretKeyEncoding, - ciphertext: dynamicSecretCfg.inputCiphertext, - tag: dynamicSecretCfg.inputTag, - iv: dynamicSecretCfg.inputIV - }) - ) as object; + + const dynamicSecretInputConfig = secretManagerDecryptor({ + cipherTextBlob: dynamicSecretCfg.encryptedConfig + }).toString(); + const decryptedStoredInput = JSON.parse(dynamicSecretInputConfig) as object; await Promise.all(dynamicSecretLeases.map(({ id }) => unsetLeaseRevocation(id))); await Promise.all( diff --git a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-service.ts b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-service.ts index 1e5487d22..eb0c6f171 100644 --- a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-service.ts +++ b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-service.ts @@ -1,14 +1,14 @@ import { ForbiddenError, subject } from "@casl/ability"; import ms from "ms"; -import { SecretKeyEncoding } from "@app/db/schemas"; import { TLicenseServiceFactory } from "@app/ee/services/license/license-service"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; import { getConfig } from "@app/lib/config/env"; -import { infisicalSymmetricDecrypt } from "@app/lib/crypto/encryption"; import { BadRequestError } from "@app/lib/errors"; import { logger } from "@app/lib/logger"; +import { TKmsServiceFactory } from "@app/services/kms/kms-service"; +import { KmsDataKey } from "@app/services/kms/kms-types"; import { TProjectDALFactory } from "@app/services/project/project-dal"; import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal"; @@ -34,6 +34,7 @@ type TDynamicSecretLeaseServiceFactoryDep = { folderDAL: Pick; permissionService: Pick; projectDAL: Pick; + kmsService: Pick; }; export type TDynamicSecretLeaseServiceFactory = ReturnType; @@ -46,7 +47,8 @@ export const dynamicSecretLeaseServiceFactory = ({ permissionService, dynamicSecretQueueService, projectDAL, - licenseService + licenseService, + kmsService }: TDynamicSecretLeaseServiceFactoryDep) => { const create = async ({ environmentSlug, @@ -94,14 +96,12 @@ export const dynamicSecretLeaseServiceFactory = ({ throw new BadRequestError({ message: `Max lease limit reached. Limit: ${appCfg.MAX_LEASE_LIMIT}` }); const selectedProvider = dynamicSecretProviders[dynamicSecretCfg.type as DynamicSecretProviders]; - const decryptedStoredInput = JSON.parse( - infisicalSymmetricDecrypt({ - keyEncoding: dynamicSecretCfg.keyEncoding as SecretKeyEncoding, - ciphertext: dynamicSecretCfg.inputCiphertext, - tag: dynamicSecretCfg.inputTag, - iv: dynamicSecretCfg.inputIV - }) - ) as object; + const { decryptor: kmsDecryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId + }); + const decryptedStoredInputJson = kmsDecryptor({ cipherTextBlob: dynamicSecretCfg.encryptedConfig }).toString(); + const decryptedStoredInput = JSON.parse(decryptedStoredInputJson) as object; const selectedTTL = ttl ?? dynamicSecretCfg.defaultTTL; const { maxTTL } = dynamicSecretCfg; @@ -164,14 +164,12 @@ export const dynamicSecretLeaseServiceFactory = ({ const dynamicSecretCfg = dynamicSecretLease.dynamicSecret; const selectedProvider = dynamicSecretProviders[dynamicSecretCfg.type as DynamicSecretProviders]; - const decryptedStoredInput = JSON.parse( - infisicalSymmetricDecrypt({ - keyEncoding: dynamicSecretCfg.keyEncoding as SecretKeyEncoding, - ciphertext: dynamicSecretCfg.inputCiphertext, - tag: dynamicSecretCfg.inputTag, - iv: dynamicSecretCfg.inputIV - }) - ) as object; + const { decryptor: kmsDecryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId + }); + const decryptedStoredInputJson = kmsDecryptor({ cipherTextBlob: dynamicSecretCfg.encryptedConfig }).toString(); + const decryptedStoredInput = JSON.parse(decryptedStoredInputJson) as object; const selectedTTL = ttl ?? dynamicSecretCfg.defaultTTL; const { maxTTL } = dynamicSecretCfg; @@ -231,14 +229,12 @@ export const dynamicSecretLeaseServiceFactory = ({ const dynamicSecretCfg = dynamicSecretLease.dynamicSecret; const selectedProvider = dynamicSecretProviders[dynamicSecretCfg.type as DynamicSecretProviders]; - const decryptedStoredInput = JSON.parse( - infisicalSymmetricDecrypt({ - keyEncoding: dynamicSecretCfg.keyEncoding as SecretKeyEncoding, - ciphertext: dynamicSecretCfg.inputCiphertext, - tag: dynamicSecretCfg.inputTag, - iv: dynamicSecretCfg.inputIV - }) - ) as object; + const { decryptor: kmsDecryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId + }); + const decryptedStoredInputJson = kmsDecryptor({ cipherTextBlob: dynamicSecretCfg.encryptedConfig }).toString(); + const decryptedStoredInput = JSON.parse(decryptedStoredInputJson) as object; const revokeResponse = await selectedProvider .revoke(decryptedStoredInput, dynamicSecretLease.externalEntityId) diff --git a/backend/src/ee/services/dynamic-secret/dynamic-secret-service.ts b/backend/src/ee/services/dynamic-secret/dynamic-secret-service.ts index 1aef3cc86..508c263ee 100644 --- a/backend/src/ee/services/dynamic-secret/dynamic-secret-service.ts +++ b/backend/src/ee/services/dynamic-secret/dynamic-secret-service.ts @@ -1,11 +1,11 @@ import { ForbiddenError, subject } from "@casl/ability"; -import { SecretKeyEncoding } from "@app/db/schemas"; import { TLicenseServiceFactory } from "@app/ee/services/license/license-service"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; -import { infisicalSymmetricDecrypt, infisicalSymmetricEncypt } from "@app/lib/crypto/encryption"; import { BadRequestError } from "@app/lib/errors"; +import { TKmsServiceFactory } from "@app/services/kms/kms-service"; +import { KmsDataKey } from "@app/services/kms/kms-types"; import { TProjectDALFactory } from "@app/services/project/project-dal"; import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal"; @@ -34,6 +34,7 @@ type TDynamicSecretServiceFactoryDep = { folderDAL: Pick; projectDAL: Pick; permissionService: Pick; + kmsService: Pick; }; export type TDynamicSecretServiceFactory = ReturnType; @@ -46,7 +47,8 @@ export const dynamicSecretServiceFactory = ({ dynamicSecretProviders, permissionService, dynamicSecretQueueService, - projectDAL + projectDAL, + kmsService }: TDynamicSecretServiceFactoryDep) => { const create = async ({ path, @@ -96,16 +98,16 @@ export const dynamicSecretServiceFactory = ({ const isConnected = await selectedProvider.validateConnection(provider.inputs); if (!isConnected) throw new BadRequestError({ message: "Provider connection failed" }); + const { encryptor: secretManagerEncryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId + }); - const encryptedInput = infisicalSymmetricEncypt(JSON.stringify(inputs)); + const encryptedConfig = secretManagerEncryptor({ plainText: Buffer.from(JSON.stringify(inputs)) }).cipherTextBlob; const dynamicSecretCfg = await dynamicSecretDAL.create({ type: provider.type, version: 1, - inputIV: encryptedInput.iv, - inputTag: encryptedInput.tag, - inputCiphertext: encryptedInput.ciphertext, - algorithm: encryptedInput.algorithm, - keyEncoding: encryptedInput.encoding, + encryptedConfig, maxTTL, defaultTTL, folderId: folder.id, @@ -165,27 +167,28 @@ export const dynamicSecretServiceFactory = ({ } const selectedProvider = dynamicSecretProviders[dynamicSecretCfg.type as DynamicSecretProviders]; - const decryptedStoredInput = JSON.parse( - infisicalSymmetricDecrypt({ - keyEncoding: dynamicSecretCfg.keyEncoding as SecretKeyEncoding, - ciphertext: dynamicSecretCfg.inputCiphertext, - tag: dynamicSecretCfg.inputTag, - iv: dynamicSecretCfg.inputIV - }) - ) as object; + const { encryptor: secretManagerEncryptor, decryptor: secretManagerDecryptor } = + await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId + }); + const dynamicSecretInputConfig = secretManagerDecryptor({ + cipherTextBlob: dynamicSecretCfg.encryptedConfig + }).toString(); + + const decryptedStoredInput = JSON.parse(dynamicSecretInputConfig) as object; const newInput = { ...decryptedStoredInput, ...(inputs || {}) }; const updatedInput = await selectedProvider.validateProviderInputs(newInput); const isConnected = await selectedProvider.validateConnection(newInput); if (!isConnected) throw new BadRequestError({ message: "Provider connection failed" }); - const encryptedInput = infisicalSymmetricEncypt(JSON.stringify(updatedInput)); + const encryptedConfig = secretManagerEncryptor({ + plainText: Buffer.from(JSON.stringify(updatedInput)) + }).cipherTextBlob; + const updatedDynamicCfg = await dynamicSecretDAL.updateById(dynamicSecretCfg.id, { - inputIV: encryptedInput.iv, - inputTag: encryptedInput.tag, - inputCiphertext: encryptedInput.ciphertext, - algorithm: encryptedInput.algorithm, - keyEncoding: encryptedInput.encoding, + encryptedConfig, maxTTL, defaultTTL, name: newName ?? name, @@ -286,14 +289,16 @@ export const dynamicSecretServiceFactory = ({ const dynamicSecretCfg = await dynamicSecretDAL.findOne({ name, folderId: folder.id }); if (!dynamicSecretCfg) throw new BadRequestError({ message: "Dynamic secret not found" }); - const decryptedStoredInput = JSON.parse( - infisicalSymmetricDecrypt({ - keyEncoding: dynamicSecretCfg.keyEncoding as SecretKeyEncoding, - ciphertext: dynamicSecretCfg.inputCiphertext, - tag: dynamicSecretCfg.inputTag, - iv: dynamicSecretCfg.inputIV - }) - ) as object; + const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId + }); + + const dynamicSecretInputConfig = secretManagerDecryptor({ + cipherTextBlob: dynamicSecretCfg.encryptedConfig + }).toString(); + + const decryptedStoredInput = JSON.parse(dynamicSecretInputConfig) as object; const selectedProvider = dynamicSecretProviders[dynamicSecretCfg.type as DynamicSecretProviders]; const providerInputs = (await selectedProvider.validateProviderInputs(decryptedStoredInput)) as object; return { ...dynamicSecretCfg, inputs: providerInputs }; diff --git a/backend/src/ee/services/external-kms/external-kms-dal.ts b/backend/src/ee/services/external-kms/external-kms-dal.ts index bb9a6ce8d..7077b4aa9 100644 --- a/backend/src/ee/services/external-kms/external-kms-dal.ts +++ b/backend/src/ee/services/external-kms/external-kms-dal.ts @@ -31,6 +31,8 @@ export const externalKmsDALFactory = (db: TDbClient) => { isReserved: el.isReserved, orgId: el.orgId, slug: el.slug, + createdAt: el.createdAt, + updatedAt: el.updatedAt, externalKms: { id: el.externalKmsId, provider: el.externalKmsProvider, diff --git a/backend/src/ee/services/external-kms/external-kms-service.ts b/backend/src/ee/services/external-kms/external-kms-service.ts index a2d314844..491971a04 100644 --- a/backend/src/ee/services/external-kms/external-kms-service.ts +++ b/backend/src/ee/services/external-kms/external-kms-service.ts @@ -5,7 +5,9 @@ import { BadRequestError } from "@app/lib/errors"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { TKmsKeyDALFactory } from "@app/services/kms/kms-key-dal"; import { TKmsServiceFactory } from "@app/services/kms/kms-service"; +import { KmsDataKey } from "@app/services/kms/kms-types"; +import { TLicenseServiceFactory } from "../license/license-service"; import { OrgPermissionActions, OrgPermissionSubjects } from "../permission/org-permission"; import { TPermissionServiceFactory } from "../permission/permission-service"; import { TExternalKmsDALFactory } from "./external-kms-dal"; @@ -22,9 +24,10 @@ import { ExternalKmsAwsSchema, KmsProviders } from "./providers/model"; type TExternalKmsServiceFactoryDep = { externalKmsDAL: TExternalKmsDALFactory; - kmsService: Pick; + kmsService: Pick; kmsDAL: Pick; permissionService: Pick; + licenseService: Pick; }; export type TExternalKmsServiceFactory = ReturnType; @@ -32,6 +35,7 @@ export type TExternalKmsServiceFactory = ReturnType { @@ -51,7 +55,15 @@ export const externalKmsServiceFactory = ({ actorAuthMethod, actorOrgId ); - ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Edit, OrgPermissionSubjects.Settings); + + ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Create, OrgPermissionSubjects.Kms); + const plan = await licenseService.getPlan(actorOrgId); + if (!plan.externalKms) { + throw new BadRequestError({ + message: "Failed to create external KMS due to plan restriction. Upgrade to the Enterprise plan." + }); + } + const kmsSlug = slug ? slugify(slug) : slugify(alphaNumericNanoId(8).toLowerCase()); let sanitizedProviderInput = ""; @@ -59,21 +71,23 @@ export const externalKmsServiceFactory = ({ case KmsProviders.Aws: { const externalKms = await AwsKmsProviderFactory({ inputs: provider.inputs }); - await externalKms.validateConnection(); // if missing kms key this generate a new kms key id and returns new provider input const newProviderInput = await externalKms.generateInputKmsKey(); sanitizedProviderInput = JSON.stringify(newProviderInput); + + await externalKms.validateConnection(); } break; default: throw new BadRequestError({ message: "external kms provided is invalid" }); } - const orgKmsKeyId = await kmsService.getOrgKmsKeyId(actorOrgId); - const kmsEncryptor = await kmsService.encryptWithKmsKey({ - kmsId: orgKmsKeyId + const { encryptor: orgDataKeyEncryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.Organization, + orgId: actorOrgId }); - const { cipherTextBlob: encryptedProviderInputs } = kmsEncryptor({ + + const { cipherTextBlob: encryptedProviderInputs } = orgDataKeyEncryptor({ plainText: Buffer.from(sanitizedProviderInput, "utf8") }); @@ -119,19 +133,28 @@ export const externalKmsServiceFactory = ({ actorAuthMethod, actorOrgId ); - ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Edit, OrgPermissionSubjects.Settings); + ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Edit, OrgPermissionSubjects.Kms); + + const plan = await licenseService.getPlan(kmsDoc.orgId); + if (!plan.externalKms) { + throw new BadRequestError({ + message: "Failed to update external KMS due to plan restriction. Upgrade to the Enterprise plan." + }); + } + const kmsSlug = slug ? slugify(slug) : undefined; const externalKmsDoc = await externalKmsDAL.findOne({ kmsKeyId: kmsDoc.id }); if (!externalKmsDoc) throw new BadRequestError({ message: "External kms not found" }); - const orgDefaultKmsId = await kmsService.getOrgKmsKeyId(kmsDoc.orgId); let sanitizedProviderInput = ""; - if (provider) { - const kmsDecryptor = await kmsService.decryptWithKmsKey({ - kmsId: orgDefaultKmsId + const { encryptor: orgDataKeyEncryptor, decryptor: orgDataKeyDecryptor } = + await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.Organization, + orgId: actorOrgId }); - const decryptedProviderInputBlob = kmsDecryptor({ + if (provider) { + const decryptedProviderInputBlob = orgDataKeyDecryptor({ cipherTextBlob: externalKmsDoc.encryptedProviderInputs }); @@ -154,10 +177,7 @@ export const externalKmsServiceFactory = ({ let encryptedProviderInputs: Buffer | undefined; if (sanitizedProviderInput) { - const kmsEncryptor = await kmsService.encryptWithKmsKey({ - kmsId: orgDefaultKmsId - }); - const { cipherTextBlob } = kmsEncryptor({ + const { cipherTextBlob } = orgDataKeyEncryptor({ plainText: Buffer.from(sanitizedProviderInput, "utf8") }); encryptedProviderInputs = cipherTextBlob; @@ -197,7 +217,7 @@ export const externalKmsServiceFactory = ({ actorAuthMethod, actorOrgId ); - ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Edit, OrgPermissionSubjects.Settings); + ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Delete, OrgPermissionSubjects.Kms); const externalKmsDoc = await externalKmsDAL.findOne({ kmsKeyId: kmsDoc.id }); if (!externalKmsDoc) throw new BadRequestError({ message: "External kms not found" }); @@ -218,7 +238,7 @@ export const externalKmsServiceFactory = ({ actorAuthMethod, actorOrgId ); - ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Edit, OrgPermissionSubjects.Settings); + ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Read, OrgPermissionSubjects.Kms); const externalKmsDocs = await externalKmsDAL.find({ orgId: actorOrgId }); @@ -234,16 +254,18 @@ export const externalKmsServiceFactory = ({ actorAuthMethod, actorOrgId ); - ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Edit, OrgPermissionSubjects.Settings); + + ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Read, OrgPermissionSubjects.Kms); const externalKmsDoc = await externalKmsDAL.findOne({ kmsKeyId: kmsDoc.id }); if (!externalKmsDoc) throw new BadRequestError({ message: "External kms not found" }); - const orgDefaultKmsId = await kmsService.getOrgKmsKeyId(kmsDoc.orgId); - const kmsDecryptor = await kmsService.decryptWithKmsKey({ - kmsId: orgDefaultKmsId + const { decryptor: orgDataKeyDecryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.Organization, + orgId: actorOrgId }); - const decryptedProviderInputBlob = kmsDecryptor({ + + const decryptedProviderInputBlob = orgDataKeyDecryptor({ cipherTextBlob: externalKmsDoc.encryptedProviderInputs }); switch (externalKmsDoc.provider) { @@ -273,16 +295,17 @@ export const externalKmsServiceFactory = ({ actorAuthMethod, actorOrgId ); - ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Edit, OrgPermissionSubjects.Settings); + ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Read, OrgPermissionSubjects.Kms); const externalKmsDoc = await externalKmsDAL.findOne({ kmsKeyId: kmsDoc.id }); if (!externalKmsDoc) throw new BadRequestError({ message: "External kms not found" }); - const orgDefaultKmsId = await kmsService.getOrgKmsKeyId(kmsDoc.orgId); - const kmsDecryptor = await kmsService.decryptWithKmsKey({ - kmsId: orgDefaultKmsId + const { decryptor: orgDataKeyDecryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.Organization, + orgId: actorOrgId }); - const decryptedProviderInputBlob = kmsDecryptor({ + + const decryptedProviderInputBlob = orgDataKeyDecryptor({ cipherTextBlob: externalKmsDoc.encryptedProviderInputs }); diff --git a/backend/src/ee/services/external-kms/providers/aws-kms.ts b/backend/src/ee/services/external-kms/providers/aws-kms.ts index 5a437fd2d..6d9166a3a 100644 --- a/backend/src/ee/services/external-kms/providers/aws-kms.ts +++ b/backend/src/ee/services/external-kms/providers/aws-kms.ts @@ -50,17 +50,26 @@ type TAwsKmsProviderFactoryReturn = TExternalKmsProviderFns & { }; export const AwsKmsProviderFactory = async ({ inputs }: AwsKmsProviderArgs): Promise => { - const providerInputs = await ExternalKmsAwsSchema.parseAsync(inputs); - const awsClient = await getAwsKmsClient(providerInputs); + let providerInputs = await ExternalKmsAwsSchema.parseAsync(inputs); + let awsClient = await getAwsKmsClient(providerInputs); const generateInputKmsKey = async () => { if (providerInputs.kmsKeyId) return providerInputs; const command = new CreateKeyCommand({ Tags: [{ TagKey: "author", TagValue: "infisical" }] }); const kmsKey = await awsClient.send(command); + if (!kmsKey.KeyMetadata?.KeyId) throw new Error("Failed to generate kms key"); - return { ...providerInputs, kmsKeyId: kmsKey.KeyMetadata?.KeyId }; + const updatedProviderInputs = await ExternalKmsAwsSchema.parseAsync({ + ...providerInputs, + kmsKeyId: kmsKey.KeyMetadata?.KeyId + }); + + providerInputs = updatedProviderInputs; + awsClient = await getAwsKmsClient(providerInputs); + + return updatedProviderInputs; }; const validateConnection = async () => { diff --git a/backend/src/ee/services/license/licence-fns.ts b/backend/src/ee/services/license/licence-fns.ts index 3e30276cb..fc0fb54a7 100644 --- a/backend/src/ee/services/license/licence-fns.ts +++ b/backend/src/ee/services/license/licence-fns.ts @@ -39,7 +39,8 @@ export const getDefaultOnPremFeatures = (): TFeatureSet => ({ secretApproval: false, secretRotation: true, caCrl: false, - instanceUserManagement: false + instanceUserManagement: false, + externalKms: false }); export const setupLicenceRequestWithStore = (baseURL: string, refreshUrl: string, licenseKey: string) => { diff --git a/backend/src/ee/services/license/license-types.ts b/backend/src/ee/services/license/license-types.ts index 1cdc9fa51..df85f008f 100644 --- a/backend/src/ee/services/license/license-types.ts +++ b/backend/src/ee/services/license/license-types.ts @@ -57,6 +57,7 @@ export type TFeatureSet = { secretRotation: true; caCrl: false; instanceUserManagement: false; + externalKms: false; }; export type TOrgPlansTableDTO = { diff --git a/backend/src/ee/services/permission/org-permission.ts b/backend/src/ee/services/permission/org-permission.ts index 6b7b3b2b2..77eaacd3b 100644 --- a/backend/src/ee/services/permission/org-permission.ts +++ b/backend/src/ee/services/permission/org-permission.ts @@ -21,7 +21,8 @@ export enum OrgPermissionSubjects { Groups = "groups", Billing = "billing", SecretScanning = "secret-scanning", - Identity = "identity" + Identity = "identity", + Kms = "kms" } export type OrgPermissionSet = @@ -37,7 +38,8 @@ export type OrgPermissionSet = | [OrgPermissionActions, OrgPermissionSubjects.Groups] | [OrgPermissionActions, OrgPermissionSubjects.SecretScanning] | [OrgPermissionActions, OrgPermissionSubjects.Billing] - | [OrgPermissionActions, OrgPermissionSubjects.Identity]; + | [OrgPermissionActions, OrgPermissionSubjects.Identity] + | [OrgPermissionActions, OrgPermissionSubjects.Kms]; const buildAdminPermission = () => { const { can, build } = new AbilityBuilder>(createMongoAbility); @@ -100,6 +102,11 @@ const buildAdminPermission = () => { can(OrgPermissionActions.Edit, OrgPermissionSubjects.Identity); can(OrgPermissionActions.Delete, OrgPermissionSubjects.Identity); + can(OrgPermissionActions.Read, OrgPermissionSubjects.Kms); + can(OrgPermissionActions.Create, OrgPermissionSubjects.Kms); + can(OrgPermissionActions.Edit, OrgPermissionSubjects.Kms); + can(OrgPermissionActions.Delete, OrgPermissionSubjects.Kms); + return build({ conditionsMatcher }); }; diff --git a/backend/src/ee/services/permission/project-permission.ts b/backend/src/ee/services/permission/project-permission.ts index 4853faf61..7401d8dcd 100644 --- a/backend/src/ee/services/permission/project-permission.ts +++ b/backend/src/ee/services/permission/project-permission.ts @@ -28,7 +28,8 @@ export enum ProjectPermissionSub { SecretRotation = "secret-rotation", Identity = "identity", CertificateAuthorities = "certificate-authorities", - Certificates = "certificates" + Certificates = "certificates", + Kms = "kms" } type SubjectFields = { @@ -60,7 +61,8 @@ export type ProjectPermissionSet = | [ProjectPermissionActions.Delete, ProjectPermissionSub.Project] | [ProjectPermissionActions.Edit, ProjectPermissionSub.Project] | [ProjectPermissionActions.Read, ProjectPermissionSub.SecretRollback] - | [ProjectPermissionActions.Create, ProjectPermissionSub.SecretRollback]; + | [ProjectPermissionActions.Create, ProjectPermissionSub.SecretRollback] + | [ProjectPermissionActions.Edit, ProjectPermissionSub.Kms]; const buildAdminPermissionRules = () => { const { can, rules } = new AbilityBuilder>(createMongoAbility); @@ -157,6 +159,8 @@ const buildAdminPermissionRules = () => { can(ProjectPermissionActions.Edit, ProjectPermissionSub.Project); can(ProjectPermissionActions.Delete, ProjectPermissionSub.Project); + can(ProjectPermissionActions.Edit, ProjectPermissionSub.Kms); + return rules; }; diff --git a/backend/src/ee/services/secret-approval-request/secret-approval-request-dal.ts b/backend/src/ee/services/secret-approval-request/secret-approval-request-dal.ts index 855442e6f..54c7563f6 100644 --- a/backend/src/ee/services/secret-approval-request/secret-approval-request-dal.ts +++ b/backend/src/ee/services/secret-approval-request/secret-approval-request-dal.ts @@ -356,5 +356,161 @@ export const secretApprovalRequestDALFactory = (db: TDbClient) => { } }; - return { ...secretApprovalRequestOrm, findById, findProjectRequestCount, findByProjectId }; + const findByProjectIdBridgeSecretV2 = async ( + { status, limit = 20, offset = 0, projectId, committer, environment, userId }: TFindQueryFilter, + tx?: Knex + ) => { + try { + // akhilmhdh: If ever u wanted a 1 to so many relationship connected with pagination + // this is the place u wanna look at. + const query = (tx || db.replicaNode())(TableName.SecretApprovalRequest) + .join(TableName.SecretFolder, `${TableName.SecretApprovalRequest}.folderId`, `${TableName.SecretFolder}.id`) + .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) + .join( + TableName.SecretApprovalPolicy, + `${TableName.SecretApprovalRequest}.policyId`, + `${TableName.SecretApprovalPolicy}.id` + ) + .join( + TableName.SecretApprovalPolicyApprover, + `${TableName.SecretApprovalPolicy}.id`, + `${TableName.SecretApprovalPolicyApprover}.policyId` + ) + .join( + db(TableName.Users).as("committerUser"), + `${TableName.SecretApprovalRequest}.committerUserId`, + `committerUser.id` + ) + .leftJoin( + TableName.SecretApprovalRequestReviewer, + `${TableName.SecretApprovalRequest}.id`, + `${TableName.SecretApprovalRequestReviewer}.requestId` + ) + .leftJoin( + TableName.SecretApprovalRequestSecretV2, + `${TableName.SecretApprovalRequestSecretV2}.requestId`, + `${TableName.SecretApprovalRequest}.id` + ) + .where( + stripUndefinedInWhere({ + projectId, + [`${TableName.Environment}.slug` as "slug"]: environment, + [`${TableName.SecretApprovalRequest}.status`]: status, + committerUserId: committer + }) + ) + .andWhere( + (bd) => + void bd + .where(`${TableName.SecretApprovalPolicyApprover}.approverUserId`, userId) + .orWhere(`${TableName.SecretApprovalRequest}.committerUserId`, userId) + ) + .select(selectAllTableCols(TableName.SecretApprovalRequest)) + .select( + db.ref("projectId").withSchema(TableName.Environment), + db.ref("slug").withSchema(TableName.Environment).as("environment"), + db.ref("id").withSchema(TableName.SecretApprovalRequestReviewer).as("reviewerId"), + db.ref("reviewerUserId").withSchema(TableName.SecretApprovalRequestReviewer), + db.ref("status").withSchema(TableName.SecretApprovalRequestReviewer).as("reviewerStatus"), + db.ref("id").withSchema(TableName.SecretApprovalPolicy).as("policyId"), + db.ref("name").withSchema(TableName.SecretApprovalPolicy).as("policyName"), + db.ref("op").withSchema(TableName.SecretApprovalRequestSecretV2).as("commitOp"), + db.ref("secretId").withSchema(TableName.SecretApprovalRequestSecretV2).as("commitSecretId"), + db.ref("id").withSchema(TableName.SecretApprovalRequestSecretV2).as("commitId"), + db.raw( + `DENSE_RANK() OVER (partition by ${TableName.Environment}."projectId" ORDER BY ${TableName.SecretApprovalRequest}."id" DESC) as rank` + ), + db.ref("secretPath").withSchema(TableName.SecretApprovalPolicy).as("policySecretPath"), + db.ref("approvals").withSchema(TableName.SecretApprovalPolicy).as("policyApprovals"), + db.ref("enforcementLevel").withSchema(TableName.SecretApprovalPolicy).as("policyEnforcementLevel"), + db.ref("approverUserId").withSchema(TableName.SecretApprovalPolicyApprover), + db.ref("email").withSchema("committerUser").as("committerUserEmail"), + db.ref("username").withSchema("committerUser").as("committerUserUsername"), + db.ref("firstName").withSchema("committerUser").as("committerUserFirstName"), + db.ref("lastName").withSchema("committerUser").as("committerUserLastName") + ) + .orderBy("createdAt", "desc"); + + const docs = await (tx || db) + .with("w", query) + .select("*") + .from[number]>("w") + .where("w.rank", ">=", offset) + .andWhere("w.rank", "<", offset + limit); + const formatedDoc = sqlNestRelationships({ + data: docs, + key: "id", + parentMapper: (el) => ({ + ...SecretApprovalRequestsSchema.parse(el), + environment: el.environment, + projectId: el.projectId, + policy: { + id: el.policyId, + name: el.policyName, + approvals: el.policyApprovals, + secretPath: el.policySecretPath, + enforcementLevel: el.policyEnforcementLevel + }, + committerUser: { + userId: el.committerUserId, + email: el.committerUserEmail, + firstName: el.committerUserFirstName, + lastName: el.committerUserLastName, + username: el.committerUserUsername + } + }), + childrenMapper: [ + { + key: "reviewerId", + label: "reviewers" as const, + mapper: ({ reviewerUserId, reviewerStatus: s }) => + reviewerUserId ? { userId: reviewerUserId, status: s } : undefined + }, + { + key: "approverUserId", + label: "approvers" as const, + mapper: ({ approverUserId }) => approverUserId + }, + { + key: "commitId", + label: "commits" as const, + mapper: ({ commitSecretId: secretId, commitId: id, commitOp: op }) => ({ + op, + id, + secretId + }) + } + ] + }); + return formatedDoc.map((el) => ({ + ...el, + policy: { ...el.policy, approvers: el.approvers } + })); + } catch (error) { + throw new DatabaseError({ error, name: "FindSAR" }); + } + }; + + const deleteByProjectId = async (projectId: string, tx?: Knex) => { + try { + const query = await (tx || db)(TableName.SecretApprovalRequest) + .join(TableName.SecretFolder, `${TableName.SecretApprovalRequest}.folderId`, `${TableName.SecretFolder}.id`) + .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) + .where({ projectId }) + .delete(); + + return query; + } catch (error) { + throw new DatabaseError({ error, name: "DeleteByProjectId" }); + } + }; + + return { + ...secretApprovalRequestOrm, + findById, + findProjectRequestCount, + findByProjectId, + findByProjectIdBridgeSecretV2, + deleteByProjectId + }; }; diff --git a/backend/src/ee/services/secret-approval-request/secret-approval-request-secret-dal.ts b/backend/src/ee/services/secret-approval-request/secret-approval-request-secret-dal.ts index 8dc06aaf5..b827f6a21 100644 --- a/backend/src/ee/services/secret-approval-request/secret-approval-request-secret-dal.ts +++ b/backend/src/ee/services/secret-approval-request/secret-approval-request-secret-dal.ts @@ -3,6 +3,7 @@ import { Knex } from "knex"; import { TDbClient } from "@app/db"; import { SecretApprovalRequestsSecretsSchema, + SecretApprovalRequestsSecretsV2Schema, TableName, TSecretApprovalRequestsSecrets, TSecretTags @@ -15,6 +16,8 @@ export type TSecretApprovalRequestSecretDALFactory = ReturnType { const secretApprovalRequestSecretOrm = ormify(db, TableName.SecretApprovalRequestSecret); const secretApprovalRequestSecretTagOrm = ormify(db, TableName.SecretApprovalRequestSecretTag); + const secretApprovalRequestSecretV2TagOrm = ormify(db, TableName.SecretApprovalRequestSecretTagV2); + const secretApprovalRequestSecretV2Orm = ormify(db, TableName.SecretApprovalRequestSecretV2); const bulkUpdateNoVersionIncrement = async (data: TSecretApprovalRequestsSecrets[], tx?: Knex) => { try { @@ -221,10 +224,197 @@ export const secretApprovalRequestSecretDALFactory = (db: TDbClient) => { throw new DatabaseError({ error, name: "FindByRequestId" }); } }; + + const findByRequestIdBridgeSecretV2 = async (requestId: string, tx?: Knex) => { + try { + const doc = await (tx || db.replicaNode())({ + secVerTag: TableName.SecretTag + }) + .from(TableName.SecretApprovalRequestSecretV2) + .where({ requestId }) + .leftJoin( + TableName.SecretApprovalRequestSecretTagV2, + `${TableName.SecretApprovalRequestSecretV2}.id`, + `${TableName.SecretApprovalRequestSecretTagV2}.secretId` + ) + .leftJoin( + TableName.SecretTag, + `${TableName.SecretApprovalRequestSecretTagV2}.tagId`, + `${TableName.SecretTag}.id` + ) + .leftJoin(TableName.SecretV2, `${TableName.SecretApprovalRequestSecretV2}.secretId`, `${TableName.SecretV2}.id`) + .leftJoin( + TableName.SecretVersionV2, + `${TableName.SecretVersionV2}.id`, + `${TableName.SecretApprovalRequestSecretV2}.secretVersion` + ) + .leftJoin( + TableName.SecretVersionV2Tag, + `${TableName.SecretVersionV2Tag}.${TableName.SecretVersionV2}Id`, + `${TableName.SecretVersionV2}.id` + ) + .leftJoin( + db.ref(TableName.SecretTag).as("secVerTag"), + `${TableName.SecretVersionV2Tag}.${TableName.SecretTag}Id`, + db.ref("id").withSchema("secVerTag") + ) + .select(selectAllTableCols(TableName.SecretApprovalRequestSecretV2)) + .select({ + secVerTagId: "secVerTag.id", + secVerTagColor: "secVerTag.color", + secVerTagSlug: "secVerTag.slug", + secVerTagName: "secVerTag.name" + }) + .select( + db.ref("id").withSchema(TableName.SecretTag).as("tagId"), + db.ref("id").withSchema(TableName.SecretApprovalRequestSecretTagV2).as("tagJnId"), + db.ref("color").withSchema(TableName.SecretTag).as("tagColor"), + db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug"), + db.ref("name").withSchema(TableName.SecretTag).as("tagName") + ) + .select( + db.ref("version").withSchema(TableName.SecretV2).as("orgSecVersion"), + db.ref("key").withSchema(TableName.SecretV2).as("orgSecKey"), + db.ref("encryptedValue").withSchema(TableName.SecretV2).as("orgSecValue"), + db.ref("encryptedComment").withSchema(TableName.SecretV2).as("orgSecComment") + ) + .select( + db.ref("version").withSchema(TableName.SecretVersionV2).as("secVerVersion"), + db.ref("key").withSchema(TableName.SecretVersionV2).as("secVerKey"), + db.ref("encryptedValue").withSchema(TableName.SecretVersionV2).as("secVerValue"), + db.ref("encryptedComment").withSchema(TableName.SecretVersionV2).as("secVerComment") + ); + const formatedDoc = sqlNestRelationships({ + data: doc, + key: "id", + parentMapper: (data) => SecretApprovalRequestsSecretsV2Schema.omit({ secretVersion: true }).parse(data), + childrenMapper: [ + { + key: "tagJnId", + label: "tags" as const, + mapper: ({ tagId: id, tagName: name, tagSlug: slug, tagColor: color }) => ({ + id, + name, + slug, + color + }) + }, + { + key: "secretId", + label: "secret" as const, + mapper: ({ orgSecVersion, orgSecKey, orgSecValue, orgSecComment, secretId }) => + secretId + ? { + id: secretId, + version: orgSecVersion, + key: orgSecKey, + encryptedValue: orgSecValue, + encryptedComment: orgSecComment + } + : undefined + }, + { + key: "secretVersion", + label: "secretVersion" as const, + mapper: ({ secretVersion, secVerVersion, secVerKey, secVerValue, secVerComment }) => + secretVersion + ? { + version: secVerVersion, + id: secretVersion, + key: secVerKey, + encryptedValue: secVerValue, + encryptedComment: secVerComment + } + : undefined, + childrenMapper: [ + { + key: "secVerTagId", + label: "tags" as const, + mapper: ({ secVerTagId: id, secVerTagName: name, secVerTagSlug: slug, secVerTagColor: color }) => ({ + // eslint-disable-next-line + id, + // eslint-disable-next-line + name, + // eslint-disable-next-line + slug, + // eslint-disable-next-line + color + }) + } + ] + } + ] + }); + return formatedDoc?.map(({ secret, secretVersion, ...el }) => ({ + ...el, + secret: secret?.[0], + secretVersion: secretVersion?.[0] + })); + } catch (error) { + throw new DatabaseError({ error, name: "FindByRequestId" }); + } + }; + // special query for migration to v2 secret + const findByProjectId = async (projectId: string, tx?: Knex) => { + try { + const docs = await (tx || db)(TableName.SecretApprovalRequestSecret) + .join( + TableName.SecretApprovalRequest, + `${TableName.SecretApprovalRequest}.id`, + `${TableName.SecretApprovalRequestSecret}.requestId` + ) + .join(TableName.SecretFolder, `${TableName.SecretApprovalRequest}.folderId`, `${TableName.SecretFolder}.id`) + .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) + .leftJoin( + TableName.SecretApprovalRequestSecretTag, + `${TableName.SecretApprovalRequestSecret}.id`, + `${TableName.SecretApprovalRequestSecretTag}.secretId` + ) + .where({ projectId }) + .select(selectAllTableCols(TableName.SecretApprovalRequestSecret)) + .select( + db.ref("id").withSchema(TableName.SecretApprovalRequestSecretTag).as("secretApprovalTagId"), + db.ref("secretId").withSchema(TableName.SecretApprovalRequestSecretTag).as("secretApprovalTagSecretId"), + db.ref("tagId").withSchema(TableName.SecretApprovalRequestSecretTag).as("secretApprovalTagSecretTagId"), + db.ref("createdAt").withSchema(TableName.SecretApprovalRequestSecretTag).as("secretApprovalTagCreatedAt"), + db.ref("updatedAt").withSchema(TableName.SecretApprovalRequestSecretTag).as("secretApprovalTagUpdatedAt") + ); + const formatedDoc = sqlNestRelationships({ + data: docs, + key: "id", + parentMapper: (data) => SecretApprovalRequestsSecretsSchema.parse(data), + childrenMapper: [ + { + key: "secretApprovalTagId", + label: "tags" as const, + mapper: ({ + secretApprovalTagSecretId, + secretApprovalTagId, + secretApprovalTagUpdatedAt, + secretApprovalTagCreatedAt + }) => ({ + secretApprovalTagSecretId, + secretApprovalTagId, + secretApprovalTagUpdatedAt, + secretApprovalTagCreatedAt + }) + } + ] + }); + return formatedDoc; + } catch (error) { + throw new DatabaseError({ error, name: "FindByRequestId" }); + } + }; + return { ...secretApprovalRequestSecretOrm, + insertV2Bridge: secretApprovalRequestSecretV2Orm.insertMany, findByRequestId, + findByRequestIdBridgeSecretV2, bulkUpdateNoVersionIncrement, - insertApprovalSecretTags: secretApprovalRequestSecretTagOrm.insertMany + findByProjectId, + insertApprovalSecretTags: secretApprovalRequestSecretTagOrm.insertMany, + insertApprovalSecretV2Tags: secretApprovalRequestSecretV2TagOrm.insertMany }; }; diff --git a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts index 68e0271d2..4ae4d25ea 100644 --- a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts +++ b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts @@ -5,20 +5,25 @@ import { SecretEncryptionAlgo, SecretKeyEncoding, SecretType, - TSecretApprovalRequestsSecretsInsert + TSecretApprovalRequestsSecretsInsert, + TSecretApprovalRequestsSecretsV2Insert } from "@app/db/schemas"; import { getConfig } from "@app/lib/config/env"; import { decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto"; import { BadRequestError, UnauthorizedError } from "@app/lib/errors"; import { groupBy, pick, unique } from "@app/lib/fn"; +import { setKnexStringValue } from "@app/lib/knex"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { EnforcementLevel } from "@app/lib/types"; import { ActorType } from "@app/services/auth/auth-type"; +import { TKmsServiceFactory } from "@app/services/kms/kms-service"; +import { KmsDataKey } from "@app/services/kms/kms-types"; import { TProjectDALFactory } from "@app/services/project/project-dal"; import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service"; import { TProjectEnvDALFactory } from "@app/services/project-env/project-env-dal"; import { TSecretDALFactory } from "@app/services/secret/secret-dal"; import { + decryptSecretWithBot, fnSecretBlindIndexCheck, fnSecretBlindIndexCheckV2, fnSecretBulkDelete, @@ -33,6 +38,15 @@ import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal"; import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal"; import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; +import { TSecretV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-dal"; +import { + fnSecretBulkDelete as fnSecretV2BridgeBulkDelete, + fnSecretBulkInsert as fnSecretV2BridgeBulkInsert, + fnSecretBulkUpdate as fnSecretV2BridgeBulkUpdate, + getAllNestedSecretReferences as getAllNestedSecretReferencesV2Bridge +} from "@app/services/secret-v2-bridge/secret-v2-bridge-fns"; +import { TSecretVersionV2DALFactory } from "@app/services/secret-v2-bridge/secret-version-dal"; +import { TSecretVersionV2TagDALFactory } from "@app/services/secret-v2-bridge/secret-version-tag-dal"; import { SmtpTemplates, TSmtpService } from "@app/services/smtp/smtp-service"; import { TUserDALFactory } from "@app/services/user/user-dal"; @@ -47,6 +61,7 @@ import { RequestState, TApprovalRequestCountDTO, TGenerateSecretApprovalRequestDTO, + TGenerateSecretApprovalRequestV2BridgeDTO, TListApprovalsDTO, TMergeSecretApprovalRequestDTO, TReviewRequestDTO, @@ -62,16 +77,26 @@ type TSecretApprovalRequestServiceFactoryDep = { secretApprovalRequestReviewerDAL: TSecretApprovalRequestReviewerDALFactory; folderDAL: Pick; secretDAL: TSecretDALFactory; - secretTagDAL: Pick; + secretTagDAL: Pick< + TSecretTagDALFactory, + "findManyTagsById" | "saveTagsToSecret" | "deleteTagsManySecret" | "saveTagsToSecretV2" | "deleteTagsToSecretV2" + >; secretBlindIndexDAL: Pick; snapshotService: Pick; secretVersionDAL: Pick; secretVersionTagDAL: Pick; - projectDAL: Pick; - secretQueueService: Pick; smtpService: Pick; userDAL: Pick; projectEnvDAL: Pick; + projectDAL: Pick; + secretQueueService: Pick; + kmsService: Pick; + secretV2BridgeDAL: Pick< + TSecretV2BridgeDALFactory, + "insertMany" | "upsertSecretReferences" | "findBySecretKeys" | "bulkUpdate" | "deleteMany" + >; + secretVersionV2BridgeDAL: Pick; + secretVersionTagV2BridgeDAL: Pick; }; export type TSecretApprovalRequestServiceFactory = ReturnType; @@ -93,7 +118,11 @@ export const secretApprovalRequestServiceFactory = ({ projectBotService, smtpService, userDAL, - projectEnvDAL + projectEnvDAL, + kmsService, + secretV2BridgeDAL, + secretVersionV2BridgeDAL, + secretVersionTagV2BridgeDAL }: TSecretApprovalRequestServiceFactoryDep) => { const requestCount = async ({ projectId, actor, actorId, actorOrgId, actorAuthMethod }: TApprovalRequestCountDTO) => { if (actor === ActorType.SERVICE) throw new BadRequestError({ message: "Cannot use service token" }); @@ -125,6 +154,19 @@ export const secretApprovalRequestServiceFactory = ({ if (actor === ActorType.SERVICE) throw new BadRequestError({ message: "Cannot use service token" }); await permissionService.getProjectPermission(actor, actorId, projectId, actorAuthMethod, actorOrgId); + + const { shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId); + if (shouldUseSecretV2Bridge) { + return secretApprovalRequestDAL.findByProjectIdBridgeSecretV2({ + projectId, + committer, + environment, + status, + userId: actorId, + limit, + offset + }); + } const approvals = await secretApprovalRequestDAL.findByProjectId({ projectId, committer, @@ -149,11 +191,14 @@ export const secretApprovalRequestServiceFactory = ({ const secretApprovalRequest = await secretApprovalRequestDAL.findById(id); if (!secretApprovalRequest) throw new BadRequestError({ message: "Secret approval request not found" }); + const { projectId } = secretApprovalRequest; + const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId); + const { policy } = secretApprovalRequest; const { hasRole } = await permissionService.getProjectPermission( actor, actorId, - secretApprovalRequest.projectId, + projectId, actorAuthMethod, actorOrgId ); @@ -165,7 +210,75 @@ export const secretApprovalRequestServiceFactory = ({ throw new UnauthorizedError({ message: "User has no access" }); } - const secrets = await secretApprovalRequestSecretDAL.findByRequestId(secretApprovalRequest.id); + let secrets; + if (shouldUseSecretV2Bridge) { + const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId + }); + const encrypedSecrets = await secretApprovalRequestSecretDAL.findByRequestIdBridgeSecretV2( + secretApprovalRequest.id + ); + secrets = encrypedSecrets.map((el) => ({ + ...el, + secretKey: el.key, + id: el.id, + version: el.version, + secretValue: el.encryptedValue + ? secretManagerDecryptor({ cipherTextBlob: el.encryptedValue }).toString() + : undefined, + secretComment: el.encryptedComment + ? secretManagerDecryptor({ cipherTextBlob: el.encryptedComment }).toString() + : undefined, + secret: el.secret + ? { + secretKey: el.secret.key, + id: el.secret.id, + version: el.secret.version, + secretValue: el.secret.encryptedValue + ? secretManagerDecryptor({ cipherTextBlob: el.secret.encryptedValue }).toString() + : undefined, + secretComment: el.secret.encryptedComment + ? secretManagerDecryptor({ cipherTextBlob: el.secret.encryptedComment }).toString() + : undefined + } + : undefined, + secretVersion: el.secretVersion + ? { + secretKey: el.secretVersion.key, + id: el.secretVersion.id, + version: el.secretVersion.version, + secretValue: el.secretVersion.encryptedValue + ? secretManagerDecryptor({ cipherTextBlob: el.secretVersion.encryptedValue }).toString() + : undefined, + secretComment: el.secretVersion.encryptedComment + ? secretManagerDecryptor({ cipherTextBlob: el.secretVersion.encryptedComment }).toString() + : undefined + } + : undefined + })); + } else { + if (!botKey) throw new BadRequestError({ message: "Bot key not found" }); + const encrypedSecrets = await secretApprovalRequestSecretDAL.findByRequestId(secretApprovalRequest.id); + secrets = encrypedSecrets.map((el) => ({ + ...el, + ...decryptSecretWithBot(el, botKey), + secret: el.secret + ? { + id: el.secret.id, + version: el.secret.version, + ...decryptSecretWithBot(el.secret, botKey) + } + : undefined, + secretVersion: el.secretVersion + ? { + id: el.secretVersion.id, + version: el.secretVersion.version, + ...decryptSecretWithBot(el.secretVersion, botKey) + } + : undefined + })); + } const secretPath = await folderDAL.findSecretPathByFolderIds(secretApprovalRequest.projectId, [ secretApprovalRequest.folderId ]); @@ -300,48 +413,167 @@ export const secretApprovalRequestServiceFactory = ({ secretApprovalRequest.policy.approvers.filter( ({ userId: approverId }) => reviewers[approverId.toString()] === ApprovalStatus.APPROVED ).length; - const isSoftEnforcement = secretApprovalRequest.policy.enforcementLevel === EnforcementLevel.Soft; if (!hasMinApproval && !isSoftEnforcement) throw new BadRequestError({ message: "Doesn't have minimum approvals needed" }); - const secretApprovalSecrets = await secretApprovalRequestSecretDAL.findByRequestId(secretApprovalRequest.id); - if (!secretApprovalSecrets) throw new BadRequestError({ message: "No secrets found" }); - const conflicts: Array<{ secretId: string; op: SecretOperations }> = []; - let secretCreationCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Create); - if (secretCreationCommits.length) { - const { secsGroupedByBlindIndex: conflictGroupByBlindIndex } = await fnSecretBlindIndexCheckV2({ - folderId, - secretDAL, - inputSecrets: secretCreationCommits.map(({ secretBlindIndex }) => { - if (!secretBlindIndex) { - throw new BadRequestError({ - message: "Missing secret blind index" - }); - } - return { secretBlindIndex }; - }) - }); - secretCreationCommits - .filter(({ secretBlindIndex }) => conflictGroupByBlindIndex[secretBlindIndex || ""]) - .forEach((el) => { - conflicts.push({ op: SecretOperations.Create, secretId: el.id }); - }); - secretCreationCommits = secretCreationCommits.filter( - ({ secretBlindIndex }) => !conflictGroupByBlindIndex[secretBlindIndex || ""] + const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId); + let mergeStatus; + if (shouldUseSecretV2Bridge) { + // this cycle if for bridged secrets + const secretApprovalSecrets = await secretApprovalRequestSecretDAL.findByRequestIdBridgeSecretV2( + secretApprovalRequest.id ); - } + if (!secretApprovalSecrets) throw new BadRequestError({ message: "No secrets found" }); - let secretUpdationCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Update); - if (secretUpdationCommits.length) { - const { secsGroupedByBlindIndex: conflictGroupByBlindIndex } = await fnSecretBlindIndexCheckV2({ - folderId, - secretDAL, - userId: "", - inputSecrets: secretUpdationCommits - .filter(({ secretBlindIndex, secret }) => secret && secret.secretBlindIndex !== secretBlindIndex) - .map(({ secretBlindIndex }) => { + const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId + }); + + const conflicts: Array<{ secretId: string; op: SecretOperations }> = []; + let secretCreationCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Create); + if (secretCreationCommits.length) { + const secrets = await secretV2BridgeDAL.findBySecretKeys( + folderId, + secretCreationCommits.map((el) => ({ + key: el.key, + type: SecretType.Shared + })) + ); + const creationConflictSecretsGroupByKey = groupBy(secrets, (i) => i.key); + secretCreationCommits + .filter(({ key }) => creationConflictSecretsGroupByKey[key]) + .forEach((el) => { + conflicts.push({ op: SecretOperations.Create, secretId: el.id }); + }); + secretCreationCommits = secretCreationCommits.filter(({ key }) => !creationConflictSecretsGroupByKey[key]); + } + + let secretUpdationCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Update); + if (secretUpdationCommits.length) { + const secrets = await secretV2BridgeDAL.findBySecretKeys( + folderId, + secretCreationCommits.map((el) => ({ + key: el.key, + type: SecretType.Shared + })) + ); + const updationConflictSecretsGroupByKey = groupBy(secrets, (i) => i.key); + secretUpdationCommits + .filter(({ key, secretId }) => updationConflictSecretsGroupByKey[key] || !secretId) + .forEach((el) => { + conflicts.push({ op: SecretOperations.Update, secretId: el.id }); + }); + + secretUpdationCommits = secretUpdationCommits.filter( + ({ key, secretId }) => Boolean(secretId) && !updationConflictSecretsGroupByKey[key] + ); + } + + const secretDeletionCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Delete); + mergeStatus = await secretApprovalRequestDAL.transaction(async (tx) => { + const newSecrets = secretCreationCommits.length + ? await fnSecretV2BridgeBulkInsert({ + tx, + folderId, + inputSecrets: secretCreationCommits.map((el) => ({ + tagIds: el?.tags.map(({ id }) => id), + version: 1, + encryptedComment: el.encryptedComment, + encryptedValue: el.encryptedValue, + skipMultilineEncoding: el.skipMultilineEncoding, + key: el.key, + references: el.encryptedValue + ? getAllNestedSecretReferencesV2Bridge( + secretManagerDecryptor({ + cipherTextBlob: el.encryptedValue + }).toString() + ) + : [], + type: SecretType.Shared + })), + secretDAL: secretV2BridgeDAL, + secretVersionDAL: secretVersionV2BridgeDAL, + secretTagDAL, + secretVersionTagDAL: secretVersionTagV2BridgeDAL + }) + : []; + const updatedSecrets = secretUpdationCommits.length + ? await fnSecretV2BridgeBulkUpdate({ + folderId, + tx, + inputSecrets: secretUpdationCommits.map((el) => { + const encryptedValue = + typeof el.encryptedValue !== "undefined" + ? { + encryptedValue: el.encryptedValue as Buffer, + references: el.encryptedValue + ? getAllNestedSecretReferencesV2Bridge( + secretManagerDecryptor({ + cipherTextBlob: el.encryptedValue + }).toString() + ) + : [] + } + : {}; + return { + filter: { id: el.secretId as string, type: SecretType.Shared }, + data: { + reminderRepeatDays: el.reminderRepeatDays, + encryptedComment: el.encryptedComment, + reminderNote: el.reminderNote, + skipMultilineEncoding: el.skipMultilineEncoding, + key: el.key, + tagIds: el?.tags.map(({ id }) => id), + ...encryptedValue + } + }; + }), + secretDAL: secretV2BridgeDAL, + secretVersionDAL: secretVersionV2BridgeDAL, + secretTagDAL, + secretVersionTagDAL: secretVersionTagV2BridgeDAL + }) + : []; + const deletedSecret = secretDeletionCommits.length + ? await fnSecretV2BridgeBulkDelete({ + projectId, + folderId, + tx, + actorId: "", + secretDAL: secretV2BridgeDAL, + secretQueueService, + inputSecrets: secretDeletionCommits.map(({ key }) => ({ secretKey: key, type: SecretType.Shared })) + }) + : []; + const updatedSecretApproval = await secretApprovalRequestDAL.updateById( + secretApprovalRequest.id, + { + conflicts: JSON.stringify(conflicts), + hasMerged: true, + status: RequestState.Closed, + statusChangedByUserId: actorId + }, + tx + ); + return { + secrets: { created: newSecrets, updated: updatedSecrets, deleted: deletedSecret }, + approval: updatedSecretApproval + }; + }); + } else { + const secretApprovalSecrets = await secretApprovalRequestSecretDAL.findByRequestId(secretApprovalRequest.id); + if (!secretApprovalSecrets) throw new BadRequestError({ message: "No secrets found" }); + + const conflicts: Array<{ secretId: string; op: SecretOperations }> = []; + let secretCreationCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Create); + if (secretCreationCommits.length) { + const { secsGroupedByBlindIndex: conflictGroupByBlindIndex } = await fnSecretBlindIndexCheckV2({ + folderId, + secretDAL, + inputSecrets: secretCreationCommits.map(({ secretBlindIndex }) => { if (!secretBlindIndex) { throw new BadRequestError({ message: "Missing secret blind index" @@ -349,80 +581,56 @@ export const secretApprovalRequestServiceFactory = ({ } return { secretBlindIndex }; }) - }); - secretUpdationCommits - .filter( - ({ secretBlindIndex, secretId }) => - (secretBlindIndex && conflictGroupByBlindIndex[secretBlindIndex]) || !secretId - ) - .forEach((el) => { - conflicts.push({ op: SecretOperations.Update, secretId: el.id }); }); + secretCreationCommits + .filter(({ secretBlindIndex }) => conflictGroupByBlindIndex[secretBlindIndex || ""]) + .forEach((el) => { + conflicts.push({ op: SecretOperations.Create, secretId: el.id }); + }); + secretCreationCommits = secretCreationCommits.filter( + ({ secretBlindIndex }) => !conflictGroupByBlindIndex[secretBlindIndex || ""] + ); + } - secretUpdationCommits = secretUpdationCommits.filter( - ({ secretBlindIndex, secretId }) => - Boolean(secretId) && (secretBlindIndex ? !conflictGroupByBlindIndex[secretBlindIndex] : true) - ); - } + let secretUpdationCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Update); + if (secretUpdationCommits.length) { + const { secsGroupedByBlindIndex: conflictGroupByBlindIndex } = await fnSecretBlindIndexCheckV2({ + folderId, + secretDAL, + userId: "", + inputSecrets: secretUpdationCommits + .filter(({ secretBlindIndex, secret }) => secret && secret.secretBlindIndex !== secretBlindIndex) + .map(({ secretBlindIndex }) => { + if (!secretBlindIndex) { + throw new BadRequestError({ + message: "Missing secret blind index" + }); + } + return { secretBlindIndex }; + }) + }); + secretUpdationCommits + .filter( + ({ secretBlindIndex, secretId }) => + (secretBlindIndex && conflictGroupByBlindIndex[secretBlindIndex]) || !secretId + ) + .forEach((el) => { + conflicts.push({ op: SecretOperations.Update, secretId: el.id }); + }); - const secretDeletionCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Delete); - const botKey = await projectBotService.getBotKey(projectId).catch(() => null); - const mergeStatus = await secretApprovalRequestDAL.transaction(async (tx) => { - const newSecrets = secretCreationCommits.length - ? await fnSecretBulkInsert({ - tx, - folderId, - inputSecrets: secretCreationCommits.map((el) => ({ - ...pick(el, [ - "secretCommentCiphertext", - "secretCommentTag", - "secretCommentIV", - "secretValueIV", - "secretValueTag", - "secretValueCiphertext", - "secretKeyCiphertext", - "secretKeyTag", - "secretKeyIV", - "metadata", - "skipMultilineEncoding", - "secretReminderNote", - "secretReminderRepeatDays", - "algorithm", - "keyEncoding", - "secretBlindIndex" - ]), - tags: el?.tags.map(({ id }) => id), - version: 1, - type: SecretType.Shared, - references: botKey - ? getAllNestedSecretReferences( - decryptSymmetric128BitHexKeyUTF8({ - ciphertext: el.secretValueCiphertext, - iv: el.secretValueIV, - tag: el.secretValueTag, - key: botKey - }) - ) - : undefined - })), - secretDAL, - secretVersionDAL, - secretTagDAL, - secretVersionTagDAL - }) - : []; - const updatedSecrets = secretUpdationCommits.length - ? await fnSecretBulkUpdate({ - folderId, - projectId, - tx, - inputSecrets: secretUpdationCommits.map((el) => ({ - filter: { - id: el.secretId as string, // this null check is already checked at top on conflict strategy - type: SecretType.Shared - }, - data: { - tags: el?.tags.map(({ id }) => id), + secretUpdationCommits = secretUpdationCommits.filter( + ({ secretBlindIndex, secretId }) => + Boolean(secretId) && (secretBlindIndex ? !conflictGroupByBlindIndex[secretBlindIndex] : true) + ); + } + + const secretDeletionCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Delete); + mergeStatus = await secretApprovalRequestDAL.transaction(async (tx) => { + const newSecrets = secretCreationCommits.length + ? await fnSecretBulkInsert({ + tx, + folderId, + inputSecrets: secretCreationCommits.map((el) => ({ ...pick(el, [ "secretCommentCiphertext", "secretCommentTag", @@ -437,8 +645,13 @@ export const secretApprovalRequestServiceFactory = ({ "skipMultilineEncoding", "secretReminderNote", "secretReminderRepeatDays", + "algorithm", + "keyEncoding", "secretBlindIndex" ]), + tags: el?.tags.map(({ id }) => id), + version: 1, + type: SecretType.Shared, references: botKey ? getAllNestedSecretReferences( decryptSymmetric128BitHexKeyUTF8({ @@ -449,48 +662,94 @@ export const secretApprovalRequestServiceFactory = ({ }) ) : undefined - } - })), - secretDAL, - secretVersionDAL, - secretTagDAL, - secretVersionTagDAL - }) - : []; - const deletedSecret = secretDeletionCommits.length - ? await fnSecretBulkDelete({ - projectId, - folderId, - tx, - actorId: "", - secretDAL, - secretQueueService, - inputSecrets: secretDeletionCommits.map(({ secretBlindIndex }) => { - if (!secretBlindIndex) { - throw new BadRequestError({ - message: "Missing secret blind index" - }); - } - return { secretBlindIndex, type: SecretType.Shared }; + })), + secretDAL, + secretVersionDAL, + secretTagDAL, + secretVersionTagDAL }) - }) - : []; - const updatedSecretApproval = await secretApprovalRequestDAL.updateById( - secretApprovalRequest.id, - { - conflicts: JSON.stringify(conflicts), - hasMerged: true, - status: RequestState.Closed, - statusChangedByUserId: actorId, - bypassReason - }, - tx - ); - return { - secrets: { created: newSecrets, updated: updatedSecrets, deleted: deletedSecret }, - approval: updatedSecretApproval - }; - }); + : []; + const updatedSecrets = secretUpdationCommits.length + ? await fnSecretBulkUpdate({ + folderId, + projectId, + tx, + inputSecrets: secretUpdationCommits.map((el) => ({ + filter: { + id: el.secretId as string, // this null check is already checked at top on conflict strategy + type: SecretType.Shared + }, + data: { + tags: el?.tags.map(({ id }) => id), + ...pick(el, [ + "secretCommentCiphertext", + "secretCommentTag", + "secretCommentIV", + "secretValueIV", + "secretValueTag", + "secretValueCiphertext", + "secretKeyCiphertext", + "secretKeyTag", + "secretKeyIV", + "metadata", + "skipMultilineEncoding", + "secretReminderNote", + "secretReminderRepeatDays", + "secretBlindIndex" + ]), + references: botKey + ? getAllNestedSecretReferences( + decryptSymmetric128BitHexKeyUTF8({ + ciphertext: el.secretValueCiphertext, + iv: el.secretValueIV, + tag: el.secretValueTag, + key: botKey + }) + ) + : undefined + } + })), + secretDAL, + secretVersionDAL, + secretTagDAL, + secretVersionTagDAL + }) + : []; + const deletedSecret = secretDeletionCommits.length + ? await fnSecretBulkDelete({ + projectId, + folderId, + tx, + actorId: "", + secretDAL, + secretQueueService, + inputSecrets: secretDeletionCommits.map(({ secretBlindIndex }) => { + if (!secretBlindIndex) { + throw new BadRequestError({ + message: "Missing secret blind index" + }); + } + return { secretBlindIndex, type: SecretType.Shared }; + }) + }) + : []; + const updatedSecretApproval = await secretApprovalRequestDAL.updateById( + secretApprovalRequest.id, + { + conflicts: JSON.stringify(conflicts), + hasMerged: true, + status: RequestState.Closed, + statusChangedByUserId: actorId + }, + tx + ); + return { + secrets: { created: newSecrets, updated: updatedSecrets, deleted: deletedSecret }, + approval: updatedSecretApproval + }; + }); + } + await snapshotService.performSnapshot(folderId); const [folder] = await folderDAL.findSecretPathByFolderIds(projectId, [folderId]); if (!folder) throw new BadRequestError({ message: "Folder not found" }); @@ -779,8 +1038,262 @@ export const secretApprovalRequestServiceFactory = ({ }); return secretApprovalRequest; }; + + const generateSecretApprovalRequestV2Bridge = async ({ + data, + actorId, + actor, + actorOrgId, + actorAuthMethod, + policy, + projectId, + secretPath, + environment + }: TGenerateSecretApprovalRequestV2BridgeDTO) => { + if (actor === ActorType.SERVICE || actor === ActorType.Machine) + throw new BadRequestError({ message: "Cannot use service token or machine token over protected branches" }); + + const { permission } = await permissionService.getProjectPermission( + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId + ); + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionActions.Read, + subject(ProjectPermissionSub.Secrets, { environment, secretPath }) + ); + + const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); + if (!folder) + throw new BadRequestError({ + message: "Folder not found for the given environment slug & secret path", + name: "GenSecretApproval" + }); + const folderId = folder.id; + + const commits: Omit[] = []; + const commitTagIds: Record = {}; + + const { encryptor: secretManagerEncryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId + }); + + // for created secret approval change + const createdSecrets = data[SecretOperations.Create]; + if (createdSecrets && createdSecrets?.length) { + const secrets = await secretV2BridgeDAL.findBySecretKeys( + folderId, + createdSecrets.map((el) => ({ + key: el.secretKey, + type: SecretType.Shared + })) + ); + if (secrets.length) + throw new BadRequestError({ message: `Secret already exist: ${secrets.map((el) => el.key).join(",")}` }); + + commits.push( + ...createdSecrets.map((createdSecret) => ({ + op: SecretOperations.Create, + version: 1, + encryptedComment: setKnexStringValue( + createdSecret.secretComment, + (value) => secretManagerEncryptor({ plainText: Buffer.from(value) }).cipherTextBlob + ), + encryptedValue: setKnexStringValue( + createdSecret.secretValue, + (value) => secretManagerEncryptor({ plainText: Buffer.from(value) }).cipherTextBlob + ), + skipMultilineEncoding: createdSecret.skipMultilineEncoding, + key: createdSecret.secretKey, + type: SecretType.Shared + })) + ); + createdSecrets.forEach(({ tagIds, secretKey }) => { + if (tagIds?.length) commitTagIds[secretKey] = tagIds; + }); + } + // not secret approval for update operations + const secretsToUpdate = data[SecretOperations.Update]; + if (secretsToUpdate && secretsToUpdate?.length) { + const secretsToUpdateStoredInDB = await secretV2BridgeDAL.findBySecretKeys( + folderId, + secretsToUpdate.map((el) => ({ + key: el.secretKey, + type: SecretType.Shared + })) + ); + if (secretsToUpdateStoredInDB.length !== secretsToUpdate.length) + throw new BadRequestError({ + message: `Secret not exist: ${secretsToUpdateStoredInDB.map((el) => el.key).join(",")}` + }); + + // now find any secret that needs to update its name + // same process as above + const secretsWithNewName = secretsToUpdate.filter(({ newSecretName }) => Boolean(newSecretName)); + if (secretsWithNewName.length) { + const secrets = await secretV2BridgeDAL.findBySecretKeys( + folderId, + secretsWithNewName.map((el) => ({ + key: el.secretKey, + type: SecretType.Shared + })) + ); + if (secrets.length) + throw new BadRequestError({ + message: `Secret not exist: ${secretsToUpdateStoredInDB.map((el) => el.key).join(",")}` + }); + } + + const updatingSecretsGroupByKey = groupBy(secretsToUpdateStoredInDB, (el) => el.key); + const latestSecretVersions = await secretVersionV2BridgeDAL.findLatestVersionMany( + folderId, + secretsToUpdateStoredInDB.map(({ id }) => id) + ); + commits.push( + ...secretsToUpdate.map( + ({ + newSecretName, + secretKey, + tagIds, + secretValue, + reminderRepeatDays, + reminderNote, + secretComment, + metadata, + skipMultilineEncoding + }) => { + const secretId = updatingSecretsGroupByKey[secretKey][0].id; + if (tagIds?.length) commitTagIds[secretKey] = tagIds; + return { + ...latestSecretVersions[secretId], + key: newSecretName || secretKey, + encryptedComment: setKnexStringValue( + secretComment, + (value) => secretManagerEncryptor({ plainText: Buffer.from(value) }).cipherTextBlob + ), + encryptedValue: setKnexStringValue( + secretValue, + (value) => secretManagerEncryptor({ plainText: Buffer.from(value) }).cipherTextBlob + ), + reminderRepeatDays, + reminderNote, + metadata, + skipMultilineEncoding, + op: SecretOperations.Update as const, + secret: secretId, + secretVersion: latestSecretVersions[secretId].id, + version: updatingSecretsGroupByKey[secretKey][0].version || 1 + }; + } + ) + ); + } + // deleted secrets + const deletedSecrets = data[SecretOperations.Delete]; + if (deletedSecrets && deletedSecrets.length) { + const secretsToDeleteInDB = await secretV2BridgeDAL.findBySecretKeys( + folderId, + deletedSecrets.map((el) => ({ + key: el.secretKey, + type: SecretType.Shared + })) + ); + if (secretsToDeleteInDB.length !== deletedSecrets.length) + throw new BadRequestError({ + message: `Secret not exist: ${secretsToDeleteInDB.map((el) => el.key).join(",")}` + }); + const secretsGroupedByKey = groupBy(secretsToDeleteInDB, (i) => i.key); + const deletedSecretIds = deletedSecrets.map((el) => secretsGroupedByKey[el.secretKey][0].id); + const latestSecretVersions = await secretVersionV2BridgeDAL.findLatestVersionMany(folderId, deletedSecretIds); + commits.push( + ...deletedSecrets.map(({ secretKey }) => { + const secretId = secretsGroupedByKey[secretKey][0].id; + return { + op: SecretOperations.Delete as const, + ...latestSecretVersions[secretId], + key: secretKey, + secret: secretId, + secretVersion: latestSecretVersions[secretId].id + }; + }) + ); + } + + if (!commits.length) throw new BadRequestError({ message: "Empty commits" }); + + const tagIds = unique(Object.values(commitTagIds).flat()); + const tags = tagIds.length ? await secretTagDAL.findManyTagsById(projectId, tagIds) : []; + if (tagIds.length !== tags.length) throw new BadRequestError({ message: "Tag not found" }); + + const secretApprovalRequest = await secretApprovalRequestDAL.transaction(async (tx) => { + const doc = await secretApprovalRequestDAL.create( + { + folderId, + slug: alphaNumericNanoId(), + policyId: policy.id, + status: "open", + hasMerged: false, + committerUserId: actorId + }, + tx + ); + const approvalCommits = await secretApprovalRequestSecretDAL.insertV2Bridge( + commits.map( + ({ + version, + op, + key, + encryptedComment, + skipMultilineEncoding, + metadata, + reminderNote, + reminderRepeatDays, + encryptedValue, + secretId, + secretVersion + }) => ({ + version, + requestId: doc.id, + op, + secretId, + metadata, + secretVersion, + skipMultilineEncoding, + encryptedValue, + reminderRepeatDays, + reminderNote, + encryptedComment, + key + }) + ), + tx + ); + + const commitsGroupByKey = groupBy(approvalCommits, (i) => i.key); + if (tagIds.length) { + await secretApprovalRequestSecretDAL.insertApprovalSecretV2Tags( + Object.keys(commitTagIds).flatMap((blindIndex) => + commitTagIds[blindIndex] + ? commitTagIds[blindIndex].map((tagId) => ({ + secretId: commitsGroupByKey[blindIndex][0].id, + tagId + })) + : [] + ), + tx + ); + } + return { ...doc, commits: approvalCommits }; + }); + return secretApprovalRequest; + }; + return { generateSecretApprovalRequest, + generateSecretApprovalRequestV2Bridge, mergeSecretApprovalRequest, reviewApproval, updateApprovalStatus, diff --git a/backend/src/ee/services/secret-approval-request/secret-approval-request-types.ts b/backend/src/ee/services/secret-approval-request/secret-approval-request-types.ts index db3dc176b..50a70fd60 100644 --- a/backend/src/ee/services/secret-approval-request/secret-approval-request-types.ts +++ b/backend/src/ee/services/secret-approval-request/secret-approval-request-types.ts @@ -26,6 +26,23 @@ export type TApprovalUpdateSecret = Partial & { tagIds?: string[]; }; +export type TApprovalCreateSecretV2Bridge = { + secretKey: string; + secretValue?: string; + secretComment?: string; + reminderNote?: string | null; + reminderRepeatDays?: number | null; + skipMultilineEncoding?: boolean; + metadata?: Record; + tagIds?: string[]; +}; + +export type TApprovalUpdateSecretV2Bridge = Partial & { + secretKey: string; + newSecretName?: string; + tagIds?: string[]; +}; + export type TGenerateSecretApprovalRequestDTO = { environment: string; secretPath: string; @@ -37,6 +54,17 @@ export type TGenerateSecretApprovalRequestDTO = { }; } & TProjectPermission; +export type TGenerateSecretApprovalRequestV2BridgeDTO = { + environment: string; + secretPath: string; + policy: TSecretApprovalPolicies; + data: { + [SecretOperations.Create]?: TApprovalCreateSecretV2Bridge[]; + [SecretOperations.Update]?: TApprovalUpdateSecretV2Bridge[]; + [SecretOperations.Delete]?: { secretKey: string }[]; + }; +} & TProjectPermission; + export type TMergeSecretApprovalRequestDTO = { approvalId: string; bypassReason?: string; diff --git a/backend/src/ee/services/secret-replication/secret-replication-service.ts b/backend/src/ee/services/secret-replication/secret-replication-service.ts index 01f7d066c..52d10b690 100644 --- a/backend/src/ee/services/secret-replication/secret-replication-service.ts +++ b/backend/src/ee/services/secret-replication/secret-replication-service.ts @@ -1,4 +1,4 @@ -import { SecretType, TSecrets } from "@app/db/schemas"; +import { SecretType, TSecrets, TSecretsV2 } from "@app/db/schemas"; import { TSecretApprovalPolicyServiceFactory } from "@app/ee/services/secret-approval-policy/secret-approval-policy-service"; import { TSecretApprovalRequestDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-dal"; import { TSecretApprovalRequestSecretDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-secret-dal"; @@ -10,6 +10,8 @@ import { logger } from "@app/lib/logger"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { QueueName, TQueueServiceFactory } from "@app/queue"; import { ActorType } from "@app/services/auth/auth-type"; +import { TKmsServiceFactory } from "@app/services/kms/kms-service"; +import { KmsDataKey } from "@app/services/kms/kms-types"; import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service"; import { TSecretDALFactory } from "@app/services/secret/secret-dal"; import { fnSecretBulkInsert, fnSecretBulkUpdate } from "@app/services/secret/secret-fns"; @@ -17,12 +19,20 @@ import { TSecretQueueFactory, uniqueSecretQueueKey } from "@app/services/secret/ import { SecretOperations } from "@app/services/secret/secret-types"; import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal"; import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal"; -import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal"; import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal"; import { ReservedFolders } from "@app/services/secret-folder/secret-folder-types"; import { TSecretImportDALFactory } from "@app/services/secret-import/secret-import-dal"; -import { fnSecretsFromImports } from "@app/services/secret-import/secret-import-fns"; +import { fnSecretsFromImports, fnSecretsV2FromImports } from "@app/services/secret-import/secret-import-fns"; import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; +import { TSecretV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-dal"; +import { + fnSecretBulkInsert as fnSecretV2BridgeBulkInsert, + fnSecretBulkUpdate as fnSecretV2BridgeBulkUpdate, + getAllNestedSecretReferences, + getAllNestedSecretReferences as getAllNestedSecretReferencesV2Bridge +} from "@app/services/secret-v2-bridge/secret-v2-bridge-fns"; +import { TSecretVersionV2DALFactory } from "@app/services/secret-v2-bridge/secret-version-dal"; +import { TSecretVersionV2TagDALFactory } from "@app/services/secret-v2-bridge/secret-version-tag-dal"; import { MAX_REPLICATION_DEPTH } from "./secret-replication-constants"; @@ -32,24 +42,42 @@ type TSecretReplicationServiceFactoryDep = { "find" | "findByBlindIndexes" | "insertMany" | "bulkUpdate" | "delete" | "upsertSecretReferences" | "transaction" >; secretVersionDAL: Pick; + secretV2BridgeDAL: Pick< + TSecretV2BridgeDALFactory, + "find" | "findBySecretKeys" | "insertMany" | "bulkUpdate" | "delete" | "upsertSecretReferences" | "transaction" + >; + secretVersionV2BridgeDAL: Pick< + TSecretVersionV2DALFactory, + "find" | "insertMany" | "update" | "findLatestVersionMany" + >; secretImportDAL: Pick; folderDAL: Pick< TSecretFolderDALFactory, "findSecretPathByFolderIds" | "findBySecretPath" | "create" | "findOne" | "findByManySecretPath" >; secretVersionTagDAL: Pick; + secretVersionV2TagBridgeDAL: Pick; secretQueueService: Pick; queueService: Pick; secretApprovalPolicyService: Pick; keyStore: Pick; - secretBlindIndexDAL: Pick; - secretTagDAL: Pick; + secretTagDAL: Pick< + TSecretTagDALFactory, + | "findManyTagsById" + | "saveTagsToSecret" + | "deleteTagsManySecret" + | "find" + | "saveTagsToSecretV2" + | "deleteTagsToSecretV2" + >; secretApprovalRequestDAL: Pick; secretApprovalRequestSecretDAL: Pick< TSecretApprovalRequestSecretDALFactory, - "insertMany" | "insertApprovalSecretTags" + "insertMany" | "insertApprovalSecretTags" | "insertV2Bridge" >; + projectBotService: Pick; + kmsService: Pick; }; export type TSecretReplicationServiceFactory = ReturnType; @@ -90,9 +118,13 @@ export const secretReplicationServiceFactory = ({ secretApprovalRequestSecretDAL, secretApprovalRequestDAL, secretQueueService, - projectBotService + projectBotService, + secretVersionV2TagBridgeDAL, + secretVersionV2BridgeDAL, + secretV2BridgeDAL, + kmsService }: TSecretReplicationServiceFactoryDep) => { - const getReplicatedSecrets = ( + const $getReplicatedSecrets = ( botKey: string, localSecrets: TSecrets[], importedSecrets: { secrets: TSecrets[] }[] @@ -119,6 +151,25 @@ export const secretReplicationServiceFactory = ({ return secrets; }; + const $getReplicatedSecretsV2 = ( + localSecrets: (TSecretsV2 & { secretKey: string; secretValue?: string })[], + importedSecrets: { secrets: (TSecretsV2 & { secretKey: string; secretValue?: string })[] }[] + ) => { + const deDupe = new Set(); + const secrets = [...localSecrets]; + + for (let i = importedSecrets.length - 1; i >= 0; i = -1) { + importedSecrets[i].secrets.forEach((el) => { + if (deDupe.has(el.key)) { + return; + } + deDupe.add(el.key); + secrets.push(el); + }); + } + return secrets; + }; + // IMPORTANT NOTE BEFORE READING THE FUNCTION // SOURCE - Where secrets are copied from // DESTINATION - Where the replicated imports that points to SOURCE from Destination @@ -139,6 +190,7 @@ export const secretReplicationServiceFactory = ({ const folder = await folderDAL.findBySecretPath(projectId, environmentSlug, secretPath); if (!folder) return; + const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId); // the the replicated imports made to the source. These are the destinations const destinationSecretImports = await secretImportDAL.find({ @@ -191,8 +243,270 @@ export const secretReplicationServiceFactory = ({ : destinationReplicatedSecretImports; if (!destinationReplicatedSecretImports.length) return; - const botKey = await projectBotService.getBotKey(projectId); + if (shouldUseSecretV2Bridge) { + const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId + }); + // these are the secrets to be added in replicated folders + const sourceLocalSecrets = await secretV2BridgeDAL.find({ folderId: folder.id, type: SecretType.Shared }); + const sourceSecretImports = await secretImportDAL.find({ folderId: folder.id }); + const sourceImportedSecrets = await fnSecretsV2FromImports({ + allowedImports: sourceSecretImports, + secretDAL: secretV2BridgeDAL, + folderDAL, + secretImportDAL, + decryptor: (value) => (value ? secretManagerDecryptor({ cipherTextBlob: value }).toString() : undefined) + }); + // secrets that gets replicated across imports + const sourceDecryptedLocalSecrets = sourceLocalSecrets.map((el) => ({ + ...el, + secretKey: el.key, + secretValue: el.encryptedValue + ? secretManagerDecryptor({ cipherTextBlob: el.encryptedValue }).toString() + : undefined + })); + const sourceSecrets = $getReplicatedSecretsV2(sourceDecryptedLocalSecrets, sourceImportedSecrets); + const sourceSecretsGroupByKey = groupBy(sourceSecrets, (i) => i.key); + + const lock = await keyStore.acquireLock( + [getReplicationKeyLockPrefix(projectId, environmentSlug, secretPath)], + 5000 + ); + + try { + /* eslint-disable no-await-in-loop */ + for (const destinationSecretImport of destinationReplicatedSecretImports) { + try { + const hasJobCompleted = await keyStore.getItem( + keystoreReplicationSuccessKey(job.id as string, destinationSecretImport.id), + KeyStorePrefixes.SecretReplication + ); + if (hasJobCompleted) { + logger.info( + { jobId: job.id, importId: destinationSecretImport.id }, + "Skipping this job as this has been successfully replicated." + ); + // eslint-disable-next-line + continue; + } + + const [destinationFolder] = await folderDAL.findSecretPathByFolderIds(projectId, [ + destinationSecretImport.folderId + ]); + if (!destinationFolder) throw new BadRequestError({ message: "Imported folder not found" }); + + let destinationReplicationFolder = await folderDAL.findOne({ + parentId: destinationFolder.id, + name: getReplicationFolderName(destinationSecretImport.id), + isReserved: true + }); + if (!destinationReplicationFolder) { + destinationReplicationFolder = await folderDAL.create({ + parentId: destinationFolder.id, + name: getReplicationFolderName(destinationSecretImport.id), + envId: destinationFolder.envId, + isReserved: true + }); + } + const destinationReplicationFolderId = destinationReplicationFolder.id; + + const destinationLocalSecretsFromDB = await secretV2BridgeDAL.find({ + folderId: destinationReplicationFolderId + }); + const destinationLocalSecrets = destinationLocalSecretsFromDB.map((el) => ({ + ...el, + secretKey: el.key, + secretValue: el.encryptedValue + ? secretManagerDecryptor({ cipherTextBlob: el.encryptedValue }).toString() + : undefined + })); + + const destinationLocalSecretsGroupedByKey = groupBy(destinationLocalSecrets, (i) => i.key); + + const locallyCreatedSecrets = sourceSecrets + .filter(({ key }) => !destinationLocalSecretsGroupedByKey[key]?.[0]) + .map((el) => ({ ...el, operation: SecretOperations.Create })); // rewrite update ops to create + + const locallyUpdatedSecrets = sourceSecrets + .filter( + ({ key, secretKey, secretValue }) => + destinationLocalSecretsGroupedByKey[key]?.[0] && + // if key or value changed + (destinationLocalSecretsGroupedByKey[key]?.[0]?.secretKey !== secretKey || + destinationLocalSecretsGroupedByKey[key]?.[0]?.secretValue !== secretValue) + ) + .map((el) => ({ ...el, operation: SecretOperations.Update })); // rewrite update ops to create + + const locallyDeletedSecrets = destinationLocalSecrets + .filter(({ key }) => !sourceSecretsGroupByKey[key]?.[0]) + .map((el) => ({ ...el, operation: SecretOperations.Delete })); + + const isEmtpy = + locallyCreatedSecrets.length + locallyUpdatedSecrets.length + locallyDeletedSecrets.length === 0; + // eslint-disable-next-line + if (isEmtpy) continue; + + const policy = await secretApprovalPolicyService.getSecretApprovalPolicy( + projectId, + destinationFolder.environmentSlug, + destinationFolder.path + ); + // this means it should be a approval request rather than direct replication + if (policy && actor === ActorType.USER) { + const localSecretsLatestVersions = destinationLocalSecrets.map(({ id }) => id); + const latestSecretVersions = await secretVersionV2BridgeDAL.findLatestVersionMany( + destinationReplicationFolderId, + localSecretsLatestVersions + ); + await secretApprovalRequestDAL.transaction(async (tx) => { + const approvalRequestDoc = await secretApprovalRequestDAL.create( + { + folderId: destinationReplicationFolderId, + slug: alphaNumericNanoId(), + policyId: policy.id, + status: "open", + hasMerged: false, + committerUserId: actorId, + isReplicated: true + }, + tx + ); + const commits = locallyCreatedSecrets + .concat(locallyUpdatedSecrets) + .concat(locallyDeletedSecrets) + .map((doc) => { + const { operation } = doc; + const localSecret = destinationLocalSecretsGroupedByKey[doc.key]?.[0]; + + return { + op: operation, + requestId: approvalRequestDoc.id, + metadata: doc.metadata, + key: doc.key, + encryptedValue: doc.encryptedValue, + encryptedComment: doc.encryptedComment, + skipMultilineEncoding: doc.skipMultilineEncoding, + // except create operation other two needs the secret id and version id + ...(operation !== SecretOperations.Create + ? { secretId: localSecret.id, secretVersion: latestSecretVersions[localSecret.id].id } + : {}) + }; + }); + const approvalCommits = await secretApprovalRequestSecretDAL.insertV2Bridge(commits, tx); + + return { ...approvalRequestDoc, commits: approvalCommits }; + }); + } else { + await secretDAL.transaction(async (tx) => { + if (locallyCreatedSecrets.length) { + await fnSecretV2BridgeBulkInsert({ + folderId: destinationReplicationFolderId, + secretVersionDAL: secretVersionV2BridgeDAL, + secretDAL: secretV2BridgeDAL, + tx, + secretTagDAL, + secretVersionTagDAL: secretVersionV2TagBridgeDAL, + inputSecrets: locallyCreatedSecrets.map((doc) => { + return { + type: doc.type, + metadata: doc.metadata, + key: doc.key, + encryptedValue: doc.encryptedValue, + encryptedComment: doc.encryptedComment, + skipMultilineEncoding: doc.skipMultilineEncoding, + references: doc.secretValue ? getAllNestedSecretReferencesV2Bridge(doc.secretValue) : [] + }; + }) + }); + } + if (locallyUpdatedSecrets.length) { + await fnSecretV2BridgeBulkUpdate({ + folderId: destinationReplicationFolderId, + secretVersionDAL: secretVersionV2BridgeDAL, + secretDAL: secretV2BridgeDAL, + tx, + secretTagDAL, + secretVersionTagDAL: secretVersionV2TagBridgeDAL, + inputSecrets: locallyUpdatedSecrets.map((doc) => { + return { + filter: { + folderId: destinationReplicationFolderId, + id: destinationLocalSecretsGroupedByKey[doc.key][0].id + }, + data: { + type: doc.type, + metadata: doc.metadata, + key: doc.key, + encryptedValue: doc.encryptedValue as Buffer, + encryptedComment: doc.encryptedComment, + skipMultilineEncoding: doc.skipMultilineEncoding, + references: doc.secretValue ? getAllNestedSecretReferencesV2Bridge(doc.secretValue) : [] + } + }; + }) + }); + } + if (locallyDeletedSecrets.length) { + await secretDAL.delete( + { + $in: { + id: locallyDeletedSecrets.map(({ id }) => id) + }, + folderId: destinationReplicationFolderId + }, + tx + ); + } + }); + + await secretQueueService.syncSecrets({ + projectId, + secretPath: destinationFolder.path, + environmentSlug: destinationFolder.environmentSlug, + actorId, + actor, + _depth: depth + 1, + _deDupeReplicationQueue: deDupeReplicationQueue, + _deDupeQueue: deDupeQueue + }); + } + + // this is used to avoid multiple times generating secret approval by failed one + await keyStore.setItemWithExpiry( + keystoreReplicationSuccessKey(job.id as string, destinationSecretImport.id), + SECRET_IMPORT_SUCCESS_LOCK, + 1, + KeyStorePrefixes.SecretReplication + ); + + await secretImportDAL.updateById(destinationSecretImport.id, { + lastReplicated: new Date(), + replicationStatus: null, + isReplicationSuccess: true + }); + } catch (err) { + logger.error( + err, + `Failed to replicate secret with import id=[${destinationSecretImport.id}] env=[${destinationSecretImport.importEnv.slug}] path=[${destinationSecretImport.importPath}]` + ); + await secretImportDAL.updateById(destinationSecretImport.id, { + lastReplicated: new Date(), + replicationStatus: (err as Error)?.message.slice(0, 500), + isReplicationSuccess: false + }); + } + } + /* eslint-enable no-await-in-loop */ + } finally { + await lock.release(); + logger.info(job.data, "Replication finished"); + } + return; + } + + if (!botKey) throw new BadRequestError({ message: "Bot not found" }); // these are the secrets to be added in replicated folders const sourceLocalSecrets = await secretDAL.find({ folderId: folder.id, type: SecretType.Shared }); const sourceSecretImports = await secretImportDAL.find({ folderId: folder.id }); @@ -203,7 +517,7 @@ export const secretReplicationServiceFactory = ({ secretImportDAL }); // secrets that gets replicated across imports - const sourceSecrets = getReplicatedSecrets(botKey, sourceLocalSecrets, sourceImportedSecrets); + const sourceSecrets = $getReplicatedSecrets(botKey, sourceLocalSecrets, sourceImportedSecrets); const sourceSecretsGroupByBlindIndex = groupBy(sourceSecrets, (i) => i.secretBlindIndex as string); const lock = await keyStore.acquireLock( @@ -372,7 +686,8 @@ export const secretReplicationServiceFactory = ({ secretCommentIV: doc.secretCommentIV, secretCommentTag: doc.secretCommentTag, secretCommentCiphertext: doc.secretCommentCiphertext, - skipMultilineEncoding: doc.skipMultilineEncoding + skipMultilineEncoding: doc.skipMultilineEncoding, + references: getAllNestedSecretReferences(doc.secretValue) }; }) }); @@ -407,7 +722,8 @@ export const secretReplicationServiceFactory = ({ secretCommentIV: doc.secretCommentIV, secretCommentTag: doc.secretCommentTag, secretCommentCiphertext: doc.secretCommentCiphertext, - skipMultilineEncoding: doc.skipMultilineEncoding + skipMultilineEncoding: doc.skipMultilineEncoding, + references: getAllNestedSecretReferences(doc.secretValue) } }; }) diff --git a/backend/src/ee/services/secret-rotation/secret-rotation-dal.ts b/backend/src/ee/services/secret-rotation/secret-rotation-dal.ts index 57d86ff04..7f885e4f9 100644 --- a/backend/src/ee/services/secret-rotation/secret-rotation-dal.ts +++ b/backend/src/ee/services/secret-rotation/secret-rotation-dal.ts @@ -10,6 +10,7 @@ export type TSecretRotationDALFactory = ReturnType { const secretRotationOrm = ormify(db, TableName.SecretRotation); const secretRotationOutputOrm = ormify(db, TableName.SecretRotationOutput); + const secretRotationOutputV2Orm = ormify(db, TableName.SecretRotationOutputV2); const findQuery = (filter: TFindFilter, tx: Knex) => tx(TableName.SecretRotation) @@ -31,13 +32,7 @@ export const secretRotationDALFactory = (db: TDbClient) => { .select(tx.ref("version").withSchema(TableName.Secret).as("secVersion")) .select(tx.ref("secretKeyIV").withSchema(TableName.Secret)) .select(tx.ref("secretKeyTag").withSchema(TableName.Secret)) - .select(tx.ref("secretKeyCiphertext").withSchema(TableName.Secret)) - .select(tx.ref("secretValueIV").withSchema(TableName.Secret)) - .select(tx.ref("secretValueTag").withSchema(TableName.Secret)) - .select(tx.ref("secretValueCiphertext").withSchema(TableName.Secret)) - .select(tx.ref("secretCommentIV").withSchema(TableName.Secret)) - .select(tx.ref("secretCommentTag").withSchema(TableName.Secret)) - .select(tx.ref("secretCommentCiphertext").withSchema(TableName.Secret)); + .select(tx.ref("secretKeyCiphertext").withSchema(TableName.Secret)); const find = async (filter: TFindFilter, tx?: Knex) => { try { @@ -54,33 +49,65 @@ export const secretRotationDALFactory = (db: TDbClient) => { { key: "secId", label: "outputs" as const, - mapper: ({ - secId, - outputKey, - secVersion, - secretKeyIV, - secretKeyTag, - secretKeyCiphertext, - secretValueTag, - secretValueIV, - secretValueCiphertext, - secretCommentIV, - secretCommentTag, - secretCommentCiphertext - }) => ({ + mapper: ({ secId, outputKey, secVersion, secretKeyIV, secretKeyTag, secretKeyCiphertext }) => ({ key: outputKey, secret: { id: secId, version: secVersion, secretKeyIV, secretKeyTag, - secretKeyCiphertext, - secretValueTag, - secretValueIV, - secretValueCiphertext, - secretCommentIV, - secretCommentTag, - secretCommentCiphertext + secretKeyCiphertext + } + }) + } + ] + }); + } catch (error) { + throw new DatabaseError({ error, name: "SecretRotationFind" }); + } + }; + + const findQuerySecretV2 = (filter: TFindFilter, tx: Knex) => + tx(TableName.SecretRotation) + .where(filter) + .join(TableName.Environment, `${TableName.SecretRotation}.envId`, `${TableName.Environment}.id`) + .leftJoin( + TableName.SecretRotationOutputV2, + `${TableName.SecretRotation}.id`, + `${TableName.SecretRotationOutputV2}.rotationId` + ) + .join(TableName.SecretV2, `${TableName.SecretRotationOutputV2}.secretId`, `${TableName.SecretV2}.id`) + .select(selectAllTableCols(TableName.SecretRotation)) + .select(tx.ref("name").withSchema(TableName.Environment).as("envName")) + .select(tx.ref("slug").withSchema(TableName.Environment).as("envSlug")) + .select(tx.ref("id").withSchema(TableName.Environment).as("envId")) + .select(tx.ref("projectId").withSchema(TableName.Environment)) + .select(tx.ref("key").withSchema(TableName.SecretRotationOutputV2).as("outputKey")) + .select(tx.ref("id").withSchema(TableName.SecretV2).as("secId")) + .select(tx.ref("version").withSchema(TableName.SecretV2).as("secVersion")) + .select(tx.ref("key").withSchema(TableName.SecretV2).as("secretKey")); + + const findSecretV2 = async (filter: TFindFilter, tx?: Knex) => { + try { + const data = await findQuerySecretV2(filter, tx || db.replicaNode()); + return sqlNestRelationships({ + data, + key: "id", + parentMapper: (el) => ({ + ...SecretRotationsSchema.parse(el), + projectId: el.projectId, + environment: { id: el.envId, name: el.envName, slug: el.envSlug } + }), + childrenMapper: [ + { + key: "secId", + label: "outputs" as const, + mapper: ({ secId, outputKey, secVersion, secretKey }) => ({ + key: outputKey, + secret: { + id: secId, + version: secVersion, + secretKey } }) } @@ -114,12 +141,19 @@ export const secretRotationDALFactory = (db: TDbClient) => { }; const findRotationOutputsByRotationId = async (rotationId: string) => secretRotationOutputOrm.find({ rotationId }); + const findRotationOutputsV2ByRotationId = async (rotationId: string) => + secretRotationOutputV2Orm.find({ rotationId }); + + // special query return { ...secretRotationOrm, find, + findSecretV2, findById, secretOutputInsertMany: secretRotationOutputOrm.insertMany, - findRotationOutputsByRotationId + secretOutputV2InsertMany: secretRotationOutputV2Orm.insertMany, + findRotationOutputsByRotationId, + findRotationOutputsV2ByRotationId }; }; diff --git a/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts b/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts index e9eedaee8..ab419e4c9 100644 --- a/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts +++ b/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts @@ -17,9 +17,13 @@ import { BadRequestError } from "@app/lib/errors"; import { logger } from "@app/lib/logger"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; +import { TKmsServiceFactory } from "@app/services/kms/kms-service"; +import { KmsDataKey } from "@app/services/kms/kms-types"; import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service"; import { TSecretDALFactory } from "@app/services/secret/secret-dal"; import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal"; +import { TSecretV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-dal"; +import { TSecretVersionV2DALFactory } from "@app/services/secret-v2-bridge/secret-version-dal"; import { TTelemetryServiceFactory } from "@app/services/telemetry/telemetry-service"; import { PostHogEventTypes } from "@app/services/telemetry/telemetry-types"; @@ -47,8 +51,11 @@ type TSecretRotationQueueFactoryDep = { secretRotationDAL: TSecretRotationDALFactory; projectBotService: Pick; secretDAL: Pick; + secretV2BridgeDAL: Pick; secretVersionDAL: Pick; + secretVersionV2BridgeDAL: Pick; telemetryService: Pick; + kmsService: Pick; }; // These error should stop the repeatable job and ask user to reconfigure rotation @@ -70,7 +77,10 @@ export const secretRotationQueueFactory = ({ projectBotService, secretDAL, secretVersionDAL, - telemetryService + telemetryService, + secretV2BridgeDAL, + secretVersionV2BridgeDAL, + kmsService }: TSecretRotationQueueFactoryDep) => { const addToQueue = async (rotationId: string, interval: number) => { const appCfg = getConfig(); @@ -111,7 +121,13 @@ export const secretRotationQueueFactory = ({ try { if (!rotationProvider || !secretRotation) throw new DisableRotationErrors({ message: "Provider not found" }); - const rotationOutputs = await secretRotationDAL.findRotationOutputsByRotationId(rotationId); + const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(secretRotation.projectId); + let rotationOutputs; + if (shouldUseSecretV2Bridge) { + rotationOutputs = await secretRotationDAL.findRotationOutputsV2ByRotationId(rotationId); + } else { + rotationOutputs = await secretRotationDAL.findRotationOutputsByRotationId(rotationId); + } if (!rotationOutputs.length) throw new DisableRotationErrors({ message: "Secrets not found" }); // deep copy @@ -267,62 +283,112 @@ export const secretRotationQueueFactory = ({ internal: newCredential.internal }); const encVarData = infisicalSymmetricEncypt(JSON.stringify(variables)); - const key = await projectBotService.getBotKey(secretRotation.projectId); - const encryptedSecrets = rotationOutputs.map(({ key: outputKey, secretId }) => ({ - secretId, - value: encryptSymmetric128BitHexKeyUTF8( - typeof newCredential.outputs[outputKey] === "object" - ? JSON.stringify(newCredential.outputs[outputKey]) - : String(newCredential.outputs[outputKey]), - key - ) - })); - // map the final values to output keys in the board - await secretRotationDAL.transaction(async (tx) => { - await secretRotationDAL.updateById( - rotationId, - { - encryptedData: encVarData.ciphertext, - encryptedDataIV: encVarData.iv, - encryptedDataTag: encVarData.tag, - keyEncoding: encVarData.encoding, - algorithm: encVarData.algorithm, - lastRotatedAt: new Date(), - statusMessage: "Rotated successfull", - status: "success" - }, - tx - ); - const updatedSecrets = await secretDAL.bulkUpdate( - encryptedSecrets.map(({ secretId, value }) => ({ - // this secret id is validated when user is inserted - filter: { id: secretId, type: SecretType.Shared }, - data: { - secretValueCiphertext: value.ciphertext, - secretValueIV: value.iv, - secretValueTag: value.tag - } - })), - tx - ); - await secretVersionDAL.insertMany( - updatedSecrets.map(({ id, updatedAt, createdAt, ...el }) => { - if (!el.secretBlindIndex) throw new BadRequestError({ message: "Missing blind index" }); - return { - ...el, - secretId: id, - secretBlindIndex: el.secretBlindIndex - }; - }), - tx - ); + const { encryptor: secretManagerEncryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId: secretRotation.projectId }); + const numberOfSecretsRotated = rotationOutputs.length; + if (shouldUseSecretV2Bridge) { + const encryptedSecrets = rotationOutputs.map(({ key: outputKey, secretId }) => ({ + secretId, + value: + typeof newCredential.outputs[outputKey] === "object" + ? JSON.stringify(newCredential.outputs[outputKey]) + : String(newCredential.outputs[outputKey]) + })); + // map the final values to output keys in the board + await secretRotationDAL.transaction(async (tx) => { + await secretRotationDAL.updateById( + rotationId, + { + encryptedData: encVarData.ciphertext, + encryptedDataIV: encVarData.iv, + encryptedDataTag: encVarData.tag, + keyEncoding: encVarData.encoding, + algorithm: encVarData.algorithm, + lastRotatedAt: new Date(), + statusMessage: "Rotated successfull", + status: "success" + }, + tx + ); + const updatedSecrets = await secretV2BridgeDAL.bulkUpdate( + encryptedSecrets.map(({ secretId, value }) => ({ + // this secret id is validated when user is inserted + filter: { id: secretId, type: SecretType.Shared }, + data: { + encryptedValue: secretManagerEncryptor({ plainText: Buffer.from(value) }).cipherTextBlob + } + })), + tx + ); + await secretVersionV2BridgeDAL.insertMany( + updatedSecrets.map(({ id, updatedAt, createdAt, ...el }) => ({ + ...el, + secretId: id + })), + tx + ); + }); + } else { + if (!botKey) throw new BadRequestError({ message: "Bot not found" }); + const encryptedSecrets = rotationOutputs.map(({ key: outputKey, secretId }) => ({ + secretId, + value: encryptSymmetric128BitHexKeyUTF8( + typeof newCredential.outputs[outputKey] === "object" + ? JSON.stringify(newCredential.outputs[outputKey]) + : String(newCredential.outputs[outputKey]), + botKey + ) + })); + // map the final values to output keys in the board + await secretRotationDAL.transaction(async (tx) => { + await secretRotationDAL.updateById( + rotationId, + { + encryptedData: encVarData.ciphertext, + encryptedDataIV: encVarData.iv, + encryptedDataTag: encVarData.tag, + keyEncoding: encVarData.encoding, + algorithm: encVarData.algorithm, + lastRotatedAt: new Date(), + statusMessage: "Rotated successfull", + status: "success" + }, + tx + ); + const updatedSecrets = await secretDAL.bulkUpdate( + encryptedSecrets.map(({ secretId, value }) => ({ + // this secret id is validated when user is inserted + filter: { id: secretId, type: SecretType.Shared }, + data: { + secretValueCiphertext: value.ciphertext, + secretValueIV: value.iv, + secretValueTag: value.tag + } + })), + tx + ); + await secretVersionDAL.insertMany( + updatedSecrets.map(({ id, updatedAt, createdAt, ...el }) => { + if (!el.secretBlindIndex) throw new BadRequestError({ message: "Missing blind index" }); + return { + ...el, + secretId: id, + secretBlindIndex: el.secretBlindIndex + }; + }), + tx + ); + }); + } + await telemetryService.sendPostHogEvents({ event: PostHogEventTypes.SecretRotated, distinctId: "", properties: { - numberOfSecrets: encryptedSecrets.length, + numberOfSecrets: numberOfSecretsRotated, environment: secretRotation.environment.slug, secretPath: secretRotation.secretPath, workspaceId: secretRotation.projectId diff --git a/backend/src/ee/services/secret-rotation/secret-rotation-service.ts b/backend/src/ee/services/secret-rotation/secret-rotation-service.ts index 9b0109a35..d346c5bd3 100644 --- a/backend/src/ee/services/secret-rotation/secret-rotation-service.ts +++ b/backend/src/ee/services/secret-rotation/secret-rotation-service.ts @@ -1,12 +1,15 @@ import { ForbiddenError, subject } from "@casl/ability"; import Ajv from "ajv"; -import { infisicalSymmetricEncypt } from "@app/lib/crypto/encryption"; +import { ProjectVersion } from "@app/db/schemas"; +import { decryptSymmetric128BitHexKeyUTF8, infisicalSymmetricEncypt } from "@app/lib/crypto/encryption"; import { BadRequestError } from "@app/lib/errors"; import { TProjectPermission } from "@app/lib/types"; import { TProjectDALFactory } from "@app/services/project/project-dal"; +import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service"; import { TSecretDALFactory } from "@app/services/secret/secret-dal"; import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal"; +import { TSecretV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-dal"; import { TLicenseServiceFactory } from "../license/license-service"; import { TPermissionServiceFactory } from "../permission/permission-service"; @@ -22,9 +25,11 @@ type TSecretRotationServiceFactoryDep = { projectDAL: Pick; folderDAL: Pick; secretDAL: Pick; + secretV2BridgeDAL: Pick; licenseService: Pick; permissionService: Pick; secretRotationQueue: TSecretRotationQueueFactory; + projectBotService: Pick; }; export type TSecretRotationServiceFactory = ReturnType; @@ -37,7 +42,9 @@ export const secretRotationServiceFactory = ({ licenseService, projectDAL, folderDAL, - secretDAL + secretDAL, + projectBotService, + secretV2BridgeDAL }: TSecretRotationServiceFactoryDep) => { const getProviderTemplates = async ({ actor, @@ -92,15 +99,25 @@ export const secretRotationServiceFactory = ({ ProjectPermissionActions.Edit, subject(ProjectPermissionSub.Secrets, { environment, secretPath }) ); - - const selectedSecrets = await secretDAL.find({ - folderId: folder.id, - $in: { id: Object.values(outputs) } - }); - if (selectedSecrets.length !== Object.values(outputs).length) - throw new BadRequestError({ message: "Secrets not found" }); - const project = await projectDAL.findById(projectId); + const shouldUseBridge = project.version === ProjectVersion.V3; + + if (shouldUseBridge) { + const selectedSecrets = await secretV2BridgeDAL.find({ + folderId: folder.id, + $in: { id: Object.values(outputs) } + }); + if (selectedSecrets.length !== Object.values(outputs).length) + throw new BadRequestError({ message: "Secrets not found" }); + } else { + const selectedSecrets = await secretDAL.find({ + folderId: folder.id, + $in: { id: Object.values(outputs) } + }); + if (selectedSecrets.length !== Object.values(outputs).length) + throw new BadRequestError({ message: "Secrets not found" }); + } + const plan = await licenseService.getPlan(project.orgId); if (!plan.secretRotation) throw new BadRequestError({ @@ -148,10 +165,18 @@ export const secretRotationServiceFactory = ({ }, tx ); - const outputSecretMapping = await secretRotationDAL.secretOutputInsertMany( - Object.entries(outputs).map(([key, secretId]) => ({ key, secretId, rotationId: doc.id })), - tx - ); + let outputSecretMapping; + if (shouldUseBridge) { + outputSecretMapping = await secretRotationDAL.secretOutputV2InsertMany( + Object.entries(outputs).map(([key, secretId]) => ({ key, secretId, rotationId: doc.id })), + tx + ); + } else { + outputSecretMapping = await secretRotationDAL.secretOutputInsertMany( + Object.entries(outputs).map(([key, secretId]) => ({ key, secretId, rotationId: doc.id })), + tx + ); + } return { ...doc, outputs: outputSecretMapping, environment: folder.environment }; }); await secretRotationQueue.addToQueue(secretRotation.id, secretRotation.interval); @@ -167,8 +192,30 @@ export const secretRotationServiceFactory = ({ actorOrgId ); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretRotation); - const doc = await secretRotationDAL.find({ projectId }); - return doc; + const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId); + if (shouldUseSecretV2Bridge) { + const docs = await secretRotationDAL.findSecretV2({ projectId }); + return docs; + } + + if (!botKey) throw new BadRequestError({ message: "bot not found" }); + const docs = await secretRotationDAL.find({ projectId }); + return docs.map((el) => ({ + ...el, + outputs: el.outputs.map((output) => ({ + ...output, + secret: { + id: output.secret.id, + version: output.secret.version, + secretKey: decryptSymmetric128BitHexKeyUTF8({ + ciphertext: output.secret.secretKeyCiphertext, + iv: output.secret.secretKeyIV, + tag: output.secret.secretKeyTag, + key: botKey + }) + } + })) + })); }; const restartById = async ({ actor, actorId, actorOrgId, actorAuthMethod, rotationId }: TRestartDTO) => { diff --git a/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts b/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts index 3e1142969..c5d8bd4fa 100644 --- a/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts +++ b/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts @@ -1,15 +1,22 @@ import { ForbiddenError, subject } from "@casl/ability"; -import { TableName, TSecretTagJunctionInsert } from "@app/db/schemas"; +import { TableName, TSecretTagJunctionInsert, TSecretV2TagJunctionInsert } from "@app/db/schemas"; +import { decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto"; import { BadRequestError, InternalServerError } from "@app/lib/errors"; import { groupBy } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; +import { TKmsServiceFactory } from "@app/services/kms/kms-service"; +import { KmsDataKey } from "@app/services/kms/kms-types"; +import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service"; import { TSecretDALFactory } from "@app/services/secret/secret-dal"; import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal"; import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal"; import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal"; import { TSecretFolderVersionDALFactory } from "@app/services/secret-folder/secret-folder-version-dal"; import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; +import { TSecretV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-dal"; +import { TSecretVersionV2DALFactory } from "@app/services/secret-v2-bridge/secret-version-dal"; +import { TSecretVersionV2TagDALFactory } from "@app/services/secret-v2-bridge/secret-version-tag-dal"; import { TLicenseServiceFactory } from "../license/license-service"; import { TPermissionServiceFactory } from "../permission/permission-service"; @@ -23,20 +30,27 @@ import { import { TSnapshotDALFactory } from "./snapshot-dal"; import { TSnapshotFolderDALFactory } from "./snapshot-folder-dal"; import { TSnapshotSecretDALFactory } from "./snapshot-secret-dal"; +import { TSnapshotSecretV2DALFactory } from "./snapshot-secret-v2-dal"; import { getFullFolderPath } from "./snapshot-service-fns"; type TSecretSnapshotServiceFactoryDep = { snapshotDAL: TSnapshotDALFactory; snapshotSecretDAL: TSnapshotSecretDALFactory; + snapshotSecretV2BridgeDAL: TSnapshotSecretV2DALFactory; snapshotFolderDAL: TSnapshotFolderDALFactory; secretVersionDAL: Pick; + secretVersionV2BridgeDAL: Pick; folderVersionDAL: Pick; secretDAL: Pick; - secretTagDAL: Pick; + secretV2BridgeDAL: Pick; + secretTagDAL: Pick; secretVersionTagDAL: Pick; + secretVersionV2TagBridgeDAL: Pick; folderDAL: Pick; permissionService: Pick; licenseService: Pick; + kmsService: Pick; + projectBotService: Pick; }; export type TSecretSnapshotServiceFactory = ReturnType; @@ -52,7 +66,13 @@ export const secretSnapshotServiceFactory = ({ permissionService, licenseService, secretTagDAL, - secretVersionTagDAL + secretVersionTagDAL, + secretVersionV2BridgeDAL, + secretV2BridgeDAL, + snapshotSecretV2BridgeDAL, + secretVersionV2TagBridgeDAL, + kmsService, + projectBotService }: TSecretSnapshotServiceFactoryDep) => { const projectSecretSnapshotCount = async ({ environment, @@ -118,7 +138,7 @@ export const secretSnapshotServiceFactory = ({ }; const getSnapshotData = async ({ actorId, actor, actorOrgId, actorAuthMethod, id }: TGetSnapshotDataDTO) => { - const snapshot = await snapshotDAL.findSecretSnapshotDataById(id); + const snapshot = await snapshotDAL.findById(id); if (!snapshot) throw new BadRequestError({ message: "Snapshot not found" }); const { permission } = await permissionService.getProjectPermission( actor, @@ -127,31 +147,122 @@ export const secretSnapshotServiceFactory = ({ actorAuthMethod, actorOrgId ); + ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretRollback); + const shouldUseBridge = snapshot.projectVersion === 3; + let snapshotDetails; + if (shouldUseBridge) { + const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId: snapshot.projectId + }); + const encryptedSnapshotDetails = await snapshotDAL.findSecretSnapshotV2DataById(id); + snapshotDetails = { + ...encryptedSnapshotDetails, + secretVersions: encryptedSnapshotDetails.secretVersions.map((el) => ({ + ...el, + secretKey: el.key, + secretValue: el.encryptedValue + ? secretManagerDecryptor({ cipherTextBlob: el.encryptedValue }).toString() + : undefined, + secretComment: el.encryptedComment + ? secretManagerDecryptor({ cipherTextBlob: el.encryptedComment }).toString() + : undefined + })) + }; + } else { + const encryptedSnapshotDetails = await snapshotDAL.findSecretSnapshotDataById(id); + const { botKey } = await projectBotService.getBotKey(snapshot.projectId); + if (!botKey) throw new BadRequestError({ message: "bot not found" }); + snapshotDetails = { + ...encryptedSnapshotDetails, + secretVersions: encryptedSnapshotDetails.secretVersions.map((el) => ({ + ...el, + secretKey: decryptSymmetric128BitHexKeyUTF8({ + ciphertext: el.secretKeyCiphertext, + iv: el.secretKeyIV, + tag: el.secretKeyTag, + key: botKey + }), + secretValue: decryptSymmetric128BitHexKeyUTF8({ + ciphertext: el.secretValueCiphertext, + iv: el.secretValueIV, + tag: el.secretValueTag, + key: botKey + }), + secretComment: + el.secretCommentTag && el.secretCommentIV && el.secretCommentCiphertext + ? decryptSymmetric128BitHexKeyUTF8({ + ciphertext: el.secretCommentCiphertext, + iv: el.secretCommentIV, + tag: el.secretCommentTag, + key: botKey + }) + : "" + })) + }; + } const fullFolderPath = await getFullFolderPath({ folderDAL, - folderId: snapshot.folderId, - envId: snapshot.environment.id + folderId: snapshotDetails.folderId, + envId: snapshotDetails.environment.id }); // We need to check if the user has access to the secrets in the folder. If we don't do this, a user could theoretically access snapshot secret values even if they don't have read access to the secrets in the folder. ForbiddenError.from(permission).throwUnlessCan( ProjectPermissionActions.Read, - subject(ProjectPermissionSub.Secrets, { environment: snapshot.environment.slug, secretPath: fullFolderPath }) + subject(ProjectPermissionSub.Secrets, { + environment: snapshotDetails.environment.slug, + secretPath: fullFolderPath + }) ); - return snapshot; + return snapshotDetails; }; const performSnapshot = async (folderId: string) => { try { if (!licenseService.isValidLicense) throw new InternalServerError({ message: "Invalid license" }); + const folder = await folderDAL.findById(folderId); + if (!folder) throw new BadRequestError({ message: "Folder not found" }); + const shouldUseSecretV2Bridge = folder.projectVersion === 3; + + if (shouldUseSecretV2Bridge) { + const snapshot = await snapshotDAL.transaction(async (tx) => { + const secretVersions = await secretVersionV2BridgeDAL.findLatestVersionByFolderId(folderId, tx); + const folderVersions = await folderVersionDAL.findLatestVersionByFolderId(folderId, tx); + const newSnapshot = await snapshotDAL.create( + { + folderId, + envId: folder.environment.envId, + parentFolderId: folder.parentId + }, + tx + ); + const snapshotSecrets = await snapshotSecretV2BridgeDAL.insertMany( + secretVersions.map(({ id }) => ({ + secretVersionId: id, + envId: folder.environment.envId, + snapshotId: newSnapshot.id + })), + tx + ); + const snapshotFolders = await snapshotFolderDAL.insertMany( + folderVersions.map(({ id }) => ({ + folderVersionId: id, + envId: folder.environment.envId, + snapshotId: newSnapshot.id + })), + tx + ); + + return { ...newSnapshot, secrets: snapshotSecrets, folder: snapshotFolders }; + }); + return snapshot; + } const snapshot = await snapshotDAL.transaction(async (tx) => { - const folder = await folderDAL.findById(folderId, tx); - if (!folder) throw new BadRequestError({ message: "Folder not found" }); - const secretVersions = await secretVersionDAL.findLatestVersionByFolderId(folderId, tx); const folderVersions = await folderVersionDAL.findLatestVersionByFolderId(folderId, tx); const newSnapshot = await snapshotDAL.create( @@ -199,6 +310,7 @@ export const secretSnapshotServiceFactory = ({ }: TRollbackSnapshotDTO) => { const snapshot = await snapshotDAL.findById(snapshotId); if (!snapshot) throw new BadRequestError({ message: "Snapshot not found" }); + const shouldUseBridge = snapshot.projectVersion === 3; const { permission } = await permissionService.getProjectPermission( actor, @@ -212,6 +324,117 @@ export const secretSnapshotServiceFactory = ({ ProjectPermissionSub.SecretRollback ); + if (shouldUseBridge) { + const rollback = await snapshotDAL.transaction(async (tx) => { + const rollbackSnaps = await snapshotDAL.findRecursivelySnapshotsV2Bridge(snapshot.id, tx); + // this will remove all secrets in current folder + const deletedTopLevelSecs = await secretV2BridgeDAL.delete({ folderId: snapshot.folderId }, tx); + const deletedTopLevelSecsGroupById = groupBy(deletedTopLevelSecs, (item) => item.id); + // this will remove all secrets and folders on child + // due to sql foreign key and link list connection removing the folders removes everything below too + const deletedFolders = await folderDAL.delete({ parentId: snapshot.folderId, isReserved: false }, tx); + const deletedTopLevelFolders = groupBy( + deletedFolders.filter(({ parentId }) => parentId === snapshot.folderId), + (item) => item.id + ); + const folders = await folderDAL.insertMany( + rollbackSnaps.flatMap(({ folderVersion, folderId }) => + folderVersion.map(({ name, id, latestFolderVersion }) => ({ + envId: snapshot.envId, + id, + // this means don't bump up the version if not root folder + // because below ones can be same version as nothing changed + version: deletedTopLevelFolders[folderId] ? latestFolderVersion + 1 : latestFolderVersion, + name, + parentId: folderId + })) + ), + tx + ); + const secrets = await secretV2BridgeDAL.insertMany( + rollbackSnaps.flatMap(({ secretVersions, folderId }) => + secretVersions.map( + ({ latestSecretVersion, version, updatedAt, createdAt, secretId, envId, id, tags, ...el }) => ({ + ...el, + id: secretId, + version: deletedTopLevelSecsGroupById[secretId] ? latestSecretVersion + 1 : latestSecretVersion, + folderId + }) + ) + ), + tx + ); + const secretTagsToBeInsert: TSecretV2TagJunctionInsert[] = []; + const secretVerTagToBeInsert: Record = {}; + rollbackSnaps.forEach(({ secretVersions }) => { + secretVersions.forEach((secVer) => { + secVer.tags.forEach((tag) => { + secretTagsToBeInsert.push({ secrets_v2Id: secVer.secretId, secret_tagsId: tag.id }); + if (!secretVerTagToBeInsert?.[secVer.secretId]) secretVerTagToBeInsert[secVer.secretId] = []; + secretVerTagToBeInsert[secVer.secretId].push(tag.id); + }); + }); + }); + await secretTagDAL.saveTagsToSecretV2(secretTagsToBeInsert, tx); + const folderVersions = await folderVersionDAL.insertMany( + folders.map(({ version, name, id, envId }) => ({ + name, + version, + folderId: id, + envId + })), + tx + ); + const secretVersions = await secretVersionV2BridgeDAL.insertMany( + secrets.map(({ id, updatedAt, createdAt, ...el }) => ({ ...el, secretId: id })), + tx + ); + await secretVersionV2TagBridgeDAL.insertMany( + secretVersions.flatMap(({ secretId, id }) => + secretVerTagToBeInsert?.[secretId]?.length + ? secretVerTagToBeInsert[secretId].map((tagId) => ({ + [`${TableName.SecretTag}Id` as const]: tagId, + [`${TableName.SecretVersionV2}Id` as const]: id + })) + : [] + ), + tx + ); + const newSnapshot = await snapshotDAL.create( + { + folderId: snapshot.folderId, + envId: snapshot.envId, + parentFolderId: snapshot.parentFolderId + }, + tx + ); + const snapshotSecrets = await snapshotSecretV2BridgeDAL.insertMany( + secretVersions + .filter(({ secretId }) => Boolean(deletedTopLevelSecsGroupById?.[secretId])) + .map(({ id }) => ({ + secretVersionId: id, + envId: newSnapshot.envId, + snapshotId: newSnapshot.id + })), + tx + ); + const snapshotFolders = await snapshotFolderDAL.insertMany( + folderVersions + .filter(({ folderId }) => Boolean(deletedTopLevelFolders?.[folderId])) + .map(({ id }) => ({ + folderVersionId: id, + envId: newSnapshot.envId, + snapshotId: newSnapshot.id + })), + tx + ); + + return { ...newSnapshot, snapshotSecrets, snapshotFolders }; + }); + + return rollback; + } + const rollback = await snapshotDAL.transaction(async (tx) => { const rollbackSnaps = await snapshotDAL.findRecursivelySnapshots(snapshot.id, tx); // this will remove all secrets in current folder diff --git a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts index a16b4548d..8e5014fdd 100644 --- a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts +++ b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts @@ -1,14 +1,17 @@ /* eslint-disable no-await-in-loop */ import { Knex } from "knex"; +import { z } from "zod"; import { TDbClient } from "@app/db"; import { SecretVersionsSchema, + SecretVersionsV2Schema, TableName, TSecretFolderVersions, TSecretSnapshotFolders, TSecretSnapshots, - TSecretVersions + TSecretVersions, + TSecretVersionsV2 } from "@app/db/schemas"; import { DatabaseError } from "@app/lib/errors"; import { ormify, selectAllTableCols, sqlNestRelationships } from "@app/lib/knex"; @@ -24,12 +27,14 @@ export const snapshotDALFactory = (db: TDbClient) => { const data = await (tx || db.replicaNode())(TableName.Snapshot) .where(`${TableName.Snapshot}.id`, id) .join(TableName.Environment, `${TableName.Snapshot}.envId`, `${TableName.Environment}.id`) + .join(TableName.Project, `${TableName.Environment}.projectId`, `${TableName.Project}.id`) .select(selectAllTableCols(TableName.Snapshot)) .select( db.ref("id").withSchema(TableName.Environment).as("envId"), db.ref("projectId").withSchema(TableName.Environment), db.ref("name").withSchema(TableName.Environment).as("envName"), - db.ref("slug").withSchema(TableName.Environment).as("envSlug") + db.ref("slug").withSchema(TableName.Environment).as("envSlug"), + db.ref("version").withSchema(TableName.Project).as("projectVersion") ) .first(); if (data) { @@ -149,6 +154,101 @@ export const snapshotDALFactory = (db: TDbClient) => { } }; + const findSecretSnapshotV2DataById = async (snapshotId: string, tx?: Knex) => { + try { + const data = await (tx || db.replicaNode())(TableName.Snapshot) + .where(`${TableName.Snapshot}.id`, snapshotId) + .join(TableName.Environment, `${TableName.Snapshot}.envId`, `${TableName.Environment}.id`) + .leftJoin(TableName.SnapshotSecretV2, `${TableName.Snapshot}.id`, `${TableName.SnapshotSecretV2}.snapshotId`) + .leftJoin( + TableName.SecretVersionV2, + `${TableName.SnapshotSecretV2}.secretVersionId`, + `${TableName.SecretVersionV2}.id` + ) + .leftJoin( + TableName.SecretVersionV2Tag, + `${TableName.SecretVersionV2Tag}.${TableName.SecretVersionV2}Id`, + `${TableName.SecretVersionV2}.id` + ) + .leftJoin( + TableName.SecretTag, + `${TableName.SecretVersionV2Tag}.${TableName.SecretTag}Id`, + `${TableName.SecretTag}.id` + ) + .leftJoin(TableName.SnapshotFolder, `${TableName.SnapshotFolder}.snapshotId`, `${TableName.Snapshot}.id`) + .leftJoin( + TableName.SecretFolderVersion, + `${TableName.SnapshotFolder}.folderVersionId`, + `${TableName.SecretFolderVersion}.id` + ) + .select(selectAllTableCols(TableName.SecretVersionV2)) + .select( + db.ref("id").withSchema(TableName.Snapshot).as("snapshotId"), + db.ref("createdAt").withSchema(TableName.Snapshot).as("snapshotCreatedAt"), + db.ref("updatedAt").withSchema(TableName.Snapshot).as("snapshotUpdatedAt"), + db.ref("id").withSchema(TableName.Environment).as("envId"), + db.ref("name").withSchema(TableName.Environment).as("envName"), + db.ref("slug").withSchema(TableName.Environment).as("envSlug"), + db.ref("projectId").withSchema(TableName.Environment), + db.ref("name").withSchema(TableName.SecretFolderVersion).as("folderVerName"), + db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("folderVerId"), + db.ref("id").withSchema(TableName.SecretTag).as("tagId"), + db.ref("id").withSchema(TableName.SecretVersionV2Tag).as("tagVersionId"), + db.ref("color").withSchema(TableName.SecretTag).as("tagColor"), + db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug"), + db.ref("name").withSchema(TableName.SecretTag).as("tagName") + ); + return sqlNestRelationships({ + data, + key: "snapshotId", + parentMapper: ({ + snapshotId: id, + folderId, + projectId, + envId, + envSlug, + envName, + snapshotCreatedAt: createdAt, + snapshotUpdatedAt: updatedAt + }) => ({ + id, + folderId, + projectId, + createdAt, + updatedAt, + environment: { id: envId, slug: envSlug, name: envName } + }), + childrenMapper: [ + { + key: "id", + label: "secretVersions" as const, + mapper: (el) => SecretVersionsV2Schema.parse(el), + childrenMapper: [ + { + key: "tagVersionId", + label: "tags" as const, + mapper: ({ tagId: id, tagName: name, tagSlug: slug, tagColor: color, tagVersionId: vId }) => ({ + id, + name, + slug, + color, + vId + }) + } + ] + }, + { + key: "folderVerId", + label: "folderVersion" as const, + mapper: ({ folderVerId: id, folderVerName: name }) => ({ id, name }) + } + ] + })?.[0]; + } catch (error) { + throw new DatabaseError({ error, name: "FindSecretSnapshotDataById" }); + } + }; + // this is used for rollback // from a starting snapshot it will collect all the secrets and folder of that // then it will start go through recursively the below folders latest snapshots then their child folder snapshot until leaf node @@ -304,6 +404,161 @@ export const snapshotDALFactory = (db: TDbClient) => { } }; + // this is used for rollback + // from a starting snapshot it will collect all the secrets and folder of that + // then it will start go through recursively the below folders latest snapshots then their child folder snapshot until leaf node + // the recursive part find all snapshot id + // then joins with respective secrets and folder + const findRecursivelySnapshotsV2Bridge = async (snapshotId: string, tx?: Knex) => { + try { + const data = await (tx || db) + .withRecursive("parent", (qb) => { + void qb + .from(TableName.Snapshot) + .leftJoin( + TableName.SnapshotFolder, + `${TableName.SnapshotFolder}.snapshotId`, + `${TableName.Snapshot}.id` + ) + .leftJoin( + TableName.SecretFolderVersion, + `${TableName.SnapshotFolder}.folderVersionId`, + `${TableName.SecretFolderVersion}.id` + ) + .select(selectAllTableCols(TableName.Snapshot)) + .select({ depth: 1 }) + .select( + db.ref("name").withSchema(TableName.SecretFolderVersion).as("folderVerName"), + db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("folderVerId") + ) + .where(`${TableName.Snapshot}.id`, snapshotId) + .union( + (cb) => + void cb + .select(selectAllTableCols(TableName.Snapshot)) + .select({ depth: db.raw("parent.depth + 1") }) + .select( + db.ref("name").withSchema(TableName.SecretFolderVersion).as("folderVerName"), + db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("folderVerId") + ) + .from(TableName.Snapshot) + .join( + db(TableName.Snapshot).groupBy("folderId").max("createdAt").select("folderId").as("latestVersion"), + `${TableName.Snapshot}.createdAt`, + "latestVersion.max" + ) + .leftJoin( + TableName.SnapshotFolder, + `${TableName.SnapshotFolder}.snapshotId`, + `${TableName.Snapshot}.id` + ) + .leftJoin( + TableName.SecretFolderVersion, + `${TableName.SnapshotFolder}.folderVersionId`, + `${TableName.SecretFolderVersion}.id` + ) + .join("parent", "parent.folderVerId", `${TableName.Snapshot}.folderId`) + ); + }) + .orderBy("depth", "asc") + .from("parent") + .leftJoin(TableName.SnapshotSecretV2, `parent.id`, `${TableName.SnapshotSecretV2}.snapshotId`) + .leftJoin( + TableName.SecretVersionV2, + `${TableName.SnapshotSecretV2}.secretVersionId`, + `${TableName.SecretVersionV2}.id` + ) + .leftJoin( + TableName.SecretVersionV2Tag, + `${TableName.SecretVersionV2Tag}.${TableName.SecretVersionV2}Id`, + `${TableName.SecretVersionV2}.id` + ) + .leftJoin( + TableName.SecretTag, + `${TableName.SecretVersionV2Tag}.${TableName.SecretTag}Id`, + `${TableName.SecretTag}.id` + ) + .leftJoin<{ latestSecretVersion: number }>( + (tx || db)(TableName.SecretVersionV2) + .groupBy("secretId") + .select("secretId") + .max("version") + .as("secGroupByMaxVersion"), + `${TableName.SecretVersionV2}.secretId`, + "secGroupByMaxVersion.secretId" + ) + .leftJoin<{ latestFolderVersion: number }>( + (tx || db)(TableName.SecretFolderVersion) + .groupBy("folderId") + .select("folderId") + .max("version") + .as("folderGroupByMaxVersion"), + `parent.folderId`, + "folderGroupByMaxVersion.folderId" + ) + .select(selectAllTableCols(TableName.SecretVersionV2)) + .select( + db.ref("id").withSchema("parent").as("snapshotId"), + db.ref("folderId").withSchema("parent").as("snapshotFolderId"), + db.ref("parentFolderId").withSchema("parent").as("snapshotParentFolderId"), + db.ref("folderVerName").withSchema("parent"), + db.ref("folderVerId").withSchema("parent"), + db.ref("max").withSchema("secGroupByMaxVersion").as("latestSecretVersion"), + db.ref("max").withSchema("folderGroupByMaxVersion").as("latestFolderVersion"), + db.ref("id").withSchema(TableName.SecretTag).as("tagId"), + db.ref("id").withSchema(TableName.SecretVersionV2Tag).as("tagVersionId"), + db.ref("color").withSchema(TableName.SecretTag).as("tagColor"), + db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug"), + db.ref("name").withSchema(TableName.SecretTag).as("tagName") + ); + + const formated = sqlNestRelationships({ + data, + key: "snapshotId", + parentMapper: ({ snapshotId: id, snapshotFolderId: folderId, snapshotParentFolderId: parentFolderId }) => ({ + id, + folderId, + parentFolderId + }), + childrenMapper: [ + { + key: "id", + label: "secretVersions" as const, + mapper: (el) => ({ + ...SecretVersionsV2Schema.parse(el), + latestSecretVersion: el.latestSecretVersion as number + }), + childrenMapper: [ + { + key: "tagVersionId", + label: "tags" as const, + mapper: ({ tagId: id, tagName: name, tagSlug: slug, tagColor: color, tagVersionId: vId }) => ({ + id, + name, + slug, + color, + vId + }) + } + ] + }, + { + key: "folderVerId", + label: "folderVersion" as const, + mapper: ({ folderVerId: id, folderVerName: name, latestFolderVersion }) => ({ + id, + name, + latestFolderVersion: latestFolderVersion as number + }) + } + ] + }); + return formated; + } catch (error) { + throw new DatabaseError({ error, name: "FindRecursivelySnapshots" }); + } + }; + // instead of copying all child folders // we will take the latest snapshot of those folders // when we need to rollback we will pull from these snapshots @@ -465,13 +720,108 @@ export const snapshotDALFactory = (db: TDbClient) => { } }; + // special query for migration for secret v2 + const findNSecretV1SnapshotByFolderId = async (folderId: string, n = 15, tx?: Knex) => { + try { + const query = (tx || db.replicaNode())(TableName.Snapshot) + .leftJoin(TableName.SnapshotSecret, `${TableName.Snapshot}.id`, `${TableName.SnapshotSecret}.snapshotId`) + .leftJoin( + TableName.SecretVersion, + `${TableName.SnapshotSecret}.secretVersionId`, + `${TableName.SecretVersion}.id` + ) + .leftJoin( + TableName.SecretVersionTag, + `${TableName.SecretVersionTag}.${TableName.SecretVersion}Id`, + `${TableName.SecretVersion}.id` + ) + .select(selectAllTableCols(TableName.SecretVersion)) + .select( + db.ref("id").withSchema(TableName.Snapshot).as("snapshotId"), + db.ref("createdAt").withSchema(TableName.Snapshot).as("snapshotCreatedAt"), + db.ref("updatedAt").withSchema(TableName.Snapshot).as("snapshotUpdatedAt"), + db.ref("envId").withSchema(TableName.SnapshotSecret).as("snapshotEnvId"), + db.ref("id").withSchema(TableName.SecretVersionTag).as("secretVersionTagId"), + db.ref("secret_versionsId").withSchema(TableName.SecretVersionTag).as("secretVersionTagSecretId"), + db.ref("secret_tagsId").withSchema(TableName.SecretVersionTag).as("secretVersionTagSecretTagId"), + db.raw( + `DENSE_RANK() OVER (partition by ${TableName.Snapshot}."id" ORDER BY ${TableName.SecretVersion}."createdAt") as rank` + ) + ) + .orderBy(`${TableName.Snapshot}.createdAt`, "desc") + .where(`${TableName.Snapshot}.folderId`, folderId); + const data = await (tx || db) + .with("w", query) + .select("*") + .from[number]>("w") + .andWhere("w.rank", "<", n); + + return sqlNestRelationships({ + data, + key: "snapshotId", + parentMapper: ({ snapshotId: id, snapshotCreatedAt: createdAt, snapshotUpdatedAt: updatedAt }) => ({ + id, + folderId, + createdAt, + updatedAt + }), + childrenMapper: [ + { + key: "id", + label: "secretVersions" as const, + mapper: (el) => SecretVersionsSchema.extend({ snapshotEnvId: z.string() }).parse(el), + childrenMapper: [ + { + key: "secretVersionTagId", + label: "tags" as const, + mapper: ({ secretVersionTagId, secretVersionTagSecretId, secretVersionTagSecretTagId }) => ({ + id: secretVersionTagId, + secretVersionId: secretVersionTagSecretId, + secretTagId: secretVersionTagSecretTagId + }) + } + ] + } + ] + }); + } catch (error) { + throw new DatabaseError({ error, name: "FindSecretSnapshotDataById" }); + } + }; + + const deleteSnapshotsAboveLimit = async (folderId: string, n = 15, tx?: Knex) => { + try { + const query = await (tx || db) + .with("to_delete", (qb) => { + void qb + .select("id") + .from(TableName.Snapshot) + .where("folderId", folderId) + .orderBy("createdAt", "desc") + .offset(n); + }) + .from(TableName.Snapshot) + .whereIn("id", (qb) => { + void qb.select("id").from("to_delete"); + }) + .delete(); + return query; + } catch (error) { + throw new DatabaseError({ error, name: "DeleteSnapshotsAboveLimit" }); + } + }; + return { ...secretSnapshotOrm, findById, findLatestSnapshotByFolderId, findRecursivelySnapshots, + findRecursivelySnapshotsV2Bridge, countOfSnapshotsByFolderId, findSecretSnapshotDataById, - pruneExcessSnapshots + findSecretSnapshotV2DataById, + pruneExcessSnapshots, + findNSecretV1SnapshotByFolderId, + deleteSnapshotsAboveLimit }; }; diff --git a/backend/src/ee/services/secret-snapshot/snapshot-secret-v2-dal.ts b/backend/src/ee/services/secret-snapshot/snapshot-secret-v2-dal.ts new file mode 100644 index 000000000..6d5f5896d --- /dev/null +++ b/backend/src/ee/services/secret-snapshot/snapshot-secret-v2-dal.ts @@ -0,0 +1,10 @@ +import { TDbClient } from "@app/db"; +import { TableName } from "@app/db/schemas"; +import { ormify } from "@app/lib/knex"; + +export type TSnapshotSecretV2DALFactory = ReturnType; + +export const snapshotSecretV2DALFactory = (db: TDbClient) => { + const snapshotSecretOrm = ormify(db, TableName.SnapshotSecretV2); + return snapshotSecretOrm; +}; diff --git a/backend/src/keystore/keystore.ts b/backend/src/keystore/keystore.ts index ce752a1e5..d7bbb0c79 100644 --- a/backend/src/keystore/keystore.ts +++ b/backend/src/keystore/keystore.ts @@ -6,7 +6,15 @@ export type TKeyStoreFactory = ReturnType; // all the key prefixes used must be set here to avoid conflict export enum KeyStorePrefixes { - SecretReplication = "secret-replication-import-lock" + SecretReplication = "secret-replication-import-lock", + KmsProjectDataKeyCreation = "kms-project-data-key-creation-lock", + KmsProjectKeyCreation = "kms-project-key-creation-lock", + WaitUntilReadyKmsProjectDataKeyCreation = "wait-until-ready-kms-project-data-key-creation-", + WaitUntilReadyKmsProjectKeyCreation = "wait-until-ready-kms-project-key-creation-", + KmsOrgKeyCreation = "kms-org-key-creation-lock", + KmsOrgDataKeyCreation = "kms-org-data-key-creation-lock", + WaitUntilReadyKmsOrgKeyCreation = "wait-until-ready-kms-org-key-creation-", + WaitUntilReadyKmsOrgDataKeyCreation = "wait-until-ready-kms-org-data-key-creation-" } type TWaitTillReady = { @@ -32,7 +40,7 @@ export const keyStoreFactory = (redisUrl: string) => { exp: number | string, value: string | number | Buffer, prefix?: string - ) => redis.setex(prefix ? `${prefix}:${key}` : key, exp, value); + ) => redis.set(prefix ? `${prefix}:${key}` : key, value, "EX", exp); const deleteItem = async (key: string) => redis.del(key); @@ -57,7 +65,7 @@ export const keyStoreFactory = (redisUrl: string) => { }); attempts += 1; // eslint-disable-next-line - isReady = keyCheckCb(await getItem(key, "wait_till_ready")); + isReady = keyCheckCb(await getItem(key)); } }; diff --git a/backend/src/lib/api-docs/constants.ts b/backend/src/lib/api-docs/constants.ts index 5e5e330e1..e151ffb85 100644 --- a/backend/src/lib/api-docs/constants.ts +++ b/backend/src/lib/api-docs/constants.ts @@ -608,7 +608,9 @@ export const RAW_SECRETS = { skipMultilineEncoding: "Skip multiline encoding for the secret value.", type: "The type of the secret to create.", workspaceId: "The ID of the project to create the secret in.", - tagIds: "The ID of the tags to be attached to the created secret." + tagIds: "The ID of the tags to be attached to the created secret.", + secretReminderRepeatDays: "Interval for secret rotation notifications, measured in days", + secretReminderNote: "Note to be attached in notification email" }, GET: { expand: "Whether or not to expand secret references", @@ -631,7 +633,10 @@ export const RAW_SECRETS = { type: "The type of the secret to update.", projectSlug: "The slug of the project to update the secret in.", workspaceId: "The ID of the project to update the secret in.", - tagIds: "The ID of the tags to be attached to the updated secret." + tagIds: "The ID of the tags to be attached to the updated secret.", + secretReminderRepeatDays: "Interval for secret rotation notifications, measured in days", + secretReminderNote: "Note to be attached in notification email", + newSecretName: "The new name for the secret" }, DELETE: { secretName: "The name of the secret to delete.", diff --git a/backend/src/lib/crypto/encryption.ts b/backend/src/lib/crypto/encryption.ts index 6af20862b..258a6d285 100644 --- a/backend/src/lib/crypto/encryption.ts +++ b/backend/src/lib/crypto/encryption.ts @@ -116,6 +116,8 @@ export const decryptAsymmetric = ({ ciphertext, nonce, publicKey, privateKey }: export const generateSymmetricKey = (size = 32) => crypto.randomBytes(size).toString("base64"); +export const generateHash = (value: string) => crypto.createHash("sha256").update(value).digest("hex"); + export const generateAsymmetricKeyPair = () => { const pair = nacl.box.keyPair(); @@ -224,8 +226,9 @@ export const infisicalSymmetricDecrypt = ({ keyEncoding: SecretKeyEncoding; }) => { const appCfg = getConfig(); - const rootEncryptionKey = appCfg.ROOT_ENCRYPTION_KEY; - const encryptionKey = appCfg.ENCRYPTION_KEY; + // the or gate is used used in migration + const rootEncryptionKey = appCfg?.ROOT_ENCRYPTION_KEY || process.env.ROOT_ENCRYPTION_KEY; + const encryptionKey = appCfg?.ENCRYPTION_KEY || process.env.ENCRYPTION_KEY; if (rootEncryptionKey && keyEncoding === SecretKeyEncoding.BASE64) { const data = decryptSymmetric({ key: rootEncryptionKey, iv, tag, ciphertext }); return data as T; diff --git a/backend/src/lib/fn/array.ts b/backend/src/lib/fn/array.ts index 1e075101b..00819ceb8 100644 --- a/backend/src/lib/fn/array.ts +++ b/backend/src/lib/fn/array.ts @@ -17,6 +17,23 @@ export const groupBy = ( {} as Record ); +/** + * Sorts an array of items into groups. The return value is a map where the keys are + * the group ids the given getGroupId function produced and the value will be the last found one for the group key + */ +export const groupByUnique = ( + array: readonly T[], + getGroupId: (item: T) => Key +): Record => + array.reduce( + (acc, item) => { + const groupId = getGroupId(item); + acc[groupId] = item; + return acc; + }, + {} as Record + ); + /** * Given a list of items returns a new list with only * unique items. Accepts an optional identity function diff --git a/backend/src/lib/fn/index.ts b/backend/src/lib/fn/index.ts index 381ecebf3..82a4c4914 100644 --- a/backend/src/lib/fn/index.ts +++ b/backend/src/lib/fn/index.ts @@ -6,3 +6,4 @@ export * from "./array"; export * from "./dates"; export * from "./object"; export * from "./string"; +export * from "./undefined"; diff --git a/backend/src/lib/fn/undefined.ts b/backend/src/lib/fn/undefined.ts new file mode 100644 index 000000000..00f9d8e51 --- /dev/null +++ b/backend/src/lib/fn/undefined.ts @@ -0,0 +1,3 @@ +export const executeIfDefined = (func: (input: T) => R, input: T | undefined): R | undefined => { + return input === undefined ? undefined : func(input); +}; diff --git a/backend/src/lib/knex/index.ts b/backend/src/lib/knex/index.ts index bd103af9d..c01d146ec 100644 --- a/backend/src/lib/knex/index.ts +++ b/backend/src/lib/knex/index.ts @@ -104,6 +104,19 @@ export const ormify = (db: Kne throw new DatabaseError({ error, name: "Create" }); } }, + upsert: async (data: readonly Tables[Tname]["insert"][], onConflictField: keyof Tables[Tname]["base"], tx?: Knex) => { + try { + if (!data.length) return []; + const res = await (tx || db)(tableName) + .insert(data as never) + .onConflict(onConflictField as never) + .merge() + .returning("*"); + return res; + } catch (error) { + throw new DatabaseError({ error, name: "Create" }); + } + }, updateById: async ( id: string, { diff --git a/backend/src/lib/knex/select.ts b/backend/src/lib/knex/select.ts index d7dfa77f0..feccad4e3 100644 --- a/backend/src/lib/knex/select.ts +++ b/backend/src/lib/knex/select.ts @@ -12,3 +12,12 @@ export const stripUndefinedInWhere = (val: T): Exclude; }; + +// if its undefined its skipped in knex +// if its empty string its set as null +// else pass to the required one +export const setKnexStringValue = (value: string | null | undefined, cb: (arg: string) => T) => { + if (typeof value === "undefined") return; + if (value === "" || value === null) return null; + return cb(value); +}; diff --git a/backend/src/queue/queue-service.ts b/backend/src/queue/queue-service.ts index d51a8e683..ec222f0fa 100644 --- a/backend/src/queue/queue-service.ts +++ b/backend/src/queue/queue-service.ts @@ -25,7 +25,8 @@ export enum QueueName { DynamicSecretRevocation = "dynamic-secret-revocation", CaCrlRotation = "ca-crl-rotation", SecretReplication = "secret-replication", - SecretSync = "secret-sync" // parent queue to push integration sync, webhook, and secret replication + SecretSync = "secret-sync", // parent queue to push integration sync, webhook, and secret replication + ProjectV3Migration = "project-v3-migration" } export enum QueueJobs { @@ -44,7 +45,8 @@ export enum QueueJobs { DynamicSecretPruning = "dynamic-secret-pruning", CaCrlRotation = "ca-crl-rotation-job", SecretReplication = "secret-replication", - SecretSync = "secret-sync" // parent queue to push integration sync, webhook, and secret replication + SecretSync = "secret-sync", // parent queue to push integration sync, webhook, and secret replication + ProjectV3Migration = "project-v3-migration" } export type TQueueJobTypes = { @@ -136,6 +138,10 @@ export type TQueueJobTypes = { name: QueueJobs.SecretSync; payload: TSyncSecretsDTO; }; + [QueueName.ProjectV3Migration]: { + name: QueueJobs.ProjectV3Migration; + payload: { projectId: string }; + }; }; export type TQueueServiceFactory = ReturnType; @@ -210,6 +216,7 @@ export const queueServiceFactory = (redisUrl: string) => { const job = await q.getJob(jobId); if (!job) return true; if (!job.repeatJobKey) return true; + await job.remove(); return q.removeRepeatableByKey(job.repeatJobKey); }; diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 59aa8d301..4e0dd4e45 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -66,6 +66,7 @@ import { secretSnapshotServiceFactory } from "@app/ee/services/secret-snapshot/s import { snapshotDALFactory } from "@app/ee/services/secret-snapshot/snapshot-dal"; import { snapshotFolderDALFactory } from "@app/ee/services/secret-snapshot/snapshot-folder-dal"; import { snapshotSecretDALFactory } from "@app/ee/services/secret-snapshot/snapshot-secret-dal"; +import { snapshotSecretV2DALFactory } from "@app/ee/services/secret-snapshot/snapshot-secret-v2-dal"; import { trustedIpDALFactory } from "@app/ee/services/trusted-ip/trusted-ip-dal"; import { trustedIpServiceFactory } from "@app/ee/services/trusted-ip/trusted-ip-service"; import { TKeyStoreFactory } from "@app/keystore/keystore"; @@ -160,6 +161,10 @@ import { secretSharingDALFactory } from "@app/services/secret-sharing/secret-sha import { secretSharingServiceFactory } from "@app/services/secret-sharing/secret-sharing-service"; import { secretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; import { secretTagServiceFactory } from "@app/services/secret-tag/secret-tag-service"; +import { secretV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-dal"; +import { secretV2BridgeServiceFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-service"; +import { secretVersionV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-version-dal"; +import { secretVersionV2TagBridgeDALFactory } from "@app/services/secret-v2-bridge/secret-version-tag-dal"; import { serviceTokenDALFactory } from "@app/services/service-token/service-token-dal"; import { serviceTokenServiceFactory } from "@app/services/service-token/service-token-service"; import { TSmtpService } from "@app/services/smtp/smtp-service"; @@ -229,6 +234,10 @@ export const registerRoutes = async ( const secretVersionTagDAL = secretVersionTagDALFactory(db); const secretBlindIndexDAL = secretBlindIndexDALFactory(db); + const secretV2BridgeDAL = secretV2BridgeDALFactory(db); + const secretVersionV2BridgeDAL = secretVersionV2BridgeDALFactory(db); + const secretVersionTagV2BridgeDAL = secretVersionV2TagBridgeDALFactory(db); + const integrationDAL = integrationDALFactory(db); const integrationAuthDAL = integrationAuthDALFactory(db); const webhookDAL = webhookDALFactory(db); @@ -277,6 +286,7 @@ export const registerRoutes = async ( const secretRotationDAL = secretRotationDALFactory(db); const snapshotDAL = snapshotDALFactory(db); const snapshotSecretDAL = snapshotSecretDALFactory(db); + const snapshotSecretV2BridgeDAL = snapshotSecretV2DALFactory(db); const snapshotFolderDAL = snapshotFolderDALFactory(db); const gitAppInstallSessionDAL = gitAppInstallSessionDALFactory(db); @@ -316,7 +326,8 @@ export const registerRoutes = async ( kmsDAL, kmsService, permissionService, - externalKmsDAL + externalKmsDAL, + licenseService }); const trustedIpService = trustedIpServiceFactory({ @@ -609,10 +620,8 @@ export const registerRoutes = async ( permissionService, projectDAL, projectQueue: projectQueueService, - secretBlindIndexDAL, identityProjectDAL, identityOrgMembershipDAL, - projectBotDAL, projectKeyDAL, userDAL, projectEnvDAL, @@ -625,7 +634,8 @@ export const registerRoutes = async ( certificateDAL, projectUserMembershipRoleDAL, identityProjectMembershipRoleDAL, - keyStore + keyStore, + kmsService }); const projectEnvService = projectEnvServiceFactory({ @@ -655,13 +665,20 @@ export const registerRoutes = async ( secretVersionDAL, folderVersionDAL, secretTagDAL, - secretVersionTagDAL + secretVersionTagDAL, + projectBotService, + kmsService, + secretV2BridgeDAL, + secretVersionV2BridgeDAL, + snapshotSecretV2BridgeDAL, + secretVersionV2TagBridgeDAL: secretVersionTagV2BridgeDAL }); const webhookService = webhookServiceFactory({ permissionService, webhookDAL, projectEnvDAL, - projectDAL + projectDAL, + kmsService }); const secretTagService = secretTagServiceFactory({ secretTagDAL, permissionService }); @@ -678,8 +695,8 @@ export const registerRoutes = async ( integrationAuthDAL, integrationDAL, permissionService, - projectBotDAL, - projectBotService + projectBotService, + kmsService }); const secretQueueService = secretQueueFactory({ queueService, @@ -699,46 +716,51 @@ export const registerRoutes = async ( secretVersionDAL, secretBlindIndexDAL, secretTagDAL, - secretVersionTagDAL + secretVersionTagDAL, + kmsService, + secretVersionV2BridgeDAL, + secretV2BridgeDAL, + secretVersionTagV2BridgeDAL, + secretRotationDAL, + integrationAuthDAL, + snapshotDAL, + snapshotSecretV2BridgeDAL, + secretApprovalRequestDAL }); const secretImportService = secretImportServiceFactory({ licenseService, + projectBotService, projectEnvDAL, folderDAL, permissionService, secretImportDAL, projectDAL, secretDAL, - secretQueueService + secretQueueService, + secretV2BridgeDAL, + kmsService }); const secretBlindIndexService = secretBlindIndexServiceFactory({ permissionService, secretDAL, secretBlindIndexDAL }); - const secretService = secretServiceFactory({ - folderDAL, - secretVersionDAL, - secretVersionTagDAL, - secretBlindIndexDAL, - permissionService, - projectDAL, - secretDAL, - secretTagDAL, - snapshotService, - secretQueueService, - secretImportDAL, - projectEnvDAL, - projectBotService, - secretApprovalPolicyService, - secretApprovalRequestDAL, - secretApprovalRequestSecretDAL - }); - const secretSharingService = secretSharingServiceFactory({ + const secretV2BridgeService = secretV2BridgeServiceFactory({ + folderDAL, + secretVersionDAL: secretVersionV2BridgeDAL, + secretQueueService, + secretDAL: secretV2BridgeDAL, permissionService, - secretSharingDAL, - orgDAL + secretVersionTagDAL: secretVersionTagV2BridgeDAL, + secretTagDAL, + projectEnvDAL, + secretImportDAL, + secretApprovalRequestDAL, + secretApprovalPolicyService, + secretApprovalRequestSecretDAL, + kmsService, + snapshotService }); const secretApprovalRequestService = secretApprovalRequestServiceFactory({ @@ -756,9 +778,40 @@ export const registerRoutes = async ( snapshotService, secretVersionTagDAL, secretQueueService, + kmsService, + secretV2BridgeDAL, + secretVersionV2BridgeDAL, + secretVersionTagV2BridgeDAL, smtpService, - userDAL, - projectEnvDAL + projectEnvDAL, + userDAL + }); + + const secretService = secretServiceFactory({ + folderDAL, + secretVersionDAL, + secretVersionTagDAL, + secretBlindIndexDAL, + permissionService, + projectDAL, + secretDAL, + secretTagDAL, + snapshotService, + secretQueueService, + secretImportDAL, + projectEnvDAL, + projectBotService, + secretApprovalPolicyService, + secretApprovalRequestDAL, + secretApprovalRequestSecretDAL, + secretV2BridgeService, + secretApprovalRequestService + }); + + const secretSharingService = secretSharingServiceFactory({ + permissionService, + secretSharingDAL, + orgDAL }); const accessApprovalPolicyService = accessApprovalPolicyServiceFactory({ @@ -794,11 +847,14 @@ export const registerRoutes = async ( queueService, folderDAL, secretApprovalPolicyService, - secretBlindIndexDAL, secretApprovalRequestDAL, secretApprovalRequestSecretDAL, secretQueueService, - projectBotService + projectBotService, + kmsService, + secretV2BridgeDAL, + secretVersionV2TagBridgeDAL: secretVersionTagV2BridgeDAL, + secretVersionV2BridgeDAL }); const secretRotationQueue = secretRotationQueueFactory({ telemetryService, @@ -806,7 +862,10 @@ export const registerRoutes = async ( queue: queueService, secretDAL, secretVersionDAL, - projectBotService + projectBotService, + secretVersionV2BridgeDAL, + secretV2BridgeDAL, + kmsService }); const secretRotationService = secretRotationServiceFactory({ @@ -816,7 +875,9 @@ export const registerRoutes = async ( projectDAL, licenseService, secretDAL, - folderDAL + folderDAL, + projectBotService, + secretV2BridgeDAL }); const integrationService = integrationServiceFactory({ @@ -927,7 +988,9 @@ export const registerRoutes = async ( queueService, dynamicSecretLeaseDAL, dynamicSecretProviders, - dynamicSecretDAL + dynamicSecretDAL, + kmsService, + folderDAL }); const dynamicSecretService = dynamicSecretServiceFactory({ projectDAL, @@ -937,7 +1000,8 @@ export const registerRoutes = async ( dynamicSecretProviders, folderDAL, permissionService, - licenseService + licenseService, + kmsService }); const dynamicSecretLeaseService = dynamicSecretLeaseServiceFactory({ projectDAL, @@ -947,7 +1011,8 @@ export const registerRoutes = async ( dynamicSecretLeaseDAL, dynamicSecretProviders, folderDAL, - licenseService + licenseService, + kmsService }); const dailyResourceCleanUp = dailyResourceCleanUpQueueServiceFactory({ auditLogDAL, @@ -956,7 +1021,8 @@ export const registerRoutes = async ( secretFolderVersionDAL: folderVersionDAL, snapshotDAL, identityAccessTokenDAL, - secretSharingDAL + secretSharingDAL, + secretVersionV2DAL: secretVersionV2BridgeDAL }); const oidcService = oidcConfigServiceFactory({ diff --git a/backend/src/server/routes/sanitizedSchemas.ts b/backend/src/server/routes/sanitizedSchemas.ts index 5b0b754f3..ab0557e93 100644 --- a/backend/src/server/routes/sanitizedSchemas.ts +++ b/backend/src/server/routes/sanitizedSchemas.ts @@ -5,6 +5,7 @@ import { IdentityProjectAdditionalPrivilegeSchema, IntegrationAuthsSchema, ProjectRolesSchema, + ProjectsSchema, SecretApprovalPoliciesSchema, UsersSchema } from "@app/db/schemas"; @@ -62,8 +63,14 @@ export const secretRawSchema = z.object({ version: z.number(), type: z.string(), secretKey: z.string(), - secretValue: z.string(), - secretComment: z.string().optional() + secretValue: z.string().optional(), + secretComment: z.string().optional(), + secretReminderNote: z.string().nullable().optional(), + secretReminderRepeatDays: z.number().nullable().optional(), + skipMultilineEncoding: z.boolean().default(false).nullable().optional(), + metadata: z.unknown().nullable().optional(), + createdAt: z.date(), + updatedAt: z.date() }); export const ProjectPermissionSchema = z.object({ @@ -122,11 +129,7 @@ export const SanitizedRoleSchema = ProjectRolesSchema.extend({ }); export const SanitizedDynamicSecretSchema = DynamicSecretsSchema.omit({ - inputIV: true, - inputTag: true, - inputCiphertext: true, - keyEncoding: true, - algorithm: true + encryptedConfig: true }); export const SanitizedAuditLogStreamSchema = z.object({ @@ -135,3 +138,18 @@ export const SanitizedAuditLogStreamSchema = z.object({ createdAt: z.date(), updatedAt: z.date() }); + +export const SanitizedProjectSchema = ProjectsSchema.pick({ + id: true, + name: true, + slug: true, + autoCapitalization: true, + orgId: true, + createdAt: true, + updatedAt: true, + version: true, + upgradeStatus: true, + pitVersionLimit: true, + kmsCertificateKeyId: true, + auditLogsRetentionDays: true +}); diff --git a/backend/src/server/routes/v1/identity-router.ts b/backend/src/server/routes/v1/identity-router.ts index 9fc9baa49..b1cd6cd9d 100644 --- a/backend/src/server/routes/v1/identity-router.ts +++ b/backend/src/server/routes/v1/identity-router.ts @@ -1,12 +1,6 @@ import { z } from "zod"; -import { - IdentitiesSchema, - IdentityOrgMembershipsSchema, - OrgMembershipRole, - OrgRolesSchema, - ProjectsSchema -} from "@app/db/schemas"; +import { IdentitiesSchema, IdentityOrgMembershipsSchema, OrgMembershipRole, OrgRolesSchema } from "@app/db/schemas"; import { EventType } from "@app/ee/services/audit-log/audit-log-types"; import { IDENTITIES } from "@app/lib/api-docs"; import { creationLimit, readLimit, writeLimit } from "@app/server/config/rateLimiter"; @@ -15,6 +9,8 @@ import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; import { AuthMode } from "@app/services/auth/auth-type"; import { PostHogEventTypes } from "@app/services/telemetry/telemetry-types"; +import { SanitizedProjectSchema } from "../sanitizedSchemas"; + export const registerIdentityRouter = async (server: FastifyZodProvider) => { server.route({ method: "POST", @@ -307,7 +303,7 @@ export const registerIdentityRouter = async (server: FastifyZodProvider) => { }) ), identity: IdentitiesSchema.pick({ name: true, id: true, authMethod: true }), - project: ProjectsSchema.pick({ name: true, id: true }) + project: SanitizedProjectSchema.pick({ name: true, id: true }) }) ) }) diff --git a/backend/src/server/routes/v1/project-router.ts b/backend/src/server/routes/v1/project-router.ts index b6c4ce7bd..7619171da 100644 --- a/backend/src/server/routes/v1/project-router.ts +++ b/backend/src/server/routes/v1/project-router.ts @@ -1,22 +1,16 @@ import { z } from "zod"; -import { - IntegrationsSchema, - ProjectMembershipsSchema, - ProjectsSchema, - UserEncryptionKeysSchema, - UsersSchema -} from "@app/db/schemas"; +import { IntegrationsSchema, ProjectMembershipsSchema, UserEncryptionKeysSchema, UsersSchema } from "@app/db/schemas"; import { PROJECTS } from "@app/lib/api-docs"; import { readLimit, writeLimit } from "@app/server/config/rateLimiter"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; import { AuthMode } from "@app/services/auth/auth-type"; import { ProjectFilterType } from "@app/services/project/project-types"; -import { integrationAuthPubSchema } from "../sanitizedSchemas"; +import { integrationAuthPubSchema, SanitizedProjectSchema } from "../sanitizedSchemas"; import { sanitizedServiceTokenSchema } from "../v2/service-token-router"; -const projectWithEnv = ProjectsSchema.merge( +const projectWithEnv = SanitizedProjectSchema.merge( z.object({ _id: z.string(), environments: z.object({ name: z.string(), slug: z.string(), id: z.string() }).array() @@ -78,7 +72,7 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { lastName: true, id: true }).merge(UserEncryptionKeysSchema.pick({ publicKey: true })), - project: ProjectsSchema.pick({ name: true, id: true }), + project: SanitizedProjectSchema.pick({ name: true, id: true }), roles: z.array( z.object({ id: z.string(), @@ -188,7 +182,7 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { }), response: { 200: z.object({ - workspace: ProjectsSchema.optional() + workspace: SanitizedProjectSchema.optional() }) } }, @@ -224,7 +218,7 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { response: { 200: z.object({ message: z.string(), - workspace: ProjectsSchema + workspace: SanitizedProjectSchema }) } }, @@ -272,7 +266,7 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { }), response: { 200: z.object({ - workspace: ProjectsSchema + workspace: SanitizedProjectSchema }) } }, @@ -314,7 +308,7 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { response: { 200: z.object({ message: z.string(), - workspace: ProjectsSchema + workspace: SanitizedProjectSchema }) } }, @@ -351,7 +345,7 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { response: { 200: z.object({ message: z.string(), - workspace: ProjectsSchema + workspace: SanitizedProjectSchema }) } }, @@ -389,7 +383,7 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { response: { 200: z.object({ message: z.string(), - workspace: ProjectsSchema + workspace: SanitizedProjectSchema }) } }, diff --git a/backend/src/server/routes/v1/secret-import-router.ts b/backend/src/server/routes/v1/secret-import-router.ts index ca604e738..ec48803f6 100644 --- a/backend/src/server/routes/v1/secret-import-router.ts +++ b/backend/src/server/routes/v1/secret-import-router.ts @@ -8,6 +8,8 @@ import { readLimit, secretsLimit } from "@app/server/config/rateLimiter"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; import { AuthMode } from "@app/services/auth/auth-type"; +import { secretRawSchema } from "../sanitizedSchemas"; + export const registerSecretImportRouter = async (server: FastifyZodProvider) => { server.route({ method: "POST", @@ -353,4 +355,48 @@ export const registerSecretImportRouter = async (server: FastifyZodProvider) => return { secrets: importedSecrets }; } }); + + server.route({ + url: "/secrets/raw", + method: "GET", + config: { + rateLimit: secretsLimit + }, + schema: { + querystring: z.object({ + workspaceId: z.string().trim(), + environment: z.string().trim(), + path: z.string().trim().default("/").transform(removeTrailingSlash) + }), + response: { + 200: z.object({ + secrets: z + .object({ + secretPath: z.string(), + environment: z.string(), + environmentInfo: z.object({ + id: z.string(), + name: z.string(), + slug: z.string() + }), + folderId: z.string().optional(), + secrets: secretRawSchema.array() + }) + .array() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY, AuthMode.SERVICE_TOKEN, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const importedSecrets = await server.services.secretImport.getRawSecretsFromImports({ + actorId: req.permission.id, + actor: req.permission.type, + actorAuthMethod: req.permission.authMethod, + actorOrgId: req.permission.orgId, + ...req.query, + projectId: req.query.workspaceId + }); + return { secrets: importedSecrets }; + } + }); }; diff --git a/backend/src/server/routes/v1/webhook-router.ts b/backend/src/server/routes/v1/webhook-router.ts index 7423f9f3e..377af135c 100644 --- a/backend/src/server/routes/v1/webhook-router.ts +++ b/backend/src/server/routes/v1/webhook-router.ts @@ -8,25 +8,24 @@ import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; import { AuthMode } from "@app/services/auth/auth-type"; import { WebhookType } from "@app/services/webhook/webhook-types"; -export const sanitizedWebhookSchema = WebhooksSchema.omit({ - encryptedSecretKey: true, - iv: true, - tag: true, - algorithm: true, - keyEncoding: true, - urlCipherText: true, - urlIV: true, - urlTag: true -}).merge( - z.object({ - projectId: z.string(), - environment: z.object({ - id: z.string(), - name: z.string(), - slug: z.string() - }) +export const sanitizedWebhookSchema = WebhooksSchema.pick({ + id: true, + secretPath: true, + lastStatus: true, + lastRunErrorMessage: true, + isDisabled: true, + createdAt: true, + updatedAt: true, + envId: true, + type: true +}).extend({ + projectId: z.string(), + environment: z.object({ + id: z.string(), + name: z.string(), + slug: z.string() }) -); +}); export const registerWebhookRouter = async (server: FastifyZodProvider) => { server.route({ @@ -228,7 +227,7 @@ export const registerWebhookRouter = async (server: FastifyZodProvider) => { response: { 200: z.object({ message: z.string(), - webhooks: sanitizedWebhookSchema.array() + webhooks: sanitizedWebhookSchema.extend({ url: z.string() }).array() }) } }, diff --git a/backend/src/server/routes/v2/identity-project-router.ts b/backend/src/server/routes/v2/identity-project-router.ts index 806a54d99..01b51b315 100644 --- a/backend/src/server/routes/v2/identity-project-router.ts +++ b/backend/src/server/routes/v2/identity-project-router.ts @@ -5,7 +5,6 @@ import { IdentitiesSchema, IdentityProjectMembershipsSchema, ProjectMembershipRole, - ProjectsSchema, ProjectUserMembershipRolesSchema } from "@app/db/schemas"; import { PROJECT_IDENTITIES } from "@app/lib/api-docs"; @@ -15,6 +14,8 @@ import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; import { AuthMode } from "@app/services/auth/auth-type"; import { ProjectUserMembershipTemporaryMode } from "@app/services/project-membership/project-membership-types"; +import { SanitizedProjectSchema } from "../sanitizedSchemas"; + export const registerIdentityProjectRouter = async (server: FastifyZodProvider) => { server.route({ method: "POST", @@ -236,7 +237,7 @@ export const registerIdentityProjectRouter = async (server: FastifyZodProvider) }) ), identity: IdentitiesSchema.pick({ name: true, id: true, authMethod: true }), - project: ProjectsSchema.pick({ name: true, id: true }) + project: SanitizedProjectSchema.pick({ name: true, id: true }) }) .array() }) @@ -294,7 +295,7 @@ export const registerIdentityProjectRouter = async (server: FastifyZodProvider) }) ), identity: IdentitiesSchema.pick({ name: true, id: true, authMethod: true }), - project: ProjectsSchema.pick({ name: true, id: true }) + project: SanitizedProjectSchema.pick({ name: true, id: true }) }) }) } diff --git a/backend/src/server/routes/v2/project-router.ts b/backend/src/server/routes/v2/project-router.ts index e59291664..58ae9e293 100644 --- a/backend/src/server/routes/v2/project-router.ts +++ b/backend/src/server/routes/v2/project-router.ts @@ -1,7 +1,7 @@ import slugify from "@sindresorhus/slugify"; import { z } from "zod"; -import { CertificateAuthoritiesSchema, CertificatesSchema, ProjectKeysSchema, ProjectsSchema } from "@app/db/schemas"; +import { CertificateAuthoritiesSchema, CertificatesSchema, ProjectKeysSchema } from "@app/db/schemas"; import { EventType } from "@app/ee/services/audit-log/audit-log-types"; import { PROJECTS } from "@app/lib/api-docs"; import { creationLimit, readLimit, writeLimit } from "@app/server/config/rateLimiter"; @@ -12,12 +12,12 @@ import { CaStatus } from "@app/services/certificate-authority/certificate-author import { ProjectFilterType } from "@app/services/project/project-types"; import { PostHogEventTypes } from "@app/services/telemetry/telemetry-types"; -const projectWithEnv = ProjectsSchema.merge( - z.object({ - _id: z.string(), - environments: z.object({ name: z.string(), slug: z.string(), id: z.string() }).array() - }) -); +import { SanitizedProjectSchema } from "../sanitizedSchemas"; + +const projectWithEnv = SanitizedProjectSchema.extend({ + _id: z.string(), + environments: z.object({ name: z.string(), slug: z.string(), id: z.string() }).array() +}); const slugSchema = z .string() @@ -161,7 +161,8 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { message: "Slug must be a valid slug" }) .optional() - .describe(PROJECTS.CREATE.slug) + .describe(PROJECTS.CREATE.slug), + kmsKeyId: z.string().optional() }), response: { 200: z.object({ @@ -177,7 +178,8 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { actorOrgId: req.permission.orgId, actorAuthMethod: req.permission.authMethod, workspaceName: req.body.projectName, - slug: req.body.slug + slug: req.body.slug, + kmsKeyId: req.body.kmsKeyId }); await server.services.telemetry.sendPostHogEvents({ @@ -212,7 +214,7 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { slug: slugSchema.describe("The slug of the project to delete.") }), response: { - 200: ProjectsSchema + 200: SanitizedProjectSchema } }, onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), @@ -283,7 +285,7 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { autoCapitalization: z.boolean().optional().describe("The new auto-capitalization setting.") }), response: { - 200: ProjectsSchema + 200: SanitizedProjectSchema } }, diff --git a/backend/src/server/routes/v3/secret-router.ts b/backend/src/server/routes/v3/secret-router.ts index 2a4f9b464..b0776d150 100644 --- a/backend/src/server/routes/v3/secret-router.ts +++ b/backend/src/server/routes/v3/secret-router.ts @@ -18,7 +18,7 @@ import { getUserAgentType } from "@app/server/plugins/audit-log"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; import { ActorType, AuthMode } from "@app/services/auth/auth-type"; import { ProjectFilterType } from "@app/services/project/project-types"; -import { SecretOperations } from "@app/services/secret/secret-types"; +import { SecretOperations, SecretProtectionType } from "@app/services/secret/secret-types"; import { PostHogEventTypes } from "@app/services/telemetry/telemetry-types"; import { secretRawSchema } from "../sanitizedSchemas"; @@ -186,7 +186,15 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { 200: z.object({ secrets: secretRawSchema .extend({ - secretPath: z.string().optional() + secretPath: z.string().optional(), + tags: SecretTagsSchema.pick({ + id: true, + slug: true, + name: true, + color: true + }) + .array() + .optional() }) .array(), imports: z @@ -194,7 +202,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { secretPath: z.string(), environment: z.string(), folderId: z.string().optional(), - secrets: secretRawSchema.array() + secrets: secretRawSchema.omit({ createdAt: true, updatedAt: true }).array() }) .array() .optional() @@ -425,17 +433,26 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { secretComment: z.string().trim().optional().default("").describe(RAW_SECRETS.CREATE.secretComment), tagIds: z.string().array().optional().describe(RAW_SECRETS.CREATE.tagIds), skipMultilineEncoding: z.boolean().optional().describe(RAW_SECRETS.CREATE.skipMultilineEncoding), - type: z.nativeEnum(SecretType).default(SecretType.Shared).describe(RAW_SECRETS.CREATE.type) + type: z.nativeEnum(SecretType).default(SecretType.Shared).describe(RAW_SECRETS.CREATE.type), + secretReminderRepeatDays: z + .number() + .optional() + .nullable() + .describe(RAW_SECRETS.CREATE.secretReminderRepeatDays), + secretReminderNote: z.string().optional().nullable().describe(RAW_SECRETS.CREATE.secretReminderNote) }), response: { - 200: z.object({ - secret: secretRawSchema - }) + 200: z.union([ + z.object({ + secret: secretRawSchema + }), + z.object({ approval: SecretApprovalRequestsSchema }).describe("When secret protection policy is enabled") + ]) } }, onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY, AuthMode.SERVICE_TOKEN, AuthMode.IDENTITY_ACCESS_TOKEN]), handler: async (req) => { - const secret = await server.services.secret.createSecretRaw({ + const secretOperation = await server.services.secret.createSecretRaw({ actorId: req.permission.id, actor: req.permission.type, actorOrgId: req.permission.orgId, @@ -448,9 +465,15 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { secretValue: req.body.secretValue, skipMultilineEncoding: req.body.skipMultilineEncoding, secretComment: req.body.secretComment, - tagIds: req.body.tagIds + tagIds: req.body.tagIds, + secretReminderNote: req.body.secretReminderNote, + secretReminderRepeatDays: req.body.secretReminderRepeatDays }); + if (secretOperation.type === SecretProtectionType.Approval) { + return { approval: secretOperation.approval }; + } + const { secret } = secretOperation; await server.services.auditLog.createAuditLog({ projectId: req.body.workspaceId, ...req.auditLogInfo, @@ -514,17 +537,29 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { .describe(RAW_SECRETS.UPDATE.secretPath), skipMultilineEncoding: z.boolean().optional().describe(RAW_SECRETS.UPDATE.skipMultilineEncoding), type: z.nativeEnum(SecretType).default(SecretType.Shared).describe(RAW_SECRETS.UPDATE.type), - tagIds: z.string().array().optional().describe(RAW_SECRETS.UPDATE.tagIds) + tagIds: z.string().array().optional().describe(RAW_SECRETS.UPDATE.tagIds), + metadata: z.record(z.string()).optional(), + secretReminderNote: z.string().optional().nullable().describe(RAW_SECRETS.UPDATE.secretReminderNote), + secretReminderRepeatDays: z + .number() + .optional() + .nullable() + .describe(RAW_SECRETS.UPDATE.secretReminderRepeatDays), + newSecretName: z.string().min(1).optional().describe(RAW_SECRETS.UPDATE.newSecretName), + secretComment: z.string().optional().describe(RAW_SECRETS.UPDATE.secretComment) }), response: { - 200: z.object({ - secret: secretRawSchema - }) + 200: z.union([ + z.object({ + secret: secretRawSchema + }), + z.object({ approval: SecretApprovalRequestsSchema }).describe("When secret protection policy is enabled") + ]) } }, onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY, AuthMode.SERVICE_TOKEN, AuthMode.IDENTITY_ACCESS_TOKEN]), handler: async (req) => { - const secret = await server.services.secret.updateSecretRaw({ + const secretOperation = await server.services.secret.updateSecretRaw({ actorId: req.permission.id, actor: req.permission.type, actorOrgId: req.permission.orgId, @@ -536,8 +571,17 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { type: req.body.type, secretValue: req.body.secretValue, skipMultilineEncoding: req.body.skipMultilineEncoding, - tagIds: req.body.tagIds + tagIds: req.body.tagIds, + secretReminderRepeatDays: req.body.secretReminderRepeatDays, + secretReminderNote: req.body.secretReminderNote, + metadata: req.body.metadata, + newSecretName: req.body.newSecretName, + secretComment: req.body.secretComment }); + if (secretOperation.type === SecretProtectionType.Approval) { + return { approval: secretOperation.approval }; + } + const { secret } = secretOperation; await server.services.auditLog.createAuditLog({ projectId: req.body.workspaceId, @@ -598,14 +642,17 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { type: z.nativeEnum(SecretType).default(SecretType.Shared).describe(RAW_SECRETS.DELETE.type) }), response: { - 200: z.object({ - secret: secretRawSchema - }) + 200: z.union([ + z.object({ + secret: secretRawSchema + }), + z.object({ approval: SecretApprovalRequestsSchema }).describe("When secret protection policy is enabled") + ]) } }, onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY, AuthMode.SERVICE_TOKEN, AuthMode.IDENTITY_ACCESS_TOKEN]), handler: async (req) => { - const secret = await server.services.secret.deleteSecretRaw({ + const secretOperation = await server.services.secret.deleteSecretRaw({ actorId: req.permission.id, actor: req.permission.type, actorAuthMethod: req.permission.authMethod, @@ -616,6 +663,10 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { secretName: req.params.secretName, type: req.body.type }); + if (secretOperation.type === SecretProtectionType.Approval) { + return { approval: secretOperation.approval }; + } + const { secret } = secretOperation; await server.services.auditLog.createAuditLog({ projectId: req.body.workspaceId, @@ -1760,7 +1811,8 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { } ], body: z.object({ - projectSlug: z.string().trim().describe(RAW_SECRETS.CREATE.projectSlug), + projectSlug: z.string().trim().optional().describe(RAW_SECRETS.UPDATE.projectSlug), + workspaceId: z.string().trim().optional().describe(RAW_SECRETS.UPDATE.workspaceId), environment: z.string().trim().describe(RAW_SECRETS.CREATE.environment), secretPath: z .string() @@ -1776,22 +1828,27 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { .transform((val) => (val.at(-1) === "\n" ? `${val.trim()}\n` : val.trim())) .describe(RAW_SECRETS.CREATE.secretValue), secretComment: z.string().trim().optional().default("").describe(RAW_SECRETS.CREATE.secretComment), - skipMultilineEncoding: z.boolean().optional().describe(RAW_SECRETS.CREATE.skipMultilineEncoding) + skipMultilineEncoding: z.boolean().optional().describe(RAW_SECRETS.CREATE.skipMultilineEncoding), + metadata: z.record(z.string()).optional(), + tagIds: z.string().array().optional().describe(RAW_SECRETS.CREATE.tagIds) }) .array() .min(1) }), response: { - 200: z.object({ - secrets: secretRawSchema.array() - }) + 200: z.union([ + z.object({ + secrets: secretRawSchema.array() + }), + z.object({ approval: SecretApprovalRequestsSchema }).describe("When secret protection policy is enabled") + ]) } }, onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY, AuthMode.SERVICE_TOKEN, AuthMode.IDENTITY_ACCESS_TOKEN]), handler: async (req) => { const { environment, projectSlug, secretPath, secrets: inputSecrets } = req.body; - const secrets = await server.services.secret.createManySecretsRaw({ + const secretOperation = await server.services.secret.createManySecretsRaw({ actorId: req.permission.id, actor: req.permission.type, actorAuthMethod: req.permission.authMethod, @@ -1799,8 +1856,13 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { secretPath, environment, projectSlug, + projectId: req.body.workspaceId, secrets: inputSecrets }); + if (secretOperation.type === SecretProtectionType.Approval) { + return { approval: secretOperation.approval }; + } + const { secrets } = secretOperation; await server.services.auditLog.createAuditLog({ projectId: secrets[0].workspace, @@ -1810,9 +1872,9 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { metadata: { environment: req.body.environment, secretPath: req.body.secretPath, - secrets: secrets.map((secret, i) => ({ + secrets: secrets.map((secret) => ({ secretId: secret.id, - secretKey: inputSecrets[i].secretKey, + secretKey: secret.secretKey, secretVersion: secret.version })) } @@ -1849,7 +1911,8 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { } ], body: z.object({ - projectSlug: z.string().trim().describe(RAW_SECRETS.UPDATE.projectSlug), + projectSlug: z.string().trim().optional().describe(RAW_SECRETS.DELETE.projectSlug), + workspaceId: z.string().trim().optional().describe(RAW_SECRETS.DELETE.workspaceId), environment: z.string().trim().describe(RAW_SECRETS.UPDATE.environment), secretPath: z .string() @@ -1865,21 +1928,32 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { .transform((val) => (val.at(-1) === "\n" ? `${val.trim()}\n` : val.trim())) .describe(RAW_SECRETS.UPDATE.secretValue), secretComment: z.string().trim().optional().describe(RAW_SECRETS.UPDATE.secretComment), - skipMultilineEncoding: z.boolean().optional().describe(RAW_SECRETS.UPDATE.skipMultilineEncoding) + skipMultilineEncoding: z.boolean().optional().describe(RAW_SECRETS.UPDATE.skipMultilineEncoding), + newSecretName: z.string().min(1).optional().describe(RAW_SECRETS.UPDATE.newSecretName), + tagIds: z.string().array().optional().describe(RAW_SECRETS.UPDATE.tagIds), + secretReminderNote: z.string().optional().nullable().describe(RAW_SECRETS.UPDATE.secretReminderNote), + secretReminderRepeatDays: z + .number() + .optional() + .nullable() + .describe(RAW_SECRETS.UPDATE.secretReminderRepeatDays) }) .array() .min(1) }), response: { - 200: z.object({ - secrets: secretRawSchema.array() - }) + 200: z.union([ + z.object({ + secrets: secretRawSchema.array() + }), + z.object({ approval: SecretApprovalRequestsSchema }).describe("When secret protection policy is enabled") + ]) } }, onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY, AuthMode.SERVICE_TOKEN, AuthMode.IDENTITY_ACCESS_TOKEN]), handler: async (req) => { const { environment, projectSlug, secretPath, secrets: inputSecrets } = req.body; - const secrets = await server.services.secret.updateManySecretsRaw({ + const secretOperation = await server.services.secret.updateManySecretsRaw({ actorId: req.permission.id, actor: req.permission.type, actorAuthMethod: req.permission.authMethod, @@ -1887,8 +1961,13 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { secretPath, environment, projectSlug, + projectId: req.body.workspaceId, secrets: inputSecrets }); + if (secretOperation.type === SecretProtectionType.Approval) { + return { approval: secretOperation.approval }; + } + const { secrets } = secretOperation; await server.services.auditLog.createAuditLog({ projectId: secrets[0].workspace, @@ -1898,9 +1977,9 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { metadata: { environment: req.body.environment, secretPath: req.body.secretPath, - secrets: secrets.map((secret, i) => ({ + secrets: secrets.map((secret) => ({ secretId: secret.id, - secretKey: inputSecrets[i].secretKey, + secretKey: secret.secretKey, secretVersion: secret.version })) } @@ -1937,7 +2016,8 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { } ], body: z.object({ - projectSlug: z.string().trim().describe(RAW_SECRETS.DELETE.projectSlug), + projectSlug: z.string().trim().optional().describe(RAW_SECRETS.DELETE.projectSlug), + workspaceId: z.string().trim().optional().describe(RAW_SECRETS.DELETE.workspaceId), environment: z.string().trim().describe(RAW_SECRETS.DELETE.environment), secretPath: z .string() @@ -1947,21 +2027,25 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { .describe(RAW_SECRETS.DELETE.secretPath), secrets: z .object({ - secretKey: z.string().trim().describe(RAW_SECRETS.DELETE.secretName) + secretKey: z.string().trim().describe(RAW_SECRETS.DELETE.secretName), + type: z.nativeEnum(SecretType).default(SecretType.Shared) }) .array() .min(1) }), response: { - 200: z.object({ - secrets: secretRawSchema.array() - }) + 200: z.union([ + z.object({ + secrets: secretRawSchema.array() + }), + z.object({ approval: SecretApprovalRequestsSchema }).describe("When secret protection policy is enabled") + ]) } }, onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY, AuthMode.SERVICE_TOKEN, AuthMode.IDENTITY_ACCESS_TOKEN]), handler: async (req) => { const { environment, projectSlug, secretPath, secrets: inputSecrets } = req.body; - const secrets = await server.services.secret.deleteManySecretsRaw({ + const secretOperation = await server.services.secret.deleteManySecretsRaw({ actorId: req.permission.id, actor: req.permission.type, actorAuthMethod: req.permission.authMethod, @@ -1969,8 +2053,13 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { environment, projectSlug, secretPath, + projectId: req.body.workspaceId, secrets: inputSecrets }); + if (secretOperation.type === SecretProtectionType.Approval) { + return { approval: secretOperation.approval }; + } + const { secrets } = secretOperation; await server.services.auditLog.createAuditLog({ projectId: secrets[0].workspace, @@ -1980,9 +2069,9 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { metadata: { environment: req.body.environment, secretPath: req.body.secretPath, - secrets: secrets.map((secret, i) => ({ + secrets: secrets.map((secret) => ({ secretId: secret.id, - secretKey: inputSecrets[i].secretKey, + secretKey: secret.secretKey, secretVersion: secret.version })) } diff --git a/backend/src/services/certificate-authority/certificate-authority-fns.ts b/backend/src/services/certificate-authority/certificate-authority-fns.ts index 9f98dcb83..32ac4bfb1 100644 --- a/backend/src/services/certificate-authority/certificate-authority-fns.ts +++ b/backend/src/services/certificate-authority/certificate-authority-fns.ts @@ -78,7 +78,7 @@ export const getCaCredentials = async ({ const kmsDecryptor = await kmsService.decryptWithKmsKey({ kmsId: keyId }); - const decryptedPrivateKey = kmsDecryptor({ + const decryptedPrivateKey = await kmsDecryptor({ cipherTextBlob: caSecret.encryptedPrivateKey }); @@ -129,13 +129,13 @@ export const getCaCertChain = async ({ kmsId: keyId }); - const decryptedCaCert = kmsDecryptor({ + const decryptedCaCert = await kmsDecryptor({ cipherTextBlob: caCert.encryptedCertificate }); const caCertObj = new x509.X509Certificate(decryptedCaCert); - const decryptedChain = kmsDecryptor({ + const decryptedChain = await kmsDecryptor({ cipherTextBlob: caCert.encryptedCertificateChain }); @@ -176,7 +176,7 @@ export const rebuildCaCrl = async ({ kmsId: keyId }); - const privateKey = kmsDecryptor({ + const privateKey = await kmsDecryptor({ cipherTextBlob: caSecret.encryptedPrivateKey }); @@ -210,7 +210,7 @@ export const rebuildCaCrl = async ({ const kmsEncryptor = await kmsService.encryptWithKmsKey({ kmsId: keyId }); - const { cipherTextBlob: encryptedCrl } = kmsEncryptor({ + const { cipherTextBlob: encryptedCrl } = await kmsEncryptor({ plainText: Buffer.from(new Uint8Array(crl.rawData)) }); diff --git a/backend/src/services/certificate-authority/certificate-authority-queue.ts b/backend/src/services/certificate-authority/certificate-authority-queue.ts index 30da119d0..1d7021937 100644 --- a/backend/src/services/certificate-authority/certificate-authority-queue.ts +++ b/backend/src/services/certificate-authority/certificate-authority-queue.ts @@ -91,7 +91,7 @@ export const certificateAuthorityQueueFactory = ({ const kmsDecryptor = await kmsService.decryptWithKmsKey({ kmsId: keyId }); - const privateKey = kmsDecryptor({ + const privateKey = await kmsDecryptor({ cipherTextBlob: caSecret.encryptedPrivateKey }); @@ -125,7 +125,7 @@ export const certificateAuthorityQueueFactory = ({ const kmsEncryptor = await kmsService.encryptWithKmsKey({ kmsId: keyId }); - const { cipherTextBlob: encryptedCrl } = kmsEncryptor({ + const { cipherTextBlob: encryptedCrl } = await kmsEncryptor({ plainText: Buffer.from(new Uint8Array(crl.rawData)) }); diff --git a/backend/src/services/certificate-authority/certificate-authority-service.ts b/backend/src/services/certificate-authority/certificate-authority-service.ts index afc8d7efb..a1390b1f7 100644 --- a/backend/src/services/certificate-authority/certificate-authority-service.ts +++ b/backend/src/services/certificate-authority/certificate-authority-service.ts @@ -181,11 +181,11 @@ export const certificateAuthorityServiceFactory = ({ ] }); - const { cipherTextBlob: encryptedCertificate } = kmsEncryptor({ + const { cipherTextBlob: encryptedCertificate } = await kmsEncryptor({ plainText: Buffer.from(new Uint8Array(cert.rawData)) }); - const { cipherTextBlob: encryptedCertificateChain } = kmsEncryptor({ + const { cipherTextBlob: encryptedCertificateChain } = await kmsEncryptor({ plainText: Buffer.alloc(0) }); @@ -209,7 +209,7 @@ export const certificateAuthorityServiceFactory = ({ signingKey: keys.privateKey }); - const { cipherTextBlob: encryptedCrl } = kmsEncryptor({ + const { cipherTextBlob: encryptedCrl } = await kmsEncryptor({ plainText: Buffer.from(new Uint8Array(crl.rawData)) }); @@ -224,7 +224,7 @@ export const certificateAuthorityServiceFactory = ({ // https://nodejs.org/api/crypto.html#static-method-keyobjectfromkey const skObj = KeyObject.from(keys.privateKey); - const { cipherTextBlob: encryptedPrivateKey } = kmsEncryptor({ + const { cipherTextBlob: encryptedPrivateKey } = await kmsEncryptor({ plainText: skObj.export({ type: "pkcs8", format: "der" @@ -458,7 +458,7 @@ export const certificateAuthorityServiceFactory = ({ }); const caCert = await certificateAuthorityCertDAL.findOne({ caId: ca.id }); - const decryptedCaCert = kmsDecryptor({ + const decryptedCaCert = await kmsDecryptor({ cipherTextBlob: caCert.encryptedCertificate }); @@ -615,11 +615,11 @@ export const certificateAuthorityServiceFactory = ({ kmsId: certificateManagerKmsId }); - const { cipherTextBlob: encryptedCertificate } = kmsEncryptor({ + const { cipherTextBlob: encryptedCertificate } = await kmsEncryptor({ plainText: Buffer.from(new Uint8Array(certObj.rawData)) }); - const { cipherTextBlob: encryptedCertificateChain } = kmsEncryptor({ + const { cipherTextBlob: encryptedCertificateChain } = await kmsEncryptor({ plainText: Buffer.from(certificateChain) }); @@ -693,7 +693,7 @@ export const certificateAuthorityServiceFactory = ({ kmsId: certificateManagerKmsId }); - const decryptedCaCert = kmsDecryptor({ + const decryptedCaCert = await kmsDecryptor({ cipherTextBlob: caCert.encryptedCertificate }); @@ -803,7 +803,7 @@ export const certificateAuthorityServiceFactory = ({ const kmsEncryptor = await kmsService.encryptWithKmsKey({ kmsId: certificateManagerKmsId }); - const { cipherTextBlob: encryptedCertificate } = kmsEncryptor({ + const { cipherTextBlob: encryptedCertificate } = await kmsEncryptor({ plainText: Buffer.from(new Uint8Array(leafCert.rawData)) }); diff --git a/backend/src/services/certificate/certificate-service.ts b/backend/src/services/certificate/certificate-service.ts index 401a55cc9..f05ed8a87 100644 --- a/backend/src/services/certificate/certificate-service.ts +++ b/backend/src/services/certificate/certificate-service.ts @@ -173,7 +173,7 @@ export const certificateServiceFactory = ({ const kmsDecryptor = await kmsService.decryptWithKmsKey({ kmsId: certificateManagerKeyId }); - const decryptedCert = kmsDecryptor({ + const decryptedCert = await kmsDecryptor({ cipherTextBlob: certBody.encryptedCertificate }); diff --git a/backend/src/services/integration-auth/integration-auth-service.ts b/backend/src/services/integration-auth/integration-auth-service.ts index e5126532f..e16f6bb77 100644 --- a/backend/src/services/integration-auth/integration-auth-service.ts +++ b/backend/src/services/integration-auth/integration-auth-service.ts @@ -11,7 +11,8 @@ import { BadRequestError } from "@app/lib/errors"; import { TProjectPermission } from "@app/lib/types"; import { TIntegrationDALFactory } from "../integration/integration-dal"; -import { TProjectBotDALFactory } from "../project-bot/project-bot-dal"; +import { TKmsServiceFactory } from "../kms/kms-service"; +import { KmsDataKey } from "../kms/kms-types"; import { TProjectBotServiceFactory } from "../project-bot/project-bot-service"; import { getApps } from "./integration-app-list"; import { TIntegrationAuthDALFactory } from "./integration-auth-dal"; @@ -53,8 +54,8 @@ type TIntegrationAuthServiceFactoryDep = { integrationAuthDAL: TIntegrationAuthDALFactory; integrationDAL: Pick; projectBotService: Pick; - projectBotDAL: Pick; permissionService: Pick; + kmsService: Pick; }; export type TIntegrationAuthServiceFactory = ReturnType; @@ -63,8 +64,8 @@ export const integrationAuthServiceFactory = ({ permissionService, integrationAuthDAL, integrationDAL, - projectBotDAL, - projectBotService + projectBotService, + kmsService }: TIntegrationAuthServiceFactoryDep) => { const listIntegrationAuthByProjectId = async ({ actorId, @@ -122,9 +123,6 @@ export const integrationAuthServiceFactory = ({ ); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Create, ProjectPermissionSub.Integrations); - const bot = await projectBotDAL.findOne({ isActive: true, projectId }); - if (!bot) throw new BadRequestError({ message: "Bot must be enabled for oauth2 code token exchange" }); - const tokenExchange = await exchangeCode({ integration, code, url }); const updateDoc: TIntegrationAuthsInsert = { projectId, @@ -145,18 +143,38 @@ export const integrationAuthServiceFactory = ({ }; } - const key = await projectBotService.getBotKey(projectId); - if (tokenExchange.refreshToken) { - const refreshEncToken = encryptSymmetric128BitHexKeyUTF8(tokenExchange.refreshToken, key); - updateDoc.refreshIV = refreshEncToken.iv; - updateDoc.refreshTag = refreshEncToken.tag; - updateDoc.refreshCiphertext = refreshEncToken.ciphertext; - } - if (tokenExchange.accessToken) { - const accessEncToken = encryptSymmetric128BitHexKeyUTF8(tokenExchange.accessToken, key); - updateDoc.accessIV = accessEncToken.iv; - updateDoc.accessTag = accessEncToken.tag; - updateDoc.accessCiphertext = accessEncToken.ciphertext; + const { shouldUseSecretV2Bridge, botKey } = await projectBotService.getBotKey(projectId); + if (shouldUseSecretV2Bridge) { + const { encryptor: secretManagerEncryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId + }); + if (tokenExchange.refreshToken) { + const refreshEncToken = secretManagerEncryptor({ + plainText: Buffer.from(tokenExchange.refreshToken) + }).cipherTextBlob; + updateDoc.encryptedRefresh = refreshEncToken; + } + if (tokenExchange.accessToken) { + const accessToken = secretManagerEncryptor({ + plainText: Buffer.from(tokenExchange.accessToken) + }).cipherTextBlob; + updateDoc.encryptedAccess = accessToken; + } + } else { + if (!botKey) throw new BadRequestError({ message: "Bot key not found" }); + if (tokenExchange.refreshToken) { + const refreshEncToken = encryptSymmetric128BitHexKeyUTF8(tokenExchange.refreshToken, botKey); + updateDoc.refreshIV = refreshEncToken.iv; + updateDoc.refreshTag = refreshEncToken.tag; + updateDoc.refreshCiphertext = refreshEncToken.ciphertext; + } + if (tokenExchange.accessToken) { + const accessEncToken = encryptSymmetric128BitHexKeyUTF8(tokenExchange.accessToken, botKey); + updateDoc.accessIV = accessEncToken.iv; + updateDoc.accessTag = accessEncToken.tag; + updateDoc.accessCiphertext = accessEncToken.ciphertext; + } } return integrationAuthDAL.transaction(async (tx) => { const doc = await integrationAuthDAL.findOne({ projectId, integration }, tx); @@ -193,9 +211,6 @@ export const integrationAuthServiceFactory = ({ ); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Create, ProjectPermissionSub.Integrations); - const bot = await projectBotDAL.findOne({ isActive: true, projectId }); - if (!bot) throw new BadRequestError({ message: "Bot must be enabled for oauth2 code token exchange" }); - const updateDoc: TIntegrationAuthsInsert = { projectId, namespace, @@ -212,109 +227,210 @@ export const integrationAuthServiceFactory = ({ : {}) }; - const key = await projectBotService.getBotKey(projectId); - if (refreshToken) { - const tokenDetails = await exchangeRefresh( - integration, - refreshToken, - url, - updateDoc.metadata as Record - ); - const refreshEncToken = encryptSymmetric128BitHexKeyUTF8(tokenDetails.refreshToken, key); - updateDoc.refreshIV = refreshEncToken.iv; - updateDoc.refreshTag = refreshEncToken.tag; - updateDoc.refreshCiphertext = refreshEncToken.ciphertext; - const accessEncToken = encryptSymmetric128BitHexKeyUTF8(tokenDetails.accessToken, key); - updateDoc.accessIV = accessEncToken.iv; - updateDoc.accessTag = accessEncToken.tag; - updateDoc.accessCiphertext = accessEncToken.ciphertext; - updateDoc.accessExpiresAt = tokenDetails.accessExpiresAt; - } + const { shouldUseSecretV2Bridge, botKey } = await projectBotService.getBotKey(projectId); + if (shouldUseSecretV2Bridge) { + const { encryptor: secretManagerEncryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId + }); + if (refreshToken) { + const tokenDetails = await exchangeRefresh( + integration, + refreshToken, + url, + updateDoc.metadata as Record + ); + const refreshEncToken = secretManagerEncryptor({ + plainText: Buffer.from(tokenDetails.refreshToken) + }).cipherTextBlob; + updateDoc.encryptedRefresh = refreshEncToken; - if (!refreshToken && (accessId || accessToken || awsAssumeIamRoleArn)) { - if (accessToken) { - const accessEncToken = encryptSymmetric128BitHexKeyUTF8(accessToken, key); + const accessEncToken = secretManagerEncryptor({ + plainText: Buffer.from(tokenDetails.accessToken) + }).cipherTextBlob; + updateDoc.encryptedAccess = accessEncToken; + updateDoc.accessExpiresAt = tokenDetails.accessExpiresAt; + } + + if (!refreshToken && (accessId || accessToken || awsAssumeIamRoleArn)) { + if (accessToken) { + const accessEncToken = secretManagerEncryptor({ + plainText: Buffer.from(accessToken) + }).cipherTextBlob; + updateDoc.encryptedAccess = accessEncToken; + } + if (accessId) { + const accessEncToken = secretManagerEncryptor({ + plainText: Buffer.from(accessId) + }).cipherTextBlob; + updateDoc.encryptedAccessId = accessEncToken; + } + if (awsAssumeIamRoleArn) { + const awsAssumeIamRoleArnEncrypted = secretManagerEncryptor({ + plainText: Buffer.from(awsAssumeIamRoleArn) + }).cipherTextBlob; + updateDoc.encryptedAwsAssumeIamRoleArn = awsAssumeIamRoleArnEncrypted; + } + } + } else { + if (!botKey) throw new BadRequestError({ message: "Bot key not found" }); + if (refreshToken) { + const tokenDetails = await exchangeRefresh( + integration, + refreshToken, + url, + updateDoc.metadata as Record + ); + const refreshEncToken = encryptSymmetric128BitHexKeyUTF8(tokenDetails.refreshToken, botKey); + updateDoc.refreshIV = refreshEncToken.iv; + updateDoc.refreshTag = refreshEncToken.tag; + updateDoc.refreshCiphertext = refreshEncToken.ciphertext; + const accessEncToken = encryptSymmetric128BitHexKeyUTF8(tokenDetails.accessToken, botKey); updateDoc.accessIV = accessEncToken.iv; updateDoc.accessTag = accessEncToken.tag; updateDoc.accessCiphertext = accessEncToken.ciphertext; + + updateDoc.accessExpiresAt = tokenDetails.accessExpiresAt; } - if (accessId) { - const accessEncToken = encryptSymmetric128BitHexKeyUTF8(accessId, key); - updateDoc.accessIdIV = accessEncToken.iv; - updateDoc.accessIdTag = accessEncToken.tag; - updateDoc.accessIdCiphertext = accessEncToken.ciphertext; - } - if (awsAssumeIamRoleArn) { - const awsAssumeIamRoleArnEnc = encryptSymmetric128BitHexKeyUTF8(awsAssumeIamRoleArn, key); - updateDoc.awsAssumeIamRoleArnCipherText = awsAssumeIamRoleArnEnc.ciphertext; - updateDoc.awsAssumeIamRoleArnIV = awsAssumeIamRoleArnEnc.iv; - updateDoc.awsAssumeIamRoleArnTag = awsAssumeIamRoleArnEnc.tag; + + if (!refreshToken && (accessId || accessToken || awsAssumeIamRoleArn)) { + if (accessToken) { + const accessEncToken = encryptSymmetric128BitHexKeyUTF8(accessToken, botKey); + updateDoc.accessIV = accessEncToken.iv; + updateDoc.accessTag = accessEncToken.tag; + updateDoc.accessCiphertext = accessEncToken.ciphertext; + } + if (accessId) { + const accessEncToken = encryptSymmetric128BitHexKeyUTF8(accessId, botKey); + updateDoc.accessIdIV = accessEncToken.iv; + updateDoc.accessIdTag = accessEncToken.tag; + updateDoc.accessIdCiphertext = accessEncToken.ciphertext; + } + if (awsAssumeIamRoleArn) { + const awsAssumeIamRoleArnEnc = encryptSymmetric128BitHexKeyUTF8(awsAssumeIamRoleArn, botKey); + updateDoc.awsAssumeIamRoleArnCipherText = awsAssumeIamRoleArnEnc.ciphertext; + updateDoc.awsAssumeIamRoleArnIV = awsAssumeIamRoleArnEnc.iv; + updateDoc.awsAssumeIamRoleArnTag = awsAssumeIamRoleArnEnc.tag; + } } } return integrationAuthDAL.create(updateDoc); }; // helper function - const getIntegrationAccessToken = async (integrationAuth: TIntegrationAuths, botKey: string) => { + const getIntegrationAccessToken = async ( + integrationAuth: TIntegrationAuths, + shouldUseSecretV2Bridge: boolean, + botKey?: string + ) => { let accessToken: string | undefined; let accessId: string | undefined; // this means its not access token based if ( integrationAuth.integration === Integrations.AWS_SECRET_MANAGER && - integrationAuth.awsAssumeIamRoleArnCipherText + (shouldUseSecretV2Bridge + ? integrationAuth.encryptedAwsAssumeIamRoleArn + : integrationAuth.awsAssumeIamRoleArnCipherText) ) { return { accessToken: "", accessId: "" }; } + if (shouldUseSecretV2Bridge) { + const { decryptor: secretManagerDecryptor, encryptor: secretManagerEncryptor } = + await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId: integrationAuth.projectId + }); + if (integrationAuth.encryptedAccess) { + accessToken = secretManagerDecryptor({ cipherTextBlob: integrationAuth.encryptedAccess }).toString(); + } - if (integrationAuth.accessTag && integrationAuth.accessIV && integrationAuth.accessCiphertext) { - accessToken = decryptSymmetric128BitHexKeyUTF8({ - ciphertext: integrationAuth.accessCiphertext, - iv: integrationAuth.accessIV, - tag: integrationAuth.accessTag, - key: botKey - }); - } + if (integrationAuth.encryptedRefresh) { + const refreshToken = secretManagerDecryptor({ cipherTextBlob: integrationAuth.encryptedRefresh }).toString(); - if (integrationAuth.refreshCiphertext && integrationAuth.refreshIV && integrationAuth.refreshTag) { - const refreshToken = decryptSymmetric128BitHexKeyUTF8({ - key: botKey, - ciphertext: integrationAuth.refreshCiphertext, - iv: integrationAuth.refreshIV, - tag: integrationAuth.refreshTag - }); + if (integrationAuth.accessExpiresAt && integrationAuth.accessExpiresAt < new Date()) { + // refer above it contains same logic except not saving + const tokenDetails = await exchangeRefresh( + integrationAuth.integration, + refreshToken, + integrationAuth?.url, + integrationAuth.metadata as Record + ); + const encryptedRefresh = secretManagerEncryptor({ + plainText: Buffer.from(tokenDetails.refreshToken) + }).cipherTextBlob; + const encryptedAccess = secretManagerEncryptor({ + plainText: Buffer.from(tokenDetails.accessToken) + }).cipherTextBlob; + accessToken = tokenDetails.accessToken; + await integrationAuthDAL.updateById(integrationAuth.id, { + accessExpiresAt: tokenDetails.accessExpiresAt, + encryptedRefresh, + encryptedAccess + }); + } + } + if (!accessToken) throw new BadRequestError({ message: "Missing access token" }); - if (integrationAuth.accessExpiresAt && integrationAuth.accessExpiresAt < new Date()) { - // refer above it contains same logic except not saving - const tokenDetails = await exchangeRefresh( - integrationAuth.integration, - refreshToken, - integrationAuth?.url, - integrationAuth.metadata as Record - ); - const refreshEncToken = encryptSymmetric128BitHexKeyUTF8(tokenDetails.refreshToken, botKey); - const accessEncToken = encryptSymmetric128BitHexKeyUTF8(tokenDetails.accessToken, botKey); - accessToken = tokenDetails.accessToken; - await integrationAuthDAL.updateById(integrationAuth.id, { - refreshIV: refreshEncToken.iv, - refreshTag: refreshEncToken.tag, - refreshCiphertext: refreshEncToken.ciphertext, - accessExpiresAt: tokenDetails.accessExpiresAt, - accessIV: accessEncToken.iv, - accessTag: accessEncToken.tag, - accessCiphertext: accessEncToken.ciphertext + if (integrationAuth.encryptedAccessId) { + accessId = secretManagerDecryptor({ + cipherTextBlob: integrationAuth.encryptedAccessId + }).toString(); + } + + // the old bot key is else + } else { + if (!botKey) throw new BadRequestError({ message: "bot key is missing" }); + if (integrationAuth.accessTag && integrationAuth.accessIV && integrationAuth.accessCiphertext) { + accessToken = decryptSymmetric128BitHexKeyUTF8({ + ciphertext: integrationAuth.accessCiphertext, + iv: integrationAuth.accessIV, + tag: integrationAuth.accessTag, + key: botKey + }); + } + + if (integrationAuth.refreshCiphertext && integrationAuth.refreshIV && integrationAuth.refreshTag) { + const refreshToken = decryptSymmetric128BitHexKeyUTF8({ + key: botKey, + ciphertext: integrationAuth.refreshCiphertext, + iv: integrationAuth.refreshIV, + tag: integrationAuth.refreshTag + }); + + if (integrationAuth.accessExpiresAt && integrationAuth.accessExpiresAt < new Date()) { + // refer above it contains same logic except not saving + const tokenDetails = await exchangeRefresh( + integrationAuth.integration, + refreshToken, + integrationAuth?.url, + integrationAuth.metadata as Record + ); + const refreshEncToken = encryptSymmetric128BitHexKeyUTF8(tokenDetails.refreshToken, botKey); + const accessEncToken = encryptSymmetric128BitHexKeyUTF8(tokenDetails.accessToken, botKey); + accessToken = tokenDetails.accessToken; + await integrationAuthDAL.updateById(integrationAuth.id, { + refreshIV: refreshEncToken.iv, + refreshTag: refreshEncToken.tag, + refreshCiphertext: refreshEncToken.ciphertext, + accessExpiresAt: tokenDetails.accessExpiresAt, + accessIV: accessEncToken.iv, + accessTag: accessEncToken.tag, + accessCiphertext: accessEncToken.ciphertext + }); + } + } + if (!accessToken) throw new BadRequestError({ message: "Missing access token" }); + + if (integrationAuth.accessIdTag && integrationAuth.accessIdIV && integrationAuth.accessIdCiphertext) { + accessId = decryptSymmetric128BitHexKeyUTF8({ + key: botKey, + ciphertext: integrationAuth.accessIdCiphertext, + iv: integrationAuth.accessIdIV, + tag: integrationAuth.accessIdTag }); } } - if (!accessToken) throw new BadRequestError({ message: "Missing access token" }); - if (integrationAuth.accessIdTag && integrationAuth.accessIdIV && integrationAuth.accessIdCiphertext) { - accessId = decryptSymmetric128BitHexKeyUTF8({ - key: botKey, - ciphertext: integrationAuth.accessIdCiphertext, - iv: integrationAuth.accessIdIV, - tag: integrationAuth.accessIdTag - }); - } return { accessId, accessToken }; }; @@ -339,8 +455,8 @@ export const integrationAuthServiceFactory = ({ ); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Integrations); - const botKey = await projectBotService.getBotKey(integrationAuth.projectId); - const { accessToken, accessId } = await getIntegrationAccessToken(integrationAuth, botKey); + const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(integrationAuth.projectId); + const { accessToken, accessId } = await getIntegrationAccessToken(integrationAuth, shouldUseSecretV2Bridge, botKey); const apps = await getApps({ integration: integrationAuth.integration, accessToken, @@ -371,8 +487,8 @@ export const integrationAuthServiceFactory = ({ ); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Integrations); - const botKey = await projectBotService.getBotKey(integrationAuth.projectId); - const { accessToken } = await getIntegrationAccessToken(integrationAuth, botKey); + const { shouldUseSecretV2Bridge, botKey } = await projectBotService.getBotKey(integrationAuth.projectId); + const { accessToken } = await getIntegrationAccessToken(integrationAuth, shouldUseSecretV2Bridge, botKey); const teams = await getTeams({ integration: integrationAuth.integration, accessToken, @@ -400,8 +516,8 @@ export const integrationAuthServiceFactory = ({ actorOrgId ); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Integrations); - const botKey = await projectBotService.getBotKey(integrationAuth.projectId); - const { accessToken } = await getIntegrationAccessToken(integrationAuth, botKey); + const { shouldUseSecretV2Bridge, botKey } = await projectBotService.getBotKey(integrationAuth.projectId); + const { accessToken } = await getIntegrationAccessToken(integrationAuth, shouldUseSecretV2Bridge, botKey); if (appId) { const { data } = await request.get( @@ -441,8 +557,8 @@ export const integrationAuthServiceFactory = ({ actorOrgId ); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Integrations); - const botKey = await projectBotService.getBotKey(integrationAuth.projectId); - const { accessToken } = await getIntegrationAccessToken(integrationAuth, botKey); + const { shouldUseSecretV2Bridge, botKey } = await projectBotService.getBotKey(integrationAuth.projectId); + const { accessToken } = await getIntegrationAccessToken(integrationAuth, shouldUseSecretV2Bridge, botKey); if (accountId) { const { data } = await request.get(`${IntegrationUrls.CHECKLY_API_URL}/v1/check-groups`, { headers: { @@ -468,8 +584,8 @@ export const integrationAuthServiceFactory = ({ actorOrgId ); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Integrations); - const botKey = await projectBotService.getBotKey(integrationAuth.projectId); - const { accessToken } = await getIntegrationAccessToken(integrationAuth, botKey); + const { shouldUseSecretV2Bridge, botKey } = await projectBotService.getBotKey(integrationAuth.projectId); + const { accessToken } = await getIntegrationAccessToken(integrationAuth, shouldUseSecretV2Bridge, botKey); const octokit = new Octokit({ auth: accessToken @@ -505,8 +621,8 @@ export const integrationAuthServiceFactory = ({ actorOrgId ); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Integrations); - const botKey = await projectBotService.getBotKey(integrationAuth.projectId); - const { accessToken } = await getIntegrationAccessToken(integrationAuth, botKey); + const { shouldUseSecretV2Bridge, botKey } = await projectBotService.getBotKey(integrationAuth.projectId); + const { accessToken } = await getIntegrationAccessToken(integrationAuth, shouldUseSecretV2Bridge, botKey); const octokit = new Octokit({ auth: accessToken @@ -537,8 +653,8 @@ export const integrationAuthServiceFactory = ({ actorOrgId ); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Integrations); - const botKey = await projectBotService.getBotKey(integrationAuth.projectId); - const { accessToken } = await getIntegrationAccessToken(integrationAuth, botKey); + const { shouldUseSecretV2Bridge, botKey } = await projectBotService.getBotKey(integrationAuth.projectId); + const { accessToken } = await getIntegrationAccessToken(integrationAuth, shouldUseSecretV2Bridge, botKey); const { data } = await request.get<{ results: Array<{ id: string; name: string }> }>( `${IntegrationUrls.QOVERY_API_URL}/organization`, { @@ -571,8 +687,8 @@ export const integrationAuthServiceFactory = ({ actorOrgId ); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Integrations); - const botKey = await projectBotService.getBotKey(integrationAuth.projectId); - const { accessId, accessToken } = await getIntegrationAccessToken(integrationAuth, botKey); + const { shouldUseSecretV2Bridge, botKey } = await projectBotService.getBotKey(integrationAuth.projectId); + const { accessId, accessToken } = await getIntegrationAccessToken(integrationAuth, shouldUseSecretV2Bridge, botKey); const kms = new AWS.KMS({ region, @@ -629,8 +745,8 @@ export const integrationAuthServiceFactory = ({ actorOrgId ); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Integrations); - const botKey = await projectBotService.getBotKey(integrationAuth.projectId); - const { accessToken } = await getIntegrationAccessToken(integrationAuth, botKey); + const { shouldUseSecretV2Bridge, botKey } = await projectBotService.getBotKey(integrationAuth.projectId); + const { accessToken } = await getIntegrationAccessToken(integrationAuth, shouldUseSecretV2Bridge, botKey); if (orgId) { const { data } = await request.get<{ results: Array<{ id: string; name: string }> }>( `${IntegrationUrls.QOVERY_API_URL}/organization/${orgId}/project`, @@ -665,8 +781,8 @@ export const integrationAuthServiceFactory = ({ actorOrgId ); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Integrations); - const botKey = await projectBotService.getBotKey(integrationAuth.projectId); - const { accessToken } = await getIntegrationAccessToken(integrationAuth, botKey); + const { shouldUseSecretV2Bridge, botKey } = await projectBotService.getBotKey(integrationAuth.projectId); + const { accessToken } = await getIntegrationAccessToken(integrationAuth, shouldUseSecretV2Bridge, botKey); if (projectId && projectId !== "none") { // TODO: fix const { data } = await request.get<{ results: { id: string; name: string }[] }>( @@ -706,8 +822,8 @@ export const integrationAuthServiceFactory = ({ actorOrgId ); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Integrations); - const botKey = await projectBotService.getBotKey(integrationAuth.projectId); - const { accessToken } = await getIntegrationAccessToken(integrationAuth, botKey); + const { shouldUseSecretV2Bridge, botKey } = await projectBotService.getBotKey(integrationAuth.projectId); + const { accessToken } = await getIntegrationAccessToken(integrationAuth, shouldUseSecretV2Bridge, botKey); if (environmentId) { const { data } = await request.get<{ results: { id: string; name: string }[] }>( `${IntegrationUrls.QOVERY_API_URL}/environment/${environmentId}/application`, @@ -746,8 +862,8 @@ export const integrationAuthServiceFactory = ({ actorOrgId ); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Integrations); - const botKey = await projectBotService.getBotKey(integrationAuth.projectId); - const { accessToken } = await getIntegrationAccessToken(integrationAuth, botKey); + const { shouldUseSecretV2Bridge, botKey } = await projectBotService.getBotKey(integrationAuth.projectId); + const { accessToken } = await getIntegrationAccessToken(integrationAuth, shouldUseSecretV2Bridge, botKey); if (environmentId) { const { data } = await request.get<{ results: { id: string; name: string }[] }>( `${IntegrationUrls.QOVERY_API_URL}/environment/${environmentId}/container`, @@ -786,8 +902,8 @@ export const integrationAuthServiceFactory = ({ actorOrgId ); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Integrations); - const botKey = await projectBotService.getBotKey(integrationAuth.projectId); - const { accessToken } = await getIntegrationAccessToken(integrationAuth, botKey); + const { shouldUseSecretV2Bridge, botKey } = await projectBotService.getBotKey(integrationAuth.projectId); + const { accessToken } = await getIntegrationAccessToken(integrationAuth, shouldUseSecretV2Bridge, botKey); if (environmentId) { const { data } = await request.get<{ results: { id: string; name: string }[] }>( `${IntegrationUrls.QOVERY_API_URL}/environment/${environmentId}/job`, @@ -825,8 +941,8 @@ export const integrationAuthServiceFactory = ({ actorOrgId ); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Integrations); - const botKey = await projectBotService.getBotKey(integrationAuth.projectId); - const { accessToken } = await getIntegrationAccessToken(integrationAuth, botKey); + const { shouldUseSecretV2Bridge, botKey } = await projectBotService.getBotKey(integrationAuth.projectId); + const { accessToken } = await getIntegrationAccessToken(integrationAuth, shouldUseSecretV2Bridge, botKey); const { data } = await request.get( `${IntegrationUrls.HEROKU_API_URL}/pipeline-couplings`, @@ -865,8 +981,8 @@ export const integrationAuthServiceFactory = ({ actorOrgId ); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Integrations); - const botKey = await projectBotService.getBotKey(integrationAuth.projectId); - const { accessToken } = await getIntegrationAccessToken(integrationAuth, botKey); + const { shouldUseSecretV2Bridge, botKey } = await projectBotService.getBotKey(integrationAuth.projectId); + const { accessToken } = await getIntegrationAccessToken(integrationAuth, shouldUseSecretV2Bridge, botKey); if (appId) { const query = ` query GetEnvironments($projectId: String!, $after: String, $before: String, $first: Int, $isEphemeral: Boolean, $last: Int) { @@ -933,8 +1049,8 @@ export const integrationAuthServiceFactory = ({ actorOrgId ); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Integrations); - const botKey = await projectBotService.getBotKey(integrationAuth.projectId); - const { accessToken } = await getIntegrationAccessToken(integrationAuth, botKey); + const { shouldUseSecretV2Bridge, botKey } = await projectBotService.getBotKey(integrationAuth.projectId); + const { accessToken } = await getIntegrationAccessToken(integrationAuth, shouldUseSecretV2Bridge, botKey); if (appId && appId !== "") { const query = ` @@ -1007,8 +1123,8 @@ export const integrationAuthServiceFactory = ({ actorOrgId ); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Integrations); - const botKey = await projectBotService.getBotKey(integrationAuth.projectId); - const { accessToken } = await getIntegrationAccessToken(integrationAuth, botKey); + const { shouldUseSecretV2Bridge, botKey } = await projectBotService.getBotKey(integrationAuth.projectId); + const { accessToken } = await getIntegrationAccessToken(integrationAuth, shouldUseSecretV2Bridge, botKey); const workspaces: TBitbucketWorkspace[] = []; let hasNextPage = true; let workspaceUrl = `${IntegrationUrls.BITBUCKET_API_URL}/2.0/workspaces`; @@ -1056,8 +1172,8 @@ export const integrationAuthServiceFactory = ({ actorOrgId ); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Integrations); - const botKey = await projectBotService.getBotKey(integrationAuth.projectId); - const { accessToken } = await getIntegrationAccessToken(integrationAuth, botKey); + const { shouldUseSecretV2Bridge, botKey } = await projectBotService.getBotKey(integrationAuth.projectId); + const { accessToken } = await getIntegrationAccessToken(integrationAuth, shouldUseSecretV2Bridge, botKey); const secretGroups: { name: string; groupId: string }[] = []; if (appId) { @@ -1124,8 +1240,8 @@ export const integrationAuthServiceFactory = ({ actorOrgId ); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Integrations); - const botKey = await projectBotService.getBotKey(integrationAuth.projectId); - const { accessToken } = await getIntegrationAccessToken(integrationAuth, botKey); + const { shouldUseSecretV2Bridge, botKey } = await projectBotService.getBotKey(integrationAuth.projectId); + const { accessToken } = await getIntegrationAccessToken(integrationAuth, shouldUseSecretV2Bridge, botKey); if (appId) { const { data: { buildType } diff --git a/backend/src/services/integration-auth/integration-sync-secret.ts b/backend/src/services/integration-auth/integration-sync-secret.ts index 4d32f819a..4fc69b76c 100644 --- a/backend/src/services/integration-auth/integration-sync-secret.ts +++ b/backend/src/services/integration-auth/integration-sync-secret.ts @@ -26,7 +26,7 @@ import sodium from "libsodium-wrappers"; import isEqual from "lodash.isequal"; import { z } from "zod"; -import { SecretType, TIntegrationAuths, TIntegrations, TSecrets } from "@app/db/schemas"; +import { SecretType, TIntegrationAuths, TIntegrations } from "@app/db/schemas"; import { getConfig } from "@app/lib/config/env"; import { request } from "@app/lib/config/request"; import { BadRequestError } from "@app/lib/errors"; @@ -275,8 +275,8 @@ const syncSecretsAzureKeyVault = async ({ }; secrets: Record; accessToken: string; - createManySecretsRawFn: (params: TCreateManySecretsRawFn) => Promise>; - updateManySecretsRawFn: (params: TUpdateManySecretsRawFn) => Promise>; + createManySecretsRawFn: (params: TCreateManySecretsRawFn) => Promise>; + updateManySecretsRawFn: (params: TUpdateManySecretsRawFn) => Promise>; }) => { interface GetAzureKeyVaultSecret { id: string; // secret URI @@ -903,8 +903,8 @@ const syncSecretsHeroku = async ({ secrets, accessToken }: { - createManySecretsRawFn: (params: TCreateManySecretsRawFn) => Promise>; - updateManySecretsRawFn: (params: TUpdateManySecretsRawFn) => Promise>; + createManySecretsRawFn: (params: TCreateManySecretsRawFn) => Promise>; + updateManySecretsRawFn: (params: TUpdateManySecretsRawFn) => Promise>; integration: TIntegrations & { projectId: string; environment: { @@ -2464,8 +2464,8 @@ const syncSecretsTerraformCloud = async ({ accessToken, integrationDAL }: { - createManySecretsRawFn: (params: TCreateManySecretsRawFn) => Promise>; - updateManySecretsRawFn: (params: TUpdateManySecretsRawFn) => Promise>; + createManySecretsRawFn: (params: TCreateManySecretsRawFn) => Promise>; + updateManySecretsRawFn: (params: TUpdateManySecretsRawFn) => Promise>; integration: TIntegrations & { projectId: string; environment: { @@ -3612,8 +3612,8 @@ export const syncIntegrationSecrets = async ({ appendices, projectId }: { - createManySecretsRawFn: (params: TCreateManySecretsRawFn) => Promise>; - updateManySecretsRawFn: (params: TUpdateManySecretsRawFn) => Promise>; + createManySecretsRawFn: (params: TCreateManySecretsRawFn) => Promise>; + updateManySecretsRawFn: (params: TUpdateManySecretsRawFn) => Promise>; integrationDAL: Pick; integration: TIntegrations & { projectId: string; diff --git a/backend/src/services/integration/integration-dal.ts b/backend/src/services/integration/integration-dal.ts index c98c7153a..0c365eb06 100644 --- a/backend/src/services/integration/integration-dal.ts +++ b/backend/src/services/integration/integration-dal.ts @@ -123,7 +123,11 @@ export const integrationDALFactory = (db: TDbClient) => { db.ref("keyEncoding").withSchema(TableName.IntegrationAuth).as("keyEncodingAu"), db.ref("awsAssumeIamRoleArnCipherText").withSchema(TableName.IntegrationAuth), db.ref("awsAssumeIamRoleArnIV").withSchema(TableName.IntegrationAuth), - db.ref("awsAssumeIamRoleArnTag").withSchema(TableName.IntegrationAuth) + db.ref("awsAssumeIamRoleArnTag").withSchema(TableName.IntegrationAuth), + db.ref("encryptedRefresh").withSchema(TableName.IntegrationAuth), + db.ref("encryptedAccess").withSchema(TableName.IntegrationAuth), + db.ref("encryptedAccessId").withSchema(TableName.IntegrationAuth), + db.ref("encryptedAwsAssumeIamRoleArn").withSchema(TableName.IntegrationAuth) ); return docs.map( ({ @@ -152,6 +156,10 @@ export const integrationDALFactory = (db: TDbClient) => { awsAssumeIamRoleArnIV, awsAssumeIamRoleArnCipherText, awsAssumeIamRoleArnTag, + encryptedAccess, + encryptedRefresh, + encryptedAccessId, + encryptedAwsAssumeIamRoleArn, ...el }) => ({ ...el, @@ -183,7 +191,11 @@ export const integrationDALFactory = (db: TDbClient) => { accessExpiresAt, awsAssumeIamRoleArnIV, awsAssumeIamRoleArnCipherText, - awsAssumeIamRoleArnTag + awsAssumeIamRoleArnTag, + encryptedAccess, + encryptedRefresh, + encryptedAccessId, + encryptedAwsAssumeIamRoleArn } }) ); diff --git a/backend/src/services/kms/kms-key-dal.ts b/backend/src/services/kms/kms-key-dal.ts index 8e1e17cd1..004348722 100644 --- a/backend/src/services/kms/kms-key-dal.ts +++ b/backend/src/services/kms/kms-key-dal.ts @@ -10,10 +10,13 @@ export type TKmsKeyDALFactory = ReturnType; export const kmskeyDALFactory = (db: TDbClient) => { const kmsOrm = ormify(db, TableName.KmsKey); + // akhilmhdh: this function should never be called outside kms service + // why: because the encrypted key should never be shared with another service const findByIdWithAssociatedKms = async (id: string, tx?: Knex) => { try { const result = await (tx || db.replicaNode())(TableName.KmsKey) .where({ [`${TableName.KmsKey}.id` as "id"]: id }) + .join(TableName.Organization, `${TableName.KmsKey}.orgId`, `${TableName.Organization}.id`) .leftJoin(TableName.InternalKms, `${TableName.KmsKey}.id`, `${TableName.InternalKms}.kmsKeyId`) .leftJoin(TableName.ExternalKms, `${TableName.KmsKey}.id`, `${TableName.ExternalKms}.kmsKeyId`) .first() @@ -31,11 +34,19 @@ export const kmskeyDALFactory = (db: TDbClient) => { db.ref("encryptedProviderInputs").withSchema(TableName.ExternalKms).as("externalKmsEncryptedProviderInput"), db.ref("status").withSchema(TableName.ExternalKms).as("externalKmsStatus"), db.ref("statusDetails").withSchema(TableName.ExternalKms).as("externalKmsStatusDetails") + ) + .select( + db.ref("kmsDefaultKeyId").withSchema(TableName.Organization).as("orgKmsDefaultKeyId"), + db.ref("kmsEncryptedDataKey").withSchema(TableName.Organization).as("orgKmsEncryptedDataKey") ); const data = { ...KmsKeysSchema.parse(result), isExternal: Boolean(result?.externalKmsId), + orgKms: { + id: result?.orgKmsDefaultKeyId, + encryptedDataKey: result?.orgKmsEncryptedDataKey + }, externalKms: result?.externalKmsId ? { id: result.externalKmsId, diff --git a/backend/src/services/kms/kms-service.ts b/backend/src/services/kms/kms-service.ts index dbc86b860..351098f9d 100644 --- a/backend/src/services/kms/kms-service.ts +++ b/backend/src/services/kms/kms-service.ts @@ -1,11 +1,20 @@ import slugify from "@sindresorhus/slugify"; import { Knex } from "knex"; +import { z } from "zod"; -import { TKeyStoreFactory } from "@app/keystore/keystore"; +import { KmsKeysSchema } from "@app/db/schemas"; +import { AwsKmsProviderFactory } from "@app/ee/services/external-kms/providers/aws-kms"; +import { + ExternalKmsAwsSchema, + KmsProviders, + TExternalKmsProviderFns +} from "@app/ee/services/external-kms/providers/model"; +import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore"; import { getConfig } from "@app/lib/config/env"; import { randomSecureBytes } from "@app/lib/crypto"; import { symmetricCipherService, SymmetricEncryption } from "@app/lib/crypto/cipher"; -import { BadRequestError } from "@app/lib/errors"; +import { generateHash } from "@app/lib/crypto/encryption"; +import { BadRequestError, NotFoundError } from "@app/lib/errors"; import { logger } from "@app/lib/logger"; import { alphaNumericNanoId } from "@app/lib/nanoid"; @@ -15,11 +24,15 @@ import { TInternalKmsDALFactory } from "./internal-kms-dal"; import { TKmsKeyDALFactory } from "./kms-key-dal"; import { TKmsRootConfigDALFactory } from "./kms-root-config-dal"; import { + KmsDataKey, + KmsType, TDecryptWithKeyDTO, TDecryptWithKmsDTO, TEncryptionWithKeyDTO, + TEncryptWithKmsDataKeyDTO, TEncryptWithKmsDTO, - TGenerateKMSDTO + TGenerateKMSDTO, + TUpdateProjectSecretManagerKmsKeyDTO } from "./kms-types"; type TKmsServiceFactoryDep = { @@ -41,6 +54,8 @@ const KMS_ROOT_CREATION_WAIT_TIME = 10; // akhilmhdh: Don't edit this value. This is measured for blob concatination in kms const KMS_VERSION = "v01"; const KMS_VERSION_BLOB_LENGTH = 3; +const KmsSanitizedSchema = KmsKeysSchema.extend({ isExternal: z.boolean() }); + export const kmsServiceFactory = ({ kmsDAL, kmsRootConfigDAL, @@ -51,7 +66,11 @@ export const kmsServiceFactory = ({ }: TKmsServiceFactoryDep) => { let ROOT_ENCRYPTION_KEY = Buffer.alloc(0); - // this is used symmetric encryption + /* + * Generate KMS Key + * This function is responsibile for generating the infisical internal KMS for various entities + * Like for secret manager, cert manager or for organization + */ const generateKmsKey = async ({ orgId, isReserved = true, tx, slug }: TGenerateKMSDTO) => { const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256); const kmsKeyMaterial = randomSecureBytes(32); @@ -83,22 +102,18 @@ export const kmsServiceFactory = ({ return doc; }; - const encryptWithKmsKey = async ({ kmsId }: Omit) => { - const kmsDoc = await kmsDAL.findByIdWithAssociatedKms(kmsId); - if (!kmsDoc) throw new BadRequestError({ message: "KMS ID not found" }); - // akhilmhdh: as more encryption are added do a check here on kmsDoc.encryptionAlgorithm - const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256); - return ({ plainText }: Pick) => { - const kmsKey = cipher.decrypt(kmsDoc.internalKms?.encryptedKey as Buffer, ROOT_ENCRYPTION_KEY); - const encryptedPlainTextBlob = cipher.encrypt(plainText, kmsKey); - - // Buffer#1 encrypted text + Buffer#2 version number - const versionBlob = Buffer.from(KMS_VERSION, "utf8"); // length is 3 - const cipherTextBlob = Buffer.concat([encryptedPlainTextBlob, versionBlob]); - return { cipherTextBlob }; - }; + const deleteInternalKms = async (kmsId: string, orgId: string, tx?: Knex) => { + const kms = await kmsDAL.findByIdWithAssociatedKms(kmsId, tx); + if (kms.isExternal) return; + if (kms.orgId !== orgId) throw new BadRequestError({ message: "KMS doesn't belong to organization" }); + return kmsDAL.deleteById(kmsId, tx); }; + /* + * Simple encryption service function to do all the encryption tasks in infisical + * This can be even later exposed directly as api for encryption as function + * The encrypted binary even has everything into it. The IV, the version etc + */ const encryptWithInputKey = async ({ key }: Omit) => { // akhilmhdh: as more encryption are added do a check here on kmsDoc.encryptionAlgorithm const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256); @@ -111,19 +126,10 @@ export const kmsServiceFactory = ({ }; }; - const decryptWithKmsKey = async ({ kmsId }: Omit) => { - const kmsDoc = await kmsDAL.findByIdWithAssociatedKms(kmsId); - if (!kmsDoc) throw new BadRequestError({ message: "KMS ID not found" }); - const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256); - const kmsKey = cipher.decrypt(kmsDoc.internalKms?.encryptedKey as Buffer, ROOT_ENCRYPTION_KEY); - - return ({ cipherTextBlob: versionedCipherTextBlob }: Pick) => { - const cipherTextBlob = versionedCipherTextBlob.subarray(0, -KMS_VERSION_BLOB_LENGTH); - const decryptedBlob = cipher.decrypt(cipherTextBlob, kmsKey); - return decryptedBlob; - }; - }; - + /* + * Simple decryption service function to do all the encryption tasks in infisical + * This can be even later exposed directly as api for encryption as function + */ const decryptWithInputKey = async ({ key }: Omit) => { const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256); @@ -134,70 +140,621 @@ export const kmsServiceFactory = ({ }; }; + /* + * Function to generate a KMS for an org + * We handle concurrent with redis locking and waitReady + * What happens is first we check kms is assigned else first we acquire lock and create the kms with connection + * In mean time the rest of the request will wait until creation is finished followed by getting the created on + * In real time this would be milliseconds + */ const getOrgKmsKeyId = async (orgId: string) => { - const keyId = await orgDAL.transaction(async (tx) => { - const org = await orgDAL.findById(orgId, tx); - if (!org) { - throw new BadRequestError({ message: "Org not found" }); + let org = await orgDAL.findById(orgId); + + if (!org) { + throw new NotFoundError({ message: "Org not found" }); + } + + if (!org.kmsDefaultKeyId) { + const lock = await keyStore + .acquireLock([KeyStorePrefixes.KmsOrgKeyCreation, orgId], 3000, { retryCount: 3 }) + .catch(() => null); + + try { + if (!lock) { + await keyStore.waitTillReady({ + key: `${KeyStorePrefixes.WaitUntilReadyKmsOrgKeyCreation}${orgId}`, + keyCheckCb: (val) => val === "true", + waitingCb: () => logger.info("KMS. Waiting for org key to be created") + }); + + org = await orgDAL.findById(orgId); + } else { + const keyId = await orgDAL.transaction(async (tx) => { + org = await orgDAL.findById(orgId, tx); + if (org.kmsDefaultKeyId) { + return org.kmsDefaultKeyId; + } + + const key = await generateKmsKey({ + isReserved: true, + orgId: org.id, + tx + }); + + await orgDAL.updateById( + org.id, + { + kmsDefaultKeyId: key.id + }, + tx + ); + + await keyStore.setItemWithExpiry(`${KeyStorePrefixes.WaitUntilReadyKmsOrgKeyCreation}${orgId}`, 10, "true"); + + return key.id; + }); + + return keyId; + } + } finally { + await lock?.release(); + } + } + + if (!org.kmsDefaultKeyId) { + throw new Error("Invalid organization KMS"); + } + + return org.kmsDefaultKeyId; + }; + + const decryptWithKmsKey = async ({ + kmsId, + depth = 0 + }: Omit & { depth?: number }) => { + if (depth > 2) throw new BadRequestError({ message: "KMS depth max limit" }); + + const kmsDoc = await kmsDAL.findByIdWithAssociatedKms(kmsId); + if (!kmsDoc) { + throw new NotFoundError({ message: "KMS ID not found" }); + } + + if (kmsDoc.externalKms) { + let externalKms: TExternalKmsProviderFns; + + if (!kmsDoc.orgKms.id || !kmsDoc.orgKms.encryptedDataKey) { + throw new Error("Invalid organization KMS"); } - if (!org.kmsDefaultKeyId) { - // create default kms key for certificate service - const key = await generateKmsKey({ - isReserved: true, - orgId: org.id, - tx - }); + // The idea is external kms connection info is encrypted by an org default KMS + // This could be external kms(in future) but at the end of the day, the end KMS will be an infisical internal one + // we put a limit of depth to avoid too many cycles + const orgKmsDecryptor = await decryptWithKmsKey({ + kmsId: kmsDoc.orgKms.id, + depth: depth + 1 + }); - await orgDAL.updateById( - org.id, - { - kmsDefaultKeyId: key.id - }, - tx - ); + const orgKmsDataKey = await orgKmsDecryptor({ + cipherTextBlob: kmsDoc.orgKms.encryptedDataKey + }); - return key.id; + const kmsDecryptor = await decryptWithInputKey({ + key: orgKmsDataKey + }); + + const decryptedProviderInputBlob = kmsDecryptor({ + cipherTextBlob: kmsDoc.externalKms.encryptedProviderInput + }); + + switch (kmsDoc.externalKms.provider) { + case KmsProviders.Aws: { + const decryptedProviderInput = await ExternalKmsAwsSchema.parseAsync( + JSON.parse(decryptedProviderInputBlob.toString("utf8")) + ); + + externalKms = await AwsKmsProviderFactory({ + inputs: decryptedProviderInput + }); + break; + } + default: + throw new Error("Invalid KMS provider."); } - return org.kmsDefaultKeyId; + return async ({ cipherTextBlob }: Pick) => { + const { data } = await externalKms.decrypt(cipherTextBlob); + + return data; + }; + } + + // internal KMS + const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256); + const kmsKey = cipher.decrypt(kmsDoc.internalKms?.encryptedKey as Buffer, ROOT_ENCRYPTION_KEY); + + return ({ cipherTextBlob: versionedCipherTextBlob }: Pick) => { + const cipherTextBlob = versionedCipherTextBlob.subarray(0, -KMS_VERSION_BLOB_LENGTH); + const decryptedBlob = cipher.decrypt(cipherTextBlob, kmsKey); + return Promise.resolve(decryptedBlob); + }; + }; + + const encryptWithKmsKey = async ({ kmsId }: Omit, tx?: Knex) => { + const kmsDoc = await kmsDAL.findByIdWithAssociatedKms(kmsId, tx); + if (!kmsDoc) { + throw new NotFoundError({ message: "KMS ID not found" }); + } + + if (kmsDoc.externalKms) { + let externalKms: TExternalKmsProviderFns; + if (!kmsDoc.orgKms.id || !kmsDoc.orgKms.encryptedDataKey) { + throw new Error("Invalid organization KMS"); + } + + const orgKmsDecryptor = await decryptWithKmsKey({ + kmsId: kmsDoc.orgKms.id + }); + + const orgKmsDataKey = await orgKmsDecryptor({ + cipherTextBlob: kmsDoc.orgKms.encryptedDataKey + }); + + const kmsDecryptor = await decryptWithInputKey({ + key: orgKmsDataKey + }); + + const decryptedProviderInputBlob = kmsDecryptor({ + cipherTextBlob: kmsDoc.externalKms.encryptedProviderInput + }); + + switch (kmsDoc.externalKms.provider) { + case KmsProviders.Aws: { + const decryptedProviderInput = await ExternalKmsAwsSchema.parseAsync( + JSON.parse(decryptedProviderInputBlob.toString("utf8")) + ); + + externalKms = await AwsKmsProviderFactory({ + inputs: decryptedProviderInput + }); + break; + } + default: + throw new Error("Invalid KMS provider."); + } + + return async ({ plainText }: Pick) => { + const { encryptedBlob } = await externalKms.encrypt(plainText); + + return { cipherTextBlob: encryptedBlob }; + }; + } + + // internal KMS + // akhilmhdh: as more encryption are added do a check here on kmsDoc.encryptionAlgorithm + const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256); + return ({ plainText }: Pick) => { + const kmsKey = cipher.decrypt(kmsDoc.internalKms?.encryptedKey as Buffer, ROOT_ENCRYPTION_KEY); + const encryptedPlainTextBlob = cipher.encrypt(plainText, kmsKey); + + // Buffer#1 encrypted text + Buffer#2 version number + const versionBlob = Buffer.from(KMS_VERSION, "utf8"); // length is 3 + const cipherTextBlob = Buffer.concat([encryptedPlainTextBlob, versionBlob]); + + return Promise.resolve({ cipherTextBlob }); + }; + }; + + const $getOrgKmsDataKey = async (orgId: string) => { + const kmsKeyId = await getOrgKmsKeyId(orgId); + let org = await orgDAL.findById(orgId); + + if (!org) { + throw new NotFoundError({ message: "Org not found" }); + } + + if (!org.kmsEncryptedDataKey) { + const lock = await keyStore + .acquireLock([KeyStorePrefixes.KmsOrgDataKeyCreation, orgId], 500, { retryCount: 0 }) + .catch(() => null); + + try { + if (!lock) { + await keyStore.waitTillReady({ + key: `${KeyStorePrefixes.WaitUntilReadyKmsOrgDataKeyCreation}${orgId}`, + keyCheckCb: (val) => val === "true", + waitingCb: () => logger.info("KMS. Waiting for org data key to be created") + }); + + org = await orgDAL.findById(orgId); + } else { + const orgDataKey = await orgDAL.transaction(async (tx) => { + org = await orgDAL.findById(orgId, tx); + if (org.kmsEncryptedDataKey) { + return; + } + + const dataKey = randomSecureBytes(); + const kmsEncryptor = await encryptWithKmsKey( + { + kmsId: kmsKeyId + }, + tx + ); + + const { cipherTextBlob } = await kmsEncryptor({ + plainText: dataKey + }); + + await orgDAL.updateById( + org.id, + { + kmsEncryptedDataKey: cipherTextBlob + }, + tx + ); + + await keyStore.setItemWithExpiry( + `${KeyStorePrefixes.WaitUntilReadyKmsOrgDataKeyCreation}${orgId}`, + 10, + "true" + ); + + return dataKey; + }); + + if (orgDataKey) { + return orgDataKey; + } + } + } finally { + await lock?.release(); + } + } + + if (!org.kmsEncryptedDataKey) { + throw new Error("Invalid organization KMS"); + } + + const kmsDecryptor = await decryptWithKmsKey({ + kmsId: kmsKeyId }); - return keyId; + return kmsDecryptor({ + cipherTextBlob: org.kmsEncryptedDataKey + }); }; const getProjectSecretManagerKmsKeyId = async (projectId: string) => { - const keyId = await projectDAL.transaction(async (tx) => { - const project = await projectDAL.findById(projectId, tx); + let project = await projectDAL.findById(projectId); + if (!project) { + throw new NotFoundError({ message: "Project not found" }); + } + + if (!project.kmsSecretManagerKeyId) { + const lock = await keyStore + .acquireLock([KeyStorePrefixes.KmsProjectKeyCreation, projectId], 3000, { retryCount: 0 }) + .catch(() => null); + + try { + if (!lock) { + await keyStore.waitTillReady({ + key: `${KeyStorePrefixes.WaitUntilReadyKmsProjectKeyCreation}${projectId}`, + keyCheckCb: (val) => val === "true", + waitingCb: () => logger.debug("KMS. Waiting for project key to be created"), + delay: 500 + }); + + project = await projectDAL.findById(projectId); + } else { + const kmsKeyId = await projectDAL.transaction(async (tx) => { + project = await projectDAL.findById(projectId, tx); + if (project.kmsSecretManagerKeyId) { + return project.kmsSecretManagerKeyId; + } + + const key = await generateKmsKey({ + isReserved: true, + orgId: project.orgId, + tx + }); + + await projectDAL.updateById( + projectId, + { + kmsSecretManagerKeyId: key.id + }, + tx + ); + + return key.id; + }); + + await keyStore.setItemWithExpiry( + `${KeyStorePrefixes.WaitUntilReadyKmsProjectKeyCreation}${projectId}`, + 10, + "true" + ); + + return kmsKeyId; + } + } finally { + await lock?.release(); + } + } + + if (!project.kmsSecretManagerKeyId) { + throw new Error("Missing project KMS key ID"); + } + + return project.kmsSecretManagerKeyId; + }; + + const $getProjectSecretManagerKmsDataKey = async (projectId: string) => { + const kmsKeyId = await getProjectSecretManagerKmsKeyId(projectId); + let project = await projectDAL.findById(projectId); + + if (!project.kmsSecretManagerEncryptedDataKey) { + const lock = await keyStore + .acquireLock([KeyStorePrefixes.KmsProjectDataKeyCreation, projectId], 3000, { retryCount: 0 }) + .catch(() => null); + + try { + if (!lock) { + await keyStore.waitTillReady({ + key: `${KeyStorePrefixes.WaitUntilReadyKmsProjectDataKeyCreation}${projectId}`, + keyCheckCb: (val) => val === "true", + waitingCb: () => logger.debug("KMS. Waiting for secret manager data key to be created"), + delay: 500 + }); + + project = await projectDAL.findById(projectId); + } else { + const projectDataKey = await projectDAL.transaction(async (tx) => { + project = await projectDAL.findById(projectId, tx); + if (project.kmsSecretManagerEncryptedDataKey) { + return; + } + + const dataKey = randomSecureBytes(); + const kmsEncryptor = await encryptWithKmsKey({ + kmsId: kmsKeyId + }); + + const { cipherTextBlob } = await kmsEncryptor({ + plainText: dataKey + }); + + await projectDAL.updateById( + projectId, + { + kmsSecretManagerEncryptedDataKey: cipherTextBlob + }, + tx + ); + + await keyStore.setItemWithExpiry( + `${KeyStorePrefixes.WaitUntilReadyKmsProjectDataKeyCreation}${projectId}`, + 10, + "true" + ); + return dataKey; + }); + + if (projectDataKey) { + return projectDataKey; + } + } + } finally { + await lock?.release(); + } + } + + if (!project.kmsSecretManagerEncryptedDataKey) { + throw new Error("Missing project data key"); + } + + const kmsDecryptor = await decryptWithKmsKey({ + kmsId: kmsKeyId + }); + + return kmsDecryptor({ + cipherTextBlob: project.kmsSecretManagerEncryptedDataKey + }); + }; + + const $getDataKey = async (dto: TEncryptWithKmsDataKeyDTO) => { + switch (dto.type) { + case KmsDataKey.SecretManager: { + return $getProjectSecretManagerKmsDataKey(dto.projectId); + } + default: { + return $getOrgKmsDataKey(dto.orgId); + } + } + }; + + // by keeping the decrypted data key in inner scope + // none of the entities outside can interact directly or expose the data key + // NOTICE: If changing here update migrations/utils/kms + const createCipherPairWithDataKey = async (encryptionContext: TEncryptWithKmsDataKeyDTO) => { + const dataKey = await $getDataKey(encryptionContext); + const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256); + + return { + encryptor: ({ plainText }: Pick) => { + const encryptedPlainTextBlob = cipher.encrypt(plainText, dataKey); + + // Buffer#1 encrypted text + Buffer#2 version number + const versionBlob = Buffer.from(KMS_VERSION, "utf8"); // length is 3 + const cipherTextBlob = Buffer.concat([encryptedPlainTextBlob, versionBlob]); + return { cipherTextBlob }; + }, + decryptor: ({ cipherTextBlob: versionedCipherTextBlob }: Pick) => { + const cipherTextBlob = versionedCipherTextBlob.subarray(0, -KMS_VERSION_BLOB_LENGTH); + const decryptedBlob = cipher.decrypt(cipherTextBlob, dataKey); + return decryptedBlob; + } + }; + }; + + const updateProjectSecretManagerKmsKey = async ({ projectId, kms }: TUpdateProjectSecretManagerKmsKeyDTO) => { + const kmsKeyId = await getProjectSecretManagerKmsKeyId(projectId); + const currentKms = await kmsDAL.findById(kmsKeyId); + + // case: internal kms -> internal kms. no change needed + if (kms.type === KmsType.Internal && currentKms.isReserved) { + return KmsSanitizedSchema.parseAsync({ isExternal: false, ...currentKms }); + } + + if (kms.type === KmsType.External) { + // validate kms is scoped in org + const { kmsId } = kms; + const project = await projectDAL.findById(projectId); if (!project) { - throw new BadRequestError({ message: "Project not found" }); + throw new NotFoundError({ + message: "Project not found." + }); + } + const kmsDoc = await kmsDAL.findByIdWithAssociatedKms(kmsId); + if (!kmsDoc) { + throw new NotFoundError({ message: "KMS ID not found." }); } - if (!project.kmsSecretManagerKeyId) { - // create default kms key for certificate service - const key = await generateKmsKey({ + if (kmsDoc.orgId !== project.orgId) { + throw new BadRequestError({ + message: "KMS ID does not belong in the organization." + }); + } + } + + const dataKey = await $getProjectSecretManagerKmsDataKey(projectId); + return kmsDAL.transaction(async (tx) => { + const project = await projectDAL.findById(projectId, tx); + let kmsId; + if (kms.type === KmsType.Internal) { + const internalKms = await generateKmsKey({ isReserved: true, orgId: project.orgId, tx }); - - await projectDAL.updateById( - projectId, - { - kmsSecretManagerKeyId: key.id - }, - tx - ); - - return key.id; + kmsId = internalKms.id; + } else { + kmsId = kms.kmsId; } - return project.kmsSecretManagerKeyId; + const kmsEncryptor = await encryptWithKmsKey({ kmsId }, tx); + const { cipherTextBlob } = await kmsEncryptor({ plainText: dataKey }); + await projectDAL.updateById( + projectId, + { + kmsSecretManagerKeyId: kmsId, + kmsSecretManagerEncryptedDataKey: cipherTextBlob + }, + tx + ); + if (currentKms.isReserved) { + await kmsDAL.deleteById(currentKms.id, tx); + } + const newKms = await kmsDAL.findById(kmsId, tx); + return KmsSanitizedSchema.parseAsync({ isExternal: !currentKms.isReserved, ...newKms }); }); - - return keyId; }; + const getProjectKeyBackup = async (projectId: string) => { + const project = await projectDAL.findById(projectId); + if (!project) { + throw new NotFoundError({ + message: "Project not found" + }); + } + + const secretManagerDataKey = await $getProjectSecretManagerKmsDataKey(projectId); + const kmsKeyIdForEncrypt = await getOrgKmsKeyId(project.orgId); + const kmsEncryptor = await encryptWithKmsKey({ kmsId: kmsKeyIdForEncrypt }); + const { cipherTextBlob: encryptedSecretManagerDataKeyWithOrgKms } = await kmsEncryptor({ + plainText: secretManagerDataKey + }); + + // backup format: version.projectId.kmsFunction.kmsId.Base64(encryptedDataKey).verificationHash + let secretManagerBackup = `v1.${projectId}.secretManager.${kmsKeyIdForEncrypt}.${encryptedSecretManagerDataKeyWithOrgKms.toString( + "base64" + )}`; + + const verificationHash = generateHash(secretManagerBackup); + secretManagerBackup = `${secretManagerBackup}.${verificationHash}`; + + return { + secretManager: secretManagerBackup + }; + }; + + const loadProjectKeyBackup = async (projectId: string, backup: string) => { + const project = await projectDAL.findById(projectId); + if (!project) { + throw new NotFoundError({ + message: "Project not found" + }); + } + + const [, backupProjectId, , backupKmsKeyId, backupBase64EncryptedDataKey, backupHash] = backup.split("."); + const computedHash = generateHash(backup.substring(0, backup.lastIndexOf("."))); + if (computedHash !== backupHash) { + throw new BadRequestError({ + message: "Invalid backup" + }); + } + + if (backupProjectId !== projectId) { + throw new BadRequestError({ + message: "Invalid backup for project" + }); + } + + const kmsDecryptor = await decryptWithKmsKey({ kmsId: backupKmsKeyId }); + const dataKey = await kmsDecryptor({ + cipherTextBlob: Buffer.from(backupBase64EncryptedDataKey, "base64") + }); + + const newKms = await kmsDAL.transaction(async (tx) => { + const key = await generateKmsKey({ + isReserved: true, + orgId: project.orgId, + tx + }); + + const kmsEncryptor = await encryptWithKmsKey({ kmsId: key.id }, tx); + const { cipherTextBlob } = await kmsEncryptor({ plainText: dataKey }); + + await projectDAL.updateById( + projectId, + { + kmsSecretManagerKeyId: key.id, + kmsSecretManagerEncryptedDataKey: cipherTextBlob + }, + tx + ); + + return kmsDAL.findByIdWithAssociatedKms(key.id, tx); + }); + + return { + secretManagerKmsKey: newKms + }; + }; + + const getKmsById = async (kmsKeyId: string, tx?: Knex) => { + const kms = await kmsDAL.findByIdWithAssociatedKms(kmsKeyId, tx); + + if (!kms.id) { + throw new NotFoundError({ + message: "KMS not found" + }); + } + const { id, slug, orgId, isExternal } = kms; + return { id, slug, orgId, isExternal }; + }; + + // akhilmhdh: a copy of this is made in migrations/utils/kms const startService = async () => { const appCfg = getConfig(); // This will switch to a seal process and HMS flow in future @@ -246,11 +803,17 @@ export const kmsServiceFactory = ({ return { startService, generateKmsKey, + deleteInternalKms, encryptWithKmsKey, - encryptWithInputKey, decryptWithKmsKey, + encryptWithInputKey, decryptWithInputKey, getOrgKmsKeyId, - getProjectSecretManagerKmsKeyId + getProjectSecretManagerKmsKeyId, + updateProjectSecretManagerKmsKey, + getProjectKeyBackup, + loadProjectKeyBackup, + getKmsById, + createCipherPairWithDataKey }; }; diff --git a/backend/src/services/kms/kms-types.ts b/backend/src/services/kms/kms-types.ts index 5ba6c1343..dc7de10a4 100644 --- a/backend/src/services/kms/kms-types.ts +++ b/backend/src/services/kms/kms-types.ts @@ -1,5 +1,25 @@ import { Knex } from "knex"; +export enum KmsDataKey { + Organization, + SecretManager + // CertificateManager +} + +export enum KmsType { + External = "external", + Internal = "internal" +} + +export type TEncryptWithKmsDataKeyDTO = + | { type: KmsDataKey.Organization; orgId: string } + | { type: KmsDataKey.SecretManager; projectId: string }; +// akhilmhdh: not implemented yet +// | { +// type: KmsDataKey.CertificateManager; +// projectId: string; +// }; + export type TGenerateKMSDTO = { orgId: string; isReserved?: boolean; @@ -26,3 +46,8 @@ export type TDecryptWithKeyDTO = { key: Buffer; cipherTextBlob: Buffer; }; + +export type TUpdateProjectSecretManagerKmsKeyDTO = { + projectId: string; + kms: { type: KmsType.Internal } | { type: KmsType.External; kmsId: string }; +}; diff --git a/backend/src/services/project-bot/project-bot-dal.ts b/backend/src/services/project-bot/project-bot-dal.ts index e25ebbd09..81c177d21 100644 --- a/backend/src/services/project-bot/project-bot-dal.ts +++ b/backend/src/services/project-bot/project-bot-dal.ts @@ -1,7 +1,7 @@ import { Knex } from "knex"; import { TDbClient } from "@app/db"; -import { TableName, TProjectBots } from "@app/db/schemas"; +import { TableName, TProjectBots, TUserEncryptionKeys } from "@app/db/schemas"; import { DatabaseError } from "@app/lib/errors"; import { ormify, selectAllTableCols } from "@app/lib/knex"; @@ -41,5 +41,43 @@ export const projectBotDALFactory = (db: TDbClient) => { } }; - return { ...projectBotOrm, findOne, findProjectByBotId }; + const findProjectUserWorkspaceKey = async (projectId: string) => { + try { + const doc = await db + .replicaNode()(TableName.ProjectMembership) + .where(`${TableName.ProjectMembership}.projectId` as "projectId", projectId) + .where(`${TableName.Users}.isGhost` as "isGhost", false) + .join(TableName.Users, `${TableName.ProjectMembership}.userId`, `${TableName.Users}.id`) + .join(TableName.ProjectKeys, `${TableName.ProjectMembership}.userId`, `${TableName.ProjectKeys}.receiverId`) + .join( + TableName.UserEncryptionKey, + `${TableName.UserEncryptionKey}.userId`, + `${TableName.Users}.id` + ) + .join( + db(TableName.UserEncryptionKey).as("senderUserEncryption"), + `${TableName.ProjectKeys}.senderId`, + `senderUserEncryption.userId` + ) + .whereNotNull(`${TableName.UserEncryptionKey}.serverEncryptedPrivateKey`) + .whereNotNull(`${TableName.UserEncryptionKey}.serverEncryptedPrivateKeyIV`) + .whereNotNull(`${TableName.UserEncryptionKey}.serverEncryptedPrivateKeyTag`) + .select( + db.ref("serverEncryptedPrivateKey").withSchema(TableName.UserEncryptionKey), + db.ref("serverEncryptedPrivateKeyTag").withSchema(TableName.UserEncryptionKey), + db.ref("serverEncryptedPrivateKeyIV").withSchema(TableName.UserEncryptionKey), + db.ref("serverEncryptedPrivateKeyEncoding").withSchema(TableName.UserEncryptionKey), + db.ref("encryptedKey").withSchema(TableName.ProjectKeys).as("projectEncryptedKey"), + db.ref("nonce").withSchema(TableName.ProjectKeys).as("projectKeyNonce"), + db.ref("publicKey").withSchema("senderUserEncryption").as("senderPublicKey"), + db.ref("id").withSchema(TableName.Users).as("userId") + ) + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "Find all project members" }); + } + }; + + return { ...projectBotOrm, findOne, findProjectByBotId, findProjectUserWorkspaceKey }; }; diff --git a/backend/src/services/project-bot/project-bot-fns.ts b/backend/src/services/project-bot/project-bot-fns.ts index 0f5d229d7..d37d620b2 100644 --- a/backend/src/services/project-bot/project-bot-fns.ts +++ b/backend/src/services/project-bot/project-bot-fns.ts @@ -1,5 +1,11 @@ import { SecretKeyEncoding } from "@app/db/schemas"; -import { decryptAsymmetric, infisicalSymmetricDecrypt } from "@app/lib/crypto/encryption"; +import { + decryptAsymmetric, + encryptAsymmetric, + generateAsymmetricKeyPair, + infisicalSymmetricDecrypt, + infisicalSymmetricEncypt +} from "@app/lib/crypto/encryption"; import { BadRequestError } from "@app/lib/errors"; import { TProjectBotDALFactory } from "@app/services/project-bot/project-bot-dal"; @@ -22,21 +28,75 @@ export const getBotKeyFnFactory = ( const project = await projectDAL.findById(projectId); if (!project) throw new BadRequestError({ message: "Project not found during bot lookup." }); - const bot = await projectBotDAL.findOne({ projectId: project.id }); + if (project.version === 3) { + return { project, shouldUseSecretV2Bridge: true }; + } - if (!bot) throw new BadRequestError({ message: "Failed to find bot key", name: "bot_not_found_error" }); - if (!bot.isActive) throw new BadRequestError({ message: "Bot is not active", name: "bot_not_found_error" }); - if (!bot.encryptedProjectKeyNonce || !bot.encryptedProjectKey) - throw new BadRequestError({ message: "Encryption key missing", name: "bot_not_found_error" }); + const bot = await projectBotDAL.findOne({ projectId: project.id }); + if (!bot || !bot.isActive || !bot.encryptedProjectKey || !bot.encryptedProjectKeyNonce) { + // trying to set bot automatically + const projectV1Keys = await projectBotDAL.findProjectUserWorkspaceKey(projectId); + if (!projectV1Keys) throw new BadRequestError({ message: "Bot not found. Please ask admin user to login" }); + + let userPrivateKey = ""; + if ( + projectV1Keys?.serverEncryptedPrivateKey && + projectV1Keys.serverEncryptedPrivateKeyIV && + projectV1Keys.serverEncryptedPrivateKeyTag && + projectV1Keys.serverEncryptedPrivateKeyEncoding + ) { + userPrivateKey = infisicalSymmetricDecrypt({ + iv: projectV1Keys.serverEncryptedPrivateKeyIV, + tag: projectV1Keys.serverEncryptedPrivateKeyTag, + ciphertext: projectV1Keys.serverEncryptedPrivateKey, + keyEncoding: projectV1Keys.serverEncryptedPrivateKeyEncoding as SecretKeyEncoding + }); + } + const workspaceKey = decryptAsymmetric({ + ciphertext: projectV1Keys.projectEncryptedKey, + nonce: projectV1Keys.projectKeyNonce, + publicKey: projectV1Keys.senderPublicKey, + privateKey: userPrivateKey + }); + const botKey = generateAsymmetricKeyPair(); + const { iv, tag, ciphertext, encoding, algorithm } = infisicalSymmetricEncypt(botKey.privateKey); + const encryptedWorkspaceKey = encryptAsymmetric(workspaceKey, botKey.publicKey, userPrivateKey); + + if (!bot) { + await projectBotDAL.create({ + name: "Infisical Bot (Ghost)", + projectId, + tag, + iv, + encryptedPrivateKey: ciphertext, + isActive: true, + publicKey: botKey.publicKey, + algorithm, + keyEncoding: encoding, + encryptedProjectKey: encryptedWorkspaceKey.ciphertext, + encryptedProjectKeyNonce: encryptedWorkspaceKey.nonce, + senderId: projectV1Keys.userId + }); + } else { + await projectBotDAL.updateById(bot.id, { + isActive: true, + encryptedProjectKey: encryptedWorkspaceKey.ciphertext, + encryptedProjectKeyNonce: encryptedWorkspaceKey.nonce, + senderId: projectV1Keys.userId + }); + } + return { botKey: workspaceKey, project, shouldUseSecretV2Bridge: false }; + } const botPrivateKey = getBotPrivateKey({ bot }); - return decryptAsymmetric({ + const botKey = decryptAsymmetric({ ciphertext: bot.encryptedProjectKey, privateKey: botPrivateKey, nonce: bot.encryptedProjectKeyNonce, publicKey: bot.sender.publicKey }); + return { botKey, project, shouldUseSecretV2Bridge: false }; }; return getBotKeyFn; diff --git a/backend/src/services/project-bot/project-bot-service.ts b/backend/src/services/project-bot/project-bot-service.ts index ce7782a80..1df29e6b7 100644 --- a/backend/src/services/project-bot/project-bot-service.ts +++ b/backend/src/services/project-bot/project-bot-service.ts @@ -60,7 +60,7 @@ export const projectBotServiceFactory = ({ const project = await projectDAL.findById(projectId, tx); - if (project.version === ProjectVersion.V2) { + if (project.version === ProjectVersion.V2 || project.version === ProjectVersion.V3) { throw new BadRequestError({ message: "Failed to create bot, project is upgraded." }); } diff --git a/backend/src/services/project/project-service.ts b/backend/src/services/project/project-service.ts index b1a53408a..d1f393e7b 100644 --- a/backend/src/services/project/project-service.ts +++ b/backend/src/services/project/project-service.ts @@ -8,8 +8,6 @@ import { TPermissionServiceFactory } from "@app/ee/services/permission/permissio import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; import { TKeyStoreFactory } from "@app/keystore/keystore"; import { isAtLeastAsPrivileged } from "@app/lib/casl"; -import { getConfig } from "@app/lib/config/env"; -import { createSecretBlindIndex } from "@app/lib/crypto"; import { infisicalSymmetricEncypt } from "@app/lib/crypto/encryption"; import { BadRequestError, ForbiddenRequestError, NotFoundError } from "@app/lib/errors"; import { alphaNumericNanoId } from "@app/lib/nanoid"; @@ -21,14 +19,13 @@ import { TCertificateAuthorityDALFactory } from "../certificate-authority/certif import { TIdentityOrgDALFactory } from "../identity/identity-org-dal"; import { TIdentityProjectDALFactory } from "../identity-project/identity-project-dal"; import { TIdentityProjectMembershipRoleDALFactory } from "../identity-project/identity-project-membership-role-dal"; +import { TKmsServiceFactory } from "../kms/kms-service"; import { TOrgDALFactory } from "../org/org-dal"; import { TOrgServiceFactory } from "../org/org-service"; -import { TProjectBotDALFactory } from "../project-bot/project-bot-dal"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TProjectKeyDALFactory } from "../project-key/project-key-dal"; import { TProjectMembershipDALFactory } from "../project-membership/project-membership-dal"; import { TProjectUserMembershipRoleDALFactory } from "../project-membership/project-user-membership-role-dal"; -import { TSecretBlindIndexDALFactory } from "../secret-blind-index/secret-blind-index-dal"; import { ROOT_FOLDER_NAME, TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; import { TUserDALFactory } from "../user/user-dal"; import { TProjectDALFactory } from "./project-dal"; @@ -38,11 +35,14 @@ import { TCreateProjectDTO, TDeleteProjectDTO, TGetProjectDTO, + TGetProjectKmsKey, TListProjectCasDTO, TListProjectCertsDTO, + TLoadProjectKmsBackupDTO, TToggleProjectAutoCapitalizationDTO, TUpdateAuditLogsRetentionDTO, TUpdateProjectDTO, + TUpdateProjectKmsDTO, TUpdateProjectNameDTO, TUpdateProjectVersionLimitDTO, TUpgradeProjectDTO @@ -65,10 +65,8 @@ type TProjectServiceFactoryDep = { identityProjectDAL: TIdentityProjectDALFactory; identityProjectMembershipRoleDAL: Pick; projectKeyDAL: Pick; - projectBotDAL: Pick; projectMembershipDAL: Pick; projectUserMembershipRoleDAL: Pick; - secretBlindIndexDAL: Pick; certificateAuthorityDAL: Pick; certificateDAL: Pick; permissionService: TPermissionServiceFactory; @@ -76,6 +74,15 @@ type TProjectServiceFactoryDep = { licenseService: Pick; orgDAL: Pick; keyStore: Pick; + kmsService: Pick< + TKmsServiceFactory, + | "updateProjectSecretManagerKmsKey" + | "getProjectKeyBackup" + | "loadProjectKeyBackup" + | "getKmsById" + | "getProjectSecretManagerKmsKeyId" + | "deleteInternalKms" + >; }; export type TProjectServiceFactory = ReturnType; @@ -90,9 +97,7 @@ export const projectServiceFactory = ({ folderDAL, orgService, identityProjectDAL, - projectBotDAL, identityOrgMembershipDAL, - secretBlindIndexDAL, projectMembershipDAL, projectEnvDAL, licenseService, @@ -100,7 +105,8 @@ export const projectServiceFactory = ({ identityProjectMembershipRoleDAL, certificateAuthorityDAL, certificateDAL, - keyStore + keyStore, + kmsService }: TProjectServiceFactoryDep) => { /* * Create workspace. Make user the admin @@ -111,7 +117,8 @@ export const projectServiceFactory = ({ actorOrgId, actorAuthMethod, workspaceName, - slug: projectSlug + slug: projectSlug, + kmsKeyId }: TCreateProjectDTO) => { const organization = await orgDAL.findOne({ id: actorOrgId }); @@ -124,9 +131,6 @@ export const projectServiceFactory = ({ ); ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Create, OrgPermissionSubjects.Workspace); - const appCfg = getConfig(); - const blindIndex = createSecretBlindIndex(appCfg.ROOT_ENCRYPTION_KEY, appCfg.ENCRYPTION_KEY); - const plan = await licenseService.getPlan(organization.id); if (plan.workspaceLimit !== null && plan.workspacesUsed >= plan.workspaceLimit) { // case: limit imposed on number of workspaces allowed @@ -139,16 +143,28 @@ export const projectServiceFactory = ({ const results = await projectDAL.transaction(async (tx) => { const ghostUser = await orgService.addGhostUser(organization.id, tx); + if (kmsKeyId) { + const kms = await kmsService.getKmsById(kmsKeyId, tx); + + if (kms.orgId !== organization.id) { + throw new BadRequestError({ + message: "KMS does not belong in the organization" + }); + } + } + const project = await projectDAL.create( { name: workspaceName, orgId: organization.id, slug: projectSlug || slugify(`${workspaceName}-${alphaNumericNanoId(4)}`), - version: ProjectVersion.V2, + kmsSecretManagerKeyId: kmsKeyId, + version: ProjectVersion.V3, pitVersionLimit: 10 }, tx ); + // set ghost user as admin of project const projectMembership = await projectMembershipDAL.create( { @@ -162,18 +178,6 @@ export const projectServiceFactory = ({ tx ); - // generate the blind index for project - await secretBlindIndexDAL.create( - { - projectId: project.id, - keyEncoding: blindIndex.keyEncoding, - saltIV: blindIndex.iv, - saltTag: blindIndex.tag, - algorithm: blindIndex.algorithm, - encryptedSaltCipherText: blindIndex.ciphertext - }, - tx - ); // set default environments and root folder for provided environments const envs = await projectEnvDAL.insertMany( DEFAULT_PROJECT_ENVS.map((el, i) => ({ ...el, projectId: project.id, position: i + 1 })), @@ -202,26 +206,7 @@ export const projectServiceFactory = ({ tx ); - const { iv, tag, ciphertext, encoding, algorithm } = infisicalSymmetricEncypt(ghostUser.keys.plainPrivateKey); - - // 5. Create & a bot for the project - await projectBotDAL.create( - { - name: "Infisical Bot (Ghost)", - projectId: project.id, - tag, - iv, - encryptedProjectKey, - encryptedProjectKeyNonce: encryptedProjectKeyIv, - encryptedPrivateKey: ciphertext, - isActive: true, - publicKey: ghostUser.keys.publicKey, - senderId: ghostUser.user.id, - algorithm, - keyEncoding: encoding - }, - tx - ); + // const { iv, tag, ciphertext, encoding, algorithm } = infisicalSymmetricEncypt(ghostUser.keys.plainPrivateKey); // Find the ghost users latest key const latestKey = await projectKeyDAL.findLatestProjectKey(ghostUser.user.id, project.id, tx); @@ -353,7 +338,12 @@ export const projectServiceFactory = ({ const deletedProject = await projectDAL.transaction(async (tx) => { const delProject = await projectDAL.deleteById(project.id, tx); const projectGhostUser = await projectMembershipDAL.findProjectGhostUser(project.id, tx).catch(() => null); - + if (delProject.kmsCertificateKeyId) { + await kmsService.deleteInternalKms(delProject.kmsCertificateKeyId, delProject.orgId, tx); + } + if (delProject.kmsSecretManagerKeyId) { + await kmsService.deleteInternalKms(delProject.kmsSecretManagerKeyId, delProject.orgId, tx); + } // Delete the org membership for the ghost user if it's found. if (projectGhostUser) { await userDAL.deleteById(projectGhostUser.id, tx); @@ -664,6 +654,112 @@ export const projectServiceFactory = ({ }; }; + const updateProjectKmsKey = async ({ + projectId, + kms, + actor, + actorId, + actorAuthMethod, + actorOrgId + }: TUpdateProjectKmsDTO) => { + const { permission } = await permissionService.getProjectPermission( + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId + ); + + ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Edit, ProjectPermissionSub.Kms); + + const secretManagerKmsKey = await kmsService.updateProjectSecretManagerKmsKey({ + projectId, + kms + }); + + return { + secretManagerKmsKey + }; + }; + + const getProjectKmsBackup = async ({ + projectId, + actor, + actorId, + actorAuthMethod, + actorOrgId + }: TProjectPermission) => { + const { permission } = await permissionService.getProjectPermission( + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId + ); + + ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Edit, ProjectPermissionSub.Kms); + + const plan = await licenseService.getPlan(actorOrgId); + if (!plan.externalKms) { + throw new BadRequestError({ + message: "Failed to get KMS backup due to plan restriction. Upgrade to the enterprise plan." + }); + } + + const kmsBackup = await kmsService.getProjectKeyBackup(projectId); + return kmsBackup; + }; + + const loadProjectKmsBackup = async ({ + projectId, + actor, + actorId, + actorAuthMethod, + actorOrgId, + backup + }: TLoadProjectKmsBackupDTO) => { + const { permission } = await permissionService.getProjectPermission( + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId + ); + + ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Edit, ProjectPermissionSub.Kms); + + const plan = await licenseService.getPlan(actorOrgId); + if (!plan.externalKms) { + throw new BadRequestError({ + message: "Failed to load KMS backup due to plan restriction. Upgrade to the enterprise plan." + }); + } + + const kmsBackup = await kmsService.loadProjectKeyBackup(projectId, backup); + return kmsBackup; + }; + + const getProjectKmsKeys = async ({ projectId, actor, actorId, actorAuthMethod, actorOrgId }: TGetProjectKmsKey) => { + const { membership } = await permissionService.getProjectPermission( + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId + ); + + if (!membership) { + throw new ForbiddenRequestError({ + message: "User is not a member of the project" + }); + } + + const kmsKeyId = await kmsService.getProjectSecretManagerKmsKeyId(projectId); + const kmsKey = await kmsService.getKmsById(kmsKeyId); + + return { secretManagerKmsKey: kmsKey }; + }; + return { createProject, deleteProject, @@ -677,6 +773,10 @@ export const projectServiceFactory = ({ listProjectCas, listProjectCertificates, updateVersionLimit, - updateAuditLogsRetention + updateAuditLogsRetention, + updateProjectKmsKey, + getProjectKmsBackup, + loadProjectKmsBackup, + getProjectKmsKeys }; }; diff --git a/backend/src/services/project/project-types.ts b/backend/src/services/project/project-types.ts index c49e51143..1c2279c7a 100644 --- a/backend/src/services/project/project-types.ts +++ b/backend/src/services/project/project-types.ts @@ -3,6 +3,7 @@ import { TProjectPermission } from "@app/lib/types"; import { ActorAuthMethod, ActorType } from "../auth/auth-type"; import { CaStatus } from "../certificate-authority/certificate-authority-types"; +import { KmsType } from "../kms/kms-types"; export enum ProjectFilterType { ID = "id", @@ -27,6 +28,7 @@ export type TCreateProjectDTO = { actorOrgId?: string; workspaceName: string; slug?: string; + kmsKeyId?: string; }; export type TDeleteProjectBySlugDTO = { @@ -103,3 +105,13 @@ export type TListProjectCertsDTO = { friendlyName?: string; commonName?: string; } & Omit; + +export type TUpdateProjectKmsDTO = { + kms: { type: KmsType.Internal } | { type: KmsType.External; kmsId: string }; +} & TProjectPermission; + +export type TLoadProjectKmsBackupDTO = { + backup: string; +} & TProjectPermission; + +export type TGetProjectKmsKey = TProjectPermission; diff --git a/backend/src/services/resource-cleanup/resource-cleanup-queue.ts b/backend/src/services/resource-cleanup/resource-cleanup-queue.ts index 2e01e3549..cb9756dd4 100644 --- a/backend/src/services/resource-cleanup/resource-cleanup-queue.ts +++ b/backend/src/services/resource-cleanup/resource-cleanup-queue.ts @@ -7,11 +7,13 @@ import { TIdentityAccessTokenDALFactory } from "../identity-access-token/identit import { TSecretVersionDALFactory } from "../secret/secret-version-dal"; import { TSecretFolderVersionDALFactory } from "../secret-folder/secret-folder-version-dal"; import { TSecretSharingDALFactory } from "../secret-sharing/secret-sharing-dal"; +import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-dal"; type TDailyResourceCleanUpQueueServiceFactoryDep = { auditLogDAL: Pick; identityAccessTokenDAL: Pick; secretVersionDAL: Pick; + secretVersionV2DAL: Pick; secretFolderVersionDAL: Pick; snapshotDAL: Pick; secretSharingDAL: Pick; @@ -27,7 +29,8 @@ export const dailyResourceCleanUpQueueServiceFactory = ({ secretVersionDAL, secretFolderVersionDAL, identityAccessTokenDAL, - secretSharingDAL + secretSharingDAL, + secretVersionV2DAL }: TDailyResourceCleanUpQueueServiceFactoryDep) => { queueService.start(QueueName.DailyResourceCleanUp, async () => { logger.info(`${QueueName.DailyResourceCleanUp}: queue task started`); @@ -36,6 +39,7 @@ export const dailyResourceCleanUpQueueServiceFactory = ({ await secretSharingDAL.pruneExpiredSharedSecrets(); await snapshotDAL.pruneExcessSnapshots(); await secretVersionDAL.pruneExcessVersions(); + await secretVersionV2DAL.pruneExcessVersions(); await secretFolderVersionDAL.pruneExcessVersions(); logger.info(`${QueueName.DailyResourceCleanUp}: queue task completed`); }); diff --git a/backend/src/services/secret-folder/secret-folder-dal.ts b/backend/src/services/secret-folder/secret-folder-dal.ts index 1803f8977..a2a01be68 100644 --- a/backend/src/services/secret-folder/secret-folder-dal.ts +++ b/backend/src/services/secret-folder/secret-folder-dal.ts @@ -312,12 +312,14 @@ export const secretFolderDALFactory = (db: TDbClient) => { const folder = await (tx || db.replicaNode())(TableName.SecretFolder) .where({ [`${TableName.SecretFolder}.id` as "id"]: id }) .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) + .join(TableName.Project, `${TableName.Environment}.projectId`, `${TableName.Project}.id`) .select(selectAllTableCols(TableName.SecretFolder)) .select( db.ref("id").withSchema(TableName.Environment).as("envId"), db.ref("slug").withSchema(TableName.Environment).as("envSlug"), db.ref("name").withSchema(TableName.Environment).as("envName"), - db.ref("projectId").withSchema(TableName.Environment) + db.ref("projectId").withSchema(TableName.Environment), + db.ref("version").withSchema(TableName.Project).as("projectVersion") ) .first(); if (folder) { @@ -329,6 +331,27 @@ export const secretFolderDALFactory = (db: TDbClient) => { } }; + // special query for project migration + const findByProjectId = async (projectId: string, tx?: Knex) => { + try { + const folders = await (tx || db.replicaNode())(TableName.SecretFolder) + .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) + .join(TableName.Project, `${TableName.Environment}.projectId`, `${TableName.Project}.id`) + .select(selectAllTableCols(TableName.SecretFolder)) + .where({ projectId }) + .select( + db.ref("id").withSchema(TableName.Environment).as("envId"), + db.ref("slug").withSchema(TableName.Environment).as("envSlug"), + db.ref("name").withSchema(TableName.Environment).as("envName"), + db.ref("projectId").withSchema(TableName.Environment), + db.ref("version").withSchema(TableName.Project).as("projectVersion") + ); + return folders; + } catch (error) { + throw new DatabaseError({ error, name: "Find by id" }); + } + }; + return { ...secretFolderOrm, update, @@ -336,6 +359,7 @@ export const secretFolderDALFactory = (db: TDbClient) => { findById, findByManySecretPath, findSecretPathByFolderIds, - findClosestFolder + findClosestFolder, + findByProjectId }; }; diff --git a/backend/src/services/secret-import/secret-import-fns.ts b/backend/src/services/secret-import/secret-import-fns.ts index f1a3a3464..1ccaaefc2 100644 --- a/backend/src/services/secret-import/secret-import-fns.ts +++ b/backend/src/services/secret-import/secret-import-fns.ts @@ -1,8 +1,9 @@ -import { SecretType, TSecretImports, TSecrets } from "@app/db/schemas"; -import { groupBy } from "@app/lib/fn"; +import { SecretType, TSecretImports, TSecrets, TSecretsV2 } from "@app/db/schemas"; +import { groupBy, unique } from "@app/lib/fn"; import { TSecretDALFactory } from "../secret/secret-dal"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; +import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal"; import { TSecretImportDALFactory } from "./secret-import-dal"; type TSecretImportSecrets = { @@ -18,6 +19,28 @@ type TSecretImportSecrets = { secrets: (TSecrets & { workspace: string; environment: string; _id: string })[]; }; +type TSecretImportSecretsV2 = { + secretPath: string; + environment: string; + environmentInfo: { + id: string; + slug: string; + name: string; + }; + folderId: string | undefined; + importFolderId: string; + secrets: (TSecretsV2 & { + workspace: string; + environment: string; + _id: string; + secretKey: string; + // akhilmhdh: yes i know you can put ?. + // But for somereason ts consider ? and undefined explicit as different just ts things + secretValue: string | undefined; + secretComment: string | undefined; + })[]; +}; + const LEVEL_BREAK = 10; const getImportUniqKey = (envSlug: string, path: string) => `${envSlug}=${path}`; export const fnSecretsFromImports = async ({ @@ -115,3 +138,133 @@ export const fnSecretsFromImports = async ({ return secrets; }; + +export const fnSecretsV2FromImports = async ({ + allowedImports: possibleCyclicImports, + folderDAL, + secretDAL, + secretImportDAL, + depth = 0, + cyclicDetector = new Set(), + decryptor, + expandSecretReferences +}: { + allowedImports: (Omit & { + importEnv: { id: string; slug: string; name: string }; + })[]; + folderDAL: Pick; + secretDAL: Pick; + secretImportDAL: Pick; + depth?: number; + cyclicDetector?: Set; + decryptor: (value?: Buffer | null) => string | undefined; + expandSecretReferences?: ( + secrets: Record + ) => Promise>; +}) => { + // avoid going more than a depth + if (depth >= LEVEL_BREAK) return []; + + const allowedImports = possibleCyclicImports.filter( + ({ importPath, importEnv }) => !cyclicDetector.has(getImportUniqKey(importEnv.slug, importPath)) + ); + + const importedFolders = ( + await folderDAL.findByManySecretPath( + allowedImports.map(({ importEnv, importPath }) => ({ + envId: importEnv.id, + secretPath: importPath + })) + ) + ).filter(Boolean); // remove undefined ones + if (!importedFolders.length) { + return []; + } + + const importedFolderIds = importedFolders.map((el) => el?.id) as string[]; + const importedFolderGroupBySourceImport = groupBy(importedFolders, (i) => `${i?.envId}-${i?.path}`); + const importedSecrets = await secretDAL.find( + { + $in: { folderId: importedFolderIds }, + type: SecretType.Shared + }, + { + sort: [["id", "asc"]] + } + ); + + const importedSecretsGroupByFolderId = groupBy(importedSecrets, (i) => i.folderId); + + allowedImports.forEach(({ importPath, importEnv }) => { + cyclicDetector.add(getImportUniqKey(importEnv.slug, importPath)); + }); + // now we need to check recursively deeper imports made inside other imports + // we go level wise meaning we take all imports of a tree level and then go deeper ones level by level + const deeperImports = await secretImportDAL.findByFolderIds(importedFolderIds); + let secretsFromDeeperImports: TSecretImportSecretsV2[] = []; + if (deeperImports.length) { + secretsFromDeeperImports = await fnSecretsV2FromImports({ + allowedImports: deeperImports.filter(({ isReplication }) => !isReplication), + secretImportDAL, + folderDAL, + secretDAL, + depth: depth + 1, + cyclicDetector, + decryptor, + expandSecretReferences + }); + } + const secretsFromdeeperImportGroupedByFolderId = groupBy(secretsFromDeeperImports, (i) => i.importFolderId); + + const processedImports = allowedImports.map(({ importPath, importEnv, id, folderId }, i) => { + const sourceImportFolder = importedFolderGroupBySourceImport[`${importEnv.id}-${importPath}`][0]; + const folderDeeperImportSecrets = + secretsFromdeeperImportGroupedByFolderId?.[sourceImportFolder?.id || ""]?.[0]?.secrets || []; + const secretsWithDuplicate = (importedSecretsGroupByFolderId?.[importedFolders?.[i]?.id as string] || []) + .map((item) => ({ + ...item, + secretKey: item.key, + secretValue: decryptor(item.encryptedValue), + secretComment: decryptor(item.encryptedComment), + environment: importEnv.slug, + workspace: "", // This field should not be used, it's only here to keep the older Python SDK versions backwards compatible with the new Postgres backend. + _id: item.id // The old Python SDK depends on the _id field being returned. We return this to keep the older Python SDK versions backwards compatible with the new Postgres backend. + })) + .concat(folderDeeperImportSecrets); + return { + secretPath: importPath, + environment: importEnv.slug, + environmentInfo: importEnv, + folderId: importedFolders?.[i]?.id, + id, + importFolderId: folderId, + secrets: unique(secretsWithDuplicate, (el) => el.secretKey) + }; + }); + + if (expandSecretReferences) { + await Promise.all( + processedImports.map(async (processedImport) => { + const secretsGroupByKey = processedImport.secrets.reduce( + (acc, item) => { + acc[item.secretKey] = { + value: item.secretValue, + comment: item.secretComment, + skipMultilineEncoding: item.skipMultilineEncoding + }; + return acc; + }, + {} as Record + ); + // eslint-disable-next-line + await expandSecretReferences(secretsGroupByKey); + processedImport.secrets.forEach((decryptedSecret) => { + // eslint-disable-next-line no-param-reassign + decryptedSecret.secretValue = secretsGroupByKey[decryptedSecret.secretKey].value; + }); + }) + ); + } + + return processedImports; +}; diff --git a/backend/src/services/secret-import/secret-import-service.ts b/backend/src/services/secret-import/secret-import-service.ts index 237c7cfe4..c69fec6b6 100644 --- a/backend/src/services/secret-import/secret-import-service.ts +++ b/backend/src/services/secret-import/secret-import-service.ts @@ -9,13 +9,18 @@ import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services import { getReplicationFolderName } from "@app/ee/services/secret-replication/secret-replication-service"; import { BadRequestError } from "@app/lib/errors"; +import { TKmsServiceFactory } from "../kms/kms-service"; +import { KmsDataKey } from "../kms/kms-types"; import { TProjectDALFactory } from "../project/project-dal"; +import { TProjectBotServiceFactory } from "../project-bot/project-bot-service"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TSecretDALFactory } from "../secret/secret-dal"; +import { decryptSecretRaw } from "../secret/secret-fns"; import { TSecretQueueFactory } from "../secret/secret-queue"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; +import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal"; import { TSecretImportDALFactory } from "./secret-import-dal"; -import { fnSecretsFromImports } from "./secret-import-fns"; +import { fnSecretsFromImports, fnSecretsV2FromImports } from "./secret-import-fns"; import { TCreateSecretImportDTO, TDeleteSecretImportDTO, @@ -29,11 +34,14 @@ type TSecretImportServiceFactoryDep = { secretImportDAL: TSecretImportDALFactory; folderDAL: TSecretFolderDALFactory; secretDAL: Pick; + secretV2BridgeDAL: Pick; + projectBotService: Pick; projectDAL: Pick; projectEnvDAL: TProjectEnvDALFactory; permissionService: Pick; secretQueueService: Pick; licenseService: Pick; + kmsService: Pick; }; const ERR_SEC_IMP_NOT_FOUND = new BadRequestError({ message: "Secret import not found" }); @@ -48,7 +56,10 @@ export const secretImportServiceFactory = ({ projectDAL, secretDAL, secretQueueService, - licenseService + licenseService, + projectBotService, + secretV2BridgeDAL, + kmsService }: TSecretImportServiceFactoryDep) => { const createImport = async ({ environment, @@ -449,12 +460,76 @@ export const secretImportServiceFactory = ({ return fnSecretsFromImports({ allowedImports, folderDAL, secretDAL, secretImportDAL }); }; + const getRawSecretsFromImports = async ({ + path: secretPath, + environment, + projectId, + actor, + actorAuthMethod, + actorId, + actorOrgId + }: TGetSecretsFromImportDTO) => { + const { permission } = await permissionService.getProjectPermission( + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId + ); + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionActions.Read, + subject(ProjectPermissionSub.Secrets, { environment, secretPath }) + ); + const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); + if (!folder) return []; + // this will already order by position + // so anything based on this order will also be in right position + const secretImports = await secretImportDAL.find({ folderId: folder.id, isReplication: false }); + + const allowedImports = secretImports.filter(({ importEnv, importPath }) => + permission.can( + ProjectPermissionActions.Read, + subject(ProjectPermissionSub.Secrets, { + environment: importEnv.slug, + secretPath: importPath + }) + ) + ); + + const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId); + if (shouldUseSecretV2Bridge) { + const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId + }); + const importedSecrets = await fnSecretsV2FromImports({ + allowedImports, + folderDAL, + secretDAL: secretV2BridgeDAL, + secretImportDAL, + decryptor: (value) => (value ? secretManagerDecryptor({ cipherTextBlob: value }).toString() : undefined) + }); + return importedSecrets; + } + + if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" }); + + const importedSecrets = await fnSecretsFromImports({ allowedImports, folderDAL, secretDAL, secretImportDAL }); + return importedSecrets.map((el) => ({ + ...el, + secrets: el.secrets.map((encryptedSecret) => + decryptSecretRaw({ ...encryptedSecret, workspace: projectId, environment, secretPath }, botKey) + ) + })); + }; + return { createImport, updateImport, deleteImport, getImports, getSecretsFromImports, + getRawSecretsFromImports, resyncSecretImportReplication, fnSecretsFromImports }; diff --git a/backend/src/services/secret-tag/secret-tag-dal.ts b/backend/src/services/secret-tag/secret-tag-dal.ts index 98cd9af22..d1f149f35 100644 --- a/backend/src/services/secret-tag/secret-tag-dal.ts +++ b/backend/src/services/secret-tag/secret-tag-dal.ts @@ -3,13 +3,14 @@ import { Knex } from "knex"; import { TDbClient } from "@app/db"; import { TableName } from "@app/db/schemas"; import { DatabaseError } from "@app/lib/errors"; -import { ormify } from "@app/lib/knex"; +import { ormify, selectAllTableCols } from "@app/lib/knex"; export type TSecretTagDALFactory = ReturnType; export const secretTagDALFactory = (db: TDbClient) => { const secretTagOrm = ormify(db, TableName.SecretTag); const secretJnTagOrm = ormify(db, TableName.JnSecretTag); + const secretV2JnTagOrm = ormify(db, TableName.SecretV2JnTag); const findManyTagsById = async (projectId: string, ids: string[], tx?: Knex) => { try { @@ -34,10 +35,25 @@ export const secretTagDALFactory = (db: TDbClient) => { } }; + // special query for migration + const findSecretTagsByProjectId = async (projectId: string, tx?: Knex) => { + try { + const tags = await (tx || db.replicaNode())(TableName.JnSecretTag) + .join(TableName.SecretTag, `${TableName.JnSecretTag}.secret_tagsId`, `${TableName.SecretTag}.id`) + .where({ projectId }) + .select(selectAllTableCols(TableName.JnSecretTag)); + return tags; + } catch (error) { + throw new DatabaseError({ error, name: "Find all by ids" }); + } + }; return { ...secretTagOrm, saveTagsToSecret: secretJnTagOrm.insertMany, deleteTagsToSecret: secretJnTagOrm.delete, + saveTagsToSecretV2: secretV2JnTagOrm.insertMany, + deleteTagsToSecretV2: secretV2JnTagOrm.delete, + findSecretTagsByProjectId, deleteTagsManySecret, findManyTagsById }; diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts new file mode 100644 index 000000000..b41e2c250 --- /dev/null +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts @@ -0,0 +1,393 @@ +import { Knex } from "knex"; +import { validate as uuidValidate } from "uuid"; + +import { TDbClient } from "@app/db"; +import { SecretsV2Schema, SecretType, TableName, TSecretsV2, TSecretsV2Update } from "@app/db/schemas"; +import { BadRequestError, DatabaseError } from "@app/lib/errors"; +import { ormify, selectAllTableCols, sqlNestRelationships } from "@app/lib/knex"; + +export type TSecretV2BridgeDALFactory = ReturnType; + +export const secretV2BridgeDALFactory = (db: TDbClient) => { + const secretOrm = ormify(db, TableName.SecretV2); + + const update = async (filter: Partial, data: Omit, tx?: Knex) => { + try { + const sec = await (tx || db)(TableName.SecretV2) + .where(filter) + .update(data) + .increment("version", 1) + .returning("*"); + return sec; + } catch (error) { + throw new DatabaseError({ error, name: "update secret" }); + } + }; + + const bulkUpdate = async ( + data: Array<{ filter: Partial; data: TSecretsV2Update }>, + + tx?: Knex + ) => { + try { + const secs = await Promise.all( + data.map(async ({ filter, data: updateData }) => { + const [doc] = await (tx || db)(TableName.SecretV2) + .where(filter) + .update(updateData) + .increment("version", 1) + .returning("*"); + if (!doc) throw new BadRequestError({ message: "Failed to update document" }); + return doc; + }) + ); + return secs; + } catch (error) { + throw new DatabaseError({ error, name: "bulk update secret" }); + } + }; + + const bulkUpdateNoVersionIncrement = async (data: TSecretsV2[], tx?: Knex) => { + try { + const existingSecrets = await secretOrm.find( + { + $in: { + id: data.map((el) => el.id) + } + }, + { tx } + ); + + if (existingSecrets.length !== data.length) { + throw new BadRequestError({ message: "Some of the secrets do not exist" }); + } + + if (data.length === 0) return []; + + const updatedSecrets = await (tx || db)(TableName.SecretV2) + .insert(data) + .onConflict("id") // this will cause a conflict then merge the data + .merge() // Merge the data with the existing data + .returning("*"); + + return updatedSecrets; + } catch (error) { + throw new DatabaseError({ error, name: "bulk update secret" }); + } + }; + + const deleteMany = async ( + data: Array<{ key: string; type: SecretType }>, + folderId: string, + userId: string, + tx?: Knex + ) => { + try { + const deletedSecrets = await (tx || db)(TableName.SecretV2) + .where({ folderId }) + .where((bd) => { + data.forEach((el) => { + void bd.orWhere({ + key: el.key, + type: el.type, + ...(el.type === SecretType.Personal ? { userId } : {}) + }); + // if shared is getting deleted then personal ones also should be deleted + if (el.type === SecretType.Shared) { + void bd.orWhere({ + key: el.key, + type: SecretType.Personal + }); + } + }); + }) + .delete() + .returning("*"); + return deletedSecrets; + } catch (error) { + throw new DatabaseError({ error, name: "delete many secret" }); + } + }; + + const findByFolderId = async (folderId: string, userId?: string, tx?: Knex) => { + try { + // check if not uui then userId id is null (corner case because service token's ID is not UUI in effort to keep backwards compatibility from mongo) + if (userId && !uuidValidate(userId)) { + // eslint-disable-next-line + userId = undefined; + } + + const secs = await (tx || db.replicaNode())(TableName.SecretV2) + .where({ folderId }) + .where((bd) => { + void bd.whereNull("userId").orWhere({ userId: userId || null }); + }) + .leftJoin( + TableName.SecretV2JnTag, + `${TableName.SecretV2}.id`, + `${TableName.SecretV2JnTag}.${TableName.SecretV2}Id` + ) + .leftJoin( + TableName.SecretTag, + `${TableName.SecretV2JnTag}.${TableName.SecretTag}Id`, + `${TableName.SecretTag}.id` + ) + .select(selectAllTableCols(TableName.SecretV2)) + .select(db.ref("id").withSchema(TableName.SecretTag).as("tagId")) + .select(db.ref("color").withSchema(TableName.SecretTag).as("tagColor")) + .select(db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug")) + .select(db.ref("name").withSchema(TableName.SecretTag).as("tagName")) + .orderBy("id", "asc"); + + const data = sqlNestRelationships({ + data: secs, + key: "id", + parentMapper: (el) => ({ _id: el.id, ...SecretsV2Schema.parse(el) }), + childrenMapper: [ + { + key: "tagId", + label: "tags" as const, + mapper: ({ tagId: id, tagColor: color, tagSlug: slug, tagName: name }) => ({ + id, + color, + slug, + name + }) + } + ] + }); + return data; + } catch (error) { + throw new DatabaseError({ error, name: "get all secret" }); + } + }; + + const getSecretTags = async (secretId: string, tx?: Knex) => { + try { + const tags = await (tx || db.replicaNode())(TableName.SecretV2JnTag) + .join(TableName.SecretTag, `${TableName.SecretV2JnTag}.${TableName.SecretTag}Id`, `${TableName.SecretTag}.id`) + .where({ [`${TableName.SecretV2}Id` as const]: secretId }) + .select(db.ref("id").withSchema(TableName.SecretTag).as("tagId")) + .select(db.ref("color").withSchema(TableName.SecretTag).as("tagColor")) + .select(db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug")) + .select(db.ref("name").withSchema(TableName.SecretTag).as("tagName")); + + return tags.map((el) => ({ + id: el.tagId, + color: el.tagColor, + slug: el.tagSlug, + name: el.tagName + })); + } catch (error) { + throw new DatabaseError({ error, name: "get secret tags" }); + } + }; + + const findByFolderIds = async (folderIds: string[], userId?: string, tx?: Knex) => { + try { + // check if not uui then userId id is null (corner case because service token's ID is not UUI in effort to keep backwards compatibility from mongo) + if (userId && !uuidValidate(userId)) { + // eslint-disable-next-line no-param-reassign + userId = undefined; + } + + const secs = await (tx || db.replicaNode())(TableName.SecretV2) + .whereIn("folderId", folderIds) + .where((bd) => { + void bd.whereNull("userId").orWhere({ userId: userId || null }); + }) + .leftJoin( + TableName.SecretV2JnTag, + `${TableName.SecretV2}.id`, + `${TableName.SecretV2JnTag}.${TableName.SecretV2}Id` + ) + .leftJoin( + TableName.SecretTag, + `${TableName.SecretV2JnTag}.${TableName.SecretTag}Id`, + `${TableName.SecretTag}.id` + ) + .select(selectAllTableCols(TableName.SecretV2)) + .select(db.ref("id").withSchema(TableName.SecretTag).as("tagId")) + .select(db.ref("color").withSchema(TableName.SecretTag).as("tagColor")) + .select(db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug")) + .select(db.ref("name").withSchema(TableName.SecretTag).as("tagName")) + .orderBy("id", "asc"); + + const data = sqlNestRelationships({ + data: secs, + key: "id", + parentMapper: (el) => ({ _id: el.id, ...SecretsV2Schema.parse(el) }), + childrenMapper: [ + { + key: "tagId", + label: "tags" as const, + mapper: ({ tagId: id, tagColor: color, tagSlug: slug, tagName: name }) => ({ + id, + color, + slug, + name + }) + } + ] + }); + return data; + } catch (error) { + throw new DatabaseError({ error, name: "get all secret" }); + } + }; + + const findBySecretKeys = async ( + folderId: string, + query: Array<{ key: string; type: SecretType.Shared } | { key: string; type: SecretType.Personal; userId: string }>, + tx?: Knex + ) => { + if (!query.length) return []; + try { + const secrets = await (tx || db.replicaNode())(TableName.SecretV2) + .where({ folderId }) + .where((bd) => { + query.forEach((el) => { + if (el.type === SecretType.Personal && !el.userId) { + throw new BadRequestError({ message: "Missing personal user id" }); + } + void bd.orWhere({ + key: el.key, + type: el.type, + userId: el.type === SecretType.Personal ? el.userId : null + }); + }); + }); + return secrets; + } catch (error) { + throw new DatabaseError({ error, name: "find by blind indexes" }); + } + }; + + const upsertSecretReferences = async ( + data: { + secretId: string; + references: Array<{ environment: string; secretPath: string; secretKey: string }>; + }[] = [], + tx?: Knex + ) => { + try { + if (!data.length) return; + + await (tx || db)(TableName.SecretReferenceV2) + .whereIn( + "secretId", + data.map(({ secretId }) => secretId) + ) + .delete(); + const newSecretReferences = data + .filter(({ references }) => references.length) + .flatMap(({ secretId, references }) => + references.map(({ environment, secretPath, secretKey }) => ({ + secretPath, + secretId, + environment, + secretKey + })) + ); + if (!newSecretReferences.length) return; + const secretReferences = await (tx || db)(TableName.SecretReferenceV2).insert(newSecretReferences); + return secretReferences; + } catch (error) { + throw new DatabaseError({ error, name: "UpsertSecretReference" }); + } + }; + + const findReferencedSecretReferences = async (projectId: string, envSlug: string, secretPath: string, tx?: Knex) => { + try { + const docs = await (tx || db.replicaNode())(TableName.SecretReferenceV2) + .where({ + secretPath, + environment: envSlug + }) + .join(TableName.SecretV2, `${TableName.SecretV2}.id`, `${TableName.SecretReferenceV2}.secretId`) + .join(TableName.SecretFolder, `${TableName.SecretV2}.folderId`, `${TableName.SecretFolder}.id`) + .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) + .where("projectId", projectId) + .select(selectAllTableCols(TableName.SecretReferenceV2)) + .select("folderId"); + + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindReferencedSecretReferences" }); + } + }; + + // special query to backfill secret value + const findAllProjectSecretValues = async (projectId: string, tx?: Knex) => { + try { + const docs = await (tx || db.replicaNode())(TableName.SecretV2) + .join(TableName.SecretFolder, `${TableName.SecretV2}.folderId`, `${TableName.SecretFolder}.id`) + .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) + .where("projectId", projectId) + // not empty + .whereNotNull("encryptedValue") + .select("encryptedValue", `${TableName.SecretV2}.id` as "id"); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindAllProjectSecretValues" }); + } + }; + + const findOneWithTags = async (filter: Partial, tx?: Knex) => { + try { + const rawDocs = await (tx || db.replicaNode())(TableName.SecretV2) + .where(filter) + .leftJoin( + TableName.SecretV2JnTag, + `${TableName.SecretV2}.id`, + `${TableName.SecretV2JnTag}.${TableName.SecretV2}Id` + ) + .leftJoin( + TableName.SecretTag, + `${TableName.SecretV2JnTag}.${TableName.SecretTag}Id`, + `${TableName.SecretTag}.id` + ) + .select(selectAllTableCols(TableName.SecretV2)) + .select(db.ref("id").withSchema(TableName.SecretTag).as("tagId")) + .select(db.ref("color").withSchema(TableName.SecretTag).as("tagColor")) + .select(db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug")) + .select(db.ref("name").withSchema(TableName.SecretTag).as("tagName")); + const docs = sqlNestRelationships({ + data: rawDocs, + key: "id", + parentMapper: (el) => ({ _id: el.id, ...SecretsV2Schema.parse(el) }), + childrenMapper: [ + { + key: "tagId", + label: "tags" as const, + mapper: ({ tagId: id, tagColor: color, tagSlug: slug, tagName: name }) => ({ + id, + color, + slug, + name + }) + } + ] + }); + return docs?.[0]; + } catch (error) { + throw new DatabaseError({ error, name: "FindOneWIthTags" }); + } + }; + + return { + ...secretOrm, + update, + bulkUpdate, + deleteMany, + bulkUpdateNoVersionIncrement, + getSecretTags, + findOneWithTags, + findByFolderId, + findByFolderIds, + findBySecretKeys, + upsertSecretReferences, + findReferencedSecretReferences, + findAllProjectSecretValues + }; +}; diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts new file mode 100644 index 000000000..19aef06f3 --- /dev/null +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts @@ -0,0 +1,559 @@ +import path from "node:path"; + +import { TableName, TSecretFolders, TSecretsV2 } from "@app/db/schemas"; +import { groupBy } from "@app/lib/fn"; +import { logger } from "@app/lib/logger"; + +import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; +import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; +import { TSecretV2BridgeDALFactory } from "./secret-v2-bridge-dal"; +import { TFnSecretBulkDelete, TFnSecretBulkInsert, TFnSecretBulkUpdate } from "./secret-v2-bridge-types"; + +const INTERPOLATION_SYNTAX_REG = /\${([^}]+)}/g; + +export const shouldUseSecretV2Bridge = (version: number) => version === 3; + +/** + * Grabs and processes nested secret references from a string + * + * This function looks for patterns that match the interpolation syntax in the input string. + * It filters out references that include nested paths, splits them into environment and + * secret path parts, and then returns an array of objects with the environment and the + * joined secret path. + * @example + * const value = "Hello ${dev.someFolder.OtherFolder.SECRET_NAME} and ${prod.anotherFolder.SECRET_NAME}"; + * const result = getAllNestedSecretReferences(value); + * // result will be: + * // [ + * // { environment: 'dev', secretPath: '/someFolder/OtherFolder' }, + * // { environment: 'prod', secretPath: '/anotherFolder' } + * // ] + */ +export const getAllNestedSecretReferences = (maybeSecretReference: string) => { + const references = Array.from(maybeSecretReference.matchAll(INTERPOLATION_SYNTAX_REG), (m) => m[1]); + return references + .filter((el) => el.includes(".")) + .map((el) => { + const [environment, ...secretPathList] = el.split("."); + return { + environment, + secretPath: path.join("/", ...secretPathList.slice(0, -1)), + secretKey: secretPathList[secretPathList.length - 1] + }; + }); +}; + +// these functions are special functions shared by a couple of resources +// used by secret approval, rotation or anywhere in which secret needs to modified +export const fnSecretBulkInsert = async ({ + // TODO: Pick types here + folderId, + inputSecrets, + secretDAL, + secretVersionDAL, + secretTagDAL, + secretVersionTagDAL, + tx +}: TFnSecretBulkInsert) => { + const sanitizedInputSecrets = inputSecrets.map( + ({ + skipMultilineEncoding, + type, + key, + userId, + encryptedComment, + version, + metadata, + reminderNote, + encryptedValue, + reminderRepeatDays + }) => ({ + skipMultilineEncoding, + type, + key, + userId, + encryptedComment, + version, + metadata, + reminderNote, + encryptedValue, + reminderRepeatDays + }) + ); + + const newSecrets = await secretDAL.insertMany(sanitizedInputSecrets.map((el) => ({ ...el, folderId }))); + const newSecretGroupedByKeyName = groupBy(newSecrets, (item) => item.key); + const newSecretTags = inputSecrets.flatMap(({ tagIds: secretTags = [], key }) => + secretTags.map((tag) => ({ + [`${TableName.SecretTag}Id` as const]: tag, + [`${TableName.SecretV2}Id` as const]: newSecretGroupedByKeyName[key][0].id + })) + ); + const secretVersions = await secretVersionDAL.insertMany( + sanitizedInputSecrets.map((el) => ({ + ...el, + folderId, + secretId: newSecretGroupedByKeyName[el.key][0].id + })), + tx + ); + await secretDAL.upsertSecretReferences( + inputSecrets.map(({ references = [], key }) => ({ + secretId: newSecretGroupedByKeyName[key][0].id, + references + })), + tx + ); + if (newSecretTags.length) { + const secTags = await secretTagDAL.saveTagsToSecretV2(newSecretTags, tx); + const secVersionsGroupBySecId = groupBy(secretVersions, (i) => i.secretId); + const newSecretVersionTags = secTags.flatMap(({ secrets_v2Id, secret_tagsId }) => ({ + [`${TableName.SecretVersionV2}Id` as const]: secVersionsGroupBySecId[secrets_v2Id][0].id, + [`${TableName.SecretTag}Id` as const]: secret_tagsId + })); + await secretVersionTagDAL.insertMany(newSecretVersionTags, tx); + } + + return newSecrets.map((secret) => ({ ...secret, _id: secret.id })); +}; + +export const fnSecretBulkUpdate = async ({ + tx, + inputSecrets, + folderId, + secretDAL, + secretVersionDAL, + secretTagDAL, + secretVersionTagDAL +}: TFnSecretBulkUpdate) => { + const sanitizedInputSecrets = inputSecrets.map( + ({ + filter, + data: { + skipMultilineEncoding, + type, + key, + encryptedValue, + userId, + encryptedComment, + metadata, + reminderNote, + reminderRepeatDays + } + }) => ({ + filter: { ...filter, folderId }, + data: { + skipMultilineEncoding, + type, + key, + userId, + encryptedComment, + metadata, + reminderNote, + encryptedValue, + reminderRepeatDays + } + }) + ); + + const newSecrets = await secretDAL.bulkUpdate(sanitizedInputSecrets, tx); + const secretVersions = await secretVersionDAL.insertMany( + newSecrets.map( + ({ + skipMultilineEncoding, + type, + key, + userId, + encryptedComment, + version, + metadata, + reminderNote, + encryptedValue, + reminderRepeatDays, + id: secretId + }) => ({ + skipMultilineEncoding, + type, + key, + userId, + encryptedComment, + version, + metadata, + reminderNote, + encryptedValue, + reminderRepeatDays, + folderId, + secretId + }) + ), + tx + ); + await secretDAL.upsertSecretReferences( + inputSecrets + .filter(({ data: { references } }) => Boolean(references)) + .map(({ data: { references = [] } }, i) => ({ + secretId: newSecrets[i].id, + references + })), + tx + ); + const secsUpdatedTag = inputSecrets.flatMap(({ data: { tags } }, i) => + tags !== undefined ? { tags, secretId: newSecrets[i].id } : [] + ); + if (secsUpdatedTag.length) { + await secretTagDAL.deleteTagsToSecretV2( + { $in: { secrets_v2Id: secsUpdatedTag.map(({ secretId }) => secretId) } }, + tx + ); + const newSecretTags = secsUpdatedTag.flatMap(({ tags: secretTags = [], secretId }) => + secretTags.map((tag) => ({ + [`${TableName.SecretTag}Id` as const]: tag, + [`${TableName.SecretV2}Id` as const]: secretId + })) + ); + if (newSecretTags.length) { + const secTags = await secretTagDAL.saveTagsToSecretV2(newSecretTags, tx); + const secVersionsGroupBySecId = groupBy(secretVersions, (i) => i.secretId); + const newSecretVersionTags = secTags.flatMap(({ secrets_v2Id, secret_tagsId }) => ({ + [`${TableName.SecretVersionV2}Id` as const]: secVersionsGroupBySecId[secrets_v2Id][0].id, + [`${TableName.SecretTag}Id` as const]: secret_tagsId + })); + await secretVersionTagDAL.insertMany(newSecretVersionTags, tx); + } + } + + return newSecrets.map((secret) => ({ ...secret, _id: secret.id })); +}; + +export const fnSecretBulkDelete = async ({ + folderId, + inputSecrets, + tx, + actorId, + secretDAL, + secretQueueService +}: TFnSecretBulkDelete) => { + const deletedSecrets = await secretDAL.deleteMany( + inputSecrets.map(({ type, secretKey }) => ({ + key: secretKey, + type + })), + folderId, + actorId, + tx + ); + + await Promise.allSettled( + deletedSecrets + .filter(({ reminderRepeatDays }) => Boolean(reminderRepeatDays)) + .map(({ id, reminderRepeatDays }) => + secretQueueService.removeSecretReminder({ secretId: id, repeatDays: reminderRepeatDays as number }) + ) + ); + + return deletedSecrets; +}; + +// Introduce a new interface for mapping parent IDs to their children +interface FolderMap { + [parentId: string]: TSecretFolders[]; +} +const buildHierarchy = (folders: TSecretFolders[]): FolderMap => { + const map: FolderMap = {}; + map.null = []; // Initialize mapping for root directory + + folders.forEach((folder) => { + const parentId = folder.parentId || "null"; + if (!map[parentId]) { + map[parentId] = []; + } + map[parentId].push(folder); + }); + + return map; +}; + +const generatePaths = ( + map: FolderMap, + parentId: string = "null", + basePath: string = "", + currentDepth: number = 0 +): { path: string; folderId: string }[] => { + const children = map[parentId || "null"] || []; + let paths: { path: string; folderId: string }[] = []; + + children.forEach((child) => { + // Determine if this is the root folder of the environment. If no parentId is present and the name is root, it's the root folder + const isRootFolder = child.name === "root" && !child.parentId; + + // Form the current path based on the base path and the current child + // eslint-disable-next-line no-nested-ternary + const currPath = basePath === "" ? (isRootFolder ? "/" : `/${child.name}`) : `${basePath}/${child.name}`; + + // Add the current path + paths.push({ + path: currPath, + folderId: child.id + }); + + // We make sure that the recursion depth doesn't exceed 20. + // We do this to create "circuit break", basically to ensure that we can't encounter any potential memory leaks. + if (currentDepth >= 20) { + logger.info(`generatePaths: Recursion depth exceeded 20, breaking out of recursion [map=${JSON.stringify(map)}]`); + return; + } + // Recursively generate paths for children, passing down the formatted path + const childPaths = generatePaths(map, child.id, currPath, currentDepth + 1); + paths = paths.concat( + childPaths.map((p) => ({ + path: p.path, + folderId: p.folderId + })) + ); + }); + + return paths; +}; + +type TRecursivelyFetchSecretsFromFoldersArg = { + folderDAL: Pick; + projectEnvDAL: Pick; + projectId: string; + environment: string; + currentPath: string; + hasAccess: (environment: string, secretPath: string) => boolean; +}; + +export const recursivelyGetSecretPaths = async ({ + folderDAL, + projectEnvDAL, + projectId, + environment, + currentPath, + hasAccess +}: TRecursivelyFetchSecretsFromFoldersArg) => { + const env = await projectEnvDAL.findOne({ + projectId, + slug: environment + }); + + if (!env) { + throw new Error(`'${environment}' environment not found in project with ID ${projectId}`); + } + + // Fetch all folders in env once with a single query + const folders = await folderDAL.find({ + envId: env.id, + isReserved: false + }); + + // Build the folder hierarchy map + const folderMap = buildHierarchy(folders); + + // Generate the paths paths and normalize the root path to / + const paths = generatePaths(folderMap).map((p) => ({ + path: p.path === "/" ? p.path : p.path.substring(1), + folderId: p.folderId + })); + + // Filter out paths that the user does not have permission to access, and paths that are not in the current path + const allowedPaths = paths.filter( + (folder) => hasAccess(environment, folder.path) && folder.path.startsWith(currentPath === "/" ? "" : currentPath) + ); + + return allowedPaths; +}; +// used to convert multi line ones to quotes ones with \n +const formatMultiValueEnv = (val?: string) => { + if (!val) return ""; + if (!val.match("\n")) return val; + return `"${val.replace(/\n/g, "\\n")}"`; +}; + +type TInterpolateSecretArg = { + projectId: string; + decryptSecretValue: (encryptedValue?: Buffer | null) => string | undefined; + secretDAL: Pick; + folderDAL: Pick; +}; + +export const expandSecretReferencesFactory = ({ + projectId, + decryptSecretValue: decryptSecret, + secretDAL, + folderDAL +}: TInterpolateSecretArg) => { + const fetchSecretFactory = () => { + const secretCache: Record> = {}; + + return async (secRefEnv: string, secRefPath: string[], secRefKey: string) => { + const referredSecretPathURL = path.join("/", ...secRefPath); + const uniqueKey = `${secRefEnv}-${referredSecretPathURL}`; + + if (secretCache?.[uniqueKey]) { + return secretCache[uniqueKey][secRefKey]; + } + + const folder = await folderDAL.findBySecretPath(projectId, secRefEnv, referredSecretPathURL); + if (!folder) return ""; + const secrets = await secretDAL.findByFolderId(folder.id); + + const decryptedSecret = secrets.reduce>((prev, secret) => { + // eslint-disable-next-line + prev[secret.key] = decryptSecret(secret.encryptedValue) || ""; + return prev; + }, {}); + + secretCache[uniqueKey] = decryptedSecret; + + return secretCache[uniqueKey][secRefKey]; + }; + }; + + const recursivelyExpandSecret = async ( + expandedSec: Record, + interpolatedSec: Record, + fetchSecret: (env: string, secPath: string[], secKey: string) => Promise, + recursionChainBreaker: Record, + key: string + ): Promise => { + if (expandedSec?.[key] !== undefined) { + return expandedSec[key]; + } + if (recursionChainBreaker?.[key]) { + return ""; + } + // eslint-disable-next-line + recursionChainBreaker[key] = true; + + let interpolatedValue = interpolatedSec[key]; + if (!interpolatedValue) { + // eslint-disable-next-line no-console + console.error(`Couldn't find referenced value - ${key}`); + return ""; + } + + const refs = interpolatedValue.match(INTERPOLATION_SYNTAX_REG); + if (refs) { + for (const interpolationSyntax of refs) { + const interpolationKey = interpolationSyntax.slice(2, interpolationSyntax.length - 1); + const entities = interpolationKey.trim().split("."); + + if (entities.length === 1) { + // eslint-disable-next-line + const val = await recursivelyExpandSecret( + expandedSec, + interpolatedSec, + fetchSecret, + recursionChainBreaker, + interpolationKey + ); + if (val) { + interpolatedValue = interpolatedValue.replaceAll(interpolationSyntax, val); + } + // eslint-disable-next-line + continue; + } + + if (entities.length > 1) { + const secRefEnv = entities[0]; + const secRefPath = entities.slice(1, entities.length - 1); + const secRefKey = entities[entities.length - 1]; + + // eslint-disable-next-line + const val = await fetchSecret(secRefEnv, secRefPath, secRefKey); + if (val) { + interpolatedValue = interpolatedValue.replaceAll(interpolationSyntax, val); + } + } + } + } + + // eslint-disable-next-line + expandedSec[key] = interpolatedValue; + return interpolatedValue; + }; + + const fetchSecret = fetchSecretFactory(); + const expandSecrets = async ( + inputSecrets: Record + ) => { + const expandedSecrets: Record = {}; + const toBeExpandedSecrets: Record = {}; + + Object.keys(inputSecrets).forEach((key) => { + if (inputSecrets[key].value?.match(INTERPOLATION_SYNTAX_REG)) { + toBeExpandedSecrets[key] = inputSecrets[key].value; + } else { + expandedSecrets[key] = inputSecrets[key].value; + } + }); + + for (const key of Object.keys(inputSecrets)) { + if (expandedSecrets?.[key]) { + // should not do multi line encoding if user has set it to skip + // eslint-disable-next-line + inputSecrets[key].value = inputSecrets[key].skipMultilineEncoding + ? formatMultiValueEnv(expandedSecrets[key]) + : expandedSecrets[key]; + // eslint-disable-next-line + continue; + } + + // this is to avoid recursion loop. So the graph should be direct graph rather than cyclic + // so for any recursion building if there is an entity two times same key meaning it will be looped + const recursionChainBreaker: Record = {}; + // eslint-disable-next-line + const expandedVal = await recursivelyExpandSecret( + expandedSecrets, + toBeExpandedSecrets, + fetchSecret, + recursionChainBreaker, + key + ); + + // eslint-disable-next-line + inputSecrets[key].value = inputSecrets[key].skipMultilineEncoding + ? formatMultiValueEnv(expandedVal) + : expandedVal; + } + + return inputSecrets; + }; + return expandSecrets; +}; + +export const reshapeBridgeSecret = ( + workspaceId: string, + environment: string, + secretPath: string, + secret: Omit & { + value?: string; + comment?: string; + tags?: { + id: string; + slug: string; + color?: string | null; + name: string; + }[]; + } +) => ({ + secretKey: secret.key, + secretPath, + workspace: workspaceId, + environment, + secretValue: secret.value, + secretComment: secret.comment, + version: secret.version, + type: secret.type, + _id: secret.id, + id: secret.id, + user: secret.userId, + tags: secret.tags, + skipMultilineEncoding: secret.skipMultilineEncoding, + secretReminderRepeatDays: secret.reminderRepeatDays, + secretReminderNote: secret.reminderNote, + metadata: secret.metadata, + createdAt: secret.createdAt, + updatedAt: secret.updatedAt +}); diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts new file mode 100644 index 000000000..34e568ca7 --- /dev/null +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts @@ -0,0 +1,1417 @@ +import { ForbiddenError, subject } from "@casl/ability"; + +import { ProjectMembershipRole, SecretsV2Schema, SecretType } from "@app/db/schemas"; +import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; +import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; +import { TSecretApprovalPolicyServiceFactory } from "@app/ee/services/secret-approval-policy/secret-approval-policy-service"; +import { TSecretApprovalRequestDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-dal"; +import { TSecretApprovalRequestSecretDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-secret-dal"; +import { TSecretSnapshotServiceFactory } from "@app/ee/services/secret-snapshot/secret-snapshot-service"; +import { BadRequestError, NotFoundError } from "@app/lib/errors"; +import { groupBy } from "@app/lib/fn"; +import { setKnexStringValue } from "@app/lib/knex"; +import { logger } from "@app/lib/logger"; +import { alphaNumericNanoId } from "@app/lib/nanoid"; + +import { ActorType } from "../auth/auth-type"; +import { TKmsServiceFactory } from "../kms/kms-service"; +import { KmsDataKey } from "../kms/kms-types"; +import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; +import { TSecretQueueFactory } from "../secret/secret-queue"; +import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; +import { TSecretImportDALFactory } from "../secret-import/secret-import-dal"; +import { fnSecretsV2FromImports } from "../secret-import/secret-import-fns"; +import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal"; +import { TSecretV2BridgeDALFactory } from "./secret-v2-bridge-dal"; +import { + expandSecretReferencesFactory, + fnSecretBulkDelete, + fnSecretBulkInsert, + fnSecretBulkUpdate, + getAllNestedSecretReferences, + recursivelyGetSecretPaths, + reshapeBridgeSecret +} from "./secret-v2-bridge-fns"; +import { + SecretOperations, + TBackFillSecretReferencesDTO, + TCreateManySecretDTO, + TCreateSecretDTO, + TDeleteManySecretDTO, + TDeleteSecretDTO, + TGetASecretDTO, + TGetSecretsDTO, + TGetSecretVersionsDTO, + TMoveSecretsDTO, + TUpdateManySecretDTO, + TUpdateSecretDTO +} from "./secret-v2-bridge-types"; +import { TSecretVersionV2DALFactory } from "./secret-version-dal"; +import { TSecretVersionV2TagDALFactory } from "./secret-version-tag-dal"; + +type TSecretV2BridgeServiceFactoryDep = { + secretDAL: TSecretV2BridgeDALFactory; + secretVersionDAL: TSecretVersionV2DALFactory; + kmsService: Pick; + secretVersionTagDAL: Pick; + secretTagDAL: TSecretTagDALFactory; + permissionService: Pick; + projectEnvDAL: Pick; + folderDAL: Pick< + TSecretFolderDALFactory, + "findBySecretPath" | "updateById" | "findById" | "findByManySecretPath" | "find" + >; + secretImportDAL: Pick; + secretQueueService: Pick; + secretApprovalPolicyService: Pick; + secretApprovalRequestDAL: Pick; + secretApprovalRequestSecretDAL: Pick< + TSecretApprovalRequestSecretDALFactory, + "insertV2Bridge" | "insertApprovalSecretV2Tags" + >; + snapshotService: Pick; +}; + +export type TSecretV2BridgeServiceFactory = ReturnType; + +/* + * This service is a bridge from our old architecture towards the new architecture + */ +export const secretV2BridgeServiceFactory = ({ + secretDAL, + projectEnvDAL, + secretTagDAL, + secretVersionDAL, + folderDAL, + permissionService, + snapshotService, + secretQueueService, + secretImportDAL, + secretVersionTagDAL, + secretApprovalPolicyService, + secretApprovalRequestDAL, + secretApprovalRequestSecretDAL, + kmsService +}: TSecretV2BridgeServiceFactoryDep) => { + const createSecret = async ({ + actor, + actorId, + actorOrgId, + environment, + actorAuthMethod, + projectId, + secretPath, + ...inputSecret + }: TCreateSecretDTO) => { + const { permission } = await permissionService.getProjectPermission( + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId + ); + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionActions.Create, + subject(ProjectPermissionSub.Secrets, { environment, secretPath }) + ); + + const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); + if (!folder) + throw new BadRequestError({ + message: "Folder not found for the given environment slug & secret path", + name: "Create secret" + }); + const folderId = folder.id; + + if (ActorType.USER !== actor && inputSecret.type === SecretType.Personal) { + throw new BadRequestError({ message: "Must be user to create personal secret" }); + } + + const doesSecretExist = await secretDAL.findOne({ + key: inputSecret.secretName, + type: SecretType.Shared, + folderId + }); + if (inputSecret.type === SecretType.Shared && doesSecretExist) + throw new BadRequestError({ message: "Secret already exist" }); + // if user creating personal check its shared also exist + if (inputSecret.type === SecretType.Personal && !doesSecretExist) { + throw new BadRequestError({ + message: "Failed to create personal secret override for no corresponding shared secret" + }); + } + + // validate tags + // fetch all tags and if not same count throw error meaning one was invalid tags + const tags = inputSecret.tagIds ? await secretTagDAL.find({ projectId, $in: { id: inputSecret.tagIds } }) : []; + if ((inputSecret.tagIds || []).length !== tags.length) throw new BadRequestError({ message: "Tag not found" }); + + const { secretName, type, ...el } = inputSecret; + const references = getAllNestedSecretReferences(inputSecret.secretValue); + const { encryptor: secretManagerEncryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId + }); + + const secret = await secretDAL.transaction((tx) => + fnSecretBulkInsert({ + folderId, + inputSecrets: [ + { + version: 1, + type, + reminderRepeatDays: el.secretReminderRepeatDays, + encryptedComment: setKnexStringValue( + el.secretComment, + (value) => secretManagerEncryptor({ plainText: Buffer.from(value) }).cipherTextBlob + ), + encryptedValue: el.secretValue + ? secretManagerEncryptor({ plainText: Buffer.from(el.secretValue) }).cipherTextBlob + : undefined, + reminderNote: el.secretReminderNote, + skipMultilineEncoding: el.skipMultilineEncoding, + key: secretName, + userId: inputSecret.type === SecretType.Personal ? actorId : null, + tagIds: inputSecret.tagIds, + references + } + ], + secretDAL, + secretVersionDAL, + secretTagDAL, + secretVersionTagDAL, + tx + }) + ); + + await snapshotService.performSnapshot(folderId); + await secretQueueService.syncSecrets({ + secretPath, + actorId, + actor, + projectId, + environmentSlug: folder.environment.slug + }); + + return reshapeBridgeSecret(projectId, environment, secretPath, { + ...secret[0], + value: inputSecret.secretValue, + comment: inputSecret.secretComment + }); + }; + + const updateSecret = async ({ + actor, + actorId, + actorOrgId, + environment, + actorAuthMethod, + projectId, + secretPath, + ...inputSecret + }: TUpdateSecretDTO) => { + const { permission } = await permissionService.getProjectPermission( + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId + ); + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionActions.Edit, + subject(ProjectPermissionSub.Secrets, { environment, secretPath }) + ); + + if (inputSecret.newSecretName === "") { + throw new BadRequestError({ message: "New secret name cannot be empty" }); + } + + const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); + if (!folder) + throw new BadRequestError({ + message: "Folder not found for the given environment slug & secret path", + name: "UpdateSecret" + }); + const folderId = folder.id; + + if (ActorType.USER !== actor && inputSecret.type === SecretType.Personal) { + throw new BadRequestError({ message: "Must be user to create personal secret" }); + } + + if (inputSecret.newSecretName && inputSecret.type === SecretType.Personal) { + throw new BadRequestError({ message: "Personal secret cannot change the key name" }); + } + + let secret; + let secretId: string; + if (inputSecret.type === SecretType.Personal) { + const personalSecretToModify = await secretDAL.findOne({ + key: inputSecret.secretName, + type: SecretType.Personal, + folderId, + userId: actorId + }); + if (!personalSecretToModify) throw new BadRequestError({ message: "Secret not found" }); + secretId = personalSecretToModify.id; + secret = personalSecretToModify; + } else { + const sharedSecretToModify = await secretDAL.findOne({ + key: inputSecret.secretName, + type: SecretType.Shared, + folderId + }); + if (!sharedSecretToModify) throw new BadRequestError({ message: "Secret not found" }); + secretId = sharedSecretToModify.id; + secret = sharedSecretToModify; + } + + if (inputSecret.newSecretName) { + const doesNewNameSecretExist = await secretDAL.findOne({ + key: inputSecret.newSecretName, + type: SecretType.Shared, + folderId + }); + if (doesNewNameSecretExist) throw new BadRequestError({ message: "Secret with the new name already exist" }); + } + + // validate tags + // fetch all tags and if not same count throw error meaning one was invalid tags + const tags = inputSecret.tagIds ? await secretTagDAL.find({ projectId, $in: { id: inputSecret.tagIds } }) : []; + if ((inputSecret.tagIds || []).length !== tags.length) throw new BadRequestError({ message: "Tag not found" }); + + const { secretName, secretValue } = inputSecret; + + const { encryptor: secretManagerEncryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId + }); + const encryptedValue = + typeof secretValue !== "undefined" + ? { + encryptedValue: secretManagerEncryptor({ plainText: Buffer.from(secretValue) }).cipherTextBlob, + references: getAllNestedSecretReferences(secretValue) + } + : {}; + + const updatedSecret = await secretDAL.transaction(async (tx) => + fnSecretBulkUpdate({ + folderId, + inputSecrets: [ + { + filter: { id: secretId }, + data: { + reminderRepeatDays: inputSecret.secretReminderRepeatDays, + encryptedComment: setKnexStringValue( + inputSecret.secretComment, + (value) => secretManagerEncryptor({ plainText: Buffer.from(value) }).cipherTextBlob + ), + reminderNote: inputSecret.secretReminderNote, + skipMultilineEncoding: inputSecret.skipMultilineEncoding, + key: inputSecret.newSecretName || secretName, + tags: inputSecret.tagIds, + ...encryptedValue + } + } + ], + secretDAL, + secretVersionDAL, + secretTagDAL, + secretVersionTagDAL, + tx + }) + ); + await secretQueueService.handleSecretReminder({ + newSecret: { + id: updatedSecret[0].id, + ...inputSecret + }, + oldSecret: secret, + projectId + }); + + await snapshotService.performSnapshot(folderId); + await secretQueueService.syncSecrets({ + actor, + actorId, + secretPath, + projectId, + environmentSlug: folder.environment.slug + }); + return reshapeBridgeSecret(projectId, environment, secretPath, { + ...updatedSecret[0], + value: inputSecret.secretValue, + comment: inputSecret.secretComment + }); + }; + + const deleteSecret = async ({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + environment, + projectId, + secretPath, + ...inputSecret + }: TDeleteSecretDTO) => { + const { permission } = await permissionService.getProjectPermission( + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId + ); + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionActions.Delete, + subject(ProjectPermissionSub.Secrets, { environment, secretPath }) + ); + + const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); + if (!folder) + throw new BadRequestError({ + message: "Folder not found for the given environment slug & secret path", + name: "Delete secret" + }); + const folderId = folder.id; + + if (ActorType.USER !== actor && inputSecret.type === SecretType.Personal) { + throw new BadRequestError({ message: "Must be user to delete personal secret" }); + } + + const deletedSecret = await secretDAL.transaction(async (tx) => + fnSecretBulkDelete({ + projectId, + folderId, + actorId, + secretDAL, + secretQueueService, + inputSecrets: [ + { + type: inputSecret.type as SecretType, + secretKey: inputSecret.secretName + } + ], + tx + }) + ); + + await snapshotService.performSnapshot(folderId); + await secretQueueService.syncSecrets({ + actor, + actorId, + secretPath, + projectId, + environmentSlug: folder.environment.slug + }); + + const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId + }); + return reshapeBridgeSecret(projectId, environment, secretPath, { + ...deletedSecret[0], + value: deletedSecret[0].encryptedValue + ? secretManagerDecryptor({ cipherTextBlob: deletedSecret[0].encryptedValue }).toString() + : undefined, + comment: deletedSecret[0].encryptedComment + ? secretManagerDecryptor({ cipherTextBlob: deletedSecret[0].encryptedComment }).toString() + : undefined + }); + }; + + const getSecrets = async ({ + actorId, + path, + environment, + projectId, + actor, + actorOrgId, + actorAuthMethod, + includeImports, + recursive, + expandSecretReferences: shouldExpandSecretReferences + }: TGetSecretsDTO) => { + const { permission } = await permissionService.getProjectPermission( + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId + ); + + let paths: { folderId: string; path: string }[] = []; + + if (recursive) { + const deepPaths = await recursivelyGetSecretPaths({ + folderDAL, + projectEnvDAL, + projectId, + environment, + currentPath: path, + hasAccess: (permissionEnvironment, permissionSecretPath) => + permission.can( + ProjectPermissionActions.Read, + subject(ProjectPermissionSub.Secrets, { + environment: permissionEnvironment, + secretPath: permissionSecretPath + }) + ) + }); + + if (!deepPaths) return { secrets: [], imports: [] }; + + paths = deepPaths.map(({ folderId, path: p }) => ({ folderId, path: p })); + } else { + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionActions.Read, + subject(ProjectPermissionSub.Secrets, { environment, secretPath: path }) + ); + + const folder = await folderDAL.findBySecretPath(projectId, environment, path); + if (!folder) return { secrets: [], imports: [] }; + + paths = [{ folderId: folder.id, path }]; + } + + const groupedPaths = groupBy(paths, (p) => p.folderId); + + const secrets = await secretDAL.findByFolderIds( + paths.map((p) => p.folderId), + actorId + ); + + const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId + }); + + const decryptedSecrets = secrets.map((secret) => + reshapeBridgeSecret(projectId, environment, groupedPaths[secret.folderId][0].path, { + ...secret, + value: secret.encryptedValue + ? secretManagerDecryptor({ cipherTextBlob: secret.encryptedValue }).toString() + : undefined, + comment: secret.encryptedComment + ? secretManagerDecryptor({ cipherTextBlob: secret.encryptedComment }).toString() + : undefined + }) + ); + const expandSecretReferences = expandSecretReferencesFactory({ + projectId, + folderDAL, + secretDAL, + decryptSecretValue: (value) => (value ? secretManagerDecryptor({ cipherTextBlob: value }).toString() : undefined) + }); + + if (shouldExpandSecretReferences) { + const secretsGroupByPath = groupBy(decryptedSecrets, (i) => i.secretPath); + for (const secretPathKey in secretsGroupByPath) { + if (Object.hasOwn(secretsGroupByPath, secretPathKey)) { + const secretsGroupByKey = secretsGroupByPath[secretPathKey].reduce( + (acc, item) => { + acc[item.secretKey] = { + value: item.secretValue, + comment: item.secretComment, + skipMultilineEncoding: item.skipMultilineEncoding + }; + return acc; + }, + {} as Record + ); + // eslint-disable-next-line + await expandSecretReferences(secretsGroupByKey); + secretsGroupByPath[secretPathKey].forEach((decryptedSecret) => { + // eslint-disable-next-line no-param-reassign + decryptedSecret.secretValue = secretsGroupByKey[decryptedSecret.secretKey].value; + }); + } + } + } + + if (!includeImports) { + return { + secrets: decryptedSecrets + }; + } + + const secretImports = await secretImportDAL.findByFolderIds(paths.map((p) => p.folderId)); + const allowedImports = secretImports.filter(({ importEnv, importPath, isReplication }) => + !isReplication && + // if its service token allow full access over imported one + actor === ActorType.SERVICE + ? true + : permission.can( + ProjectPermissionActions.Read, + subject(ProjectPermissionSub.Secrets, { + environment: importEnv.slug, + secretPath: importPath + }) + ) + ); + const importedSecrets = await fnSecretsV2FromImports({ + allowedImports, + secretDAL, + folderDAL, + secretImportDAL, + expandSecretReferences, + decryptor: (value) => (value ? secretManagerDecryptor({ cipherTextBlob: value }).toString() : undefined) + }); + + return { + secrets: decryptedSecrets, + imports: importedSecrets + }; + }; + + const getSecretByName = async ({ + actorId, + actor, + actorOrgId, + actorAuthMethod, + projectId, + environment, + path, + type, + secretName, + version, + includeImports, + expandSecretReferences: shouldExpandSecretReferences + }: TGetASecretDTO) => { + const { permission } = await permissionService.getProjectPermission( + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId + ); + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionActions.Read, + subject(ProjectPermissionSub.Secrets, { environment, secretPath: path }) + ); + const folder = await folderDAL.findBySecretPath(projectId, environment, path); + if (!folder) + throw new BadRequestError({ + message: "Folder not found for the given environment slug & secret path", + name: "Create secret" + }); + const folderId = folder.id; + + let secretType = type; + if (actor === ActorType.SERVICE) { + logger.info( + `secretServiceFactory: overriding secret type for service token [projectId=${projectId}] [factoryFunctionName=getSecretByName]` + ); + secretType = SecretType.Shared; + } + + const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId + }); + + const secret = await (version === undefined + ? secretDAL.findOneWithTags({ + folderId, + type: secretType, + key: secretName, + userId: secretType === SecretType.Personal ? actorId : null + }) + : secretVersionDAL + .findOne({ + folderId, + type: secretType, + userId: secretType === SecretType.Personal ? actorId : null, + key: secretName + }) + .then((el) => SecretsV2Schema.parse({ ...el, id: el.secretId }))); + + const expandSecretReferences = expandSecretReferencesFactory({ + projectId, + folderDAL, + secretDAL, + decryptSecretValue: (value) => (value ? secretManagerDecryptor({ cipherTextBlob: value }).toString() : undefined) + }); + + // now if secret is not found + // then search for imported secrets + // here we consider the import order also thus starting from bottom + if (!secret && includeImports) { + const secretImports = await secretImportDAL.find({ folderId, isReplication: false }); + const allowedImports = secretImports.filter(({ importEnv, importPath }) => + // if its service token allow full access over imported one + actor === ActorType.SERVICE + ? true + : permission.can( + ProjectPermissionActions.Read, + subject(ProjectPermissionSub.Secrets, { + environment: importEnv.slug, + secretPath: importPath + }) + ) + ); + const importedSecrets = await fnSecretsV2FromImports({ + allowedImports, + secretDAL, + folderDAL, + secretImportDAL, + decryptor: (value) => (value ? secretManagerDecryptor({ cipherTextBlob: value }).toString() : undefined), + expandSecretReferences: shouldExpandSecretReferences ? expandSecretReferences : undefined + }); + + for (let i = importedSecrets.length - 1; i >= 0; i -= 1) { + for (let j = 0; j < importedSecrets[i].secrets.length; j += 1) { + const importedSecret = importedSecrets[i].secrets[j]; + if (secretName === importedSecret.key) { + return reshapeBridgeSecret( + projectId, + importedSecrets[i].environment, + importedSecrets[i].secretPath, + importedSecret + ); + } + } + } + } + if (!secret) throw new BadRequestError({ message: "Secret not found" }); + + let secretValue = secret.encryptedValue + ? secretManagerDecryptor({ cipherTextBlob: secret.encryptedValue }).toString() + : undefined; + if (shouldExpandSecretReferences && secretValue) { + const secretReferenceExpandedRecord = { + [secret.key]: { value: secretValue } + }; + // eslint-disable-next-line + await expandSecretReferences(secretReferenceExpandedRecord); + secretValue = secretReferenceExpandedRecord[secret.key].value; + } + + return reshapeBridgeSecret(projectId, environment, path, { + ...secret, + value: secretValue, + comment: secret.encryptedComment + ? secretManagerDecryptor({ cipherTextBlob: secret.encryptedComment }).toString() + : undefined + }); + }; + + const createManySecret = async ({ + secretPath, + actor, + actorId, + actorAuthMethod, + actorOrgId, + environment, + projectId, + secrets: inputSecrets + }: TCreateManySecretDTO) => { + const { permission } = await permissionService.getProjectPermission( + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId + ); + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionActions.Create, + subject(ProjectPermissionSub.Secrets, { environment, secretPath }) + ); + + const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); + if (!folder) + throw new BadRequestError({ + message: "Folder not found for the given environment slug & secret path", + name: "Create secret" + }); + const folderId = folder.id; + + const secrets = await secretDAL.findBySecretKeys( + folderId, + inputSecrets.map((el) => ({ + key: el.secretKey, + type: SecretType.Shared + })) + ); + if (secrets.length) + throw new BadRequestError({ message: `Secret already exist: ${secrets.map((el) => el.key).join(",")}` }); + + // get all tags + const sanitizedTagIds = inputSecrets.flatMap(({ tagIds = [] }) => tagIds); + const tags = sanitizedTagIds.length ? await secretTagDAL.findManyTagsById(projectId, sanitizedTagIds) : []; + if (tags.length !== sanitizedTagIds.length) throw new BadRequestError({ message: "Tag not found" }); + + const { encryptor: secretManagerEncryptor, decryptor: secretManagerDecryptor } = + await kmsService.createCipherPairWithDataKey({ type: KmsDataKey.SecretManager, projectId }); + + const newSecrets = await secretDAL.transaction(async (tx) => + fnSecretBulkInsert({ + inputSecrets: inputSecrets.map((el) => ({ + version: 1, + encryptedComment: setKnexStringValue( + el.secretComment, + (value) => secretManagerEncryptor({ plainText: Buffer.from(value) }).cipherTextBlob + ), + encryptedValue: el.secretValue + ? secretManagerEncryptor({ plainText: Buffer.from(el.secretValue) }).cipherTextBlob + : undefined, + skipMultilineEncoding: el.skipMultilineEncoding, + key: el.secretKey, + tagIds: el.tagIds, + references: getAllNestedSecretReferences(el.secretValue), + type: SecretType.Shared + })), + folderId, + secretDAL, + secretVersionDAL, + secretTagDAL, + secretVersionTagDAL, + tx + }) + ); + + await snapshotService.performSnapshot(folderId); + await secretQueueService.syncSecrets({ + actor, + actorId, + secretPath, + projectId, + environmentSlug: folder.environment.slug + }); + + return newSecrets.map((el) => + reshapeBridgeSecret(projectId, environment, secretPath, { + ...el, + value: el.encryptedValue ? secretManagerDecryptor({ cipherTextBlob: el.encryptedValue }).toString() : undefined, + comment: el.encryptedComment + ? secretManagerDecryptor({ cipherTextBlob: el.encryptedComment }).toString() + : undefined + }) + ); + }; + + const updateManySecret = async ({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + environment, + projectId, + secretPath, + secrets: inputSecrets + }: TUpdateManySecretDTO) => { + const { permission } = await permissionService.getProjectPermission( + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId + ); + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionActions.Edit, + subject(ProjectPermissionSub.Secrets, { environment, secretPath }) + ); + + const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); + if (!folder) + throw new BadRequestError({ + message: "Folder not found for the given environment slug & secret path", + name: "Update secret" + }); + const folderId = folder.id; + + const secretsToUpdate = await secretDAL.findBySecretKeys( + folderId, + inputSecrets.map((el) => ({ + key: el.secretKey, + type: SecretType.Shared + })) + ); + if (secretsToUpdate.length !== inputSecrets.length) + throw new BadRequestError({ message: `Secret not exist: ${secretsToUpdate.map((el) => el.key).join(",")}` }); + const secretsToUpdateInDBGroupedByKey = groupBy(secretsToUpdate, (i) => i.key); + + // now find any secret that needs to update its name + // same process as above + const secretsWithNewName = inputSecrets.filter(({ newSecretName }) => Boolean(newSecretName)); + if (secretsWithNewName.length) { + const secrets = await secretDAL.findBySecretKeys( + folderId, + secretsWithNewName.map((el) => ({ + key: el.newSecretName as string, + type: SecretType.Shared + })) + ); + if (secrets.length) + throw new BadRequestError({ + message: `Secret with new name exist: ${secretsWithNewName.map((el) => el.newSecretName).join(",")}` + }); + } + + // get all tags + const sanitizedTagIds = inputSecrets.flatMap(({ tagIds = [] }) => tagIds); + const tags = sanitizedTagIds.length ? await secretTagDAL.findManyTagsById(projectId, sanitizedTagIds) : []; + if (tags.length !== sanitizedTagIds.length) throw new BadRequestError({ message: "Tag not found" }); + + const { encryptor: secretManagerEncryptor, decryptor: secretManagerDecryptor } = + await kmsService.createCipherPairWithDataKey({ type: KmsDataKey.SecretManager, projectId }); + + const secrets = await secretDAL.transaction(async (tx) => + fnSecretBulkUpdate({ + folderId, + tx, + inputSecrets: inputSecrets.map((el) => { + const originalSecret = secretsToUpdateInDBGroupedByKey[el.secretKey][0]; + const encryptedValue = + typeof el.secretValue !== "undefined" + ? { + encryptedValue: secretManagerEncryptor({ plainText: Buffer.from(el.secretValue) }).cipherTextBlob, + references: getAllNestedSecretReferences(el.secretValue) + } + : {}; + return { + filter: { id: originalSecret.id, type: SecretType.Shared }, + data: { + reminderRepeatDays: el.secretReminderRepeatDays, + encryptedComment: setKnexStringValue( + el.secretComment, + (value) => secretManagerEncryptor({ plainText: Buffer.from(value) }).cipherTextBlob + ), + reminderNote: el.secretReminderNote, + skipMultilineEncoding: el.skipMultilineEncoding, + key: el.newSecretName || el.secretKey, + tags: el.tagIds, + ...encryptedValue + } + }; + }), + secretDAL, + secretVersionDAL, + secretTagDAL, + secretVersionTagDAL + }) + ); + await snapshotService.performSnapshot(folderId); + await secretQueueService.syncSecrets({ + actor, + actorId, + secretPath, + projectId, + environmentSlug: folder.environment.slug + }); + + return secrets.map((el) => + reshapeBridgeSecret(projectId, environment, secretPath, { + ...el, + value: el.encryptedValue ? secretManagerDecryptor({ cipherTextBlob: el.encryptedValue }).toString() : undefined, + comment: el.encryptedComment + ? secretManagerDecryptor({ cipherTextBlob: el.encryptedComment }).toString() + : undefined + }) + ); + }; + + const deleteManySecret = async ({ + secrets: inputSecrets, + secretPath, + environment, + projectId, + actor, + actorId, + actorAuthMethod, + actorOrgId + }: TDeleteManySecretDTO) => { + const { permission } = await permissionService.getProjectPermission( + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId + ); + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionActions.Delete, + subject(ProjectPermissionSub.Secrets, { environment, secretPath }) + ); + + const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); + if (!folder) + throw new BadRequestError({ + message: "Folder not found for the given environment slug & secret path", + name: "Create secret" + }); + const folderId = folder.id; + + const secretsToDelete = await secretDAL.findBySecretKeys( + folderId, + inputSecrets.map((el) => ({ + key: el.secretKey, + type: SecretType.Shared + })) + ); + if (secretsToDelete.length !== inputSecrets.length) + throw new BadRequestError({ message: `Secret not exist: ${secretsToDelete.map((el) => el.key).join(",")}` }); + + const secretsDeleted = await secretDAL.transaction(async (tx) => + fnSecretBulkDelete({ + secretDAL, + secretQueueService, + inputSecrets: inputSecrets.map(({ type, secretKey }) => ({ + secretKey, + type: type || SecretType.Shared + })), + projectId, + folderId, + actorId, + tx + }) + ); + + // await snapshotService.performSnapshot(folderId); + await secretQueueService.syncSecrets({ + actor, + actorId, + secretPath, + projectId, + environmentSlug: folder.environment.slug + }); + + const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId + }); + return secretsDeleted.map((el) => + reshapeBridgeSecret(projectId, environment, secretPath, { + ...el, + value: el.encryptedValue ? secretManagerDecryptor({ cipherTextBlob: el.encryptedValue }).toString() : undefined, + comment: el.encryptedComment + ? secretManagerDecryptor({ cipherTextBlob: el.encryptedComment }).toString() + : undefined + }) + ); + }; + + const getSecretVersions = async ({ + actorId, + actor, + actorOrgId, + actorAuthMethod, + limit = 20, + offset = 0, + secretId + }: TGetSecretVersionsDTO) => { + const secret = await secretDAL.findById(secretId); + if (!secret) throw new BadRequestError({ message: "Failed to find secret" }); + + const folder = await folderDAL.findById(secret.folderId); + if (!folder) throw new BadRequestError({ message: "Failed to find secret" }); + + const { permission } = await permissionService.getProjectPermission( + actor, + actorId, + folder.projectId, + actorAuthMethod, + actorOrgId + ); + ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretRollback); + const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId: folder.projectId + }); + const secretVersions = await secretVersionDAL.find({ secretId }, { offset, limit, sort: [["createdAt", "desc"]] }); + return secretVersions.map((el) => + reshapeBridgeSecret(folder.projectId, folder.environment.envSlug, "/", { + ...el, + value: el.encryptedValue ? secretManagerDecryptor({ cipherTextBlob: el.encryptedValue }).toString() : undefined, + comment: el.encryptedComment + ? secretManagerDecryptor({ cipherTextBlob: el.encryptedComment }).toString() + : undefined + }) + ); + }; + + // this is a backfilling API for secret references + // what it does is it will go through all the secret values and parse all references + // populate the secret reference to do sync integrations + const backfillSecretReferences = async ({ + projectId, + actor, + actorId, + actorOrgId, + actorAuthMethod + }: TBackFillSecretReferencesDTO) => { + const { hasRole } = await permissionService.getProjectPermission( + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId + ); + + if (!hasRole(ProjectMembershipRole.Admin)) + throw new BadRequestError({ message: "Only admins are allowed to take this action" }); + + const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId + }); + await secretDAL.transaction(async (tx) => { + const secrets = await secretDAL.findAllProjectSecretValues(projectId, tx); + await secretDAL.upsertSecretReferences( + secrets + .filter((el) => Boolean(el.encryptedValue)) + .map(({ id, encryptedValue }) => ({ + secretId: id, + references: encryptedValue + ? getAllNestedSecretReferences(secretManagerDecryptor({ cipherTextBlob: encryptedValue }).toString()) + : [] + })), + tx + ); + }); + + return { message: "Successfully backfilled secret references" }; + }; + + const moveSecrets = async ({ + sourceEnvironment, + sourceSecretPath, + destinationEnvironment, + destinationSecretPath, + secretIds, + projectId, + shouldOverwrite, + actor, + actorId, + actorAuthMethod, + actorOrgId + }: TMoveSecretsDTO) => { + const { permission } = await permissionService.getProjectPermission( + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId + ); + + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionActions.Delete, + subject(ProjectPermissionSub.Secrets, { environment: sourceEnvironment, secretPath: sourceSecretPath }) + ); + + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionActions.Create, + subject(ProjectPermissionSub.Secrets, { environment: destinationEnvironment, secretPath: destinationSecretPath }) + ); + + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionActions.Edit, + subject(ProjectPermissionSub.Secrets, { environment: destinationEnvironment, secretPath: destinationSecretPath }) + ); + + const sourceFolder = await folderDAL.findBySecretPath(projectId, sourceEnvironment, sourceSecretPath); + if (!sourceFolder) { + throw new NotFoundError({ + message: "Source path does not exist." + }); + } + + const destinationFolder = await folderDAL.findBySecretPath( + projectId, + destinationEnvironment, + destinationSecretPath + ); + + if (!destinationFolder) { + throw new NotFoundError({ + message: "Destination path does not exist." + }); + } + + const sourceSecrets = await secretDAL.find({ + type: SecretType.Shared, + $in: { + id: secretIds + } + }); + + if (sourceSecrets.length !== secretIds.length) { + throw new BadRequestError({ + message: "Invalid secrets" + }); + } + + const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId + }); + const decryptedSourceSecrets = sourceSecrets.map((secret) => ({ + ...secret, + value: secret.encryptedValue + ? secretManagerDecryptor({ cipherTextBlob: secret.encryptedValue }).toString() + : undefined + })); + + let isSourceUpdated = false; + let isDestinationUpdated = false; + + // Moving secrets is a two-step process. + await secretDAL.transaction(async (tx) => { + // First step is to create/update the secret in the destination: + const destinationSecretsFromDB = await secretDAL.find( + { + folderId: destinationFolder.id + }, + { tx } + ); + + const decryptedDestinationSecrets = destinationSecretsFromDB.map((secret) => { + return { + ...secret, + value: secret.encryptedValue + ? secretManagerDecryptor({ cipherTextBlob: secret.encryptedValue }).toString() + : undefined + }; + }); + + const destinationSecretsGroupedByKey = groupBy(decryptedDestinationSecrets, (i) => i.key); + + const locallyCreatedSecrets = decryptedSourceSecrets + .filter(({ key }) => !destinationSecretsGroupedByKey[key]?.[0]) + .map((el) => ({ ...el, operation: SecretOperations.Create })); + + const locallyUpdatedSecrets = decryptedSourceSecrets + .filter( + ({ key, value }) => + destinationSecretsGroupedByKey[key]?.[0] && destinationSecretsGroupedByKey[key]?.[0]?.value !== value + ) + .map((el) => ({ ...el, operation: SecretOperations.Update })); + + if (locallyUpdatedSecrets.length > 0 && !shouldOverwrite) { + const existingKeys = locallyUpdatedSecrets.map((s) => s.key); + + throw new BadRequestError({ + message: `Failed to move secrets. The following secrets already exist in the destination: ${existingKeys.join( + "," + )}` + }); + } + + const isEmpty = locallyCreatedSecrets.length + locallyUpdatedSecrets.length === 0; + + if (isEmpty) { + throw new BadRequestError({ + message: "Selected secrets already exist in the destination." + }); + } + const destinationFolderPolicy = await secretApprovalPolicyService.getSecretApprovalPolicy( + projectId, + destinationFolder.environment.slug, + destinationFolder.path + ); + + if (destinationFolderPolicy && actor === ActorType.USER) { + // if secret approval policy exists for destination, we create the secret approval request + const localSecretsIds = decryptedDestinationSecrets.map(({ id }) => id); + const latestSecretVersions = await secretVersionDAL.findLatestVersionMany( + destinationFolder.id, + localSecretsIds, + tx + ); + + const approvalRequestDoc = await secretApprovalRequestDAL.create( + { + folderId: destinationFolder.id, + slug: alphaNumericNanoId(), + policyId: destinationFolderPolicy.id, + status: "open", + hasMerged: false, + committerUserId: actorId + }, + tx + ); + + const commits = locallyCreatedSecrets.concat(locallyUpdatedSecrets).map((doc) => { + const { operation } = doc; + const localSecret = destinationSecretsGroupedByKey[doc.key]?.[0]; + + return { + op: operation, + requestId: approvalRequestDoc.id, + metadata: doc.metadata, + key: doc.key, + encryptedValue: doc.encryptedValue, + encryptedComment: doc.encryptedComment, + skipMultilineEncoding: doc.skipMultilineEncoding, + // except create operation other two needs the secret id and version id + ...(operation !== SecretOperations.Create + ? { secretId: localSecret.id, secretVersion: latestSecretVersions[localSecret.id].id } + : {}) + }; + }); + await secretApprovalRequestSecretDAL.insertV2Bridge(commits, tx); + } else { + // apply changes directly + if (locallyCreatedSecrets.length) { + await fnSecretBulkInsert({ + folderId: destinationFolder.id, + secretVersionDAL, + secretDAL, + tx, + secretTagDAL, + secretVersionTagDAL, + inputSecrets: locallyCreatedSecrets.map((doc) => { + return { + type: doc.type, + metadata: doc.metadata, + key: doc.key, + encryptedValue: doc.encryptedValue, + encryptedComment: doc.encryptedComment, + skipMultilineEncoding: doc.skipMultilineEncoding, + reminderNote: doc.reminderNote, + reminderRepeatDays: doc.reminderRepeatDays, + references: doc.value ? getAllNestedSecretReferences(doc.value) : [] + }; + }) + }); + } + if (locallyUpdatedSecrets.length) { + await fnSecretBulkUpdate({ + folderId: destinationFolder.id, + secretVersionDAL, + secretDAL, + tx, + secretTagDAL, + secretVersionTagDAL, + inputSecrets: locallyUpdatedSecrets.map((doc) => { + return { + filter: { + folderId: destinationFolder.id, + id: destinationSecretsGroupedByKey[doc.key][0].id + }, + data: { + metadata: doc.metadata, + key: doc.key, + encryptedComment: doc.encryptedComment, + skipMultilineEncoding: doc.skipMultilineEncoding, + reminderNote: doc.reminderNote, + reminderRepeatDays: doc.reminderRepeatDays, + ...(doc.encryptedValue + ? { + encryptedValue: doc.encryptedValue, + references: doc.value ? getAllNestedSecretReferences(doc.value) : [] + } + : { + encryptedValue: undefined, + references: undefined + }) + } + }; + }) + }); + } + + isDestinationUpdated = true; + } + + // Next step is to delete the secrets from the source folder: + const sourceSecretsGroupByKey = groupBy(sourceSecrets, (i) => i.key); + const locallyDeletedSecrets = decryptedSourceSecrets.map((el) => ({ ...el, operation: SecretOperations.Delete })); + + const sourceFolderPolicy = await secretApprovalPolicyService.getSecretApprovalPolicy( + projectId, + sourceFolder.environment.slug, + sourceFolder.path + ); + + if (sourceFolderPolicy && actor === ActorType.USER) { + // if secret approval policy exists for source, we create the secret approval request + const localSecretsIds = decryptedSourceSecrets.map(({ id }) => id); + const latestSecretVersions = await secretVersionDAL.findLatestVersionMany(sourceFolder.id, localSecretsIds, tx); + const approvalRequestDoc = await secretApprovalRequestDAL.create( + { + folderId: sourceFolder.id, + slug: alphaNumericNanoId(), + policyId: sourceFolderPolicy.id, + status: "open", + hasMerged: false, + committerUserId: actorId + }, + tx + ); + + const commits = locallyDeletedSecrets.map((doc) => { + const { operation } = doc; + const localSecret = sourceSecretsGroupByKey[doc.key]?.[0]; + + return { + op: operation, + requestId: approvalRequestDoc.id, + metadata: doc.metadata, + key: doc.key, + encryptedComment: doc.encryptedComment, + encryptedValue: doc.encryptedValue, + skipMultilineEncoding: doc.skipMultilineEncoding, + secretId: localSecret.id, + secretVersion: latestSecretVersions[localSecret.id].id + }; + }); + + await secretApprovalRequestSecretDAL.insertV2Bridge(commits, tx); + } else { + // if no secret approval policy is present, we delete directly. + await secretDAL.delete( + { + $in: { + id: locallyDeletedSecrets.map(({ id }) => id) + }, + folderId: sourceFolder.id + }, + tx + ); + + isSourceUpdated = true; + } + }); + + if (isDestinationUpdated) { + await snapshotService.performSnapshot(destinationFolder.id); + await secretQueueService.syncSecrets({ + projectId, + secretPath: destinationFolder.path, + environmentSlug: destinationFolder.environment.slug, + actorId, + actor + }); + } + + if (isSourceUpdated) { + await snapshotService.performSnapshot(sourceFolder.id); + await secretQueueService.syncSecrets({ + projectId, + secretPath: sourceFolder.path, + environmentSlug: sourceFolder.environment.slug, + actorId, + actor + }); + } + + return { + projectId, + isSourceUpdated, + isDestinationUpdated + }; + }; + + return { + createSecret, + deleteSecret, + updateSecret, + createManySecret, + updateManySecret, + deleteManySecret, + getSecretByName, + getSecrets, + getSecretVersions, + backfillSecretReferences, + moveSecrets + }; +}; diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts new file mode 100644 index 000000000..1964aac05 --- /dev/null +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts @@ -0,0 +1,267 @@ +import { Knex } from "knex"; + +import { SecretType, TSecretsV2, TSecretsV2Insert, TSecretsV2Update } from "@app/db/schemas"; +import { TProjectPermission } from "@app/lib/types"; +import { TProjectDALFactory } from "@app/services/project/project-dal"; +import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal"; +import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; + +import { TSecretV2BridgeDALFactory } from "./secret-v2-bridge-dal"; +import { TSecretVersionV2DALFactory } from "./secret-version-dal"; +import { TSecretVersionV2TagDALFactory } from "./secret-version-tag-dal"; + +type TPartialSecret = Pick; + +type TPartialInputSecret = Pick; + +export type TGetSecretsDTO = { + expandSecretReferences?: boolean; + path: string; + environment: string; + includeImports?: boolean; + recursive?: boolean; +} & TProjectPermission; + +export type TGetASecretDTO = { + secretName: string; + path: string; + environment: string; + expandSecretReferences?: boolean; + type: "shared" | "personal"; + includeImports?: boolean; + version?: number; + projectId: string; +} & Omit; + +export type TCreateSecretDTO = TProjectPermission & { + secretName: string; + secretPath: string; + environment: string; + secretValue: string; + type: SecretType; + tagIds?: string[]; + secretComment?: string; + skipMultilineEncoding?: boolean; + secretReminderRepeatDays?: number | null; + secretReminderNote?: string | null; +}; + +export type TUpdateSecretDTO = TProjectPermission & { + secretPath: string; + environment: string; + secretName: string; + secretValue?: string; + newSecretName?: string; + secretComment?: string; + type: SecretType; + tagIds?: string[]; + skipMultilineEncoding?: boolean; + secretReminderRepeatDays?: number | null; + secretReminderNote?: string | null; + metadata?: { + source?: string; + }; +}; + +export type TDeleteSecretDTO = TProjectPermission & { + secretPath: string; + environment: string; + secretName: string; + type: SecretType; +}; + +export type TCreateManySecretDTO = Omit & { + secretPath: string; + projectId: string; + environment: string; + secrets: { + secretKey: string; + secretValue: string; + secretComment?: string; + skipMultilineEncoding?: boolean; + tagIds?: string[]; + metadata?: { + source?: string; + }; + }[]; +}; + +export type TUpdateManySecretDTO = Omit & { + secretPath: string; + projectId: string; + environment: string; + secrets: { + secretKey: string; + newSecretName?: string; + secretValue: string; + secretComment?: string; + skipMultilineEncoding?: boolean; + tagIds?: string[]; + secretReminderRepeatDays?: number | null; + secretReminderNote?: string | null; + }[]; +}; + +export type TDeleteManySecretDTO = Omit & { + secretPath: string; + projectId: string; + environment: string; + secrets: { + secretKey: string; + type?: SecretType; + }[]; +}; + +export type TGetSecretVersionsDTO = Omit & { + limit?: number; + offset?: number; + secretId: string; +}; + +export type TSecretReference = { environment: string; secretPath: string; secretKey: string }; + +export type TFnSecretBulkInsert = { + folderId: string; + tx?: Knex; + inputSecrets: Array & { tagIds?: string[]; references: TSecretReference[] }>; + secretDAL: Pick; + secretVersionDAL: Pick; + secretTagDAL: Pick; + secretVersionTagDAL: Pick; +}; + +type TRequireReferenceIfValue = + | (Omit & { + encryptedValue: Buffer | null; + references: TSecretReference[]; + }) + | (Omit & { + encryptedValue?: never; + references?: never; + }); + +export type TFnSecretBulkUpdate = { + folderId: string; + inputSecrets: { + filter: Partial; + data: TRequireReferenceIfValue & { tags?: string[] }; + }[]; + secretDAL: Pick; + secretVersionDAL: Pick; + secretTagDAL: Pick; + secretVersionTagDAL: Pick; + tx?: Knex; +}; + +export type TFnSecretBulkDelete = { + folderId: string; + projectId: string; + inputSecrets: Array<{ type: SecretType; secretKey: string }>; + actorId: string; + tx?: Knex; + secretDAL: Pick; + secretQueueService: { + removeSecretReminder: (data: TRemoveSecretReminderDTO) => Promise; + }; +}; + +export type THandleReminderDTO = { + newSecret: TPartialInputSecret; + oldSecret: TPartialSecret; + projectId: string; +}; + +export type TCreateSecretReminderDTO = { + oldSecret: TPartialSecret; + newSecret: TPartialSecret; + projectId: string; +}; + +export type TRemoveSecretReminderDTO = { + secretId: string; + repeatDays: number; +}; + +export type TBackFillSecretReferencesDTO = TProjectPermission; + +export type TCreateManySecretsFnFactory = { + projectDAL: TProjectDALFactory; + secretDAL: TSecretV2BridgeDALFactory; + secretVersionDAL: TSecretVersionV2DALFactory; + secretTagDAL: TSecretTagDALFactory; + secretVersionTagDAL: TSecretVersionV2TagDALFactory; + folderDAL: TSecretFolderDALFactory; +}; + +export type TCreateManySecretsFn = { + projectId: string; + environment: string; + path: string; + secrets: { + secretName: string; + secretValue: string; + type: SecretType; + secretComment?: string; + skipMultilineEncoding?: boolean; + tags?: string[]; + metadata?: { + source?: string; + }; + }[]; + userId?: string; // only relevant for personal secret(s) +}; + +export type TUpdateManySecretsFnFactory = { + projectDAL: TProjectDALFactory; + secretDAL: TSecretV2BridgeDALFactory; + secretVersionDAL: TSecretVersionV2DALFactory; + secretTagDAL: TSecretTagDALFactory; + secretVersionTagDAL: TSecretVersionV2TagDALFactory; + folderDAL: TSecretFolderDALFactory; +}; + +export type TUpdateManySecretsFn = { + projectId: string; + environment: string; + path: string; + secrets: { + secretName: string; + newSecretName?: string; + secretValue: string; + type: SecretType; + secretComment?: string; + skipMultilineEncoding?: boolean; + secretReminderRepeatDays?: number | null; + secretReminderNote?: string | null; + tags?: string[]; + metadata?: { + source?: string; + }; + }[]; + userId?: string; +}; + +export enum SecretOperations { + Create = "create", + Update = "update", + Delete = "delete" +} + +export type TMoveSecretsDTO = { + projectId: string; + sourceEnvironment: string; + sourceSecretPath: string; + destinationEnvironment: string; + destinationSecretPath: string; + secretIds: string[]; + shouldOverwrite: boolean; +} & Omit; + +export type TAttachSecretTagsDTO = { + projectId: string; + secretName: string; + tagSlugs: string[]; + environment: string; + secretPath: string; + type: SecretType; +} & Omit; diff --git a/backend/src/services/secret-v2-bridge/secret-version-dal.ts b/backend/src/services/secret-v2-bridge/secret-version-dal.ts new file mode 100644 index 000000000..e4922a57d --- /dev/null +++ b/backend/src/services/secret-v2-bridge/secret-version-dal.ts @@ -0,0 +1,124 @@ +import { Knex } from "knex"; + +import { TDbClient } from "@app/db"; +import { TableName, TSecretVersionsV2, TSecretVersionsV2Update } from "@app/db/schemas"; +import { BadRequestError, DatabaseError } from "@app/lib/errors"; +import { ormify, selectAllTableCols } from "@app/lib/knex"; + +export type TSecretVersionV2DALFactory = ReturnType; + +export const secretVersionV2BridgeDALFactory = (db: TDbClient) => { + const secretVersionV2Orm = ormify(db, TableName.SecretVersionV2); + + // This will fetch all latest secret versions from a folder + const findLatestVersionByFolderId = async (folderId: string, tx?: Knex) => { + try { + const docs = await (tx || db.replicaNode())(TableName.SecretVersionV2) + .where(`${TableName.SecretVersionV2}.folderId`, folderId) + .join(TableName.SecretV2, `${TableName.SecretV2}.id`, `${TableName.SecretVersionV2}.secretId`) + .join( + (tx || db)(TableName.SecretVersionV2) + .groupBy("folderId", "secretId") + .max("version") + .select("secretId") + .as("latestVersion"), + (bd) => { + bd.on(`${TableName.SecretVersionV2}.secretId`, "latestVersion.secretId").andOn( + `${TableName.SecretVersionV2}.version`, + "latestVersion.max" + ); + } + ) + .select(selectAllTableCols(TableName.SecretVersionV2)); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindLatestVersionByFolderId" }); + } + }; + + const bulkUpdate = async ( + data: Array<{ filter: Partial; data: TSecretVersionsV2Update }>, + tx?: Knex + ) => { + try { + const secs = await Promise.all( + data.map(async ({ filter, data: updateData }) => { + const [doc] = await (tx || db)(TableName.SecretVersionV2) + .where(filter) + .update(updateData) + .increment("version", 1) // TODO: Is this really needed? + .returning("*"); + if (!doc) throw new BadRequestError({ message: "Failed to update document" }); + return doc; + }) + ); + return secs; + } catch (error) { + throw new DatabaseError({ error, name: "bulk update secret" }); + } + }; + + const findLatestVersionMany = async (folderId: string, secretIds: string[], tx?: Knex) => { + try { + if (!secretIds.length) return {}; + const docs: Array = await (tx || db.replicaNode())(TableName.SecretVersionV2) + .where("folderId", folderId) + .whereIn(`${TableName.SecretVersionV2}.secretId`, secretIds) + .join( + (tx || db)(TableName.SecretVersionV2) + .groupBy("secretId") + .max("version") + .select("secretId") + .as("latestVersion"), + (bd) => { + bd.on(`${TableName.SecretVersionV2}.secretId`, "latestVersion.secretId").andOn( + `${TableName.SecretVersionV2}.version`, + "latestVersion.max" + ); + } + ); + return docs.reduce>( + (prev, curr) => ({ ...prev, [curr.secretId || ""]: curr }), + {} + ); + } catch (error) { + throw new DatabaseError({ error, name: "FindLatestVersinMany" }); + } + }; + + const pruneExcessVersions = async () => { + try { + await db(TableName.SecretVersionV2) + .with("version_cte", (qb) => { + void qb + .from(TableName.SecretVersionV2) + .select( + "id", + "folderId", + db.raw( + `ROW_NUMBER() OVER (PARTITION BY ${TableName.SecretVersionV2}."secretId" ORDER BY ${TableName.SecretVersionV2}."createdAt" DESC) AS row_num` + ) + ); + }) + .join(TableName.SecretFolder, `${TableName.SecretFolder}.id`, `${TableName.SecretVersionV2}.folderId`) + .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolder}.envId`) + .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) + .join("version_cte", "version_cte.id", `${TableName.SecretVersionV2}.id`) + .whereRaw(`version_cte.row_num > ${TableName.Project}."pitVersionLimit"`) + .delete(); + } catch (error) { + throw new DatabaseError({ + error, + name: "Secret Version Prune" + }); + } + }; + + return { + ...secretVersionV2Orm, + pruneExcessVersions, + findLatestVersionMany, + bulkUpdate, + findLatestVersionByFolderId + }; +}; diff --git a/backend/src/services/secret-v2-bridge/secret-version-tag-dal.ts b/backend/src/services/secret-v2-bridge/secret-version-tag-dal.ts new file mode 100644 index 000000000..f1a3531e9 --- /dev/null +++ b/backend/src/services/secret-v2-bridge/secret-version-tag-dal.ts @@ -0,0 +1,10 @@ +import { TDbClient } from "@app/db"; +import { TableName } from "@app/db/schemas"; +import { ormify } from "@app/lib/knex"; + +export type TSecretVersionV2TagDALFactory = ReturnType; + +export const secretVersionV2TagBridgeDALFactory = (db: TDbClient) => { + const secretVersionTagDAL = ormify(db, TableName.SecretVersionV2Tag); + return secretVersionTagDAL; +}; diff --git a/backend/src/services/secret/secret-dal.ts b/backend/src/services/secret/secret-dal.ts index c26880e38..aac01d393 100644 --- a/backend/src/services/secret/secret-dal.ts +++ b/backend/src/services/secret/secret-dal.ts @@ -20,8 +20,6 @@ export const secretDALFactory = (db: TDbClient) => { } }; - // the idea is to use postgres specific function - // insert with id this will cause a conflict then merge the data const bulkUpdate = async ( data: Array<{ filter: Partial; data: TSecretsUpdate }>, diff --git a/backend/src/services/secret/secret-fns.ts b/backend/src/services/secret/secret-fns.ts index aa112e6b0..662067645 100644 --- a/backend/src/services/secret/secret-fns.ts +++ b/backend/src/services/secret/secret-fns.ts @@ -22,8 +22,14 @@ import { import { BadRequestError } from "@app/lib/errors"; import { groupBy, unique } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; +import { + fnSecretBulkInsert as fnSecretV2BridgeBulkInsert, + fnSecretBulkUpdate as fnSecretV2BridgeBulkUpdate, + getAllNestedSecretReferences as getAllNestedSecretReferencesV2Bridge +} from "@app/services/secret-v2-bridge/secret-v2-bridge-fns"; import { ActorAuthMethod, ActorType } from "../auth/auth-type"; +import { KmsDataKey } from "../kms/kms-types"; import { getBotKeyFnFactory } from "../project-bot/project-bot-fns"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; @@ -407,7 +413,12 @@ export const decryptSecretRaw = ( id: secret.id, user: secret.userId, tags: secret.tags, - skipMultilineEncoding: secret.skipMultilineEncoding + skipMultilineEncoding: secret.skipMultilineEncoding, + secretReminderRepeatDays: secret.secretReminderRepeatDays, + secretReminderNote: secret.secretReminderNote, + metadata: secret.metadata, + createdAt: secret.createdAt, + updatedAt: secret.updatedAt }; }; @@ -536,10 +547,51 @@ export const fnSecretBulkInsert = async ({ secretVersionTagDAL, tx }: TFnSecretBulkInsert) => { - const newSecrets = await secretDAL.insertMany( - inputSecrets.map(({ tags, references, ...el }) => ({ ...el, folderId })), - tx + const sanitizedInputSecrets = inputSecrets.map( + ({ + skipMultilineEncoding, + type, + userId, + version, + metadata, + algorithm, + secretKeyIV, + secretKeyTag, + secretValueIV, + keyEncoding, + secretValueTag, + secretCommentIV, + secretBlindIndex, + secretCommentTag, + secretKeyCiphertext, + secretReminderNote, + secretValueCiphertext, + secretCommentCiphertext, + secretReminderRepeatDays + }) => ({ + skipMultilineEncoding, + folderId, + type, + userId, + version, + metadata, + algorithm, + secretKeyIV, + secretKeyTag, + secretValueIV, + keyEncoding, + secretValueTag, + secretCommentIV, + secretBlindIndex, + secretCommentTag, + secretKeyCiphertext, + secretReminderNote, + secretValueCiphertext, + secretCommentCiphertext, + secretReminderRepeatDays + }) ); + const newSecrets = await secretDAL.insertMany(sanitizedInputSecrets, tx); const newSecretGroupByBlindIndex = groupBy(newSecrets, (item) => item.secretBlindIndex as string); const newSecretTags = inputSecrets.flatMap(({ tags: secretTags = [], secretBlindIndex }) => secretTags.map((tag) => ({ @@ -548,9 +600,8 @@ export const fnSecretBulkInsert = async ({ })) ); const secretVersions = await secretVersionDAL.insertMany( - inputSecrets.map(({ tags, references, ...el }) => ({ + sanitizedInputSecrets.map((el) => ({ ...el, - folderId, secretId: newSecretGroupByBlindIndex[el.secretBlindIndex as string][0].id })), tx @@ -585,13 +636,55 @@ export const fnSecretBulkUpdate = async ({ secretTagDAL, secretVersionTagDAL }: TFnSecretBulkUpdate) => { - const newSecrets = await secretDAL.bulkUpdate( - inputSecrets.map(({ filter, data: { tags, references, ...data } }) => ({ + const sanitizedInputSecrets = inputSecrets.map( + ({ + filter, + data: { + skipMultilineEncoding, + type, + userId, + metadata, + algorithm, + secretKeyIV, + secretKeyTag, + secretValueIV, + keyEncoding, + secretValueTag, + secretCommentIV, + secretBlindIndex, + secretCommentTag, + secretKeyCiphertext, + secretReminderNote, + secretValueCiphertext, + secretCommentCiphertext, + secretReminderRepeatDays + } + }) => ({ filter: { ...filter, folderId }, - data - })), - tx + data: { + skipMultilineEncoding, + type, + userId, + metadata, + algorithm, + secretKeyIV, + secretKeyTag, + secretValueIV, + keyEncoding, + secretValueTag, + secretCommentIV, + secretBlindIndex, + secretCommentTag, + secretKeyCiphertext, + secretReminderNote, + secretValueCiphertext, + secretCommentCiphertext, + secretReminderRepeatDays + } + }) ); + + const newSecrets = await secretDAL.bulkUpdate(sanitizedInputSecrets, tx); const secretVersions = await secretVersionDAL.insertMany( newSecrets.map(({ id, createdAt, updatedAt, ...el }) => ({ ...el, @@ -674,7 +767,11 @@ export const createManySecretsRawFnFactory = ({ secretBlindIndexDAL, secretTagDAL, secretVersionTagDAL, - folderDAL + folderDAL, + secretVersionV2BridgeDAL, + secretV2BridgeDAL, + secretVersionTagV2BridgeDAL, + kmsService }: TCreateManySecretsRawFnFactory) => { const getBotKeyFn = getBotKeyFnFactory(projectBotDAL, projectDAL); const createManySecretsRawFn = async ({ @@ -684,10 +781,7 @@ export const createManySecretsRawFnFactory = ({ secrets, userId }: TCreateManySecretsRawFn) => { - const botKey = await getBotKeyFn(projectId); - if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" }); - - await projectDAL.checkProjectUpgradeStatus(projectId); + const { botKey, shouldUseSecretV2Bridge } = await getBotKeyFn(projectId); const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); if (!folder) @@ -696,6 +790,62 @@ export const createManySecretsRawFnFactory = ({ name: "Create secret" }); const folderId = folder.id; + if (shouldUseSecretV2Bridge) { + const { encryptor: secretManagerEncryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId + }); + + const secretsStoredInDB = await secretV2BridgeDAL.findBySecretKeys( + folderId, + secrets.map((el) => ({ + key: el.secretName, + type: SecretType.Shared + })) + ); + if (secretsStoredInDB.length) + throw new BadRequestError({ + message: `Secret already exist: ${secretsStoredInDB.map((el) => el.key).join(",")}` + }); + + const inputSecrets = secrets.map((secret) => { + return { + type: secret.type, + userId: secret.type === SecretType.Personal ? userId : null, + key: secret.secretName, + encryptedValue: secretManagerEncryptor({ plainText: Buffer.from(secret.secretValue) }).cipherTextBlob, + encryptedComent: secret.secretComment + ? secretManagerEncryptor({ plainText: Buffer.from(secret.secretComment) }).cipherTextBlob + : null, + skipMultilineEncoding: secret.skipMultilineEncoding, + tags: secret.tags, + references: getAllNestedSecretReferencesV2Bridge(secret.secretValue) + }; + }); + + // get all tags + const tagIds = inputSecrets.flatMap(({ tags = [] }) => tags); + const tags = tagIds.length ? await secretTagDAL.findManyTagsById(projectId, tagIds) : []; + if (tags.length !== tagIds.length) throw new BadRequestError({ message: "Tag not found" }); + + const newSecrets = await secretDAL.transaction(async (tx) => + fnSecretV2BridgeBulkInsert({ + inputSecrets: inputSecrets.map((el) => ({ + ...el, + version: 1, + tagIds: el.tags + })), + folderId, + secretDAL: secretV2BridgeDAL, + secretVersionDAL: secretVersionV2BridgeDAL, + secretTagDAL, + secretVersionTagDAL: secretVersionTagV2BridgeDAL, + tx + }) + ); + + return newSecrets; + } const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId }); if (!blindIndexCfg) throw new BadRequestError({ message: "Blind index not found", name: "Create secret" }); @@ -710,6 +860,7 @@ export const createManySecretsRawFnFactory = ({ secretDAL }); + if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" }); const inputSecrets = secrets.map((secret) => { const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey); const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey); @@ -742,7 +893,7 @@ export const createManySecretsRawFnFactory = ({ const newSecrets = await secretDAL.transaction(async (tx) => fnSecretBulkInsert({ - inputSecrets: inputSecrets.map(({ secretName, ...el }) => ({ + inputSecrets: inputSecrets.map(({ secretName, tags: _, ...el }) => ({ ...el, version: 0, secretBlindIndex: keyName2BlindIndex[secretName], @@ -772,7 +923,11 @@ export const updateManySecretsRawFnFactory = ({ secretBlindIndexDAL, secretTagDAL, secretVersionTagDAL, - folderDAL + folderDAL, + secretVersionTagV2BridgeDAL, + secretVersionV2BridgeDAL, + secretV2BridgeDAL, + kmsService }: TUpdateManySecretsRawFnFactory) => { const getBotKeyFn = getBotKeyFnFactory(projectBotDAL, projectDAL); const updateManySecretsRawFn = async ({ @@ -781,11 +936,8 @@ export const updateManySecretsRawFnFactory = ({ path: secretPath, secrets, // consider accepting instead ciphertext secrets userId - }: TUpdateManySecretsRawFn): Promise> => { - const botKey = await getBotKeyFn(projectId); - if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" }); - - await projectDAL.checkProjectUpgradeStatus(projectId); + }: TUpdateManySecretsRawFn): Promise> => { + const { botKey, shouldUseSecretV2Bridge } = await getBotKeyFn(projectId); const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); if (!folder) @@ -794,7 +946,82 @@ export const updateManySecretsRawFnFactory = ({ name: "Update secret" }); const folderId = folder.id; + if (shouldUseSecretV2Bridge) { + const { encryptor: secretManagerEncryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId + }); + const secretsToUpdate = await secretV2BridgeDAL.findBySecretKeys( + folderId, + secrets.map((el) => ({ + key: el.secretName, + type: SecretType.Shared + })) + ); + if (secretsToUpdate.length !== secrets.length) + throw new BadRequestError({ message: `Secret not exist: ${secretsToUpdate.map((el) => el.key).join(",")}` }); + + // now find any secret that needs to update its name + // same process as above + const secretsWithNewName = secrets.filter(({ newSecretName }) => Boolean(newSecretName)); + if (secretsWithNewName.length) { + const secretsWithNewNameInDB = await secretV2BridgeDAL.findBySecretKeys( + folderId, + secrets.map((el) => ({ + key: el.secretName, + type: SecretType.Shared + })) + ); + if (secretsWithNewNameInDB.length) + throw new BadRequestError({ + message: `Secret not exist: ${secretsWithNewName.map((el) => el.newSecretName).join(",")}` + }); + } + + const secretsToUpdateInDBGroupedByKey = groupBy(secretsToUpdate, (i) => i.key); + const inputSecrets = secrets.map((secret) => { + if (secret.newSecretName === "") { + throw new BadRequestError({ message: "New secret name cannot be empty" }); + } + + return { + type: secret.type, + userId: secret.type === SecretType.Personal ? userId : null, + key: secret.newSecretName || secret.secretName, + encryptedValue: secretManagerEncryptor({ plainText: Buffer.from(secret.secretValue) }).cipherTextBlob, + encryptedComent: secret.secretComment + ? secretManagerEncryptor({ plainText: Buffer.from(secret.secretComment) }).cipherTextBlob + : null, + skipMultilineEncoding: secret.skipMultilineEncoding, + tags: secret.tags, + references: getAllNestedSecretReferencesV2Bridge(secret.secretValue) + }; + }); + + const tagIds = inputSecrets.flatMap(({ tags = [] }) => tags); + const tags = tagIds.length ? await secretTagDAL.findManyTagsById(projectId, tagIds) : []; + if (tagIds.length !== tags.length) throw new BadRequestError({ message: "Tag not found" }); + + const updatedSecrets = await secretDAL.transaction(async (tx) => + fnSecretV2BridgeBulkUpdate({ + folderId, + tx, + inputSecrets: inputSecrets.map((el) => ({ + filter: { id: secretsToUpdateInDBGroupedByKey[el.key][0].id, type: SecretType.Shared }, + data: el + })), + secretDAL: secretV2BridgeDAL, + secretVersionDAL: secretVersionV2BridgeDAL, + secretTagDAL, + secretVersionTagDAL: secretVersionTagV2BridgeDAL + }) + ); + + return updatedSecrets; + } + + if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" }); const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId }); if (!blindIndexCfg) throw new BadRequestError({ message: "Blind index not found", name: "Update secret" }); @@ -882,3 +1109,50 @@ export const updateManySecretsRawFnFactory = ({ return updateManySecretsRawFn; }; + +export const decryptSecretWithBot = ( + secret: Pick< + TSecrets, + | "secretKeyIV" + | "secretKeyTag" + | "secretKeyCiphertext" + | "secretValueIV" + | "secretValueTag" + | "secretValueCiphertext" + | "secretCommentIV" + | "secretCommentTag" + | "secretCommentCiphertext" + >, + key: string +) => { + const secretKey = decryptSymmetric128BitHexKeyUTF8({ + ciphertext: secret.secretKeyCiphertext, + iv: secret.secretKeyIV, + tag: secret.secretKeyTag, + key + }); + + const secretValue = decryptSymmetric128BitHexKeyUTF8({ + ciphertext: secret.secretValueCiphertext, + iv: secret.secretValueIV, + tag: secret.secretValueTag, + key + }); + + let secretComment = ""; + + if (secret.secretCommentCiphertext && secret.secretCommentIV && secret.secretCommentTag) { + secretComment = decryptSymmetric128BitHexKeyUTF8({ + ciphertext: secret.secretCommentCiphertext, + iv: secret.secretCommentIV, + tag: secret.secretCommentTag, + key + }); + } + + return { + secretKey, + secretValue, + secretComment + }; +}; diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index 0f0645f58..6626dff7f 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -1,6 +1,11 @@ /* eslint-disable no-await-in-loop */ import { AxiosError } from "axios"; +import { ProjectUpgradeStatus, ProjectVersion, TSecretSnapshotSecretsV2, TSecretVersionsV2 } from "@app/db/schemas"; +import { TSecretApprovalRequestDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-dal"; +import { TSecretRotationDALFactory } from "@app/ee/services/secret-rotation/secret-rotation-dal"; +import { TSnapshotDALFactory } from "@app/ee/services/secret-snapshot/snapshot-dal"; +import { TSnapshotSecretV2DALFactory } from "@app/ee/services/secret-snapshot/snapshot-secret-v2-dal"; import { getConfig } from "@app/lib/config/env"; import { decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto"; import { daysToMillisecond, secondsToMillis } from "@app/lib/dates"; @@ -16,8 +21,11 @@ import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/se import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; import { TIntegrationDALFactory } from "../integration/integration-dal"; +import { TIntegrationAuthDALFactory } from "../integration-auth/integration-auth-dal"; import { TIntegrationAuthServiceFactory } from "../integration-auth/integration-auth-service"; import { syncIntegrationSecrets } from "../integration-auth/integration-sync-secret"; +import { TKmsServiceFactory } from "../kms/kms-service"; +import { KmsDataKey } from "../kms/kms-types"; import { TOrgDALFactory } from "../org/org-dal"; import { TProjectDALFactory } from "../project/project-dal"; import { TProjectBotServiceFactory } from "../project-bot/project-bot-service"; @@ -25,6 +33,11 @@ import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TProjectMembershipDALFactory } from "../project-membership/project-membership-dal"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; import { TSecretImportDALFactory } from "../secret-import/secret-import-dal"; +import { fnSecretsV2FromImports } from "../secret-import/secret-import-fns"; +import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal"; +import { expandSecretReferencesFactory, getAllNestedSecretReferences } from "../secret-v2-bridge/secret-v2-bridge-fns"; +import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-dal"; +import { TSecretVersionV2TagDALFactory } from "../secret-v2-bridge/secret-version-tag-dal"; import { SmtpTemplates, TSmtpService } from "../smtp/smtp-service"; import { TWebhookDALFactory } from "../webhook/webhook-dal"; import { fnTriggerWebhook } from "../webhook/webhook-fns"; @@ -41,13 +54,14 @@ export type TSecretQueueFactory = ReturnType; type TSecretQueueFactoryDep = { queueService: TQueueServiceFactory; integrationDAL: Pick; + integrationAuthDAL: Pick; projectBotService: Pick; integrationAuthService: Pick; folderDAL: TSecretFolderDALFactory; secretDAL: TSecretDALFactory; - secretImportDAL: Pick; + secretImportDAL: Pick; webhookDAL: Pick; - projectEnvDAL: Pick; + projectEnvDAL: Pick; projectDAL: TProjectDALFactory; projectBotDAL: TProjectBotDALFactory; projectMembershipDAL: Pick; @@ -57,6 +71,14 @@ type TSecretQueueFactoryDep = { secretBlindIndexDAL: TSecretBlindIndexDALFactory; secretTagDAL: TSecretTagDALFactory; secretVersionTagDAL: TSecretVersionTagDALFactory; + kmsService: Pick; + secretV2BridgeDAL: TSecretV2BridgeDALFactory; + secretVersionV2BridgeDAL: Pick; + secretVersionTagV2BridgeDAL: Pick; + secretRotationDAL: Pick; + secretApprovalRequestDAL: Pick; + snapshotDAL: Pick; + snapshotSecretV2BridgeDAL: Pick; }; export type TGetSecrets = { @@ -76,6 +98,7 @@ type TIntegrationSecret = Record< export const secretQueueFactory = ({ queueService, integrationDAL, + integrationAuthDAL, projectBotService, integrationAuthService, secretDAL, @@ -91,7 +114,15 @@ export const secretQueueFactory = ({ secretVersionDAL, secretBlindIndexDAL, secretTagDAL, - secretVersionTagDAL + secretVersionTagDAL, + secretV2BridgeDAL, + secretVersionV2BridgeDAL, + kmsService, + secretVersionTagV2BridgeDAL, + secretRotationDAL, + snapshotDAL, + snapshotSecretV2BridgeDAL, + secretApprovalRequestDAL }: TSecretQueueFactoryDep) => { const removeSecretReminder = async (dto: TRemoveSecretReminderDTO) => { const appCfg = getConfig(); @@ -195,7 +226,11 @@ export const secretQueueFactory = ({ secretBlindIndexDAL, secretTagDAL, secretVersionTagDAL, - folderDAL + folderDAL, + kmsService, + secretVersionV2BridgeDAL, + secretV2BridgeDAL, + secretVersionTagV2BridgeDAL }); const updateManySecretsRawFn = updateManySecretsRawFnFactory({ @@ -206,9 +241,84 @@ export const secretQueueFactory = ({ secretBlindIndexDAL, secretTagDAL, secretVersionTagDAL, - folderDAL + folderDAL, + kmsService, + secretVersionV2BridgeDAL, + secretV2BridgeDAL, + secretVersionTagV2BridgeDAL }); + /** + * Return the secrets in a given [folderId] including secrets from + * nested imported folders recursively. + */ + const getIntegrationSecretsV2 = async (dto: { + projectId: string; + environment: string; + folderId: string; + depth: number; + decryptor: (value: Buffer | null | undefined) => string; + }) => { + const content: TIntegrationSecret = {}; + if (dto.depth > MAX_SYNC_SECRET_DEPTH) { + logger.info( + `getIntegrationSecrets: secret depth exceeded for [projectId=${dto.projectId}] [folderId=${dto.folderId}] [depth=${dto.depth}]` + ); + return content; + } + + // process secrets in current folder + const secrets = await secretV2BridgeDAL.findByFolderId(dto.folderId); + secrets.forEach((secret) => { + const secretKey = secret.key; + const secretValue = dto.decryptor(secret.encryptedValue); + content[secretKey] = { value: secretValue }; + + if (secret.encryptedComment) { + const commentValue = dto.decryptor(secret.encryptedComment); + content[secretKey].comment = commentValue; + } + + content[secretKey].skipMultilineEncoding = Boolean(secret.skipMultilineEncoding); + }); + + const expandSecretReferences = expandSecretReferencesFactory({ + decryptSecretValue: dto.decryptor, + secretDAL: secretV2BridgeDAL, + folderDAL, + projectId: dto.projectId + }); + + await expandSecretReferences(content); + // check if current folder has any imports from other folders + const secretImports = await secretImportDAL.find({ folderId: dto.folderId, isReplication: false }); + + // if no imports then return secrets in the current folder + if (!secretImports.length) return content; + const importedSecrets = await fnSecretsV2FromImports({ + decryptor: dto.decryptor, + folderDAL, + secretDAL: secretV2BridgeDAL, + expandSecretReferences, + secretImportDAL, + allowedImports: secretImports + }); + + for (let i = importedSecrets.length - 1; i >= 0; i -= 1) { + for (let j = 0; j < importedSecrets[i].secrets.length; j += 1) { + const importedSecret = importedSecrets[i].secrets[j]; + if (!content[importedSecret.key]) { + content[importedSecret.key] = { + skipMultilineEncoding: importedSecret.skipMultilineEncoding, + comment: importedSecret.secretComment, + value: importedSecret.secretValue || "" + }; + } + } + } + return content; + }; + /** * Return the secrets in a given [folderId] including secrets from * nested imported folders recursively. @@ -467,24 +577,39 @@ export const secretQueueFactory = ({ ); } - const secretReferences = await secretDAL.findReferencedSecretReferences( - projectId, - folder.environment.slug, - secretPath - ); - if (secretReferences.length) { - const referencedFolderIds = unique(secretReferences, (i) => i.folderId).map(({ folderId }) => folderId); + const { shouldUseSecretV2Bridge, botKey } = await projectBotService.getBotKey(projectId); + const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId + }); + let referencedFolderIds; + if (shouldUseSecretV2Bridge) { + const secretReferences = await secretV2BridgeDAL.findReferencedSecretReferences( + projectId, + folder.environment.slug, + secretPath + ); + referencedFolderIds = unique(secretReferences, (i) => i.folderId).map(({ folderId }) => folderId); + } else { + const secretReferences = await secretDAL.findReferencedSecretReferences( + projectId, + folder.environment.slug, + secretPath + ); + referencedFolderIds = unique(secretReferences, (i) => i.folderId).map(({ folderId }) => folderId); + } + if (referencedFolderIds.length) { const referencedFolders = await folderDAL.findSecretPathByFolderIds(projectId, referencedFolderIds); const referencedFoldersGroupedById = groupBy(referencedFolders.filter(Boolean), (i) => i?.id as string); logger.info( `getIntegrationSecrets: Syncing secret due to reference change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}] [secretPath=${job.data.secretPath}] [depth=${depth}]` ); await Promise.all( - secretReferences - .filter(({ folderId }) => Boolean(referencedFoldersGroupedById[folderId][0]?.path)) + referencedFolderIds + .filter((folderId) => Boolean(referencedFoldersGroupedById[folderId][0]?.path)) // filter out already synced ones .filter( - ({ folderId }) => + (folderId) => !deDupeQueue[ uniqueSecretQueueKey( referencedFoldersGroupedById[folderId][0]?.environmentSlug as string, @@ -492,7 +617,7 @@ export const secretQueueFactory = ({ ) ] ) - .map(({ folderId }) => + .map((folderId) => syncSecrets({ projectId, secretPath: referencedFoldersGroupedById[folderId][0]?.path as string, @@ -515,6 +640,23 @@ export const secretQueueFactory = ({ logger.info( `getIntegrationSecrets: secret integration sync started [jobId=${job.id}] [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}] [secretPath=${job.data.secretPath}] [depth=${job.data.depth}]` ); + + const secrets = shouldUseSecretV2Bridge + ? await getIntegrationSecretsV2({ + environment, + projectId, + folderId: folder.id, + depth: 1, + decryptor: (value) => (value ? secretManagerDecryptor({ cipherTextBlob: value }).toString() : "") + }) + : await getIntegrationSecrets({ + environment, + projectId, + folderId: folder.id, + key: botKey as string, + depth: 1 + }); + for (const integration of toBeSyncedIntegrations) { const integrationAuth = { ...integration.integrationAuth, @@ -523,27 +665,31 @@ export const secretQueueFactory = ({ projectId: integration.projectId }; - const botKey = await projectBotService.getBotKey(projectId); - const { accessToken, accessId } = await integrationAuthService.getIntegrationAccessToken(integrationAuth, botKey); - const awsAssumeRoleArn = + const { accessToken, accessId } = await integrationAuthService.getIntegrationAccessToken( + integrationAuth, + shouldUseSecretV2Bridge, + botKey + ); + let awsAssumeRoleArn = null; + if (shouldUseSecretV2Bridge) { + if (integrationAuth.encryptedAwsAssumeIamRoleArn) { + awsAssumeRoleArn = secretManagerDecryptor({ + cipherTextBlob: Buffer.from(integrationAuth.encryptedAwsAssumeIamRoleArn) + }).toString(); + } + } else if ( integrationAuth.awsAssumeIamRoleArnTag && integrationAuth.awsAssumeIamRoleArnIV && integrationAuth.awsAssumeIamRoleArnCipherText - ? decryptSymmetric128BitHexKeyUTF8({ - ciphertext: integrationAuth.awsAssumeIamRoleArnCipherText, - iv: integrationAuth.awsAssumeIamRoleArnIV, - tag: integrationAuth.awsAssumeIamRoleArnTag, - key: botKey - }) - : null; + ) { + awsAssumeRoleArn = decryptSymmetric128BitHexKeyUTF8({ + ciphertext: integrationAuth.awsAssumeIamRoleArnCipherText, + iv: integrationAuth.awsAssumeIamRoleArnIV, + tag: integrationAuth.awsAssumeIamRoleArnTag, + key: botKey as string + }); + } - const secrets = await getIntegrationSecrets({ - environment, - projectId, - folderId: folder.id, - key: botKey, - depth: 1 - }); const suffixedSecrets: typeof secrets = {}; const metadata = integration.metadata as Record; if (metadata) { @@ -637,17 +783,363 @@ export const secretQueueFactory = ({ }); }); + const startSecretV2Migration = async (projectId: string) => { + await queueService.queue( + QueueName.ProjectV3Migration, + QueueJobs.ProjectV3Migration, + { projectId }, + { + removeOnComplete: true, + removeOnFail: true + } + ); + }; + + queueService.start(QueueName.ProjectV3Migration, async (job) => { + const { projectId } = job.data; + const { + botKey, + shouldUseSecretV2Bridge: isProjectUpgradedToV3, + project + } = await projectBotService.getBotKey(projectId); + if (isProjectUpgradedToV3 || project.upgradeStatus === ProjectUpgradeStatus.InProgress) { + return; + } + if (!botKey) throw new BadRequestError({ message: "Bot not found" }); + await projectDAL.updateById(projectId, { upgradeStatus: ProjectUpgradeStatus.InProgress }); + + const { encryptor: secretManagerEncryptor } = await kmsService.createCipherPairWithDataKey({ + projectId, + type: KmsDataKey.SecretManager + }); + + const folders = await folderDAL.findByProjectId(projectId); + // except secret version and snapshot migrate rest of everything first in a transaction + await secretDAL.transaction(async (tx) => { + for (const folder of folders) { + const folderId = folder.id; + /* + * Secrets Migration + * */ + // eslint-disable-next-line no-await-in-loop + const projectV1Secrets = await secretDAL.find({ folderId }, { tx }); + if (projectV1Secrets.length) { + const secretReferences: { + secretId: string; + references: { environment: string; secretPath: string; secretKey: string }[]; + }[] = []; + await secretV2BridgeDAL.insertMany( + projectV1Secrets.map((el) => { + const key = decryptSymmetric128BitHexKeyUTF8({ + ciphertext: el.secretKeyCiphertext, + iv: el.secretKeyIV, + tag: el.secretKeyTag, + key: botKey + }); + const value = decryptSymmetric128BitHexKeyUTF8({ + ciphertext: el.secretValueCiphertext, + iv: el.secretValueIV, + tag: el.secretValueTag, + key: botKey + }); + const comment = + el.secretCommentCiphertext && el.secretCommentTag && el.secretCommentIV + ? decryptSymmetric128BitHexKeyUTF8({ + ciphertext: el.secretCommentCiphertext, + iv: el.secretCommentIV, + tag: el.secretCommentTag, + key: botKey + }) + : ""; + const encryptedValue = secretManagerEncryptor({ plainText: Buffer.from(value) }).cipherTextBlob; + // create references + const references = getAllNestedSecretReferences(value); + secretReferences.push({ secretId: el.id, references }); + + const encryptedComment = comment + ? secretManagerEncryptor({ plainText: Buffer.from(comment) }).cipherTextBlob + : null; + return { + id: el.id, + createdAt: el.createdAt, + updatedAt: el.updatedAt, + skipMultilineEncoding: el.skipMultilineEncoding, + encryptedComment, + encryptedValue, + key, + version: el.version, + type: el.type, + userId: el.userId, + folderId: el.folderId, + metadata: el.metadata, + reminderNote: el.secretReminderNote, + reminderRepeatDays: el.secretReminderRepeatDays + }; + }), + tx + ); + await secretV2BridgeDAL.upsertSecretReferences(secretReferences, tx); + } + + const SNAPSHOT_BATCH_SIZE = 10; + const snapshots = await snapshotDAL.findNSecretV1SnapshotByFolderId(folderId, SNAPSHOT_BATCH_SIZE, tx); + const projectV3SecretVersionsGroupById: Record = {}; + const projectV3SecretVersionTags: { secret_versions_v2Id: string; secret_tagsId: string }[] = []; + const projectV3SnapshotSecrets: Omit[] = []; + snapshots.forEach(({ secretVersions = [], ...snapshot }) => { + secretVersions.forEach((el) => { + projectV3SnapshotSecrets.push({ + secretVersionId: el.id, + snapshotId: snapshot.id, + createdAt: snapshot.createdAt, + updatedAt: snapshot.updatedAt, + envId: el.snapshotEnvId + }); + if (projectV3SecretVersionsGroupById[el.id]) return; + + const key = decryptSymmetric128BitHexKeyUTF8({ + ciphertext: el.secretKeyCiphertext, + iv: el.secretKeyIV, + tag: el.secretKeyTag, + key: botKey + }); + const value = decryptSymmetric128BitHexKeyUTF8({ + ciphertext: el.secretValueCiphertext, + iv: el.secretValueIV, + tag: el.secretValueTag, + key: botKey + }); + const comment = + el.secretCommentCiphertext && el.secretCommentTag && el.secretCommentIV + ? decryptSymmetric128BitHexKeyUTF8({ + ciphertext: el.secretCommentCiphertext, + iv: el.secretCommentIV, + tag: el.secretCommentTag, + key: botKey + }) + : ""; + const encryptedValue = secretManagerEncryptor({ plainText: Buffer.from(value) }).cipherTextBlob; + + const encryptedComment = comment + ? secretManagerEncryptor({ plainText: Buffer.from(comment) }).cipherTextBlob + : null; + projectV3SecretVersionsGroupById[el.id] = { + id: el.id, + createdAt: el.createdAt, + updatedAt: el.updatedAt, + skipMultilineEncoding: el.skipMultilineEncoding, + encryptedComment, + encryptedValue, + key, + version: el.version, + type: el.type, + userId: el.userId, + folderId: el.folderId, + metadata: el.metadata, + reminderNote: el.secretReminderNote, + reminderRepeatDays: el.secretReminderRepeatDays, + secretId: el.secretId, + envId: el.envId + }; + el.tags.forEach(({ secretTagId }) => { + projectV3SecretVersionTags.push({ secret_tagsId: secretTagId, secret_versions_v2Id: el.id }); + }); + }); + }); + // this is corner case in which some times the snapshot may not have the secret version of an existing secret + // example: on some integration it will pull values from 3rd party on integration but snapshot is not taken + // Thus it won't have secret version + const latestSecretVersionByFolder = await secretVersionDAL.findLatestVersionMany( + folderId, + projectV1Secrets.map((el) => el.id), + tx + ); + Object.values(latestSecretVersionByFolder).forEach((el) => { + if (projectV3SecretVersionsGroupById[el.id]) return; + const key = decryptSymmetric128BitHexKeyUTF8({ + ciphertext: el.secretKeyCiphertext, + iv: el.secretKeyIV, + tag: el.secretKeyTag, + key: botKey + }); + const value = decryptSymmetric128BitHexKeyUTF8({ + ciphertext: el.secretValueCiphertext, + iv: el.secretValueIV, + tag: el.secretValueTag, + key: botKey + }); + const comment = + el.secretCommentCiphertext && el.secretCommentTag && el.secretCommentIV + ? decryptSymmetric128BitHexKeyUTF8({ + ciphertext: el.secretCommentCiphertext, + iv: el.secretCommentIV, + tag: el.secretCommentTag, + key: botKey + }) + : ""; + const encryptedValue = secretManagerEncryptor({ plainText: Buffer.from(value) }).cipherTextBlob; + + const encryptedComment = comment + ? secretManagerEncryptor({ plainText: Buffer.from(comment) }).cipherTextBlob + : null; + projectV3SecretVersionsGroupById[el.id] = { + id: el.id, + createdAt: el.createdAt, + updatedAt: el.updatedAt, + skipMultilineEncoding: el.skipMultilineEncoding, + encryptedComment, + encryptedValue, + key, + version: el.version, + type: el.type, + userId: el.userId, + folderId: el.folderId, + metadata: el.metadata, + reminderNote: el.secretReminderNote, + reminderRepeatDays: el.secretReminderRepeatDays, + secretId: el.secretId, + envId: el.envId + }; + }); + + const projectV3SecretVersions = Object.values(projectV3SecretVersionsGroupById); + if (projectV3SecretVersions.length) { + await secretVersionV2BridgeDAL.insertMany(projectV3SecretVersions, tx); + } + if (projectV3SecretVersionTags.length) { + await secretVersionTagV2BridgeDAL.insertMany(projectV3SecretVersionTags, tx); + } + + if (projectV3SnapshotSecrets.length) { + await snapshotSecretV2BridgeDAL.insertMany(projectV3SnapshotSecrets, tx); + } + await snapshotDAL.deleteSnapshotsAboveLimit(folderId, SNAPSHOT_BATCH_SIZE, tx); + } + /* + * Secret Tag Migration + * */ + // eslint-disable-next-line no-await-in-loop + const projectV1SecretTags = await secretTagDAL.findSecretTagsByProjectId(projectId, tx); + if (projectV1SecretTags.length) { + await secretTagDAL.saveTagsToSecretV2( + projectV1SecretTags.map((el) => ({ + secrets_v2Id: el.secretsId, + secret_tagsId: el.secret_tagsId + })), + tx + ); + } + + /* + * Integration Auth Migration + * Saving the new encrypted colum + * */ + // eslint-disable-next-line no-await-in-loop + const projectV1IntegrationAuths = await integrationAuthDAL.find({ projectId }, { tx }); + await integrationAuthDAL.upsert( + projectV1IntegrationAuths.map((el) => { + const accessToken = + el.accessIV && el.accessTag && el.accessCiphertext + ? decryptSymmetric128BitHexKeyUTF8({ + ciphertext: el.accessCiphertext, + iv: el.accessIV, + tag: el.accessTag, + key: botKey + }) + : undefined; + const accessId = + el.accessIdIV && el.accessIdTag && el.accessIdCiphertext + ? decryptSymmetric128BitHexKeyUTF8({ + ciphertext: el.accessIdCiphertext, + iv: el.accessIdIV, + tag: el.accessIdTag, + key: botKey + }) + : undefined; + const refreshToken = + el.refreshIV && el.refreshTag && el.refreshCiphertext + ? decryptSymmetric128BitHexKeyUTF8({ + ciphertext: el.refreshCiphertext, + iv: el.refreshIV, + tag: el.refreshTag, + key: botKey + }) + : undefined; + const awsAssumeRoleArn = + el.awsAssumeIamRoleArnCipherText && el.awsAssumeIamRoleArnIV && el.awsAssumeIamRoleArnTag + ? decryptSymmetric128BitHexKeyUTF8({ + ciphertext: el.awsAssumeIamRoleArnCipherText, + iv: el.awsAssumeIamRoleArnIV, + tag: el.awsAssumeIamRoleArnTag, + key: botKey + }) + : undefined; + + const encryptedAccess = accessToken + ? secretManagerEncryptor({ plainText: Buffer.from(accessToken) }).cipherTextBlob + : null; + const encryptedAccessId = accessId + ? secretManagerEncryptor({ plainText: Buffer.from(accessId) }).cipherTextBlob + : null; + const encryptedRefresh = refreshToken + ? secretManagerEncryptor({ plainText: Buffer.from(refreshToken) }).cipherTextBlob + : null; + const encryptedAwsAssumeIamRoleArn = awsAssumeRoleArn + ? secretManagerEncryptor({ plainText: Buffer.from(awsAssumeRoleArn) }).cipherTextBlob + : null; + return { + ...el, + encryptedAccess, + encryptedRefresh, + encryptedAccessId, + encryptedAwsAssumeIamRoleArn + }; + }), + "id", + tx + ); + /* + * Secret Rotation Secret Migration + * Saving the new encrypted colum + * */ + const projectV1SecretRotations = await secretRotationDAL.find({ projectId }, tx); + await secretRotationDAL.secretOutputV2InsertMany( + projectV1SecretRotations.flatMap((el) => + el.outputs.map((output) => ({ rotationId: el.id, key: output.key, secretId: output.secret.id })) + ), + tx + ); + + /* + * approvals: we will delete all approvals this is because some secret versions may not be added yet + * Thus doesn't make sense for rest to be there + * */ + await secretApprovalRequestDAL.deleteByProjectId(projectId, tx); + await projectDAL.updateById(projectId, { upgradeStatus: null, version: ProjectVersion.V3 }, tx); + }); + }); + + // eslint-disable-next-line + queueService.listen(QueueName.ProjectV3Migration, "failed", async (job, err) => { + if (job?.data) { + const { projectId } = job.data; + await projectDAL.updateById(projectId, { upgradeStatus: ProjectUpgradeStatus.Failed }); + logger.error(err, `Failed to migrate project to v3: ${projectId}`); + } + }); + queueService.listen(QueueName.IntegrationSync, "failed", (job, err) => { logger.error(err, "Failed to sync integration %s", job?.id); }); queueService.start(QueueName.SecretWebhook, async (job) => { - await fnTriggerWebhook({ ...job.data, projectEnvDAL, webhookDAL, projectDAL }); + await fnTriggerWebhook({ ...job.data, projectEnvDAL, webhookDAL, projectDAL, kmsService }); }); return { // depth is internal only field thus no need to make it available outside syncSecrets, + startSecretV2Migration, syncIntegrations, addSecretReminder, removeSecretReminder, diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index efe5af03e..3a14f0e30 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -4,6 +4,7 @@ import { ForbiddenError, subject } from "@casl/ability"; import { ProjectMembershipRole, + ProjectUpgradeStatus, SecretEncryptionAlgo, SecretKeyEncoding, SecretsSchema, @@ -14,6 +15,7 @@ import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services import { TSecretApprovalPolicyServiceFactory } from "@app/ee/services/secret-approval-policy/secret-approval-policy-service"; import { TSecretApprovalRequestDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-dal"; import { TSecretApprovalRequestSecretDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-secret-dal"; +import { TSecretApprovalRequestServiceFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-service"; import { TSecretSnapshotServiceFactory } from "@app/ee/services/secret-snapshot/secret-snapshot-service"; import { getConfig } from "@app/lib/config/env"; import { @@ -35,6 +37,7 @@ import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; import { TSecretImportDALFactory } from "../secret-import/secret-import-dal"; import { fnSecretsFromImports } from "../secret-import/secret-import-fns"; import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal"; +import { TSecretV2BridgeServiceFactory } from "../secret-v2-bridge/secret-v2-bridge-service"; import { TSecretDALFactory } from "./secret-dal"; import { decryptSecretRaw, @@ -49,6 +52,7 @@ import { import { TSecretQueueFactory } from "./secret-queue"; import { SecretOperations, + SecretProtectionType, TAttachSecretTagsDTO, TBackFillSecretReferencesDTO, TCreateBulkSecretDTO, @@ -65,6 +69,7 @@ import { TGetSecretsRawDTO, TGetSecretVersionsDTO, TMoveSecretsDTO, + TStartSecretsV2MigrationDTO, TUpdateBulkSecretDTO, TUpdateManySecretRawDTO, TUpdateSecretDTO, @@ -83,14 +88,22 @@ type TSecretServiceFactoryDep = { TSecretFolderDALFactory, "findBySecretPath" | "updateById" | "findById" | "findByManySecretPath" | "find" >; + secretV2BridgeService: TSecretV2BridgeServiceFactory; secretBlindIndexDAL: TSecretBlindIndexDALFactory; permissionService: Pick; snapshotService: Pick; - secretQueueService: Pick; + secretQueueService: Pick< + TSecretQueueFactory, + "syncSecrets" | "handleSecretReminder" | "removeSecretReminder" | "startSecretV2Migration" + >; projectBotService: Pick; secretImportDAL: Pick; secretVersionTagDAL: Pick; secretApprovalPolicyService: Pick; + secretApprovalRequestService: Pick< + TSecretApprovalRequestServiceFactory, + "generateSecretApprovalRequest" | "generateSecretApprovalRequestV2Bridge" + >; secretApprovalRequestDAL: Pick; secretApprovalRequestSecretDAL: Pick< TSecretApprovalRequestSecretDALFactory, @@ -115,19 +128,21 @@ export const secretServiceFactory = ({ secretVersionTagDAL, secretApprovalPolicyService, secretApprovalRequestDAL, - secretApprovalRequestSecretDAL + secretApprovalRequestSecretDAL, + secretV2BridgeService, + secretApprovalRequestService }: TSecretServiceFactoryDep) => { const getSecretReference = async (projectId: string) => { // if bot key missing means e2e still exist - const botKey = await projectBotService.getBotKey(projectId).catch(() => null); + const projectBot = await projectBotService.getBotKey(projectId).catch(() => null); return (el: { ciphertext?: string; iv: string; tag: string }) => - botKey + projectBot?.botKey ? getAllNestedSecretReferences( decryptSymmetric128BitHexKeyUTF8({ ciphertext: el.ciphertext || "", iv: el.iv, tag: el.tag, - key: botKey + key: projectBot.botKey }) ) : undefined; @@ -951,7 +966,23 @@ export const secretServiceFactory = ({ expandSecretReferences, recursive }: TGetSecretsRawDTO) => { - const botKey = await projectBotService.getBotKey(projectId); + const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId); + if (shouldUseSecretV2Bridge) { + const { secrets, imports } = await secretV2BridgeService.getSecrets({ + projectId, + expandSecretReferences, + actorId, + actor, + actorOrgId, + environment, + path, + recursive, + actorAuthMethod, + includeImports + }); + return { secrets, imports }; + } + if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" }); const { secrets, imports } = await getSecrets({ @@ -1103,9 +1134,23 @@ export const secretServiceFactory = ({ version }: TGetASecretRawDTO) => { const projectId = workspaceId || (await projectDAL.findProjectBySlug(projectSlug as string, actorOrgId)).id; - - const botKey = await projectBotService.getBotKey(projectId); - if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" }); + const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId); + if (shouldUseSecretV2Bridge) { + const secret = await secretV2BridgeService.getSecretByName({ + environment, + projectId, + includeImports, + actorAuthMethod, + path, + actorOrgId, + actor, + actorId, + expandSecretReferences, + type, + secretName + }); + return secret; + } const encryptedSecret = await getSecretByName({ actorId, @@ -1121,6 +1166,8 @@ export const secretServiceFactory = ({ version }); + if (!botKey) + throw new BadRequestError({ message: "Please upgrade your project first", name: "bot_not_found_error" }); const decryptedSecret = decryptSecretRaw(encryptedSecret, botKey); if (expandSecretReferences) { @@ -1176,14 +1223,98 @@ export const secretServiceFactory = ({ secretValue, secretComment, skipMultilineEncoding, - tagIds + tagIds, + secretReminderNote, + secretReminderRepeatDays }: TCreateSecretRawDTO) => { - const botKey = await projectBotService.getBotKey(projectId); - if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" }); + const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId); + const policy = + actor === ActorType.USER && type === SecretType.Shared + ? await secretApprovalPolicyService.getSecretApprovalPolicy(projectId, environment, secretPath) + : undefined; + if (shouldUseSecretV2Bridge) { + if (policy) { + const approval = await secretApprovalRequestService.generateSecretApprovalRequestV2Bridge({ + policy, + secretPath, + environment, + projectId, + actor, + actorId, + actorOrgId, + actorAuthMethod, + data: { + [SecretOperations.Create]: [ + { + secretKey: secretName, + skipMultilineEncoding, + secretComment, + secretValue, + tagIds, + reminderNote: secretReminderNote, + reminderRepeatDays: secretReminderRepeatDays + } + ] + } + }); + return { type: SecretProtectionType.Approval as const, approval }; + } + const secret = await secretV2BridgeService.createSecret({ + secretName, + type, + actorId, + actor, + actorOrgId, + actorAuthMethod, + projectId, + environment, + secretPath, + secretComment, + secretValue, + tagIds, + secretReminderNote, + skipMultilineEncoding, + secretReminderRepeatDays + }); + return { secret, type: SecretProtectionType.Direct as const }; + } + + if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" }); const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secretName, botKey); const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secretValue || "", botKey); const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secretComment || "", botKey); + if (policy) { + const approval = await secretApprovalRequestService.generateSecretApprovalRequest({ + policy, + secretPath, + environment, + projectId, + actor, + actorId, + actorOrgId, + actorAuthMethod, + data: { + [SecretOperations.Create]: [ + { + secretName, + secretKeyCiphertext: secretKeyEncrypted.ciphertext, + secretKeyIV: secretKeyEncrypted.iv, + secretKeyTag: secretKeyEncrypted.tag, + secretValueCiphertext: secretValueEncrypted.ciphertext, + secretValueIV: secretValueEncrypted.iv, + secretValueTag: secretValueEncrypted.tag, + secretCommentCiphertext: secretCommentEncrypted.ciphertext, + secretCommentIV: secretCommentEncrypted.iv, + secretCommentTag: secretCommentEncrypted.tag, + skipMultilineEncoding, + tagIds + } + ] + } + }); + return { type: SecretProtectionType.Approval as const, approval }; + } const secret = await createSecret({ secretName, @@ -1205,10 +1336,12 @@ export const secretServiceFactory = ({ secretCommentIV: secretCommentEncrypted.iv, secretCommentTag: secretCommentEncrypted.tag, skipMultilineEncoding, + secretReminderRepeatDays, + secretReminderNote, tags: tagIds }); - return decryptSecretRaw(secret, botKey); + return { type: SecretProtectionType.Direct as const, secret: decryptSecretRaw(secret, botKey) }; }; const updateSecretRaw = async ({ @@ -1223,12 +1356,108 @@ export const secretServiceFactory = ({ secretPath, secretValue, skipMultilineEncoding, - tagIds + tagIds, + secretReminderNote, + secretReminderRepeatDays, + metadata, + secretComment, + newSecretName }: TUpdateSecretRawDTO) => { - const botKey = await projectBotService.getBotKey(projectId); + const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId); + const policy = + actor === ActorType.USER && type === SecretType.Shared + ? await secretApprovalPolicyService.getSecretApprovalPolicy(projectId, environment, secretPath) + : undefined; + if (shouldUseSecretV2Bridge) { + if (policy) { + const approval = await secretApprovalRequestService.generateSecretApprovalRequestV2Bridge({ + policy, + secretPath, + environment, + projectId, + actor, + actorId, + actorOrgId, + actorAuthMethod, + data: { + [SecretOperations.Update]: [ + { + secretKey: secretName, + newSecretName, + skipMultilineEncoding, + secretComment, + secretValue, + tagIds, + reminderNote: secretReminderNote, + reminderRepeatDays: secretReminderRepeatDays + } + ] + } + }); + return { type: SecretProtectionType.Approval as const, approval }; + } + const secret = await secretV2BridgeService.updateSecret({ + secretReminderRepeatDays, + skipMultilineEncoding, + secretReminderNote, + tagIds, + secretComment, + secretPath, + environment, + projectId, + actorAuthMethod, + actorOrgId, + actor, + actorId, + type, + secretName, + newSecretName, + metadata, + secretValue + }); + return { type: SecretProtectionType.Direct as const, secret }; + } + if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" }); const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secretValue || "", botKey); + const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secretComment || "", botKey); + const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(newSecretName || secretName, botKey); + + if (policy) { + const approval = await secretApprovalRequestService.generateSecretApprovalRequest({ + policy, + secretPath, + environment, + projectId, + actor, + actorId, + actorOrgId, + actorAuthMethod, + data: { + [SecretOperations.Update]: [ + { + secretName, + newSecretName, + skipMultilineEncoding, + secretKeyCiphertext: secretKeyEncrypted.ciphertext, + secretKeyIV: secretKeyEncrypted.iv, + secretKeyTag: secretKeyEncrypted.tag, + secretValueCiphertext: secretValueEncrypted.ciphertext, + secretValueIV: secretValueEncrypted.iv, + secretValueTag: secretValueEncrypted.tag, + secretCommentCiphertext: secretCommentEncrypted.ciphertext, + secretCommentIV: secretCommentEncrypted.iv, + secretCommentTag: secretCommentEncrypted.tag, + tagIds, + secretReminderNote, + secretReminderRepeatDays + } + ] + } + }); + return { approval, type: SecretProtectionType.Approval as const }; + } const secret = await updateSecret({ secretName, @@ -1244,11 +1473,21 @@ export const secretServiceFactory = ({ secretValueIV: secretValueEncrypted.iv, secretValueTag: secretValueEncrypted.tag, skipMultilineEncoding, - tags: tagIds + tags: tagIds, + metadata, + secretReminderRepeatDays, + secretReminderNote, + newSecretName, + secretKeyIV: secretKeyEncrypted.iv, + secretKeyTag: secretKeyEncrypted.tag, + secretKeyCiphertext: secretKeyEncrypted.ciphertext, + secretCommentIV: secretCommentEncrypted.iv, + secretCommentTag: secretCommentEncrypted.tag, + secretCommentCiphertext: secretCommentEncrypted.ciphertext }); await snapshotService.performSnapshot(secret.folderId); - return decryptSecretRaw(secret, botKey); + return { type: SecretProtectionType.Direct as const, secret: decryptSecretRaw(secret, botKey) }; }; const deleteSecretRaw = async ({ @@ -1262,9 +1501,66 @@ export const secretServiceFactory = ({ type, secretPath }: TDeleteSecretRawDTO) => { - const botKey = await projectBotService.getBotKey(projectId); + const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId); + const policy = + actor === ActorType.USER && type === SecretType.Shared + ? await secretApprovalPolicyService.getSecretApprovalPolicy(projectId, environment, secretPath) + : undefined; + if (shouldUseSecretV2Bridge) { + if (policy) { + const approval = await secretApprovalRequestService.generateSecretApprovalRequestV2Bridge({ + policy, + actorAuthMethod, + actorOrgId, + actorId, + actor, + projectId, + environment, + secretPath, + data: { + [SecretOperations.Delete]: [ + { + secretKey: secretName + } + ] + } + }); + return { type: SecretProtectionType.Approval as const, approval }; + } + const secret = await secretV2BridgeService.deleteSecret({ + secretName, + type, + actorId, + actor, + actorOrgId, + actorAuthMethod, + projectId, + environment, + secretPath + }); + return { type: SecretProtectionType.Direct as const, secret }; + } if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" }); - + if (policy) { + const approval = await secretApprovalRequestService.generateSecretApprovalRequest({ + policy, + actorAuthMethod, + actorOrgId, + actorId, + actor, + projectId, + environment, + secretPath, + data: { + [SecretOperations.Delete]: [ + { + secretName + } + ] + } + }); + return { type: SecretProtectionType.Approval as const, approval }; + } const secret = await deleteSecret({ secretName, projectId, @@ -1277,12 +1573,13 @@ export const secretServiceFactory = ({ actorAuthMethod }); - return decryptSecretRaw(secret, botKey); + return { type: SecretProtectionType.Direct as const, secret: decryptSecretRaw(secret, botKey) }; }; const createManySecretsRaw = async ({ actorId, projectSlug, + projectId: optionalProjectId, environment, actor, actorOrgId, @@ -1290,22 +1587,62 @@ export const secretServiceFactory = ({ secretPath, secrets: inputSecrets = [] }: TCreateManySecretRawDTO) => { - const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId); - if (!project) throw new BadRequestError({ message: "Project not found" }); - const projectId = project.id; + if (!projectSlug && !optionalProjectId) + throw new BadRequestError({ message: "Must provide either project slug or projectId" }); + + let projectId = optionalProjectId as string; + // pick either project slug or projectid + if (!optionalProjectId && projectSlug) { + const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId); + if (!project) throw new BadRequestError({ message: "Project not found" }); + projectId = project.id; + } + + const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId); + const policy = + actor === ActorType.USER + ? await secretApprovalPolicyService.getSecretApprovalPolicy(projectId, environment, secretPath) + : undefined; + if (shouldUseSecretV2Bridge) { + if (policy) { + const approval = await secretApprovalRequestService.generateSecretApprovalRequestV2Bridge({ + policy, + secretPath, + environment, + projectId, + actor, + actorId, + actorOrgId, + actorAuthMethod, + data: { + [SecretOperations.Create]: inputSecrets.map((el) => ({ + tagIds: el.tagIds, + secretValue: el.secretValue, + secretComment: el.secretComment, + metadata: el.metadata, + skipMultilineEncoding: el.skipMultilineEncoding, + secretKey: el.secretKey + })) + } + }); + return { type: SecretProtectionType.Approval as const, approval }; + } + const secrets = await secretV2BridgeService.createManySecret({ + secretPath, + environment, + projectId, + actorAuthMethod, + actorOrgId, + actor, + actorId, + secrets: inputSecrets + }); + return { secrets, type: SecretProtectionType.Direct as const }; + } - const botKey = await projectBotService.getBotKey(projectId); if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" }); - - const secrets = await createManySecret({ - projectId, - environment, - path: secretPath, - actor, - actorId, - actorOrgId, - actorAuthMethod, - secrets: inputSecrets.map(({ secretComment, secretKey, secretValue, skipMultilineEncoding }) => { + const sanitizedSecrets = inputSecrets.map( + ({ secretComment, secretKey, metadata, tagIds, secretValue, skipMultilineEncoding }) => { const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secretKey, botKey); const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secretValue || "", botKey); const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secretComment || "", botKey); @@ -1320,34 +1657,31 @@ export const secretServiceFactory = ({ secretValueTag: secretValueEncrypted.tag, secretCommentCiphertext: secretCommentEncrypted.ciphertext, secretCommentIV: secretCommentEncrypted.iv, - secretCommentTag: secretCommentEncrypted.tag + secretCommentTag: secretCommentEncrypted.tag, + tags: tagIds, + tagIds, + metadata }; - }) - }); - - return secrets.map((secret) => - decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey) + } ); - }; + if (policy) { + const approval = await secretApprovalRequestService.generateSecretApprovalRequest({ + policy, + secretPath, + environment, + projectId, + actor, + actorId, + actorOrgId, + actorAuthMethod, + data: { + [SecretOperations.Create]: sanitizedSecrets + } + }); + return { type: SecretProtectionType.Approval as const, approval }; + } - const updateManySecretsRaw = async ({ - actorId, - projectSlug, - environment, - actor, - actorOrgId, - actorAuthMethod, - secretPath, - secrets: inputSecrets = [] - }: TUpdateManySecretRawDTO) => { - const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId); - if (!project) throw new BadRequestError({ message: "Project not found" }); - const projectId = project.id; - - const botKey = await projectBotService.getBotKey(projectId); - if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" }); - - const secrets = await updateManySecret({ + const secrets = await createManySecret({ projectId, environment, path: secretPath, @@ -1355,12 +1689,101 @@ export const secretServiceFactory = ({ actorId, actorOrgId, actorAuthMethod, - secrets: inputSecrets.map(({ secretComment, secretKey, secretValue, skipMultilineEncoding }) => { - const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secretKey, botKey); + secrets: sanitizedSecrets + }); + + return { + type: SecretProtectionType.Direct as const, + secrets: secrets.map((secret) => + decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey) + ) + }; + }; + + const updateManySecretsRaw = async ({ + actorId, + projectSlug, + projectId: optionalProjectId, + environment, + actor, + actorOrgId, + actorAuthMethod, + secretPath, + secrets: inputSecrets = [] + }: TUpdateManySecretRawDTO) => { + if (!projectSlug && !optionalProjectId) + throw new BadRequestError({ message: "Must provide either project slug or projectId" }); + + let projectId = optionalProjectId as string; + if (!optionalProjectId && projectSlug) { + const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId); + if (!project) throw new BadRequestError({ message: "Project not found" }); + projectId = project.id; + } + + const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId); + const policy = + actor === ActorType.USER + ? await secretApprovalPolicyService.getSecretApprovalPolicy(projectId, environment, secretPath) + : undefined; + if (shouldUseSecretV2Bridge) { + if (policy) { + const approval = await secretApprovalRequestService.generateSecretApprovalRequestV2Bridge({ + policy, + secretPath, + environment, + projectId, + actor, + actorId, + actorOrgId, + actorAuthMethod, + data: { + [SecretOperations.Update]: inputSecrets.map((el) => ({ + tagIds: el.tagIds, + secretValue: el.secretValue, + secretComment: el.secretComment, + skipMultilineEncoding: el.skipMultilineEncoding, + secretKey: el.secretKey + })) + } + }); + return { type: SecretProtectionType.Approval as const, approval }; + } + const secrets = await secretV2BridgeService.updateManySecret({ + secretPath, + environment, + projectId, + actorAuthMethod, + actorOrgId, + actor, + actorId, + secrets: inputSecrets + }); + return { type: SecretProtectionType.Direct as const, secrets }; + } + + if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" }); + const sanitizedSecrets = inputSecrets.map( + ({ + secretComment, + secretKey, + secretValue, + skipMultilineEncoding, + tagIds: tags, + newSecretName, + secretReminderNote, + secretReminderRepeatDays + }) => { + const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(newSecretName || secretKey, botKey); const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secretValue || "", botKey); const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secretComment || "", botKey); return { secretName: secretKey, + newSecretName, + tags, + tagIds: tags, + secretReminderRepeatDays, + secretReminderNote, type: SecretType.Shared, skipMultilineEncoding, secretKeyCiphertext: secretKeyEncrypted.ciphertext, @@ -1373,17 +1796,48 @@ export const secretServiceFactory = ({ secretCommentIV: secretCommentEncrypted.iv, secretCommentTag: secretCommentEncrypted.tag }; - }) + } + ); + if (policy) { + const approval = await secretApprovalRequestService.generateSecretApprovalRequest({ + policy, + secretPath, + environment, + projectId, + actor, + actorId, + actorOrgId, + actorAuthMethod, + data: { + [SecretOperations.Update]: sanitizedSecrets + } + }); + + return { type: SecretProtectionType.Approval as const, approval }; + } + const secrets = await updateManySecret({ + projectId, + environment, + path: secretPath, + actor, + actorId, + actorOrgId, + actorAuthMethod, + secrets: sanitizedSecrets }); - return secrets.map((secret) => - decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey) - ); + return { + type: SecretProtectionType.Direct as const, + secrets: secrets.map((secret) => + decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey) + ) + }; }; const deleteManySecretsRaw = async ({ actorId, projectSlug, + projectId: optionalProjectId, environment, actor, actorOrgId, @@ -1391,13 +1845,69 @@ export const secretServiceFactory = ({ secretPath, secrets: inputSecrets = [] }: TDeleteManySecretRawDTO) => { - const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId); - if (!project) throw new BadRequestError({ message: "Project not found" }); - const projectId = project.id; + if (!projectSlug && !optionalProjectId) + throw new BadRequestError({ message: "Must provide either project slug or projectId" }); + + let projectId = optionalProjectId as string; + if (!optionalProjectId && projectSlug) { + const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId); + if (!project) throw new BadRequestError({ message: "Project not found" }); + projectId = project.id; + } + + const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId); + const policy = + actor === ActorType.USER + ? await secretApprovalPolicyService.getSecretApprovalPolicy(projectId, environment, secretPath) + : undefined; + if (shouldUseSecretV2Bridge) { + if (policy) { + const approval = await secretApprovalRequestService.generateSecretApprovalRequestV2Bridge({ + policy, + actorAuthMethod, + actorOrgId, + actorId, + actor, + projectId, + environment, + secretPath, + data: { + [SecretOperations.Delete]: inputSecrets + } + }); + return { type: SecretProtectionType.Approval as const, approval }; + } + const secrets = await secretV2BridgeService.deleteManySecret({ + secretPath, + environment, + projectId, + actorAuthMethod, + actorOrgId, + actor, + actorId, + secrets: inputSecrets + }); + return { type: SecretProtectionType.Direct as const, secrets }; + } - const botKey = await projectBotService.getBotKey(projectId); if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" }); + if (policy) { + const approval = await secretApprovalRequestService.generateSecretApprovalRequest({ + policy, + actorAuthMethod, + actorOrgId, + actorId, + actor, + projectId, + environment, + secretPath, + data: { + [SecretOperations.Delete]: inputSecrets.map((el) => ({ secretName: el.secretKey })) + } + }); + return { type: SecretProtectionType.Approval as const, approval }; + } const secrets = await deleteManySecret({ projectId, environment, @@ -1406,12 +1916,15 @@ export const secretServiceFactory = ({ actorId, actorOrgId, actorAuthMethod, - secrets: inputSecrets.map(({ secretKey }) => ({ secretName: secretKey, type: SecretType.Shared })) + secrets: inputSecrets.map(({ secretKey, type = SecretType.Shared }) => ({ secretName: secretKey, type })) }); - return secrets.map((secret) => - decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey) - ); + return { + type: SecretProtectionType.Direct as const, + secrets: secrets.map((secret) => + decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey) + ) + }; }; const getSecretVersions = async ({ @@ -1423,12 +1936,31 @@ export const secretServiceFactory = ({ offset = 0, secretId }: TGetSecretVersionsDTO) => { + const secretVersionV2 = await secretV2BridgeService + .getSecretVersions({ + actorId, + actor, + actorOrgId, + actorAuthMethod, + limit, + offset, + secretId + }) + .catch((err) => { + if ((err as Error).message === "BadRequest: Failed to find secret") { + return null; + } + }); + if (secretVersionV2) return secretVersionV2; + const secret = await secretDAL.findById(secretId); if (!secret) throw new BadRequestError({ message: "Failed to find secret" }); - const folder = await folderDAL.findById(secret.folderId); if (!folder) throw new BadRequestError({ message: "Failed to find secret" }); + const { botKey } = await projectBotService.getBotKey(folder.projectId); + if (!botKey) throw new BadRequestError({ message: "bot not found" }); + const { permission } = await permissionService.getProjectPermission( actor, actorId, @@ -1437,9 +1969,18 @@ export const secretServiceFactory = ({ actorOrgId ); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretRollback); - const secretVersions = await secretVersionDAL.find({ secretId }, { offset, limit, sort: [["createdAt", "desc"]] }); - return secretVersions; + return secretVersions.map((el) => + decryptSecretRaw( + { + ...el, + workspace: folder.projectId, + environment: folder.environment.envSlug, + secretPath: "/" + }, + botKey + ) + ); }; const attachTags = async ({ @@ -1455,7 +1996,6 @@ export const secretServiceFactory = ({ actorId }: TAttachSecretTagsDTO) => { const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId); - const { permission } = await permissionService.getProjectPermission( actor, actorId, @@ -1558,7 +2098,6 @@ export const secretServiceFactory = ({ actorId }: TAttachSecretTagsDTO) => { const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId); - const { permission } = await permissionService.getProjectPermission( actor, actorId, @@ -1673,7 +2212,17 @@ export const secretServiceFactory = ({ if (!hasRole(ProjectMembershipRole.Admin)) throw new BadRequestError({ message: "Only admins are allowed to take this action" }); - const botKey = await projectBotService.getBotKey(projectId); + const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId); + if (shouldUseSecretV2Bridge) { + return secretV2BridgeService.backfillSecretReferences({ + projectId, + actor, + actorId, + actorOrgId, + actorAuthMethod + }); + } + if (!botKey) throw new BadRequestError({ message: "Please upgrade your project first", name: "bot_not_found_error" }); @@ -1717,6 +2266,21 @@ export const secretServiceFactory = ({ message: "Project not found." }); } + if (project.version === 3) { + return secretV2BridgeService.moveSecrets({ + sourceEnvironment, + sourceSecretPath, + destinationEnvironment, + destinationSecretPath, + secretIds, + projectId: project.id, + shouldOverwrite, + actor, + actorId, + actorAuthMethod, + actorOrgId + }); + } const { permission } = await permissionService.getProjectPermission( actor, @@ -1741,7 +2305,7 @@ export const secretServiceFactory = ({ subject(ProjectPermissionSub.Secrets, { environment: destinationEnvironment, secretPath: destinationSecretPath }) ); - const botKey = await projectBotService.getBotKey(project.id); + const { botKey } = await projectBotService.getBotKey(project.id); if (!botKey) { throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" }); } @@ -2085,6 +2649,33 @@ export const secretServiceFactory = ({ }; }; + const startSecretV2Migration = async ({ + projectId, + actor, + actorId, + actorOrgId, + actorAuthMethod + }: TStartSecretsV2MigrationDTO) => { + const { hasRole } = await permissionService.getProjectPermission( + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId + ); + + if (!hasRole(ProjectMembershipRole.Admin)) + throw new BadRequestError({ message: "Only admins are allowed to take this action" }); + + const { shouldUseSecretV2Bridge: isProjectV3, project } = await projectBotService.getBotKey(projectId); + if (isProjectV3) throw new BadRequestError({ message: "project is already in v3" }); + if (project.upgradeStatus === ProjectUpgradeStatus.InProgress) + throw new BadRequestError({ message: "project is upgrading" }); + + await secretQueueService.startSecretV2Migration(projectId); + return { message: "Migrating project to new KMS architecture" }; + }; + return { attachTags, detachTags, @@ -2106,6 +2697,7 @@ export const secretServiceFactory = ({ deleteManySecretsRaw, getSecretVersions, backfillSecretReferences, - moveSecrets + moveSecrets, + startSecretV2Migration }; }; diff --git a/backend/src/services/secret/secret-types.ts b/backend/src/services/secret/secret-types.ts index d806eab11..99bde05f2 100644 --- a/backend/src/services/secret/secret-types.ts +++ b/backend/src/services/secret/secret-types.ts @@ -12,6 +12,10 @@ import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-fold import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; import { ActorType } from "../auth/auth-type"; +import { TKmsServiceFactory } from "../kms/kms-service"; +import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal"; +import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-dal"; +import { TSecretVersionV2TagDALFactory } from "../secret-v2-bridge/secret-version-tag-dal"; type TPartialSecret = Pick; @@ -160,14 +164,16 @@ export type TGetASecretRawDTO = { } & Omit; export type TCreateSecretRawDTO = TProjectPermission & { + secretName: string; secretPath: string; environment: string; - secretName: string; secretValue: string; type: SecretType; tagIds?: string[]; secretComment?: string; skipMultilineEncoding?: boolean; + secretReminderRepeatDays?: number | null; + secretReminderNote?: string | null; }; export type TUpdateSecretRawDTO = TProjectPermission & { @@ -175,11 +181,16 @@ export type TUpdateSecretRawDTO = TProjectPermission & { environment: string; secretName: string; secretValue?: string; + newSecretName?: string; + secretComment?: string; type: SecretType; tagIds?: string[]; skipMultilineEncoding?: boolean; secretReminderRepeatDays?: number | null; secretReminderNote?: string | null; + metadata?: { + source?: string; + }; }; export type TDeleteSecretRawDTO = TProjectPermission & { @@ -191,34 +202,46 @@ export type TDeleteSecretRawDTO = TProjectPermission & { export type TCreateManySecretRawDTO = Omit & { secretPath: string; - projectSlug: string; + projectId?: string; + projectSlug?: string; environment: string; secrets: { secretKey: string; secretValue: string; secretComment?: string; skipMultilineEncoding?: boolean; + tagIds?: string[]; + metadata?: { + source?: string; + }; }[]; }; export type TUpdateManySecretRawDTO = Omit & { secretPath: string; - projectSlug: string; + projectId?: string; + projectSlug?: string; environment: string; secrets: { secretKey: string; + newSecretName?: string; secretValue: string; secretComment?: string; skipMultilineEncoding?: boolean; + tagIds?: string[]; + secretReminderRepeatDays?: number | null; + secretReminderNote?: string | null; }[]; }; export type TDeleteManySecretRawDTO = Omit & { secretPath: string; - projectSlug: string; + projectId?: string; + projectSlug?: string; environment: string; secrets: { secretKey: string; + type?: SecretType; }[]; }; @@ -322,6 +345,13 @@ export type TCreateManySecretsRawFnFactory = { secretTagDAL: TSecretTagDALFactory; secretVersionTagDAL: TSecretVersionTagDALFactory; folderDAL: TSecretFolderDALFactory; + kmsService: Pick; + secretV2BridgeDAL: Pick< + TSecretV2BridgeDALFactory, + "insertMany" | "upsertSecretReferences" | "findBySecretKeys" | "bulkUpdate" | "deleteMany" + >; + secretVersionV2BridgeDAL: Pick; + secretVersionTagV2BridgeDAL: Pick; }; export type TCreateManySecretsRawFn = { @@ -351,6 +381,13 @@ export type TUpdateManySecretsRawFnFactory = { secretTagDAL: TSecretTagDALFactory; secretVersionTagDAL: TSecretVersionTagDALFactory; folderDAL: TSecretFolderDALFactory; + kmsService: Pick; + secretV2BridgeDAL: Pick< + TSecretV2BridgeDALFactory, + "insertMany" | "upsertSecretReferences" | "findBySecretKeys" | "bulkUpdate" | "deleteMany" + >; + secretVersionV2BridgeDAL: Pick; + secretVersionTagV2BridgeDAL: Pick; }; export type TUpdateManySecretsRawFn = { @@ -407,3 +444,10 @@ export type TMoveSecretsDTO = { secretIds: string[]; shouldOverwrite: boolean; } & Omit; + +export enum SecretProtectionType { + Approval = "approval", + Direct = "direct" +} + +export type TStartSecretsV2MigrationDTO = TProjectPermission; diff --git a/backend/src/services/webhook/webhook-dal.ts b/backend/src/services/webhook/webhook-dal.ts index 14d30a35e..ba2745737 100644 --- a/backend/src/services/webhook/webhook-dal.ts +++ b/backend/src/services/webhook/webhook-dal.ts @@ -20,7 +20,7 @@ export const webhookDALFactory = (db: TDbClient) => { .select(tx.ref("projectId").withSchema(TableName.Environment)) .select(selectAllTableCols(TableName.Webhook)); - const find = async (filter: Partial, tx?: Knex) => { + const find = async (filter: Partial, tx?: Knex) => { try { const docs = await webhookFindQuery(tx || db.replicaNode(), filter); return docs.map(({ envId, envSlug, envName, ...el }) => ({ diff --git a/backend/src/services/webhook/webhook-fns.ts b/backend/src/services/webhook/webhook-fns.ts index 4690edba9..7f91ca2db 100644 --- a/backend/src/services/webhook/webhook-fns.ts +++ b/backend/src/services/webhook/webhook-fns.ts @@ -3,12 +3,12 @@ import crypto from "node:crypto"; import { AxiosError } from "axios"; import picomatch from "picomatch"; -import { SecretKeyEncoding, TWebhooks } from "@app/db/schemas"; import { request } from "@app/lib/config/request"; -import { infisicalSymmetricDecrypt } from "@app/lib/crypto/encryption"; import { BadRequestError } from "@app/lib/errors"; import { logger } from "@app/lib/logger"; +import { TKmsServiceFactory } from "../kms/kms-service"; +import { KmsDataKey } from "../kms/kms-types"; import { TProjectDALFactory } from "../project/project-dal"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TWebhookDALFactory } from "./webhook-dal"; @@ -16,40 +16,12 @@ import { WebhookType } from "./webhook-types"; const WEBHOOK_TRIGGER_TIMEOUT = 15 * 1000; -export const decryptWebhookDetails = (webhook: TWebhooks) => { - const { keyEncoding, iv, encryptedSecretKey, tag, urlCipherText, urlIV, urlTag, url } = webhook; - - let decryptedSecretKey = ""; - let decryptedUrl = url; - - if (encryptedSecretKey) { - decryptedSecretKey = infisicalSymmetricDecrypt({ - keyEncoding: keyEncoding as SecretKeyEncoding, - ciphertext: encryptedSecretKey, - iv: iv as string, - tag: tag as string - }); - } - - if (urlCipherText) { - decryptedUrl = infisicalSymmetricDecrypt({ - keyEncoding: keyEncoding as SecretKeyEncoding, - ciphertext: urlCipherText, - iv: urlIV as string, - tag: urlTag as string - }); - } - - return { - secretKey: decryptedSecretKey, - url: decryptedUrl - }; -}; - -export const triggerWebhookRequest = async (webhook: TWebhooks, data: Record) => { +export const triggerWebhookRequest = async ( + { webhookSecretKey: secretKey, webhookUrl: url }: { webhookSecretKey?: string; webhookUrl: string }, + data: Record +) => { const headers: Record = {}; const payload = { ...data, timestamp: Date.now() }; - const { secretKey, url } = decryptWebhookDetails(webhook); if (secretKey) { const webhookSign = crypto.createHmac("sha256", secretKey).update(JSON.stringify(payload)).digest("hex"); @@ -124,6 +96,7 @@ export type TFnTriggerWebhookDTO = { webhookDAL: Pick; projectEnvDAL: Pick; projectDAL: Pick; + kmsService: Pick; }; // this is reusable function @@ -134,7 +107,8 @@ export const fnTriggerWebhook = async ({ projectId, webhookDAL, projectEnvDAL, - projectDAL + projectDAL, + kmsService }: TFnTriggerWebhookDTO) => { const webhooks = await webhookDAL.findAllWebhooks(projectId, environment); const toBeTriggeredHooks = webhooks.filter( @@ -144,10 +118,20 @@ export const fnTriggerWebhook = async ({ if (!toBeTriggeredHooks.length) return; logger.info("Secret webhook job started", { environment, secretPath, projectId }); const project = await projectDAL.findById(projectId); + const { decryptor: kmsDataKeyDecryptor } = await kmsService.createCipherPairWithDataKey({ + projectId, + type: KmsDataKey.SecretManager + }); + const webhooksTriggered = await Promise.allSettled( - toBeTriggeredHooks.map((hook) => - triggerWebhookRequest( - hook, + toBeTriggeredHooks.map((hook) => { + const webhookUrl = kmsDataKeyDecryptor({ cipherTextBlob: hook.encryptedUrl }).toString(); + const webhookSecretKey = hook.encryptedSecretKeyWithKms + ? kmsDataKeyDecryptor({ cipherTextBlob: hook.encryptedSecretKeyWithKms }).toString() + : undefined; + + return triggerWebhookRequest( + { webhookUrl, webhookSecretKey }, getWebhookPayload("secrets.modified", { workspaceName: project.name, workspaceId: projectId, @@ -155,8 +139,8 @@ export const fnTriggerWebhook = async ({ secretPath, type: hook.type }) - ) - ) + ); + }) ); // filter hooks by status diff --git a/backend/src/services/webhook/webhook-service.ts b/backend/src/services/webhook/webhook-service.ts index 41dacd34b..2698b1ea2 100644 --- a/backend/src/services/webhook/webhook-service.ts +++ b/backend/src/services/webhook/webhook-service.ts @@ -1,15 +1,15 @@ import { ForbiddenError } from "@casl/ability"; -import { TWebhooksInsert } from "@app/db/schemas"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; -import { infisicalSymmetricEncypt } from "@app/lib/crypto/encryption"; import { BadRequestError } from "@app/lib/errors"; +import { TKmsServiceFactory } from "../kms/kms-service"; +import { KmsDataKey } from "../kms/kms-types"; import { TProjectDALFactory } from "../project/project-dal"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TWebhookDALFactory } from "./webhook-dal"; -import { decryptWebhookDetails, getWebhookPayload, triggerWebhookRequest } from "./webhook-fns"; +import { getWebhookPayload, triggerWebhookRequest } from "./webhook-fns"; import { TCreateWebhookDTO, TDeleteWebhookDTO, @@ -23,6 +23,7 @@ type TWebhookServiceFactoryDep = { projectEnvDAL: TProjectEnvDALFactory; projectDAL: Pick; permissionService: Pick; + kmsService: Pick; }; export type TWebhookServiceFactory = ReturnType; @@ -31,7 +32,8 @@ export const webhookServiceFactory = ({ webhookDAL, projectEnvDAL, permissionService, - projectDAL + projectDAL, + kmsService }: TWebhookServiceFactoryDep) => { const createWebhook = async ({ actor, @@ -56,33 +58,28 @@ export const webhookServiceFactory = ({ const env = await projectEnvDAL.findOne({ projectId, slug: environment }); if (!env) throw new BadRequestError({ message: "Env not found" }); - const insertDoc: TWebhooksInsert = { - url: "", // deprecated - we are moving away from plaintext URLs + const { encryptor: secretManagerEncryptor } = await kmsService.createCipherPairWithDataKey({ + projectId, + type: KmsDataKey.SecretManager + }); + + const encryptedSecretKeyWithKms = webhookSecretKey + ? secretManagerEncryptor({ + plainText: Buffer.from(webhookSecretKey) + }).cipherTextBlob + : null; + const encryptedUrl = secretManagerEncryptor({ + plainText: Buffer.from(webhookUrl) + }).cipherTextBlob; + + const webhook = await webhookDAL.create({ + encryptedUrl, + encryptedSecretKeyWithKms, envId: env.id, isDisabled: false, secretPath: secretPath || "/", type - }; - - if (webhookSecretKey) { - const { ciphertext, iv, tag, algorithm, encoding } = infisicalSymmetricEncypt(webhookSecretKey); - insertDoc.encryptedSecretKey = ciphertext; - insertDoc.iv = iv; - insertDoc.tag = tag; - insertDoc.algorithm = algorithm; - insertDoc.keyEncoding = encoding; - } - - if (webhookUrl) { - const { ciphertext, iv, tag, algorithm, encoding } = infisicalSymmetricEncypt(webhookUrl); - insertDoc.urlCipherText = ciphertext; - insertDoc.urlIV = iv; - insertDoc.urlTag = tag; - insertDoc.algorithm = algorithm; - insertDoc.keyEncoding = encoding; - } - - const webhook = await webhookDAL.create(insertDoc); + }); return { ...webhook, projectId, environment: env }; }; @@ -136,9 +133,18 @@ export const webhookServiceFactory = ({ ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Webhooks); let webhookError: string | undefined; + const { decryptor: kmsDataKeyDecryptor } = await kmsService.createCipherPairWithDataKey({ + projectId: project.id, + type: KmsDataKey.SecretManager + }); + const webhookUrl = kmsDataKeyDecryptor({ cipherTextBlob: webhook.encryptedUrl }).toString(); + const webhookSecretKey = webhook.encryptedSecretKeyWithKms + ? kmsDataKeyDecryptor({ cipherTextBlob: webhook.encryptedSecretKeyWithKms }).toString() + : undefined; + try { await triggerWebhookRequest( - webhook, + { webhookUrl, webhookSecretKey }, getWebhookPayload("test", { workspaceName: project.name, workspaceId: webhook.projectId, @@ -177,11 +183,15 @@ export const webhookServiceFactory = ({ ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Webhooks); const webhooks = await webhookDAL.findAllWebhooks(projectId, environment, secretPath); + const { decryptor: kmsDataKeyDecryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId + }); return webhooks.map((w) => { - const { url } = decryptWebhookDetails(w); + const decryptedUrl = kmsDataKeyDecryptor({ cipherTextBlob: w.encryptedUrl }).toString(); return { ...w, - url + url: decryptedUrl }; }); }; diff --git a/docs/documentation/platform/kms/aws-kms.mdx b/docs/documentation/platform/kms/aws-kms.mdx new file mode 100644 index 000000000..0bf33bb0d --- /dev/null +++ b/docs/documentation/platform/kms/aws-kms.mdx @@ -0,0 +1,148 @@ +--- +title: "AWS Key Management Service (KMS)" +description: "Learn how to manage encryption using AWS KMS" +--- + +You can configure your projects to use AWS KMS keys for encryption, enhancing the security and management of your secrets. + +## Prerequisites + +Depending on the AWS Authentication Method you intend to use, you will have to do either of the following: + + + + + + 1. Navigate to the [Create IAM Role](https://console.aws.amazon.com/iamv2/home#/roles/create?step=selectEntities) page in your AWS Console. + ![IAM Role Creation](../../images/integrations/aws/integration-aws-iam-assume-role.png) + + 2. Select **AWS Account** as the **Trusted Entity Type**. + 3. Choose **Another AWS Account** and enter **381492033652** (Infisical AWS Account ID). This restricts the role to be assumed only by Infisical. If self-hosting, provide your AWS account number instead. + 4. Optionally, enable **Require external ID** and enter your **project ID** to further enhance security. + + + Use the following custom policy to grant the minimum permissions required by Infisical to integrate with AWS KMS + + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowKMSAccess", + "Effect": "Allow", + "Action": [ + "kms:Decrypt", + "kms:Encrypt", + "kms:DescribeKey" + ], + "Resource": "*" + } + ] + } + ``` + + + + + + Navigate to your IAM user and add a policy to grant the following permissions: + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowKMSAccess", + "Effect": "Allow", + "Action": [ + "kms:Decrypt", + "kms:Encrypt", + "kms:DescribeKey" + ], + "Resource": "*" + } + ] + } + ``` + + + +## Setup AWS KMS in the Organization Settings + +Follow these steps to set up AWS KMS for your organization: + + + + ![Open encryption org settings](../../../images/platform/kms/aws/encryption-org-settings.png) + + + ![Add encryption org settings](../../../images/platform/kms/aws/encryption-org-settings-add.png) + Click the 'Add' button to begin adding a new external KMS. + + + ![Select Encryption Provider](../../../images/platform/kms/aws/encryption-modal-provider-select.png) + Choose 'AWS KMS' from the list of encryption providers. + + + Fill in the required details for AWS KMS: + + Name for referencing the AWS KMS key within the organization. + + + + Short description of the AWS KMS key. + + + + Authentication mode for AWS, either "AWS Assume Role" or "Access Key". + + + + ARN of the AWS role to assume for providing Infisical access to the AWS KMS Key (required if Authentication Mode is "AWS Assume Role") + + + + Custom identifier for additional validation during role assumption. + + + + AWS IAM Access Key ID for authentication (required if Authentication Mode is "Access Key"). + + + + AWS IAM Secret Access Key for authentication (required if Authentication Mode is "Access Key"). + + + + AWS region where the AWS KMS Key is located. + + + Key ID of the AWS KMS Key. If left blank, Infisical will generate and use a new AWS KMS Key in the specified region. + ![AWS KMS key ID](../../../images/platform/kms/aws/aws-kms-key-id.png) + + + + + Save your configuration to apply the settings. + + + +You now have an AWS KMS Key configured at the organization level. You can assign these keys to existing projects via the Project Settings page. + +## Assign AWS KMS Key to an Existing Project + +Follow these steps to assign an AWS KMS key to a project: + + + + ![Open encryption project + settings](../../../images/platform/kms/aws/encryption-project-settings.png) + + + ![Select encryption project + settings](../../../images/platform/kms/aws/encryption-project-settings-select.png) + Choose the AWS KMS key you configured earlier. + + + Save the changes to apply the new encryption settings to your project. + + diff --git a/docs/documentation/platform/kms/overview.mdx b/docs/documentation/platform/kms/overview.mdx new file mode 100644 index 000000000..99cfa134f --- /dev/null +++ b/docs/documentation/platform/kms/overview.mdx @@ -0,0 +1,28 @@ +--- +title: "Key Management Service (KMS)" +sidebarTitle: "Overview" +description: "Learn how to configure your project's encryption" +--- + +## Introduction + +Infisical leverages a Key Management Service (KMS) to securely encrypt and decrypt secrets in your projects. + +## Overview + +Infisical's KMS ensures the security of your project's secrets through the following mechanisms: + +- Each project is assigned a unique workspace key, which is responsible for encrypting and decrypting secret values. +- The workspace key itself is encrypted using the project's configured KMS. +- When secrets are requested, the workspace key is derived from the configured KMS. This key is then used to decrypt the secret values on-demand before sending them to the requesting client. + +## Configuration + +You can set the KMS for new projects during project creation. +![Configure KMS new](../../../images/platform/kms/configure-kms-new.png) +For existing projects, you can configure the KMS from the Project Settings page. +![Configure KMS existing](../../../images/platform/kms/configure-kms-existing.png) + +## External KMS + +Infisical supports the use of external KMS solutions to enhance security and compliance. You can configure your project to use services like [AWS Key Management Service](./aws-kms) for managing encryption. diff --git a/docs/images/platform/kms/aws/aws-kms-key-id.png b/docs/images/platform/kms/aws/aws-kms-key-id.png new file mode 100644 index 000000000..ddeac5093 Binary files /dev/null and b/docs/images/platform/kms/aws/aws-kms-key-id.png differ diff --git a/docs/images/platform/kms/aws/encryption-modal-provider-select.png b/docs/images/platform/kms/aws/encryption-modal-provider-select.png new file mode 100644 index 000000000..704043a74 Binary files /dev/null and b/docs/images/platform/kms/aws/encryption-modal-provider-select.png differ diff --git a/docs/images/platform/kms/aws/encryption-org-settings-add.png b/docs/images/platform/kms/aws/encryption-org-settings-add.png new file mode 100644 index 000000000..03c9a3eaa Binary files /dev/null and b/docs/images/platform/kms/aws/encryption-org-settings-add.png differ diff --git a/docs/images/platform/kms/aws/encryption-org-settings.png b/docs/images/platform/kms/aws/encryption-org-settings.png new file mode 100644 index 000000000..b413cd321 Binary files /dev/null and b/docs/images/platform/kms/aws/encryption-org-settings.png differ diff --git a/docs/images/platform/kms/aws/encryption-project-settings-select.png b/docs/images/platform/kms/aws/encryption-project-settings-select.png new file mode 100644 index 000000000..a6323bcb3 Binary files /dev/null and b/docs/images/platform/kms/aws/encryption-project-settings-select.png differ diff --git a/docs/images/platform/kms/aws/encryption-project-settings.png b/docs/images/platform/kms/aws/encryption-project-settings.png new file mode 100644 index 000000000..1df97abae Binary files /dev/null and b/docs/images/platform/kms/aws/encryption-project-settings.png differ diff --git a/docs/images/platform/kms/configure-kms-existing.png b/docs/images/platform/kms/configure-kms-existing.png new file mode 100644 index 000000000..8d26724aa Binary files /dev/null and b/docs/images/platform/kms/configure-kms-existing.png differ diff --git a/docs/images/platform/kms/configure-kms-new.png b/docs/images/platform/kms/configure-kms-new.png new file mode 100644 index 000000000..e18bb227c Binary files /dev/null and b/docs/images/platform/kms/configure-kms-new.png differ diff --git a/docs/mint.json b/docs/mint.json index 6628316f8..5c7012745 100644 --- a/docs/mint.json +++ b/docs/mint.json @@ -154,6 +154,13 @@ "documentation/platform/dynamic-secrets/aws-iam" ] }, + { + "group": "Key Management", + "pages": [ + "documentation/platform/kms/overview", + "documentation/platform/kms/aws-kms" + ] + }, "documentation/platform/secret-sharing" ] }, diff --git a/frontend/src/components/utilities/secrets/encryptSecrets.ts b/frontend/src/components/utilities/secrets/encryptSecrets.ts index 4eb59ae9f..121f3ab73 100644 --- a/frontend/src/components/utilities/secrets/encryptSecrets.ts +++ b/frontend/src/components/utilities/secrets/encryptSecrets.ts @@ -1,30 +1,4 @@ -import crypto from "crypto"; - -import { SecretDataProps, Tag } from "public/data/frequentInterfaces"; - -import { fetchUserWsKey } from "@app/hooks/api/keys/queries"; -import { SecretType } from "@app/hooks/api/types"; - -import { decryptAssymmetric, encryptSymmetric } from "../cryptography/crypto"; - -interface EncryptedSecretProps { - id: string; - createdAt: string; - environment: string; - secretName: string; - secretCommentCiphertext: string; - secretCommentIV: string; - secretCommentTag: string; - secretKeyCiphertext: string; - secretKeyIV: string; - secretKeyTag: string; - secretValueCiphertext: string; - secretValueIV: string; - secretValueTag: string; - type: SecretType; - tags: Tag[]; -} - +import { SecretDataProps } from "public/data/frequentInterfaces"; /** * Encypt secrets before pushing the to the DB * @param {object} obj @@ -34,7 +8,6 @@ interface EncryptedSecretProps { */ const encryptSecrets = async ({ secretsToEncrypt, - workspaceId, env }: { secretsToEncrypt: SecretDataProps[]; @@ -43,74 +16,14 @@ const encryptSecrets = async ({ }) => { let secrets; try { - // const sharedKey = await getLatestFileKey({ workspaceId }); - const wsKey = await fetchUserWsKey(workspaceId); - - const PRIVATE_KEY = localStorage.getItem("PRIVATE_KEY") as string; - - let randomBytes: string; - if (wsKey) { - // case: a (shared) key exists for the workspace - randomBytes = decryptAssymmetric({ - ciphertext: wsKey.encryptedKey, - nonce: wsKey.nonce, - publicKey: wsKey.sender.publicKey, - privateKey: PRIVATE_KEY - }); - } else { - // case: a (shared) key does not exist for the workspace - randomBytes = crypto.randomBytes(16).toString("hex"); - } - secrets = secretsToEncrypt.map((secret) => { - // encrypt key - const { - ciphertext: secretKeyCiphertext, - iv: secretKeyIV, - tag: secretKeyTag - } = encryptSymmetric({ - plaintext: secret.key, - key: randomBytes - }); - - // encrypt value - const { - ciphertext: secretValueCiphertext, - iv: secretValueIV, - tag: secretValueTag - } = encryptSymmetric({ - plaintext: secret.value ?? "", - key: randomBytes - }); - - // encrypt comment - const { - ciphertext: secretCommentCiphertext, - iv: secretCommentIV, - tag: secretCommentTag - } = encryptSymmetric({ - plaintext: secret.comment ?? "", - key: randomBytes - }); - - const result: EncryptedSecretProps = { + const result = { id: secret.id, createdAt: "", environment: env, - secretName: secret.key, - secretKeyCiphertext, - secretKeyIV, - secretKeyTag, - secretValueCiphertext, - secretValueIV, - secretValueTag, - secretCommentCiphertext, - secretCommentIV, - secretCommentTag, - type: - secret.valueOverride === undefined || secret?.value !== secret?.valueOverride - ? SecretType.Shared - : SecretType.Personal, + secretKey: secret.key, + secretValue: secret.value, + secretComment: secret.comment, tags: secret.tags }; diff --git a/frontend/src/components/v2/Checkbox/Checkbox.tsx b/frontend/src/components/v2/Checkbox/Checkbox.tsx index 0e4581372..8c7472b26 100644 --- a/frontend/src/components/v2/Checkbox/Checkbox.tsx +++ b/frontend/src/components/v2/Checkbox/Checkbox.tsx @@ -14,6 +14,7 @@ export type CheckboxProps = Omit< isChecked?: boolean; isRequired?: boolean; checkIndicatorBg?: string | undefined; + isError?: boolean; }; export const Checkbox = ({ @@ -24,6 +25,7 @@ export const Checkbox = ({ isDisabled, isRequired, checkIndicatorBg, + isError, ...props }: CheckboxProps): JSX.Element => { return ( @@ -46,7 +48,10 @@ export const Checkbox = ({ -