From d5662dfef48a4cfb93d33c2d7c2e1574bf12e069 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Mon, 30 Sep 2024 17:20:37 +0400 Subject: [PATCH 1/4] feat: allow creation of multiple project envs --- backend/src/keystore/keystore.ts | 7 + backend/src/server/routes/index.ts | 1 + .../project-env/project-env-service.ts | 174 +++++++++++++----- 3 files changed, 133 insertions(+), 49 deletions(-) diff --git a/backend/src/keystore/keystore.ts b/backend/src/keystore/keystore.ts index 3b19012e7..c78c428b0 100644 --- a/backend/src/keystore/keystore.ts +++ b/backend/src/keystore/keystore.ts @@ -16,6 +16,13 @@ export const KeyStorePrefixes = { WaitUntilReadyKmsOrgKeyCreation: "wait-until-ready-kms-org-key-creation-", WaitUntilReadyKmsOrgDataKeyCreation: "wait-until-ready-kms-org-data-key-creation-", + ProjectEnvironmentCreation: "project-environment-creation-lock", + ProjectEnvironmentUpdate: "project-environment-update-lock", + ProjectEnvironmentDelete: "project-environment-delete-lock", + WaitUntilReadyCreateProjectEnvironment: "wait-until-ready-create-project-environments-", + WaitUntilReadyUpdateProjectEnvironment: "wait-until-ready-update-project-environments-", + WaitUntilReadyDeleteProjectEnvironment: "wait-until-ready-delete-project-environments-", + SyncSecretIntegrationLock: (projectId: string, environmentSlug: string, secretPath: string) => `sync-integration-mutex-${projectId}-${environmentSlug}-${secretPath}` as const, SyncSecretIntegrationLastRunTimestamp: (projectId: string, environmentSlug: string, secretPath: string) => diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index b548e2ff5..3ad8cf750 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -748,6 +748,7 @@ export const registerRoutes = async ( const projectEnvService = projectEnvServiceFactory({ permissionService, projectEnvDAL, + keyStore, licenseService, projectDAL, folderDAL diff --git a/backend/src/services/project-env/project-env-service.ts b/backend/src/services/project-env/project-env-service.ts index b39518066..2fc2207a3 100644 --- a/backend/src/services/project-env/project-env-service.ts +++ b/backend/src/services/project-env/project-env-service.ts @@ -3,7 +3,9 @@ import { ForbiddenError } from "@casl/ability"; import { TLicenseServiceFactory } from "@app/ee/services/license/license-service"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; +import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore"; import { BadRequestError, NotFoundError } from "@app/lib/errors"; +import { logger } from "@app/lib/logger"; import { TProjectDALFactory } from "../project/project-dal"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; @@ -16,6 +18,7 @@ type TProjectEnvServiceFactoryDep = { projectDAL: Pick; permissionService: Pick; licenseService: Pick; + keyStore: Pick; }; export type TProjectEnvServiceFactory = ReturnType; @@ -24,6 +27,7 @@ export const projectEnvServiceFactory = ({ projectEnvDAL, permissionService, licenseService, + keyStore, projectDAL, folderDAL }: TProjectEnvServiceFactoryDep) => { @@ -45,32 +49,56 @@ export const projectEnvServiceFactory = ({ ); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Create, ProjectPermissionSub.Environments); - const envs = await projectEnvDAL.find({ projectId }); - const existingEnv = envs.find(({ slug: envSlug }) => envSlug === slug); - if (existingEnv) - throw new BadRequestError({ - message: "Environment with slug already exist", - name: "CreateEnvironment" + const lock = await keyStore + .acquireLock([KeyStorePrefixes.ProjectEnvironmentCreation, projectId], 5000) + .catch(() => null); + + try { + if (!lock) { + await keyStore.waitTillReady({ + key: `${KeyStorePrefixes.WaitUntilReadyCreateProjectEnvironment}${projectId}`, + keyCheckCb: (val) => val === "true", + waitingCb: () => logger.debug("Create project environment. Waiting for "), + delay: 500 + }); + } + + const envs = await projectEnvDAL.find({ projectId }); + const existingEnv = envs.find(({ slug: envSlug }) => envSlug === slug); + if (existingEnv) + throw new BadRequestError({ + message: "Environment with slug already exist", + name: "CreateEnvironment" + }); + + const project = await projectDAL.findById(projectId); + const plan = await licenseService.getPlan(project.orgId); + if (plan.environmentLimit !== null && envs.length >= plan.environmentLimit) { + // case: limit imposed on number of environments allowed + // case: number of environments used exceeds the number of environments allowed + throw new BadRequestError({ + message: + "Failed to create environment due to environment limit reached. Upgrade plan to create more environments." + }); + } + + const env = await projectEnvDAL.transaction(async (tx) => { + const lastPos = await projectEnvDAL.findLastEnvPosition(projectId, tx); + const doc = await projectEnvDAL.create({ slug, name, projectId, position: lastPos + 1 }, tx); + await folderDAL.create({ name: "root", parentId: null, envId: doc.id, version: 1 }, tx); + return doc; }); - const project = await projectDAL.findById(projectId); - const plan = await licenseService.getPlan(project.orgId); - if (plan.environmentLimit !== null && envs.length >= plan.environmentLimit) { - // case: limit imposed on number of environments allowed - // case: number of environments used exceeds the number of environments allowed - throw new BadRequestError({ - message: - "Failed to create environment due to environment limit reached. Upgrade plan to create more environments." - }); + await keyStore.setItemWithExpiry( + `${KeyStorePrefixes.WaitUntilReadyCreateProjectEnvironment}${projectId}`, + 10, + "true" + ); + + return env; + } finally { + await lock?.release(); } - - const env = await projectEnvDAL.transaction(async (tx) => { - const lastPos = await projectEnvDAL.findLastEnvPosition(projectId, tx); - const doc = await projectEnvDAL.create({ slug, name, projectId, position: lastPos + 1 }, tx); - await folderDAL.create({ name: "root", parentId: null, envId: doc.id, version: 1 }, tx); - return doc; - }); - return env; }; const updateEnvironment = async ({ @@ -93,26 +121,50 @@ export const projectEnvServiceFactory = ({ ); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Edit, ProjectPermissionSub.Environments); - const oldEnv = await projectEnvDAL.findOne({ id, projectId }); - if (!oldEnv) throw new NotFoundError({ message: "Environment not found" }); + const lock = await keyStore + .acquireLock([KeyStorePrefixes.ProjectEnvironmentUpdate, projectId], 5000) + .catch(() => null); - if (slug) { - const existingEnv = await projectEnvDAL.findOne({ slug, projectId }); - if (existingEnv && existingEnv.id !== id) { - throw new BadRequestError({ - message: "Environment with slug already exist", - name: "UpdateEnvironment" + try { + if (!lock) { + await keyStore.waitTillReady({ + key: `${KeyStorePrefixes.WaitUntilReadyUpdateProjectEnvironment}${projectId}`, + keyCheckCb: (val) => val === "true", + waitingCb: () => logger.debug("Update project environment. Waiting for project environment update"), + delay: 500 }); } - } - const env = await projectEnvDAL.transaction(async (tx) => { - if (position) { - await projectEnvDAL.updateAllPosition(projectId, oldEnv.position, position, tx); + const oldEnv = await projectEnvDAL.findOne({ id, projectId }); + if (!oldEnv) throw new NotFoundError({ message: "Environment not found", name: "UpdateEnvironment" }); + + if (slug) { + const existingEnv = await projectEnvDAL.findOne({ slug, projectId }); + if (existingEnv && existingEnv.id !== id) { + throw new BadRequestError({ + message: "Environment with slug already exist", + name: "UpdateEnvironment" + }); + } } - return projectEnvDAL.updateById(oldEnv.id, { name, slug, position }, tx); - }); - return { environment: env, old: oldEnv }; + + const env = await projectEnvDAL.transaction(async (tx) => { + if (position) { + await projectEnvDAL.updateAllPosition(projectId, oldEnv.position, position, tx); + } + return projectEnvDAL.updateById(oldEnv.id, { name, slug, position }, tx); + }); + + await keyStore.setItemWithExpiry( + `${KeyStorePrefixes.WaitUntilReadyUpdateProjectEnvironment}${projectId}`, + 10, + "true" + ); + + return { environment: env, old: oldEnv }; + } finally { + await lock?.release(); + } }; const deleteEnvironment = async ({ projectId, actor, actorId, actorOrgId, actorAuthMethod, id }: TDeleteEnvDTO) => { @@ -125,18 +177,42 @@ export const projectEnvServiceFactory = ({ ); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Delete, ProjectPermissionSub.Environments); - const env = await projectEnvDAL.transaction(async (tx) => { - const [doc] = await projectEnvDAL.delete({ id, projectId }, tx); - if (!doc) - throw new NotFoundError({ - message: "Env doesn't exist", - name: "DeleteEnvironment" - }); + const lock = await keyStore + .acquireLock([KeyStorePrefixes.ProjectEnvironmentDelete, projectId], 5000) + .catch(() => null); - await projectEnvDAL.updateAllPosition(projectId, doc.position, -1, tx); - return doc; - }); - return env; + try { + if (!lock) { + await keyStore.waitTillReady({ + key: `${KeyStorePrefixes.WaitUntilReadyDeleteProjectEnvironment}${projectId}`, + keyCheckCb: (val) => val === "true", + waitingCb: () => logger.debug("Delete project environment. Waiting for "), + delay: 500 + }); + } + + const env = await projectEnvDAL.transaction(async (tx) => { + const [doc] = await projectEnvDAL.delete({ id, projectId }, tx); + if (!doc) + throw new NotFoundError({ + message: "Environment doesn't exist", + name: "DeleteEnvironment" + }); + + await projectEnvDAL.updateAllPosition(projectId, doc.position, -1, tx); + return doc; + }); + + await keyStore.setItemWithExpiry( + `${KeyStorePrefixes.WaitUntilReadyDeleteProjectEnvironment}${projectId}`, + 10, + "true" + ); + + return env; + } finally { + await lock?.release(); + } }; const getEnvironmentById = async ({ projectId, actor, actorId, actorOrgId, actorAuthMethod, id }: TGetEnvDTO) => { From 0abab57d8320cfe5874d7a7849a1e97ce1636b85 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Mon, 30 Sep 2024 17:25:00 +0400 Subject: [PATCH 2/4] fix: variable naming --- backend/src/keystore/keystore.ts | 2 +- backend/src/services/project-env/project-env-service.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/keystore/keystore.ts b/backend/src/keystore/keystore.ts index c78c428b0..37555f375 100644 --- a/backend/src/keystore/keystore.ts +++ b/backend/src/keystore/keystore.ts @@ -16,7 +16,7 @@ export const KeyStorePrefixes = { WaitUntilReadyKmsOrgKeyCreation: "wait-until-ready-kms-org-key-creation-", WaitUntilReadyKmsOrgDataKeyCreation: "wait-until-ready-kms-org-data-key-creation-", - ProjectEnvironmentCreation: "project-environment-creation-lock", + ProjectEnvironmentCreate: "project-environment-creation-lock", ProjectEnvironmentUpdate: "project-environment-update-lock", ProjectEnvironmentDelete: "project-environment-delete-lock", WaitUntilReadyCreateProjectEnvironment: "wait-until-ready-create-project-environments-", diff --git a/backend/src/services/project-env/project-env-service.ts b/backend/src/services/project-env/project-env-service.ts index 2fc2207a3..90d8c9fc3 100644 --- a/backend/src/services/project-env/project-env-service.ts +++ b/backend/src/services/project-env/project-env-service.ts @@ -50,7 +50,7 @@ export const projectEnvServiceFactory = ({ ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Create, ProjectPermissionSub.Environments); const lock = await keyStore - .acquireLock([KeyStorePrefixes.ProjectEnvironmentCreation, projectId], 5000) + .acquireLock([KeyStorePrefixes.ProjectEnvironmentCreate, projectId], 5000) .catch(() => null); try { From 9a27873af50f0e34427228ea10d4fed9b78cdb95 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Thu, 3 Oct 2024 21:47:37 +0400 Subject: [PATCH 3/4] requested changes --- backend/src/keystore/keystore.ts | 6 +++--- .../src/services/project-env/project-env-service.ts | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/backend/src/keystore/keystore.ts b/backend/src/keystore/keystore.ts index 37555f375..be63d03d9 100644 --- a/backend/src/keystore/keystore.ts +++ b/backend/src/keystore/keystore.ts @@ -19,9 +19,9 @@ export const KeyStorePrefixes = { ProjectEnvironmentCreate: "project-environment-creation-lock", ProjectEnvironmentUpdate: "project-environment-update-lock", ProjectEnvironmentDelete: "project-environment-delete-lock", - WaitUntilReadyCreateProjectEnvironment: "wait-until-ready-create-project-environments-", - WaitUntilReadyUpdateProjectEnvironment: "wait-until-ready-update-project-environments-", - WaitUntilReadyDeleteProjectEnvironment: "wait-until-ready-delete-project-environments-", + + WaitUntilReadyProjectEnvironmentOperation: (projectId: string) => + `wait-until-ready-project-environments-operation-${projectId}`, SyncSecretIntegrationLock: (projectId: string, environmentSlug: string, secretPath: string) => `sync-integration-mutex-${projectId}-${environmentSlug}-${secretPath}` as const, diff --git a/backend/src/services/project-env/project-env-service.ts b/backend/src/services/project-env/project-env-service.ts index 90d8c9fc3..f74e7544a 100644 --- a/backend/src/services/project-env/project-env-service.ts +++ b/backend/src/services/project-env/project-env-service.ts @@ -56,7 +56,7 @@ export const projectEnvServiceFactory = ({ try { if (!lock) { await keyStore.waitTillReady({ - key: `${KeyStorePrefixes.WaitUntilReadyCreateProjectEnvironment}${projectId}`, + key: KeyStorePrefixes.WaitUntilReadyProjectEnvironmentOperation(projectId), keyCheckCb: (val) => val === "true", waitingCb: () => logger.debug("Create project environment. Waiting for "), delay: 500 @@ -90,7 +90,7 @@ export const projectEnvServiceFactory = ({ }); await keyStore.setItemWithExpiry( - `${KeyStorePrefixes.WaitUntilReadyCreateProjectEnvironment}${projectId}`, + KeyStorePrefixes.WaitUntilReadyProjectEnvironmentOperation(projectId), 10, "true" ); @@ -128,7 +128,7 @@ export const projectEnvServiceFactory = ({ try { if (!lock) { await keyStore.waitTillReady({ - key: `${KeyStorePrefixes.WaitUntilReadyUpdateProjectEnvironment}${projectId}`, + key: KeyStorePrefixes.WaitUntilReadyProjectEnvironmentOperation(projectId), keyCheckCb: (val) => val === "true", waitingCb: () => logger.debug("Update project environment. Waiting for project environment update"), delay: 500 @@ -156,7 +156,7 @@ export const projectEnvServiceFactory = ({ }); await keyStore.setItemWithExpiry( - `${KeyStorePrefixes.WaitUntilReadyUpdateProjectEnvironment}${projectId}`, + KeyStorePrefixes.WaitUntilReadyProjectEnvironmentOperation(projectId), 10, "true" ); @@ -184,7 +184,7 @@ export const projectEnvServiceFactory = ({ try { if (!lock) { await keyStore.waitTillReady({ - key: `${KeyStorePrefixes.WaitUntilReadyDeleteProjectEnvironment}${projectId}`, + key: KeyStorePrefixes.WaitUntilReadyProjectEnvironmentOperation(projectId), keyCheckCb: (val) => val === "true", waitingCb: () => logger.debug("Delete project environment. Waiting for "), delay: 500 @@ -204,7 +204,7 @@ export const projectEnvServiceFactory = ({ }); await keyStore.setItemWithExpiry( - `${KeyStorePrefixes.WaitUntilReadyDeleteProjectEnvironment}${projectId}`, + KeyStorePrefixes.WaitUntilReadyProjectEnvironmentOperation(projectId), 10, "true" ); From 787f8318fec3717808fa69a93d0acd9c1afae77f Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Thu, 3 Oct 2024 22:50:47 +0400 Subject: [PATCH 4/4] updated locks --- backend/src/keystore/keystore.ts | 6 +----- backend/src/services/project-env/project-env-service.ts | 6 +++--- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/backend/src/keystore/keystore.ts b/backend/src/keystore/keystore.ts index be63d03d9..5b2c7fba9 100644 --- a/backend/src/keystore/keystore.ts +++ b/backend/src/keystore/keystore.ts @@ -16,13 +16,9 @@ export const KeyStorePrefixes = { WaitUntilReadyKmsOrgKeyCreation: "wait-until-ready-kms-org-key-creation-", WaitUntilReadyKmsOrgDataKeyCreation: "wait-until-ready-kms-org-data-key-creation-", - ProjectEnvironmentCreate: "project-environment-creation-lock", - ProjectEnvironmentUpdate: "project-environment-update-lock", - ProjectEnvironmentDelete: "project-environment-delete-lock", - WaitUntilReadyProjectEnvironmentOperation: (projectId: string) => `wait-until-ready-project-environments-operation-${projectId}`, - + ProjectEnvironmentLock: (projectId: string) => `project-environment-lock-${projectId}` as const, SyncSecretIntegrationLock: (projectId: string, environmentSlug: string, secretPath: string) => `sync-integration-mutex-${projectId}-${environmentSlug}-${secretPath}` as const, SyncSecretIntegrationLastRunTimestamp: (projectId: string, environmentSlug: string, secretPath: string) => diff --git a/backend/src/services/project-env/project-env-service.ts b/backend/src/services/project-env/project-env-service.ts index f74e7544a..645f34ade 100644 --- a/backend/src/services/project-env/project-env-service.ts +++ b/backend/src/services/project-env/project-env-service.ts @@ -50,7 +50,7 @@ export const projectEnvServiceFactory = ({ ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Create, ProjectPermissionSub.Environments); const lock = await keyStore - .acquireLock([KeyStorePrefixes.ProjectEnvironmentCreate, projectId], 5000) + .acquireLock([KeyStorePrefixes.ProjectEnvironmentLock(projectId)], 5000) .catch(() => null); try { @@ -122,7 +122,7 @@ export const projectEnvServiceFactory = ({ ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Edit, ProjectPermissionSub.Environments); const lock = await keyStore - .acquireLock([KeyStorePrefixes.ProjectEnvironmentUpdate, projectId], 5000) + .acquireLock([KeyStorePrefixes.ProjectEnvironmentLock(projectId)], 5000) .catch(() => null); try { @@ -178,7 +178,7 @@ export const projectEnvServiceFactory = ({ ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Delete, ProjectPermissionSub.Environments); const lock = await keyStore - .acquireLock([KeyStorePrefixes.ProjectEnvironmentDelete, projectId], 5000) + .acquireLock([KeyStorePrefixes.ProjectEnvironmentLock(projectId)], 5000) .catch(() => null); try {