Merge pull request #2514 from Infisical/daniel/create-multiple-project-envs

fix: allow creation of multiple project envs
2026-09-22 13:39:35 +00:00 · 2024-10-04 00:04:10 +04:00
parent ee2ee48b47 787f8318fe
commit 6c6e932899
3 changed files with 129 additions and 49 deletions
--- a/backend/src/keystore/keystore.ts
+++ b/backend/src/keystore/keystore.ts
@@ -16,6 +16,9 @@ export const KeyStorePrefixes = {
  WaitUntilReadyKmsOrgKeyCreation: "wait-until-ready-kms-org-key-creation-",
  WaitUntilReadyKmsOrgDataKeyCreation: "wait-until-ready-kms-org-data-key-creation-",

+  WaitUntilReadyProjectEnvironmentOperation: (projectId: string) =>
+    `wait-until-ready-project-environments-operation-${projectId}`,
+  ProjectEnvironmentLock: (projectId: string) => `project-environment-lock-${projectId}` as const,
  SyncSecretIntegrationLock: (projectId: string, environmentSlug: string, secretPath: string) =>
    `sync-integration-mutex-${projectId}-${environmentSlug}-${secretPath}` as const,
  SyncSecretIntegrationLastRunTimestamp: (projectId: string, environmentSlug: string, secretPath: string) =>
--- a/backend/src/server/routes/index.ts
+++ b/backend/src/server/routes/index.ts
@@ -748,6 +748,7 @@ export const registerRoutes = async (
  const projectEnvService = projectEnvServiceFactory({
    permissionService,
    projectEnvDAL,
+    keyStore,
    licenseService,
    projectDAL,
    folderDAL
--- a/backend/src/services/project-env/project-env-service.ts
+++ b/backend/src/services/project-env/project-env-service.ts
@@ -3,7 +3,9 @@ import { ForbiddenError } from "@casl/ability";
 import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
 import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
 import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission";
+import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore";
 import { BadRequestError, NotFoundError } from "@app/lib/errors";
+import { logger } from "@app/lib/logger";

 import { TProjectDALFactory } from "../project/project-dal";
 import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
@@ -16,6 +18,7 @@ type TProjectEnvServiceFactoryDep = {
  projectDAL: Pick<TProjectDALFactory, "findById">;
  permissionService: Pick<TPermissionServiceFactory, "getProjectPermission">;
  licenseService: Pick<TLicenseServiceFactory, "getPlan">;
+  keyStore: Pick<TKeyStoreFactory, "acquireLock" | "setItemWithExpiry" | "getItem" | "waitTillReady">;
 };

 export type TProjectEnvServiceFactory = ReturnType<typeof projectEnvServiceFactory>;
@@ -24,6 +27,7 @@ export const projectEnvServiceFactory = ({
  projectEnvDAL,
  permissionService,
  licenseService,
+  keyStore,
  projectDAL,
  folderDAL
 }: TProjectEnvServiceFactoryDep) => {
@@ -45,32 +49,56 @@ export const projectEnvServiceFactory = ({
    );
    ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Create, ProjectPermissionSub.Environments);

-    const envs = await projectEnvDAL.find({ projectId });
-    const existingEnv = envs.find(({ slug: envSlug }) => envSlug === slug);
-    if (existingEnv)
-      throw new BadRequestError({
-        message: "Environment with slug already exist",
-        name: "CreateEnvironment"
+    const lock = await keyStore
+      .acquireLock([KeyStorePrefixes.ProjectEnvironmentLock(projectId)], 5000)
+      .catch(() => null);
+
+    try {
+      if (!lock) {
+        await keyStore.waitTillReady({
+          key: KeyStorePrefixes.WaitUntilReadyProjectEnvironmentOperation(projectId),
+          keyCheckCb: (val) => val === "true",
+          waitingCb: () => logger.debug("Create project environment. Waiting for "),
+          delay: 500
+        });
+      }
+
+      const envs = await projectEnvDAL.find({ projectId });
+      const existingEnv = envs.find(({ slug: envSlug }) => envSlug === slug);
+      if (existingEnv)
+        throw new BadRequestError({
+          message: "Environment with slug already exist",
+          name: "CreateEnvironment"
+        });
+
+      const project = await projectDAL.findById(projectId);
+      const plan = await licenseService.getPlan(project.orgId);
+      if (plan.environmentLimit !== null && envs.length >= plan.environmentLimit) {
+        // case: limit imposed on number of environments allowed
+        // case: number of environments used exceeds the number of environments allowed
+        throw new BadRequestError({
+          message:
+            "Failed to create environment due to environment limit reached. Upgrade plan to create more environments."
+        });
+      }
+
+      const env = await projectEnvDAL.transaction(async (tx) => {
+        const lastPos = await projectEnvDAL.findLastEnvPosition(projectId, tx);
+        const doc = await projectEnvDAL.create({ slug, name, projectId, position: lastPos + 1 }, tx);
+        await folderDAL.create({ name: "root", parentId: null, envId: doc.id, version: 1 }, tx);
+        return doc;
      });

-    const project = await projectDAL.findById(projectId);
-    const plan = await licenseService.getPlan(project.orgId);
-    if (plan.environmentLimit !== null && envs.length >= plan.environmentLimit) {
-      // case: limit imposed on number of environments allowed
-      // case: number of environments used exceeds the number of environments allowed
-      throw new BadRequestError({
-        message:
-          "Failed to create environment due to environment limit reached. Upgrade plan to create more environments."
-      });
+      await keyStore.setItemWithExpiry(
+        KeyStorePrefixes.WaitUntilReadyProjectEnvironmentOperation(projectId),
+        10,
+        "true"
+      );
+
+      return env;
+    } finally {
+      await lock?.release();
    }
-
-    const env = await projectEnvDAL.transaction(async (tx) => {
-      const lastPos = await projectEnvDAL.findLastEnvPosition(projectId, tx);
-      const doc = await projectEnvDAL.create({ slug, name, projectId, position: lastPos + 1 }, tx);
-      await folderDAL.create({ name: "root", parentId: null, envId: doc.id, version: 1 }, tx);
-      return doc;
-    });
-    return env;
  };

  const updateEnvironment = async ({
@@ -93,26 +121,50 @@ export const projectEnvServiceFactory = ({
    );
    ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Edit, ProjectPermissionSub.Environments);

-    const oldEnv = await projectEnvDAL.findOne({ id, projectId });
-    if (!oldEnv) throw new NotFoundError({ message: "Environment not found" });
+    const lock = await keyStore
+      .acquireLock([KeyStorePrefixes.ProjectEnvironmentLock(projectId)], 5000)
+      .catch(() => null);

-    if (slug) {
-      const existingEnv = await projectEnvDAL.findOne({ slug, projectId });
-      if (existingEnv && existingEnv.id !== id) {
-        throw new BadRequestError({
-          message: "Environment with slug already exist",
-          name: "UpdateEnvironment"
+    try {
+      if (!lock) {
+        await keyStore.waitTillReady({
+          key: KeyStorePrefixes.WaitUntilReadyProjectEnvironmentOperation(projectId),
+          keyCheckCb: (val) => val === "true",
+          waitingCb: () => logger.debug("Update project environment. Waiting for project environment update"),
+          delay: 500
        });
      }
-    }

-    const env = await projectEnvDAL.transaction(async (tx) => {
-      if (position) {
-        await projectEnvDAL.updateAllPosition(projectId, oldEnv.position, position, tx);
+      const oldEnv = await projectEnvDAL.findOne({ id, projectId });
+      if (!oldEnv) throw new NotFoundError({ message: "Environment not found", name: "UpdateEnvironment" });
+
+      if (slug) {
+        const existingEnv = await projectEnvDAL.findOne({ slug, projectId });
+        if (existingEnv && existingEnv.id !== id) {
+          throw new BadRequestError({
+            message: "Environment with slug already exist",
+            name: "UpdateEnvironment"
+          });
+        }
      }
-      return projectEnvDAL.updateById(oldEnv.id, { name, slug, position }, tx);
-    });
-    return { environment: env, old: oldEnv };
+
+      const env = await projectEnvDAL.transaction(async (tx) => {
+        if (position) {
+          await projectEnvDAL.updateAllPosition(projectId, oldEnv.position, position, tx);
+        }
+        return projectEnvDAL.updateById(oldEnv.id, { name, slug, position }, tx);
+      });
+
+      await keyStore.setItemWithExpiry(
+        KeyStorePrefixes.WaitUntilReadyProjectEnvironmentOperation(projectId),
+        10,
+        "true"
+      );
+
+      return { environment: env, old: oldEnv };
+    } finally {
+      await lock?.release();
+    }
  };

  const deleteEnvironment = async ({ projectId, actor, actorId, actorOrgId, actorAuthMethod, id }: TDeleteEnvDTO) => {
@@ -125,18 +177,42 @@ export const projectEnvServiceFactory = ({
    );
    ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Delete, ProjectPermissionSub.Environments);

-    const env = await projectEnvDAL.transaction(async (tx) => {
-      const [doc] = await projectEnvDAL.delete({ id, projectId }, tx);
-      if (!doc)
-        throw new NotFoundError({
-          message: "Env doesn't exist",
-          name: "DeleteEnvironment"
-        });
+    const lock = await keyStore
+      .acquireLock([KeyStorePrefixes.ProjectEnvironmentLock(projectId)], 5000)
+      .catch(() => null);

-      await projectEnvDAL.updateAllPosition(projectId, doc.position, -1, tx);
-      return doc;
-    });
-    return env;
+    try {
+      if (!lock) {
+        await keyStore.waitTillReady({
+          key: KeyStorePrefixes.WaitUntilReadyProjectEnvironmentOperation(projectId),
+          keyCheckCb: (val) => val === "true",
+          waitingCb: () => logger.debug("Delete project environment. Waiting for "),
+          delay: 500
+        });
+      }
+
+      const env = await projectEnvDAL.transaction(async (tx) => {
+        const [doc] = await projectEnvDAL.delete({ id, projectId }, tx);
+        if (!doc)
+          throw new NotFoundError({
+            message: "Environment doesn't exist",
+            name: "DeleteEnvironment"
+          });
+
+        await projectEnvDAL.updateAllPosition(projectId, doc.position, -1, tx);
+        return doc;
+      });
+
+      await keyStore.setItemWithExpiry(
+        KeyStorePrefixes.WaitUntilReadyProjectEnvironmentOperation(projectId),
+        10,
+        "true"
+      );
+
+      return env;
+    } finally {
+      await lock?.release();
+    }
  };

  const getEnvironmentById = async ({ projectId, actor, actorId, actorOrgId, actorAuthMethod, id }: TGetEnvDTO) => {