Merge pull request #2530 from Infisical/daniel/terraform-imports-prerequsuite

feat: terraform imports prerequisite / api improvements

.github/pull_request_template.md

@@ -6,6 +6,7 @@
 
 - [ ] Bug fix
 - [ ] New feature
+- [ ] Improvement
 - [ ] Breaking change
 - [ ] Documentation
 

backend/src/ee/services/audit-log/audit-log-types.ts

@@ -123,6 +123,7 @@ export enum EventType {
   UPDATE_WEBHOOK_STATUS = "update-webhook-status",
   DELETE_WEBHOOK = "delete-webhook",
   GET_SECRET_IMPORTS = "get-secret-imports",
+  GET_SECRET_IMPORT = "get-secret-import",
   CREATE_SECRET_IMPORT = "create-secret-import",
   UPDATE_SECRET_IMPORT = "update-secret-import",
   DELETE_SECRET_IMPORT = "delete-secret-import",
@@ -1011,6 +1012,14 @@ interface GetSecretImportsEvent {
   };
 }
 
+interface GetSecretImportEvent {
+  type: EventType.GET_SECRET_IMPORT;
+  metadata: {
+    secretImportId: string;
+    folderId: string;
+  };
+}
+
 interface CreateSecretImportEvent {
   type: EventType.CREATE_SECRET_IMPORT;
   metadata: {
@@ -1674,6 +1683,7 @@ export type Event =
   | UpdateWebhookStatusEvent
   | DeleteWebhookEvent
   | GetSecretImportsEvent
+  | GetSecretImportEvent
   | CreateSecretImportEvent
   | UpdateSecretImportEvent
   | DeleteSecretImportEvent

backend/src/lib/api-docs/constants.ts

@@ -675,6 +675,9 @@ export const SECRET_IMPORTS = {
     environment: "The slug of the environment to list secret imports from.",
     path: "The path to list secret imports from."
   },
+  GET: {
+    secretImportId: "The ID of the secret import to fetch."
+  },
   CREATE: {
     environment: "The slug of the environment to import into.",
     path: "The path to import into.",

backend/src/server/routes/v1/project-env-router.ts

@@ -4,7 +4,7 @@ import { z } from "zod";
 import { ProjectEnvironmentsSchema } from "@app/db/schemas";
 import { EventType } from "@app/ee/services/audit-log/audit-log-types";
 import { ENVIRONMENTS } from "@app/lib/api-docs";
-import { writeLimit } from "@app/server/config/rateLimiter";
+import { readLimit, writeLimit } from "@app/server/config/rateLimiter";
 import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
 import { AuthMode } from "@app/services/auth/auth-type";
 
@@ -23,6 +23,7 @@ export const registerProjectEnvRouter = async (server: FastifyZodProvider) => {
         }
       ],
       params: z.object({
+        // NOTE(daniel): workspaceId isn't used, but we need to keep it for backwards compatibility. The endpoint defined below, uses no project ID, and is takes a pure environment ID.
         workspaceId: z.string().trim().describe(ENVIRONMENTS.GET.workspaceId),
         envId: z.string().trim().describe(ENVIRONMENTS.GET.id)
       }),
@@ -39,7 +40,53 @@ export const registerProjectEnvRouter = async (server: FastifyZodProvider) => {
         actor: req.permission.type,
         actorOrgId: req.permission.orgId,
         actorAuthMethod: req.permission.authMethod,
-        projectId: req.params.workspaceId,
+        id: req.params.envId
+      });
+
+      await server.services.auditLog.createAuditLog({
+        ...req.auditLogInfo,
+        projectId: environment.projectId,
+        event: {
+          type: EventType.GET_ENVIRONMENT,
+          metadata: {
+            id: environment.id
+          }
+        }
+      });
+
+      return { environment };
+    }
+  });
+
+  server.route({
+    method: "GET",
+    url: "/environments/:envId",
+    config: {
+      rateLimit: readLimit
+    },
+    schema: {
+      description: "Get Environment by ID",
+      security: [
+        {
+          bearerAuth: []
+        }
+      ],
+      params: z.object({
+        envId: z.string().trim().describe(ENVIRONMENTS.GET.id)
+      }),
+      response: {
+        200: z.object({
+          environment: ProjectEnvironmentsSchema
+        })
+      }
+    },
+    onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
+    handler: async (req) => {
+      const environment = await server.services.projectEnv.getEnvironmentById({
+        actorId: req.permission.id,
+        actor: req.permission.type,
+        actorOrgId: req.permission.orgId,
+        actorAuthMethod: req.permission.authMethod,
         id: req.params.envId
       });
 

backend/src/server/routes/v1/secret-folder-router.ts

@@ -365,7 +365,15 @@ export const registerSecretFolderRouter = async (server: FastifyZodProvider) =>
       }),
       response: {
         200: z.object({
-          folder: SecretFoldersSchema
+          folder: SecretFoldersSchema.extend({
+            environment: z.object({
+              envId: z.string(),
+              envName: z.string(),
+              envSlug: z.string()
+            }),
+            path: z.string(),
+            projectId: z.string()
+          })
         })
       }
     },

backend/src/server/routes/v1/secret-import-router.ts

@@ -312,6 +312,64 @@ export const registerSecretImportRouter = async (server: FastifyZodProvider) =>
     }
   });
 
+  server.route({
+    url: "/:secretImportId",
+    method: "GET",
+    config: {
+      rateLimit: readLimit
+    },
+    schema: {
+      description: "Get single secret import",
+      security: [
+        {
+          bearerAuth: []
+        }
+      ],
+      params: z.object({
+        secretImportId: z.string().trim().describe(SECRET_IMPORTS.GET.secretImportId)
+      }),
+      response: {
+        200: z.object({
+          secretImport: SecretImportsSchema.omit({ importEnv: true }).extend({
+            environment: z.object({
+              id: z.string(),
+              name: z.string(),
+              slug: z.string()
+            }),
+            projectId: z.string(),
+            importEnv: z.object({ name: z.string(), slug: z.string(), id: z.string() }),
+            secretPath: z.string()
+          })
+        })
+      }
+    },
+
+    onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY, AuthMode.SERVICE_TOKEN, AuthMode.IDENTITY_ACCESS_TOKEN]),
+    handler: async (req) => {
+      const secretImport = await server.services.secretImport.getImportById({
+        actorId: req.permission.id,
+        actor: req.permission.type,
+        actorAuthMethod: req.permission.authMethod,
+        actorOrgId: req.permission.orgId,
+        id: req.params.secretImportId
+      });
+
+      await server.services.auditLog.createAuditLog({
+        ...req.auditLogInfo,
+        projectId: secretImport.projectId,
+        event: {
+          type: EventType.GET_SECRET_IMPORT,
+          metadata: {
+            secretImportId: secretImport.id,
+            folderId: secretImport.folderId
+          }
+        }
+      });
+
+      return { secretImport };
+    }
+  });
+
   server.route({
     url: "/secrets",
     method: "GET",

backend/src/services/project-env/project-env-service.ts

@@ -215,29 +215,26 @@ export const projectEnvServiceFactory = ({
     }
   };
 
-  const getEnvironmentById = async ({ projectId, actor, actorId, actorOrgId, actorAuthMethod, id }: TGetEnvDTO) => {
+  const getEnvironmentById = async ({ actor, actorId, actorOrgId, actorAuthMethod, id }: TGetEnvDTO) => {
+    const environment = await projectEnvDAL.findById(id);
+
+    if (!environment) {
+      throw new NotFoundError({
+        message: "Environment does not exist"
+      });
+    }
+
     const { permission } = await permissionService.getProjectPermission(
       actor,
       actorId,
-      projectId,
+      environment.projectId,
       actorAuthMethod,
       actorOrgId
     );
 
     ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Environments);
 
-    const [env] = await projectEnvDAL.find({
+    return environment;
-      id,
-      projectId
-    });
-
-    if (!env) {
-      throw new NotFoundError({
-        message: "Environment does not exist"
-      });
-    }
-
-    return env;
   };
 
   return {

backend/src/services/project-env/project-env-types.ts

@@ -23,4 +23,4 @@ export type TReorderEnvDTO = {
 
 export type TGetEnvDTO = {
   id: string;
-} & TProjectPermission;
+} & Omit<TProjectPermission, "projectId">;

backend/src/services/secret-folder/secret-folder-service.ts

@@ -502,12 +502,21 @@ export const secretFolderServiceFactory = ({
 
   const getFolderById = async ({ actor, actorId, actorOrgId, actorAuthMethod, id }: TGetFolderByIdDTO) => {
     const folder = await folderDAL.findById(id);
-    if (!folder) throw new NotFoundError({ message: "folder not found" });
+    if (!folder) throw new NotFoundError({ message: "Folder not found" });
     // folder list is allowed to be read by anyone
     // permission to check does user has access
     await permissionService.getProjectPermission(actor, actorId, folder.projectId, actorAuthMethod, actorOrgId);
 
-    return folder;
+    const [folderWithPath] = await folderDAL.findSecretPathByFolderIds(folder.projectId, [folder.id]);
+
+    if (!folderWithPath) {
+      throw new NotFoundError({ message: "Folder path not found" });
+    }
+
+    return {
+      ...folder,
+      path: folderWithPath.path
+    };
   };
 
   return {

backend/src/services/secret-import/secret-import-dal.ts

@@ -97,6 +97,34 @@ export const secretImportDALFactory = (db: TDbClient) => {
     }
   };
 
+  const findById = async (id: string, tx?: Knex) => {
+    try {
+      const doc = await (tx || db.replicaNode())(TableName.SecretImport)
+        .where({ [`${TableName.SecretImport}.id` as "id"]: id })
+        .join(TableName.Environment, `${TableName.SecretImport}.importEnv`, `${TableName.Environment}.id`)
+        .select(
+          db.ref("*").withSchema(TableName.SecretImport) as unknown as keyof TSecretImports,
+          db.ref("slug").withSchema(TableName.Environment),
+          db.ref("name").withSchema(TableName.Environment),
+          db.ref("id").withSchema(TableName.Environment).as("envId")
+        )
+        .first();
+
+      if (!doc) {
+        return null;
+      }
+
+      const { envId, slug, name, ...el } = doc;
+
+      return {
+        ...el,
+        importEnv: { id: envId, slug, name }
+      };
+    } catch (error) {
+      throw new DatabaseError({ error, name: "Find secret imports" });
+    }
+  };
+
   const getProjectImportCount = async (
     { search, ...filter }: Partial<TSecretImports & { projectId: string; search?: string }>,
     tx?: Knex
@@ -144,6 +172,7 @@ export const secretImportDALFactory = (db: TDbClient) => {
   return {
     ...secretImportOrm,
     find,
+    findById,
     findByFolderIds,
     findLastImportPosition,
     updateAllPosition,

backend/src/services/secret-import/secret-import-service.ts

@@ -24,6 +24,7 @@ import { fnSecretsFromImports, fnSecretsV2FromImports } from "./secret-import-fn
 import {
   TCreateSecretImportDTO,
   TDeleteSecretImportDTO,
+  TGetSecretImportByIdDTO,
   TGetSecretImportsDTO,
   TGetSecretsFromImportDTO,
   TResyncSecretImportReplicationDTO,
@@ -455,6 +456,64 @@ export const secretImportServiceFactory = ({
     return secImports;
   };
 
+  const getImportById = async ({
+    actor,
+    actorId,
+    actorAuthMethod,
+    actorOrgId,
+    id: importId
+  }: TGetSecretImportByIdDTO) => {
+    const importDoc = await secretImportDAL.findById(importId);
+
+    if (!importDoc) {
+      throw new NotFoundError({ message: "Secret import not found" });
+    }
+
+    // the folder to import into
+    const folder = await folderDAL.findById(importDoc.folderId);
+
+    if (!folder) throw new NotFoundError({ message: "Secret import folder not found" });
+
+    // the folder to import into, with path
+    const [folderWithPath] = await folderDAL.findSecretPathByFolderIds(folder.projectId, [folder.id]);
+
+    if (!folderWithPath) throw new NotFoundError({ message: "Folder path not found" });
+
+    const { permission } = await permissionService.getProjectPermission(
+      actor,
+      actorId,
+      folder.projectId,
+      actorAuthMethod,
+      actorOrgId
+    );
+
+    ForbiddenError.from(permission).throwUnlessCan(
+      ProjectPermissionActions.Read,
+      subject(ProjectPermissionSub.Secrets, {
+        environment: folder.environment.envSlug,
+        secretPath: folderWithPath.path
+      })
+    );
+
+    const importIntoEnv = await projectEnvDAL.findOne({
+      projectId: folder.projectId,
+      slug: folder.environment.envSlug
+    });
+
+    if (!importIntoEnv) throw new NotFoundError({ message: "Environment to import into not found" });
+
+    return {
+      ...importDoc,
+      projectId: folder.projectId,
+      secretPath: folderWithPath.path,
+      environment: {
+        id: importIntoEnv.id,
+        slug: importIntoEnv.slug,
+        name: importIntoEnv.name
+      }
+    };
+  };
+
   const getSecretsFromImports = async ({
     path: secretPath,
     environment,
@@ -565,6 +624,7 @@ export const secretImportServiceFactory = ({
     updateImport,
     deleteImport,
     getImports,
+    getImportById,
     getSecretsFromImports,
     getRawSecretsFromImports,
     resyncSecretImportReplication,

backend/src/services/secret-import/secret-import-types.ts

@@ -37,6 +37,10 @@ export type TGetSecretImportsDTO = {
   offset?: number;
 } & TProjectPermission;
 
+export type TGetSecretImportByIdDTO = {
+  id: string;
+} & Omit<TProjectPermission, "projectId">;
+
 export type TGetSecretsFromImportDTO = {
   environment: string;
   path: string;