Merge branch 'main' into feat/addReplicateFolderContent

backend/src/server/lib/schemas.ts

@@ -1,6 +1,8 @@
 import slugify from "@sindresorhus/slugify";
 import { z } from "zod";
 
+import { CharacterType, characterValidator } from "@app/lib/validator/validate-string";
+
 interface SlugSchemaInputs {
   min?: number;
   max?: number;
@@ -27,4 +29,13 @@ export const GenericResourceNameSchema = z
   .trim()
   .min(1, { message: "Name must be at least 1 character" })
   .max(64, { message: "Name must be 64 or fewer characters" })
-  .regex(/^[a-zA-Z0-9\-_\s]+$/, "Name can only contain alphanumeric characters, dashes, underscores, and spaces");
+  .refine(
+    (val) =>
+      characterValidator([
+        CharacterType.AlphaNumeric,
+        CharacterType.Hyphen,
+        CharacterType.Underscore,
+        CharacterType.Spaces
+      ])(val),
+    "Name can only contain alphanumeric characters, dashes, underscores, and spaces"
+  );

backend/src/server/plugins/error-handler.ts

@@ -13,6 +13,7 @@ import {
   InternalServerError,
   NotFoundError,
   OidcAuthError,
+  PermissionBoundaryError,
   RateLimitError,
   ScimRequestError,
   UnauthorizedError
@@ -117,7 +118,7 @@ export const fastifyErrHandler = fastifyPlugin(async (server: FastifyZodProvider
           conditions: el.conditions
         }))
       });
-    } else if (error instanceof ForbiddenRequestError) {
+    } else if (error instanceof ForbiddenRequestError || error instanceof PermissionBoundaryError) {
       void res.status(HttpStatusCodes.Forbidden).send({
         reqId: req.id,
         statusCode: HttpStatusCodes.Forbidden,

backend/src/server/plugins/secret-scanner.ts

@@ -65,7 +65,7 @@ export const registerSecretScannerGhApp = async (server: FastifyZodProvider) =>
           payload: JSON.stringify(req.body),
           signature: signatureSHA256
         });
-        void res.send("ok");
+        return res.send("ok");
       }
     });
   }

backend/src/server/plugins/serve-ui.ts

@@ -34,7 +34,7 @@ export const registerServeUI = async (
         TELEMETRY_CAPTURING_ENABLED: appCfg.TELEMETRY_ENABLED
       };
       const js = `window.__INFISICAL_RUNTIME_ENV__ = Object.freeze(${JSON.stringify(config)});`;
-      void res.send(js);
+      return res.send(js);
     }
   });
 
@@ -57,7 +57,7 @@ export const registerServeUI = async (
           reply.callNotFound();
           return;
         }
-        void reply.sendFile("index.html");
+        return reply.sendFile("index.html");
       }
     });
   }

backend/src/server/routes/index.ts

@@ -1,5 +1,6 @@
 import { CronJob } from "cron";
 import { Knex } from "knex";
+import { monitorEventLoopDelay } from "perf_hooks";
 import { z } from "zod";
 
 import { registerCertificateEstRouter } from "@app/ee/routes/est/certificate-est-router";
@@ -96,6 +97,7 @@ import { trustedIpDALFactory } from "@app/ee/services/trusted-ip/trusted-ip-dal"
 import { trustedIpServiceFactory } from "@app/ee/services/trusted-ip/trusted-ip-service";
 import { TKeyStoreFactory } from "@app/keystore/keystore";
 import { getConfig, TEnvConfig } from "@app/lib/config/env";
+import { logger } from "@app/lib/logger";
 import { TQueueServiceFactory } from "@app/queue";
 import { readLimit } from "@app/server/config/rateLimiter";
 import { accessTokenQueueServiceFactory } from "@app/services/access-token-queue/access-token-queue";
@@ -246,6 +248,9 @@ import { registerV1Routes } from "./v1";
 import { registerV2Routes } from "./v2";
 import { registerV3Routes } from "./v3";
 
+const histogram = monitorEventLoopDelay({ resolution: 20 });
+histogram.enable();
+
 export const registerRoutes = async (
   server: FastifyZodProvider,
   {
@@ -408,7 +413,14 @@ export const registerRoutes = async (
     serviceTokenDAL,
     projectDAL
   });
-  const licenseService = licenseServiceFactory({ permissionService, orgDAL, licenseDAL, keyStore });
+  const licenseService = licenseServiceFactory({
+    permissionService,
+    orgDAL,
+    licenseDAL,
+    keyStore,
+    identityOrgMembershipDAL,
+    projectDAL
+  });
 
   const hsmService = hsmServiceFactory({
     hsmModule,
@@ -1630,6 +1642,18 @@ export const registerRoutes = async (
       const cfg = getConfig();
       const serverCfg = await getServerCfg();
 
+      const meanLagMs = histogram.mean / 1e6;
+      const maxLagMs = histogram.max / 1e6;
+      const p99LagMs = histogram.percentile(99) / 1e6;
+
+      logger.info(
+        `Event loop stats - Mean: ${meanLagMs.toFixed(2)}ms, Max: ${maxLagMs.toFixed(2)}ms, p99: ${p99LagMs.toFixed(
+          2
+        )}ms`
+      );
+
+      logger.info(`Raw event loop stats: ${JSON.stringify(histogram, null, 2)}`);
+
       // try {
       //   await db.raw("SELECT NOW()");
       // } catch (err) {

backend/src/server/routes/sanitizedSchemas.ts

@@ -70,6 +70,19 @@ export const DefaultResponseErrorsSchema = {
   })
 };
 
+export const booleanSchema = z
+  .union([z.boolean(), z.string().trim()])
+  .transform((value) => {
+    if (typeof value === "string") {
+      // ie if not empty, 0 or false, return true
+      return Boolean(value) && Number(value) !== 0 && value.toLowerCase() !== "false";
+    }
+
+    return value;
+  })
+  .optional()
+  .default(true);
+
 export const sapPubSchema = SecretApprovalPoliciesSchema.merge(
   z.object({
     environment: z.object({

backend/src/server/routes/v1/dashboard-router.ts

@@ -16,7 +16,12 @@ import { secretsLimit } from "@app/server/config/rateLimiter";
 import { getTelemetryDistinctId } from "@app/server/lib/telemetry";
 import { getUserAgentType } from "@app/server/plugins/audit-log";
 import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
-import { SanitizedDynamicSecretSchema, SanitizedTagSchema, secretRawSchema } from "@app/server/routes/sanitizedSchemas";
+import {
+  booleanSchema,
+  SanitizedDynamicSecretSchema,
+  SanitizedTagSchema,
+  secretRawSchema
+} from "@app/server/routes/sanitizedSchemas";
 import { AuthMode } from "@app/services/auth/auth-type";
 import { ResourceMetadataSchema } from "@app/services/resource-metadata/resource-metadata-schema";
 import { SecretsOrderBy } from "@app/services/secret/secret-types";
@@ -24,20 +29,6 @@ import { PostHogEventTypes } from "@app/services/telemetry/telemetry-types";
 
 const MAX_DEEP_SEARCH_LIMIT = 500; // arbitrary limit to prevent excessive results
 
-// handle querystring boolean values
-const booleanSchema = z
-  .union([z.boolean(), z.string().trim()])
-  .transform((value) => {
-    if (typeof value === "string") {
-      // ie if not empty, 0 or false, return true
-      return Boolean(value) && Number(value) !== 0 && value.toLowerCase() !== "false";
-    }
-
-    return value;
-  })
-  .optional()
-  .default(true);
-
 const parseSecretPathSearch = (search?: string) => {
   if (!search)
     return {
@@ -109,6 +100,7 @@ export const registerDashboardRouter = async (server: FastifyZodProvider) => {
         search: z.string().trim().describe(DASHBOARD.SECRET_OVERVIEW_LIST.search).optional(),
         includeSecrets: booleanSchema.describe(DASHBOARD.SECRET_OVERVIEW_LIST.includeSecrets),
         includeFolders: booleanSchema.describe(DASHBOARD.SECRET_OVERVIEW_LIST.includeFolders),
+        includeImports: booleanSchema.describe(DASHBOARD.SECRET_OVERVIEW_LIST.includeImports),
         includeDynamicSecrets: booleanSchema.describe(DASHBOARD.SECRET_OVERVIEW_LIST.includeDynamicSecrets)
       }),
       response: {
@@ -124,9 +116,17 @@ export const registerDashboardRouter = async (server: FastifyZodProvider) => {
             })
             .array()
             .optional(),
+          imports: SecretImportsSchema.omit({ importEnv: true })
+            .extend({
+              importEnv: z.object({ name: z.string(), slug: z.string(), id: z.string() }),
+              environment: z.string()
+            })
+            .array()
+            .optional(),
           totalFolderCount: z.number().optional(),
           totalDynamicSecretCount: z.number().optional(),
           totalSecretCount: z.number().optional(),
+          totalImportCount: z.number().optional(),
           totalCount: z.number()
         })
       }
@@ -143,6 +143,7 @@ export const registerDashboardRouter = async (server: FastifyZodProvider) => {
         orderDirection,
         includeFolders,
         includeSecrets,
+        includeImports,
         includeDynamicSecrets
       } = req.query;
 
@@ -159,6 +160,7 @@ export const registerDashboardRouter = async (server: FastifyZodProvider) => {
       let remainingLimit = limit;
       let adjustedOffset = offset;
 
+      let imports: Awaited<ReturnType<typeof server.services.secretImport.getImportsMultiEnv>> | undefined;
       let folders: Awaited<ReturnType<typeof server.services.folder.getFoldersMultiEnv>> | undefined;
       let secrets: Awaited<ReturnType<typeof server.services.secret.getSecretsRawMultiEnv>> | undefined;
       let dynamicSecrets:
@@ -168,6 +170,53 @@ export const registerDashboardRouter = async (server: FastifyZodProvider) => {
       let totalFolderCount: number | undefined;
       let totalDynamicSecretCount: number | undefined;
       let totalSecretCount: number | undefined;
+      let totalImportCount: number | undefined;
+
+      if (includeImports) {
+        totalImportCount = await server.services.secretImport.getProjectImportMultiEnvCount({
+          actorId: req.permission.id,
+          actor: req.permission.type,
+          actorAuthMethod: req.permission.authMethod,
+          actorOrgId: req.permission.orgId,
+          projectId,
+          environments,
+          path: secretPath,
+          search
+        });
+
+        if (remainingLimit > 0 && totalImportCount > adjustedOffset) {
+          imports = await server.services.secretImport.getImportsMultiEnv({
+            actorId: req.permission.id,
+            actor: req.permission.type,
+            actorAuthMethod: req.permission.authMethod,
+            actorOrgId: req.permission.orgId,
+            projectId,
+            environments,
+            path: secretPath,
+            search,
+            limit: remainingLimit,
+            offset: adjustedOffset
+          });
+
+          await server.services.auditLog.createAuditLog({
+            ...req.auditLogInfo,
+            projectId: req.query.projectId,
+            event: {
+              type: EventType.GET_SECRET_IMPORTS,
+              metadata: {
+                environment: environments.join(","),
+                folderId: imports?.[0]?.folderId,
+                numberOfImports: imports.length
+              }
+            }
+          });
+
+          remainingLimit -= imports.length;
+          adjustedOffset = 0;
+        } else {
+          adjustedOffset = Math.max(0, adjustedOffset - totalImportCount);
+        }
+      }
 
       if (includeFolders) {
         // this is the unique count, ie duplicate folders across envs only count as 1
@@ -345,10 +394,13 @@ export const registerDashboardRouter = async (server: FastifyZodProvider) => {
         folders,
         dynamicSecrets,
         secrets,
+        imports,
         totalFolderCount,
         totalDynamicSecretCount,
+        totalImportCount,
         totalSecretCount,
-        totalCount: (totalFolderCount ?? 0) + (totalDynamicSecretCount ?? 0) + (totalSecretCount ?? 0)
+        totalCount:
+          (totalFolderCount ?? 0) + (totalDynamicSecretCount ?? 0) + (totalSecretCount ?? 0) + (totalImportCount ?? 0)
       };
     }
   });

backend/src/server/routes/v1/identity-kubernetes-auth-router.ts

@@ -24,7 +24,7 @@ const IdentityKubernetesAuthResponseSchema = IdentityKubernetesAuthsSchema.pick(
   allowedAudience: true
 }).extend({
   caCert: z.string(),
-  tokenReviewerJwt: z.string()
+  tokenReviewerJwt: z.string().optional().nullable()
 });
 
 export const registerIdentityKubernetesRouter = async (server: FastifyZodProvider) => {
@@ -98,7 +98,7 @@ export const registerIdentityKubernetesRouter = async (server: FastifyZodProvide
         .object({
           kubernetesHost: z.string().trim().min(1).describe(KUBERNETES_AUTH.ATTACH.kubernetesHost),
           caCert: z.string().trim().default("").describe(KUBERNETES_AUTH.ATTACH.caCert),
-          tokenReviewerJwt: z.string().trim().min(1).describe(KUBERNETES_AUTH.ATTACH.tokenReviewerJwt),
+          tokenReviewerJwt: z.string().trim().optional().describe(KUBERNETES_AUTH.ATTACH.tokenReviewerJwt),
           allowedNamespaces: z.string().describe(KUBERNETES_AUTH.ATTACH.allowedNamespaces), // TODO: validation
           allowedNames: z.string().describe(KUBERNETES_AUTH.ATTACH.allowedNames),
           allowedAudience: z.string().describe(KUBERNETES_AUTH.ATTACH.allowedAudience),
@@ -195,7 +195,7 @@ export const registerIdentityKubernetesRouter = async (server: FastifyZodProvide
         .object({
           kubernetesHost: z.string().trim().min(1).optional().describe(KUBERNETES_AUTH.UPDATE.kubernetesHost),
           caCert: z.string().trim().optional().describe(KUBERNETES_AUTH.UPDATE.caCert),
-          tokenReviewerJwt: z.string().trim().min(1).optional().describe(KUBERNETES_AUTH.UPDATE.tokenReviewerJwt),
+          tokenReviewerJwt: z.string().trim().nullable().optional().describe(KUBERNETES_AUTH.UPDATE.tokenReviewerJwt),
           allowedNamespaces: z.string().optional().describe(KUBERNETES_AUTH.UPDATE.allowedNamespaces), // TODO: validation
           allowedNames: z.string().optional().describe(KUBERNETES_AUTH.UPDATE.allowedNames),
           allowedAudience: z.string().optional().describe(KUBERNETES_AUTH.UPDATE.allowedAudience),

backend/src/server/routes/v1/identity-oidc-auth-router.ts

@@ -32,6 +32,8 @@ const IdentityOidcAuthResponseSchema = IdentityOidcAuthsSchema.pick({
   caCert: z.string()
 });
 
+const MAX_OIDC_CLAIM_SIZE = 32_768;
+
 export const registerIdentityOidcAuthRouter = async (server: FastifyZodProvider) => {
   server.route({
     method: "POST",
@@ -55,7 +57,7 @@ export const registerIdentityOidcAuthRouter = async (server: FastifyZodProvider)
       }
     },
     handler: async (req) => {
-      const { identityOidcAuth, accessToken, identityAccessToken, identityMembershipOrg } =
+      const { identityOidcAuth, accessToken, identityAccessToken, identityMembershipOrg, oidcTokenData } =
         await server.services.identityOidcAuth.login({
           identityId: req.body.identityId,
           jwt: req.body.jwt
@@ -69,7 +71,11 @@ export const registerIdentityOidcAuthRouter = async (server: FastifyZodProvider)
           metadata: {
             identityId: identityOidcAuth.identityId,
             identityAccessTokenId: identityAccessToken.id,
-            identityOidcAuthId: identityOidcAuth.id
+            identityOidcAuthId: identityOidcAuth.id,
+            oidcClaimsReceived:
+              Buffer.from(JSON.stringify(oidcTokenData), "utf8").byteLength < MAX_OIDC_CLAIM_SIZE
+                ? oidcTokenData
+                : { payload: "Error: Payload exceeds 32KB, provided oidc claim not recorded in audit log." }
           }
         }
       });

backend/src/server/routes/v1/organization-router.ts

@@ -4,7 +4,6 @@ import {
   AuditLogsSchema,
   GroupsSchema,
   IncidentContactsSchema,
-  OrganizationsSchema,
   OrgMembershipsSchema,
   OrgRolesSchema,
   UsersSchema
@@ -57,7 +56,7 @@ export const registerOrgRouter = async (server: FastifyZodProvider) => {
       }),
       response: {
         200: z.object({
-          organization: OrganizationsSchema
+          organization: sanitizedOrganizationSchema
         })
       }
     },
@@ -263,7 +262,7 @@ export const registerOrgRouter = async (server: FastifyZodProvider) => {
       response: {
         200: z.object({
           message: z.string(),
-          organization: OrganizationsSchema
+          organization: sanitizedOrganizationSchema
         })
       }
     },

backend/src/server/routes/v1/secret-folder-router.ts

@@ -9,6 +9,8 @@ import { readLimit, secretsLimit } from "@app/server/config/rateLimiter";
 import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
 import { AuthMode } from "@app/services/auth/auth-type";
 
+import { booleanSchema } from "../sanitizedSchemas";
+
 export const registerSecretFolderRouter = async (server: FastifyZodProvider) => {
   server.route({
     url: "/",
@@ -347,11 +349,14 @@ export const registerSecretFolderRouter = async (server: FastifyZodProvider) =>
           .default("/")
           .transform(prefixWithSlash)
           .transform(removeTrailingSlash)
-          .describe(FOLDERS.LIST.directory)
+          .describe(FOLDERS.LIST.directory),
+        recursive: booleanSchema.default(false).describe(FOLDERS.LIST.recursive)
       }),
       response: {
         200: z.object({
-          folders: SecretFoldersSchema.array()
+          folders: SecretFoldersSchema.extend({
+            relativePath: z.string().optional()
+          }).array()
         })
       }
     },

backend/src/server/routes/v2/organization-router.ts

@@ -1,7 +1,6 @@
 import { z } from "zod";
 
 import {
-  OrganizationsSchema,
   OrgMembershipsSchema,
   ProjectMembershipsSchema,
   ProjectsSchema,
@@ -15,6 +14,7 @@ import { readLimit, writeLimit } from "@app/server/config/rateLimiter";
 import { GenericResourceNameSchema } from "@app/server/lib/schemas";
 import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
 import { ActorType, AuthMode } from "@app/services/auth/auth-type";
+import { sanitizedOrganizationSchema } from "@app/services/org/org-schema";
 
 export const registerOrgRouter = async (server: FastifyZodProvider) => {
   server.route({
@@ -335,7 +335,7 @@ export const registerOrgRouter = async (server: FastifyZodProvider) => {
       }),
       response: {
         200: z.object({
-          organization: OrganizationsSchema
+          organization: sanitizedOrganizationSchema
         })
       }
     },
@@ -365,7 +365,7 @@ export const registerOrgRouter = async (server: FastifyZodProvider) => {
       }),
       response: {
         200: z.object({
-          organization: OrganizationsSchema,
+          organization: sanitizedOrganizationSchema,
           accessToken: z.string()
         })
       }
@@ -396,4 +396,30 @@ export const registerOrgRouter = async (server: FastifyZodProvider) => {
       return { organization, accessToken: tokens.accessToken };
     }
   });
+
+  server.route({
+    method: "POST",
+    url: "/privilege-system-upgrade",
+    config: {
+      rateLimit: writeLimit
+    },
+    schema: {
+      response: {
+        200: z.object({
+          organization: sanitizedOrganizationSchema
+        })
+      }
+    },
+    onRequest: verifyAuth([AuthMode.JWT]),
+    handler: async (req) => {
+      const organization = await server.services.org.upgradePrivilegeSystem({
+        actorId: req.permission.id,
+        actorOrgId: req.permission.orgId,
+        actorAuthMethod: req.permission.authMethod,
+        orgId: req.permission.orgId
+      });
+
+      return { organization };
+    }
+  });
 };