Merge remote-tracking branch 'origin/main' into feat/totp-dynamic-secret

backend/src/db/migrations/20240802181855_ca-cert-version.ts

@@ -64,23 +64,25 @@ export async function up(knex: Knex): Promise<void> {
   }
 
   if (await knex.schema.hasTable(TableName.Certificate)) {
-    await knex.schema.alterTable(TableName.Certificate, (t) => {
-      t.uuid("caCertId").nullable();
-      t.foreign("caCertId").references("id").inTable(TableName.CertificateAuthorityCert);
-    });
+    const hasCaCertIdColumn = await knex.schema.hasColumn(TableName.Certificate, "caCertId");
+    if (!hasCaCertIdColumn) {
+      await knex.schema.alterTable(TableName.Certificate, (t) => {
+        t.uuid("caCertId").nullable();
+        t.foreign("caCertId").references("id").inTable(TableName.CertificateAuthorityCert);
+      });
 
-    await knex.raw(`
+      await knex.raw(`
         UPDATE "${TableName.Certificate}" cert
         SET "caCertId" = (
           SELECT caCert.id
           FROM "${TableName.CertificateAuthorityCert}" caCert
           WHERE caCert."caId" = cert."caId"
-        )
-      `);
+        )`);
 
-    await knex.schema.alterTable(TableName.Certificate, (t) => {
-      t.uuid("caCertId").notNullable().alter();
-    });
+      await knex.schema.alterTable(TableName.Certificate, (t) => {
+        t.uuid("caCertId").notNullable().alter();
+      });
+    }
   }
 }
 

backend/src/db/utils.ts

@@ -2,6 +2,9 @@ import { Knex } from "knex";
 
 import { TableName } from "./schemas";
 
+interface PgTriggerResult {
+  rows: Array<{ exists: boolean }>;
+}
 export const createJunctionTable = (knex: Knex, tableName: TableName, table1Name: TableName, table2Name: TableName) =>
   knex.schema.createTable(tableName, (table) => {
     table.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
@@ -28,13 +31,26 @@ DROP FUNCTION IF EXISTS on_update_timestamp() CASCADE;
 
 // we would be using this to apply updatedAt where ever we wanta
 // remember to set `timestamps(true,true,true)` before this on schema
-export const createOnUpdateTrigger = (knex: Knex, tableName: string) =>
-  knex.raw(`
-CREATE TRIGGER "${tableName}_updatedAt"
-BEFORE UPDATE ON ${tableName}
-FOR EACH ROW
-EXECUTE PROCEDURE on_update_timestamp();
-`);
+export const createOnUpdateTrigger = async (knex: Knex, tableName: string) => {
+  const triggerExists = await knex.raw<PgTriggerResult>(`
+    SELECT EXISTS (
+      SELECT 1 
+      FROM pg_trigger 
+      WHERE tgname = '${tableName}_updatedAt'
+    );
+  `);
+
+  if (!triggerExists?.rows?.[0]?.exists) {
+    return knex.raw(`
+      CREATE TRIGGER "${tableName}_updatedAt"
+      BEFORE UPDATE ON ${tableName}
+      FOR EACH ROW
+      EXECUTE PROCEDURE on_update_timestamp();
+    `);
+  }
+
+  return null;
+};
 
 export const dropOnUpdateTrigger = (knex: Knex, tableName: string) =>
   knex.raw(`DROP TRIGGER IF EXISTS "${tableName}_updatedAt" ON ${tableName}`);

backend/src/ee/services/permission/permission-fns.ts

@@ -29,4 +29,18 @@ function validateOrgSSO(actorAuthMethod: ActorAuthMethod, isOrgSsoEnforced: TOrg
   }
 }
 
-export { isAuthMethodSaml, validateOrgSSO };
+const escapeHandlebarsMissingMetadata = (obj: Record<string, string>) => {
+  const handler = {
+    get(target: Record<string, string>, prop: string) {
+      if (!(prop in target)) {
+        // eslint-disable-next-line no-param-reassign
+        target[prop] = `{{identity.metadata.${prop}}}`; // Add missing key as an "own" property
+      }
+      return target[prop];
+    }
+  };
+
+  return new Proxy(obj, handler);
+};
+
+export { escapeHandlebarsMissingMetadata, isAuthMethodSaml, validateOrgSSO };

backend/src/ee/services/permission/permission-service.ts

@@ -21,7 +21,7 @@ import { TServiceTokenDALFactory } from "@app/services/service-token/service-tok
 
 import { orgAdminPermissions, orgMemberPermissions, orgNoAccessPermissions, OrgPermissionSet } from "./org-permission";
 import { TPermissionDALFactory } from "./permission-dal";
-import { validateOrgSSO } from "./permission-fns";
+import { escapeHandlebarsMissingMetadata, validateOrgSSO } from "./permission-fns";
 import { TBuildOrgPermissionDTO, TBuildProjectPermissionDTO } from "./permission-service-types";
 import {
   buildServiceTokenProjectPermission,
@@ -227,11 +227,13 @@ export const permissionServiceFactory = ({
       })) || [];
 
     const rules = buildProjectPermissionRules(rolePermissions.concat(additionalPrivileges));
-    const templatedRules = handlebars.compile(JSON.stringify(rules), { data: false, strict: true });
-    const metadataKeyValuePair = objectify(
-      userProjectPermission.metadata,
-      (i) => i.key,
-      (i) => i.value
+    const templatedRules = handlebars.compile(JSON.stringify(rules), { data: false });
+    const metadataKeyValuePair = escapeHandlebarsMissingMetadata(
+      objectify(
+        userProjectPermission.metadata,
+        (i) => i.key,
+        (i) => i.value
+      )
     );
     const interpolateRules = templatedRules(
       {
@@ -292,12 +294,15 @@ export const permissionServiceFactory = ({
       })) || [];
 
     const rules = buildProjectPermissionRules(rolePermissions.concat(additionalPrivileges));
-    const templatedRules = handlebars.compile(JSON.stringify(rules), { data: false, strict: true });
-    const metadataKeyValuePair = objectify(
-      identityProjectPermission.metadata,
-      (i) => i.key,
-      (i) => i.value
+    const templatedRules = handlebars.compile(JSON.stringify(rules), { data: false });
+    const metadataKeyValuePair = escapeHandlebarsMissingMetadata(
+      objectify(
+        identityProjectPermission.metadata,
+        (i) => i.key,
+        (i) => i.value
+      )
     );
+
     const interpolateRules = templatedRules(
       {
         identity: {

backend/src/server/routes/v1/dashboard-router.ts

@@ -840,4 +840,91 @@ export const registerDashboardRouter = async (server: FastifyZodProvider) => {
       };
     }
   });
+
+  server.route({
+    method: "GET",
+    url: "/secrets-by-keys",
+    config: {
+      rateLimit: secretsLimit
+    },
+    schema: {
+      security: [
+        {
+          bearerAuth: []
+        }
+      ],
+      querystring: z.object({
+        projectId: z.string().trim(),
+        environment: z.string().trim(),
+        secretPath: z.string().trim().default("/").transform(removeTrailingSlash),
+        keys: z.string().trim().transform(decodeURIComponent)
+      }),
+      response: {
+        200: z.object({
+          secrets: secretRawSchema
+            .extend({
+              secretPath: z.string().optional(),
+              tags: SecretTagsSchema.pick({
+                id: true,
+                slug: true,
+                color: true
+              })
+                .extend({ name: z.string() })
+                .array()
+                .optional()
+            })
+            .array()
+            .optional()
+        })
+      }
+    },
+    onRequest: verifyAuth([AuthMode.JWT]),
+    handler: async (req) => {
+      const { secretPath, projectId, environment } = req.query;
+
+      const keys = req.query.keys?.split(",").filter((key) => Boolean(key.trim())) ?? [];
+      if (!keys.length) throw new BadRequestError({ message: "One or more keys required" });
+
+      const { secrets } = await server.services.secret.getSecretsRaw({
+        actorId: req.permission.id,
+        actor: req.permission.type,
+        actorOrgId: req.permission.orgId,
+        environment,
+        actorAuthMethod: req.permission.authMethod,
+        projectId,
+        path: secretPath,
+        keys
+      });
+
+      await server.services.auditLog.createAuditLog({
+        projectId,
+        ...req.auditLogInfo,
+        event: {
+          type: EventType.GET_SECRETS,
+          metadata: {
+            environment,
+            secretPath,
+            numberOfSecrets: secrets.length
+          }
+        }
+      });
+
+      if (getUserAgentType(req.headers["user-agent"]) !== UserAgentType.K8_OPERATOR) {
+        await server.services.telemetry.sendPostHogEvents({
+          event: PostHogEventTypes.SecretPulled,
+          distinctId: getTelemetryDistinctId(req),
+          properties: {
+            numberOfSecrets: secrets.length,
+            workspaceId: projectId,
+            environment,
+            secretPath,
+            channel: getUserAgentType(req.headers["user-agent"]),
+            ...req.auditLogInfo
+          }
+        });
+      }
+
+      return { secrets };
+    }
+  });
 };

backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts

@@ -361,6 +361,10 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
               void bd.whereILike(`${TableName.SecretV2}.key`, `%${filters?.search}%`);
             }
           }
+
+          if (filters?.keys) {
+            void bd.whereIn(`${TableName.SecretV2}.key`, filters.keys);
+          }
         })
         .where((bd) => {
           void bd.whereNull(`${TableName.SecretV2}.userId`).orWhere({ userId: userId || null });

backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts

@@ -518,7 +518,10 @@ export const expandSecretReferencesFactory = ({
           }
 
           if (referencedSecretValue) {
-            expandedValue = expandedValue.replaceAll(interpolationSyntax, referencedSecretValue);
+            expandedValue = expandedValue.replaceAll(
+              interpolationSyntax,
+              () => referencedSecretValue // prevents special characters from triggering replacement patterns
+            );
           }
         }
       }

backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts

@@ -150,9 +150,13 @@ export const secretV2BridgeServiceFactory = ({
       }
     });
 
-    if (referredSecrets.length !== references.length)
+    if (
+      referredSecrets.length !==
+      new Set(references.map(({ secretKey, secretPath, environment }) => `${secretKey}.${secretPath}.${environment}`))
+        .size // only count unique references
+    )
       throw new BadRequestError({
-        message: `Referenced secret not found. Found only ${diff(
+        message: `Referenced secret(s) not found: ${diff(
           references.map((el) => el.secretKey),
           referredSecrets.map((el) => el.key)
         ).join(",")}`

backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts

@@ -33,6 +33,7 @@ export type TGetSecretsDTO = {
   offset?: number;
   limit?: number;
   search?: string;
+  keys?: string[];
 } & TProjectPermission;
 
 export type TGetASecretDTO = {
@@ -294,6 +295,7 @@ export type TFindSecretsByFolderIdsFilter = {
   search?: string;
   tagSlugs?: string[];
   includeTagsInSearch?: boolean;
+  keys?: string[];
 };
 
 export type TGetSecretsRawByFolderMappingsDTO = {

backend/src/services/secret/secret-types.ts

@@ -185,6 +185,7 @@ export type TGetSecretsRawDTO = {
   offset?: number;
   limit?: number;
   search?: string;
+  keys?: string[];
 } & TProjectPermission;
 
 export type TGetASecretRawDTO = {

backend/src/services/totp/totp-fns.ts

@@ -0,0 +1,3 @@
+import crypto from "node:crypto";
+
+export const generateRecoveryCode = () => String(crypto.randomInt(10 ** 7, 10 ** 8 - 1));

backend/src/services/totp/totp-service.ts

@@ -1,5 +1,3 @@
-import crypto from "node:crypto";
-
 import { authenticator } from "otplib";
 
 import { BadRequestError, ForbiddenRequestError, NotFoundError } from "@app/lib/errors";
@@ -7,6 +5,7 @@ import { BadRequestError, ForbiddenRequestError, NotFoundError } from "@app/lib/
 import { TKmsServiceFactory } from "../kms/kms-service";
 import { TUserDALFactory } from "../user/user-dal";
 import { TTotpConfigDALFactory } from "./totp-config-dal";
+import { generateRecoveryCode } from "./totp-fns";
 import {
   TCreateUserTotpRecoveryCodesDTO,
   TDeleteUserTotpConfigDTO,
@@ -25,6 +24,8 @@ type TTotpServiceFactoryDep = {
 
 export type TTotpServiceFactory = ReturnType<typeof totpServiceFactory>;
 
+const MAX_RECOVERY_CODE_LIMIT = 10;
+
 export const totpServiceFactory = ({ totpConfigDAL, kmsService, userDAL }: TTotpServiceFactoryDep) => {
   const getUserTotpConfig = async ({ userId }: TGetUserTotpConfigDTO) => {
     const totpConfig = await totpConfigDAL.findOne({
@@ -82,7 +83,7 @@ export const totpServiceFactory = ({ totpConfigDAL, kmsService, userDAL }: TTotp
       // create new TOTP configuration
       const secret = authenticator.generateSecret();
       const encryptedSecret = encryptWithRoot(Buffer.from(secret));
-      const recoveryCodes = Array.from({ length: 10 }).map(() => String(crypto.randomInt(10 ** 7, 10 ** 8 - 1)));
+      const recoveryCodes = Array.from({ length: MAX_RECOVERY_CODE_LIMIT }).map(generateRecoveryCode);
       const encryptedRecoveryCodes = encryptWithRoot(Buffer.from(recoveryCodes.join(",")));
       const newTotpConfig = await totpConfigDAL.create({
         userId,
@@ -241,16 +242,14 @@ export const totpServiceFactory = ({ totpConfigDAL, kmsService, userDAL }: TTotp
       }
 
       const recoveryCodes = decryptWithRoot(totpConfig.encryptedRecoveryCodes).toString().split(",");
-      if (recoveryCodes.length >= 10) {
+      if (recoveryCodes.length >= MAX_RECOVERY_CODE_LIMIT) {
         throw new BadRequestError({
-          message: "Cannot have more than 10 recovery codes at a time"
+          message: `Cannot have more than ${MAX_RECOVERY_CODE_LIMIT} recovery codes at a time`
         });
       }
 
-      const toGenerateCount = 10 - recoveryCodes.length;
-      const newRecoveryCodes = Array.from({ length: toGenerateCount }).map(() =>
-        String(crypto.randomInt(10 ** 7, 10 ** 8 - 1))
-      );
+      const toGenerateCount = MAX_RECOVERY_CODE_LIMIT - recoveryCodes.length;
+      const newRecoveryCodes = Array.from({ length: toGenerateCount }).map(generateRecoveryCode);
       const encryptedRecoveryCodes = encryptWithRoot(Buffer.from([...recoveryCodes, ...newRecoveryCodes].join(",")));
 
       await totpConfigDAL.updateById(totpConfig.id, {