Merge branch 'heads/main' into daniel/request-ids

backend/src/db/migrations/20241121131344_make-identity-metadata-not-nullable-again.ts

@@ -0,0 +1,20 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  if (await knex.schema.hasColumn(TableName.IdentityMetadata, "value")) {
+    await knex(TableName.IdentityMetadata).whereNull("value").delete();
+    await knex.schema.alterTable(TableName.IdentityMetadata, (t) => {
+      t.string("value", 1020).notNullable().alter();
+    });
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  if (await knex.schema.hasColumn(TableName.IdentityMetadata, "value")) {
+    await knex.schema.alterTable(TableName.IdentityMetadata, (t) => {
+      t.string("value", 1020).alter();
+    });
+  }
+}

backend/src/ee/services/dynamic-secret/providers/index.ts

@@ -13,6 +13,7 @@ import { RabbitMqProvider } from "./rabbit-mq";
 import { RedisDatabaseProvider } from "./redis";
 import { SapHanaProvider } from "./sap-hana";
 import { SqlDatabaseProvider } from "./sql-database";
+import { TotpProvider } from "./totp";
 
 export const buildDynamicSecretProviders = () => ({
   [DynamicSecretProviders.SqlDatabase]: SqlDatabaseProvider(),
@@ -27,5 +28,6 @@ export const buildDynamicSecretProviders = () => ({
   [DynamicSecretProviders.AzureEntraID]: AzureEntraIDProvider(),
   [DynamicSecretProviders.Ldap]: LdapProvider(),
   [DynamicSecretProviders.SapHana]: SapHanaProvider(),
-  [DynamicSecretProviders.Snowflake]: SnowflakeProvider()
+  [DynamicSecretProviders.Snowflake]: SnowflakeProvider(),
+  [DynamicSecretProviders.Totp]: TotpProvider()
 });

backend/src/ee/services/dynamic-secret/providers/models.ts

@@ -17,6 +17,17 @@ export enum LdapCredentialType {
   Static = "static"
 }
 
+export enum TotpConfigType {
+  URL = "url",
+  MANUAL = "manual"
+}
+
+export enum TotpAlgorithm {
+  SHA1 = "sha1",
+  SHA256 = "sha256",
+  SHA512 = "sha512"
+}
+
 export const DynamicSecretRedisDBSchema = z.object({
   host: z.string().trim().toLowerCase(),
   port: z.number(),
@@ -221,6 +232,34 @@ export const LdapSchema = z.union([
   })
 ]);
 
+export const DynamicSecretTotpSchema = z.discriminatedUnion("configType", [
+  z.object({
+    configType: z.literal(TotpConfigType.URL),
+    url: z
+      .string()
+      .url()
+      .trim()
+      .min(1)
+      .refine((val) => {
+        const urlObj = new URL(val);
+        const secret = urlObj.searchParams.get("secret");
+
+        return Boolean(secret);
+      }, "OTP URL must contain secret field")
+  }),
+  z.object({
+    configType: z.literal(TotpConfigType.MANUAL),
+    secret: z
+      .string()
+      .trim()
+      .min(1)
+      .transform((val) => val.replace(/\s+/g, "")),
+    period: z.number().optional(),
+    algorithm: z.nativeEnum(TotpAlgorithm).optional(),
+    digits: z.number().optional()
+  })
+]);
+
 export enum DynamicSecretProviders {
   SqlDatabase = "sql-database",
   Cassandra = "cassandra",
@@ -234,7 +273,8 @@ export enum DynamicSecretProviders {
   AzureEntraID = "azure-entra-id",
   Ldap = "ldap",
   SapHana = "sap-hana",
-  Snowflake = "snowflake"
+  Snowflake = "snowflake",
+  Totp = "totp"
 }
 
 export const DynamicSecretProviderSchema = z.discriminatedUnion("type", [
@@ -250,7 +290,8 @@ export const DynamicSecretProviderSchema = z.discriminatedUnion("type", [
   z.object({ type: z.literal(DynamicSecretProviders.RabbitMq), inputs: DynamicSecretRabbitMqSchema }),
   z.object({ type: z.literal(DynamicSecretProviders.AzureEntraID), inputs: AzureEntraIDSchema }),
   z.object({ type: z.literal(DynamicSecretProviders.Ldap), inputs: LdapSchema }),
-  z.object({ type: z.literal(DynamicSecretProviders.Snowflake), inputs: DynamicSecretSnowflakeSchema })
+  z.object({ type: z.literal(DynamicSecretProviders.Snowflake), inputs: DynamicSecretSnowflakeSchema }),
+  z.object({ type: z.literal(DynamicSecretProviders.Totp), inputs: DynamicSecretTotpSchema })
 ]);
 
 export type TDynamicProviderFns = {

backend/src/ee/services/dynamic-secret/providers/totp.ts

@@ -0,0 +1,92 @@
+import { authenticator } from "otplib";
+import { HashAlgorithms } from "otplib/core";
+
+import { BadRequestError } from "@app/lib/errors";
+import { alphaNumericNanoId } from "@app/lib/nanoid";
+
+import { DynamicSecretTotpSchema, TDynamicProviderFns, TotpConfigType } from "./models";
+
+export const TotpProvider = (): TDynamicProviderFns => {
+  const validateProviderInputs = async (inputs: unknown) => {
+    const providerInputs = await DynamicSecretTotpSchema.parseAsync(inputs);
+
+    return providerInputs;
+  };
+
+  const validateConnection = async () => {
+    return true;
+  };
+
+  const create = async (inputs: unknown) => {
+    const providerInputs = await validateProviderInputs(inputs);
+
+    const entityId = alphaNumericNanoId(32);
+    const authenticatorInstance = authenticator.clone();
+
+    let secret: string;
+    let period: number | null | undefined;
+    let digits: number | null | undefined;
+    let algorithm: HashAlgorithms | null | undefined;
+
+    if (providerInputs.configType === TotpConfigType.URL) {
+      const urlObj = new URL(providerInputs.url);
+      secret = urlObj.searchParams.get("secret") as string;
+      const periodFromUrl = urlObj.searchParams.get("period");
+      const digitsFromUrl = urlObj.searchParams.get("digits");
+      const algorithmFromUrl = urlObj.searchParams.get("algorithm");
+
+      if (periodFromUrl) {
+        period = +periodFromUrl;
+      }
+
+      if (digitsFromUrl) {
+        digits = +digitsFromUrl;
+      }
+
+      if (algorithmFromUrl) {
+        algorithm = algorithmFromUrl.toLowerCase() as HashAlgorithms;
+      }
+    } else {
+      secret = providerInputs.secret;
+      period = providerInputs.period;
+      digits = providerInputs.digits;
+      algorithm = providerInputs.algorithm as unknown as HashAlgorithms;
+    }
+
+    if (digits) {
+      authenticatorInstance.options = { digits };
+    }
+
+    if (algorithm) {
+      authenticatorInstance.options = { algorithm };
+    }
+
+    if (period) {
+      authenticatorInstance.options = { step: period };
+    }
+
+    return {
+      entityId,
+      data: { TOTP: authenticatorInstance.generate(secret), TIME_REMAINING: authenticatorInstance.timeRemaining() }
+    };
+  };
+
+  const revoke = async (_inputs: unknown, entityId: string) => {
+    return { entityId };
+  };
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  const renew = async (_inputs: unknown, _entityId: string) => {
+    throw new BadRequestError({
+      message: "Lease renewal is not supported for TOTPs"
+    });
+  };
+
+  return {
+    validateProviderInputs,
+    validateConnection,
+    create,
+    revoke,
+    renew
+  };
+};

backend/src/ee/services/permission/permission-dal.ts

@@ -127,14 +127,15 @@ export const permissionDALFactory = (db: TDbClient) => {
 
   const getProjectPermission = async (userId: string, projectId: string) => {
     try {
+      const subQueryUserGroups = db(TableName.UserGroupMembership).where("userId", userId).select("groupId");
       const docs = await db
         .replicaNode()(TableName.Users)
         .where(`${TableName.Users}.id`, userId)
-        .leftJoin(TableName.UserGroupMembership, `${TableName.UserGroupMembership}.userId`, `${TableName.Users}.id`)
         .leftJoin(TableName.GroupProjectMembership, (queryBuilder) => {
           void queryBuilder
             .on(`${TableName.GroupProjectMembership}.projectId`, db.raw("?", [projectId]))
-            .andOn(`${TableName.GroupProjectMembership}.groupId`, `${TableName.UserGroupMembership}.groupId`);
+            // @ts-expect-error akhilmhdh: this is valid knexjs query. Its just ts type argument is missing it
+            .andOnIn(`${TableName.GroupProjectMembership}.groupId`, subQueryUserGroups);
         })
         .leftJoin(
           TableName.GroupProjectMembershipRole,

backend/src/ee/services/permission/permission-types.ts

@@ -1,14 +1,7 @@
 import picomatch from "picomatch";
 import { z } from "zod";
 
-export enum PermissionConditionOperators {
-  $IN = "$in",
-  $ALL = "$all",
-  $REGEX = "$regex",
-  $EQ = "$eq",
-  $NEQ = "$ne",
-  $GLOB = "$glob"
-}
+import { PermissionConditionOperators } from "@app/lib/casl";
 
 export const PermissionConditionSchema = {
   [PermissionConditionOperators.$IN]: z.string().trim().min(1).array(),

backend/src/ee/services/permission/project-permission.ts

@@ -1,10 +1,10 @@
 import { AbilityBuilder, createMongoAbility, ForcedSubject, MongoAbility } from "@casl/ability";
 import { z } from "zod";
 
-import { conditionsMatcher } from "@app/lib/casl";
+import { conditionsMatcher, PermissionConditionOperators } from "@app/lib/casl";
 import { UnpackedPermissionSchema } from "@app/server/routes/santizedSchemas/permission";
 
-import { PermissionConditionOperators, PermissionConditionSchema } from "./permission-types";
+import { PermissionConditionSchema } from "./permission-types";
 
 export enum ProjectPermissionActions {
   Read = "read",

backend/src/lib/casl/index.ts

@@ -54,3 +54,12 @@ export const isAtLeastAsPrivileged = (permissions1: MongoAbility, permissions2:
 
   return set1.size >= set2.size;
 };
+
+export enum PermissionConditionOperators {
+  $IN = "$in",
+  $ALL = "$all",
+  $REGEX = "$regex",
+  $EQ = "$eq",
+  $NEQ = "$ne",
+  $GLOB = "$glob"
+}

backend/src/server/plugins/error-handler.ts

@@ -1,4 +1,4 @@
-import { ForbiddenError } from "@casl/ability";
+import { ForbiddenError, PureAbility } from "@casl/ability";
 import fastifyPlugin from "fastify-plugin";
 import jwt from "jsonwebtoken";
 import { ZodError } from "zod";
@@ -77,7 +77,13 @@ export const fastifyErrHandler = fastifyPlugin(async (server: FastifyZodProvider
         requestId: req.id,
         statusCode: HttpStatusCodes.Forbidden,
         error: "PermissionDenied",
-        message: `You are not allowed to ${error.action} on ${error.subjectType} - ${JSON.stringify(error.subject)}`
+        message: `You are not allowed to ${error.action} on ${error.subjectType}`,
+        details: (error.ability as PureAbility).rulesFor(error.action as string, error.subjectType).map((el) => ({
+          action: el.action,
+          inverted: el.inverted,
+          subject: el.subject,
+          conditions: el.conditions
+        }))
       });
     } else if (error instanceof ForbiddenRequestError) {
       void res.status(HttpStatusCodes.Forbidden).send({

backend/src/server/routes/sanitizedSchemas.ts

@@ -51,6 +51,7 @@ export const DefaultResponseErrorsSchema = {
     requestId: z.string(),
     statusCode: z.literal(403),
     message: z.string(),
+    details: z.any().optional(),
     error: z.string()
   }),
   500: z.object({

backend/src/services/identity-token-auth/identity-token-auth-service.ts

@@ -385,8 +385,8 @@ export const identityTokenAuthServiceFactory = ({
     actorOrgId
   }: TUpdateTokenAuthTokenDTO) => {
     const foundToken = await identityAccessTokenDAL.findOne({
-      id: tokenId,
-      authMethod: IdentityAuthMethod.TOKEN_AUTH
+      [`${TableName.IdentityAccessToken}.id` as "id"]: tokenId,
+      [`${TableName.IdentityAccessToken}.authMethod` as "authMethod"]: IdentityAuthMethod.TOKEN_AUTH
     });
     if (!foundToken) throw new NotFoundError({ message: `Token with ID ${tokenId} not found` });
 
@@ -444,8 +444,8 @@ export const identityTokenAuthServiceFactory = ({
   }: TRevokeTokenAuthTokenDTO) => {
     const identityAccessToken = await identityAccessTokenDAL.findOne({
       [`${TableName.IdentityAccessToken}.id` as "id"]: tokenId,
-      isAccessTokenRevoked: false,
-      authMethod: IdentityAuthMethod.TOKEN_AUTH
+      [`${TableName.IdentityAccessToken}.isAccessTokenRevoked` as "isAccessTokenRevoked"]: false,
+      [`${TableName.IdentityAccessToken}.authMethod` as "authMethod"]: IdentityAuthMethod.TOKEN_AUTH
     });
     if (!identityAccessToken)
       throw new NotFoundError({

backend/src/services/integration-auth/integration-sync-secret.ts

@@ -3075,7 +3075,7 @@ const syncSecretsTerraformCloud = async ({
 }) => {
   // get secrets from Terraform Cloud
   const terraformSecrets = (
-    await request.get<{ data: { attributes: { key: string; value: string }; id: string }[] }>(
+    await request.get<{ data: { attributes: { key: string; value: string; sensitive: boolean }; id: string }[] }>(
       `${IntegrationUrls.TERRAFORM_CLOUD_API_URL}/api/v2/workspaces/${integration.appId}/vars`,
       {
         headers: {
@@ -3089,7 +3089,7 @@ const syncSecretsTerraformCloud = async ({
       ...obj,
       [secret.attributes.key]: secret
     }),
-    {} as Record<string, { attributes: { key: string; value: string }; id: string }>
+    {} as Record<string, { attributes: { key: string; value: string; sensitive: boolean }; id: string }>
   );
 
   const secretsToAdd: { [key: string]: string } = {};
@@ -3170,7 +3170,8 @@ const syncSecretsTerraformCloud = async ({
             attributes: {
               key,
               value: secrets[key]?.value,
-              category: integration.targetService
+              category: integration.targetService,
+              sensitive: true
             }
           }
         },
@@ -3183,7 +3184,11 @@ const syncSecretsTerraformCloud = async ({
         }
       );
       // case: secret exists in Terraform Cloud
-    } else if (secrets[key]?.value !== terraformSecrets[key].attributes.value) {
+    } else if (
+      // we now set secrets to sensitive in Terraform Cloud, this checks if existing secrets are not sensitive and updates them accordingly
+      !terraformSecrets[key].attributes.sensitive ||
+      secrets[key]?.value !== terraformSecrets[key].attributes.value
+    ) {
       // -> update secret
       await request.patch(
         `${IntegrationUrls.TERRAFORM_CLOUD_API_URL}/api/v2/workspaces/${integration.appId}/vars/${terraformSecrets[key].id}`,
@@ -3193,7 +3198,8 @@ const syncSecretsTerraformCloud = async ({
             id: terraformSecrets[key].id,
             attributes: {
               ...terraformSecrets[key],
-              value: secrets[key]?.value
+              value: secrets[key]?.value,
+              sensitive: true
             }
           }
         },