feat(infisical-pg): added secret version tag

backend-pg/scripts/create-backend-file.ts

@@ -27,8 +27,8 @@ if (componentType === 1) {
     .split("-")
     .map((el, index) => (index === 0 ? el : `${el[0].toUpperCase()}${el.slice(1)}`))
     .join("");
-  const dalTypeName = `T${pascalCase}DalFactory`;
+  const dalTypeName = `T${pascalCase}DALFactory`;
-  const dalName = `${camelCase}DalFactory`;
+  const dalName = `${camelCase}DALFactory`;
   const serviceTypeName = `T${pascalCase}ServiceFactory`;
   const serviceName = `${camelCase}ServiceFactory`;
 
@@ -53,12 +53,12 @@ export const ${dalName} = (db: TDbClient) => {
     `import { ${dalTypeName} } from "./${componentName}-dal";
 
 type ${serviceTypeName}Dep = {
-  ${camelCase}Dal: ${dalTypeName};
+  ${camelCase}DAL: ${dalTypeName};
 };
 
 export type ${serviceTypeName} = ReturnType<typeof ${serviceName}>;
 
-export const ${serviceName} = ({ ${camelCase}Dal }: ${serviceTypeName}Dep) => {
+export const ${serviceName} = ({ ${camelCase}DAL }: ${serviceTypeName}Dep) => {
   return {};
 };
 `
@@ -75,8 +75,8 @@ export const ${serviceName} = ({ ${camelCase}Dal }: ${serviceTypeName}Dep) => {
     .split("-")
     .map((el, index) => (index === 0 ? el : `${el[0].toUpperCase()}${el.slice(1)}`))
     .join("");
-  const dalTypeName = `T${pascalCase}DalFactory`;
+  const dalTypeName = `T${pascalCase}DALFactory`;
-  const dalName = `${camelCase}DalFactory`;
+  const dalName = `${camelCase}DALFactory`;
 
   writeFileSync(
     path.join(__dirname, "../src/services", componentPath, `${componentName}-dal.ts`),

backend-pg/src/@types/knex.d.ts

@@ -143,6 +143,9 @@ import {
   TSecretVersions,
   TSecretVersionsInsert,
   TSecretVersionsUpdate,
+  TSecretVersionTagJunction,
+  TSecretVersionTagJunctionInsert,
+  TSecretVersionTagJunctionUpdate,
   TServiceTokens,
   TServiceTokensInsert,
   TServiceTokensUpdate,
@@ -163,8 +166,7 @@ import {
   TUsersUpdate,
   TWebhooks,
   TWebhooksInsert,
-  TWebhooksUpdate
+  TWebhooksUpdate} from "@app/db/schemas";
-} from "@app/db/schemas";
 
 declare module "knex/types/tables" {
   interface Tables {
@@ -403,5 +405,10 @@ declare module "knex/types/tables" {
       TSecretTagJunctionInsert,
       TSecretTagJunctionUpdate
     >;
+    [TableName.SecretVersionTag]: Knex.CompositeTableType<
+      TSecretVersionTagJunction,
+      TSecretVersionTagJunctionInsert,
+      TSecretVersionTagJunctionUpdate
+    >;
   }
 }

backend-pg/src/db/migrations/20231220052508_secret-version.ts

@@ -40,14 +40,14 @@ export async function up(knex: Knex): Promise<void> {
   // many to many relation between tags
   await createJunctionTable(
     knex,
-    TableName.JnSecretVersionTag,
+    TableName.SecretVersionTag,
     TableName.SecretVersion,
     TableName.SecretTag
   );
 }
 
 export async function down(knex: Knex): Promise<void> {
-  await knex.schema.dropTableIfExists(TableName.JnSecretVersionTag);
+  await knex.schema.dropTableIfExists(TableName.SecretVersionTag);
   await knex.schema.dropTableIfExists(TableName.SecretVersion);
   await dropOnUpdateTrigger(knex, TableName.SecretVersion);
 }

backend-pg/src/db/schemas/index.ts

@@ -44,6 +44,7 @@ export * from "./secret-snapshot-secrets";
 export * from "./secret-snapshots";
 export * from "./secret-tag-junction";
 export * from "./secret-tags";
+export * from "./secret-version-tag-junction";
 export * from "./secret-versions";
 export * from "./secrets";
 export * from "./service-tokens";

backend-pg/src/db/schemas/models.ts

@@ -54,9 +54,9 @@ export enum TableName {
   GitAppOrg = "git_app_org",
   SecretScanningGitRisk = "secret_scanning_git_risks",
   TrustedIps = "trusted_ips",
-  // junction tables
+  // junction tables with tags
   JnSecretTag = "secret_tag_junction",
-  JnSecretVersionTag = "secret_version_tag_junction"
+  SecretVersionTag = "secret_version_tag_junction"
 }
 
 export type TImmutableDBKeys = "id" | "createdAt" | "updatedAt";

backend-pg/src/ee/routes/v1/snapshot-router.ts

@@ -1,6 +1,6 @@
 import { z } from "zod";
 
-import { SecretSnapshotsSchema, SecretVersionsSchema } from "@app/db/schemas";
+import { SecretSnapshotsSchema, SecretTagsSchema, SecretVersionsSchema } from "@app/db/schemas";
 import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
 import { AuthMode } from "@app/services/auth/auth-type";
 
@@ -22,7 +22,18 @@ export const registerSnapshotRouter = async (server: FastifyZodProvider) => {
               slug: z.string(),
               name: z.string()
             }),
-            secretVersions: SecretVersionsSchema.omit({ secretBlindIndex: true }).array(),
+            secretVersions: SecretVersionsSchema.omit({ secretBlindIndex: true })
+              .merge(
+                z.object({
+                  tags: SecretTagsSchema.pick({
+                    id: true,
+                    slug: true,
+                    name: true,
+                    color: true
+                  }).array()
+                })
+              )
+              .array(),
             folderVersion: z.object({ id: z.string(), name: z.string() }).array(),
             createdAt: z.date(),
             updatedAt: z.date()

backend-pg/src/ee/services/secret-snapshot/snapshot-dal.ts

@@ -67,6 +67,16 @@ export const snapshotDALFactory = (db: TDbClient) => {
           `${TableName.SnapshotSecret}.secretVersionId`,
           `${TableName.SecretVersion}.id`
         )
+        .leftJoin(
+          TableName.SecretVersionTag,
+          `${TableName.SecretVersionTag}.${TableName.SecretVersion}Id`,
+          `${TableName.SecretVersion}.id`
+        )
+        .leftJoin(
+          TableName.SecretTag,
+          `${TableName.SecretVersionTag}.${TableName.SecretTag}Id`,
+          `${TableName.SecretTag}.id`
+        )
         .leftJoin(
           TableName.SnapshotFolder,
           `${TableName.SnapshotFolder}.snapshotId`,
@@ -87,7 +97,12 @@ export const snapshotDALFactory = (db: TDbClient) => {
           db.ref("slug").withSchema(TableName.Environment).as("envSlug"),
           db.ref("projectId").withSchema(TableName.Environment),
           db.ref("name").withSchema(TableName.SecretFolderVersion).as("folderVerName"),
-          db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("folderVerId")
+          db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("folderVerId"),
+          db.ref("id").withSchema(TableName.SecretTag).as("tagId"),
+          db.ref("id").withSchema(TableName.SecretVersionTag).as("tagVersionId"),
+          db.ref("color").withSchema(TableName.SecretTag).as("tagColor"),
+          db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug"),
+          db.ref("name").withSchema(TableName.SecretTag).as("tagName")
         );
       return sqlNestRelationships({
         data,
@@ -111,7 +126,20 @@ export const snapshotDALFactory = (db: TDbClient) => {
           {
             key: "id",
             label: "secretVersions" as const,
-            mapper: (el) => SecretVersionsSchema.parse(el)
+            mapper: (el) => SecretVersionsSchema.parse(el),
+            childrenMapper: [
+              {
+                key: "tagVersionId",
+                label: "tags" as const,
+                mapper: ({
+                  tagId: id,
+                  tagName: name,
+                  tagSlug: slug,
+                  tagColor: color,
+                  tagVersionId: vId
+                }) => ({ id, name, slug, color, vId })
+              }
+            ]
           },
           {
             key: "folderVerId",

backend-pg/src/lib/knex/join.ts

@@ -45,10 +45,57 @@ export type TChildMapper<T extends {}, U extends string = string, R extends unkn
   key: keyof T;
   label: U;
   mapper: (arg: T) => R;
+  childrenMapper?: TChildMapper<T, U, R>[];
 };
 
 type MappedRecord<T extends TChildMapper<any>> = {
-  [K in T["label"]]: Exclude<ReturnType<Extract<T, { label: K }>["mapper"]>, null | undefined>[];
+  [K in T["label"]]: Array<
+    Exclude<ReturnType<Extract<T, { label: K }>["mapper"]>, null | undefined> &
+      (Extract<T, { label: K }>["childrenMapper"] extends Array<infer TChild>
+        ? TChild extends TChildMapper<any>
+          ? MappedRecord<TChild>
+          : {}
+        : {})
+  >;
+};
+
+const sqlChildMapper = <
+  T extends Record<string, any> = {},
+  P extends Record<string, any> = {},
+  C extends TChildMapper<T>[] = TChildMapper<T>[]
+>(
+  doc: T,
+  docsByPk: P,
+  lookupTable: Set<string>,
+  pk: keyof P,
+  prefix: string,
+  childrenMapper: C
+) => {
+  if (!docsByPk) return;
+  type Cm = MappedRecord<(typeof childrenMapper)[number]>;
+  childrenMapper.forEach(({ label, mapper, key: childPk, childrenMapper: nestedMappers }) => {
+    // eslint-disable-next-line
+    if (!docsByPk?.[pk as keyof P]?.[label]) docsByPk[pk as keyof P][label as keyof Cm] = [] as any;
+
+    if (doc?.[childPk] !== null && typeof doc?.[childPk] !== "undefined") {
+      const ck = `${prefix}-${label}-${doc[childPk]}`;
+      if (!lookupTable.has(ck)) {
+        const val = mapper(doc);
+        if (typeof val !== "undefined" && val !== null) docsByPk[pk as keyof P][label].push(val);
+        lookupTable.add(ck);
+      }
+      if (nestedMappers) {
+        sqlChildMapper(
+          doc,
+          docsByPk[pk][label as keyof P],
+          lookupTable,
+          docsByPk[pk][label as keyof P].length - 1,
+          ck,
+          nestedMappers
+        );
+      }
+    }
+  });
 };
 
 export const sqlNestRelationships = <
@@ -61,31 +108,21 @@ export const sqlNestRelationships = <
   parentMapper,
   childrenMapper
 }: TSqlPackRelationships<T, P, C>) => {
-  const parentLookup = new Set<string>();
+  const lookupTable = new Set<string>();
-  const childLookUp = new Set<string>();
   const recordsOrder: string[] = [];
 
   type Cm = MappedRecord<(typeof childrenMapper)[number]>;
 
   const recordsGroupedByPk: Record<string, P & Cm> = {};
-  data.forEach((el) => {
+  data.forEach((doc) => {
-    const pk = el[key];
+    const pk = doc[key];
-    if (!parentLookup.has(pk)) {
+    if (!lookupTable.has(pk)) {
-      recordsGroupedByPk[pk] = parentMapper(el) as P & Cm;
+      recordsGroupedByPk[pk] = parentMapper(doc) as P & Cm;
       recordsOrder.push(pk);
-      parentLookup.add(pk);
+      lookupTable.add(pk);
     }
-    childrenMapper.forEach(({ label, mapper, key: cKey }) => {
+
-      if (!recordsGroupedByPk[pk][label]) recordsGroupedByPk[pk][label as keyof Cm] = [] as any;
+    sqlChildMapper(doc, recordsGroupedByPk, lookupTable, pk, "", childrenMapper);
-      if (el[cKey] !== null && typeof el[cKey] !== "undefined") {
-        const ck = `${pk}-${label}-${el[cKey]}`;
-        if (!childLookUp.has(ck)) {
-          const val = mapper(el);
-          if (typeof val !== "undefined" && val !== null) recordsGroupedByPk[pk][label].push(val);
-          childLookUp.add(ck);
-        }
-      }
-    });
   });
   return recordsOrder.map((pkId) => recordsGroupedByPk[pkId]);
 };

backend-pg/src/server/routes/index.ts

@@ -79,6 +79,7 @@ import { secretDALFactory } from "@app/services/secret/secret-dal";
 import { secretQueueFactory } from "@app/services/secret/secret-queue";
 import { secretServiceFactory } from "@app/services/secret/secret-service";
 import { secretVersionDALFactory } from "@app/services/secret/secret-version-dal";
+import { secretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal";
 import { secretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";
 import { secretFolderServiceFactory } from "@app/services/secret-folder/secret-folder-service";
 import { secretFolderVersionDALFactory } from "@app/services/secret-folder/secret-folder-version-dal";
@@ -138,6 +139,7 @@ export const registerRoutes = async (
   const folderVersionDAL = secretFolderVersionDALFactory(db);
   const secretImportDAL = secretImportDALFactory(db);
   const secretVersionDAL = secretVersionDALFactory(db);
+  const secretVersionTagDAL = secretVersionTagDALFactory(db);
   const secretBlindIndexDAL = secretBlindIndexDALFactory(db);
 
   const integrationDAL = integrationDALFactory(db);
@@ -351,6 +353,7 @@ export const registerRoutes = async (
   const secretService = secretServiceFactory({
     folderDAL,
     secretVersionDAL,
+    secretVersionTagDAL,
     secretBlindIndexDAL,
     permissionService,
     secretDAL,

backend-pg/src/services/secret-tag/secret-tag-dal.ts

@@ -29,7 +29,7 @@ export const secretTagDALFactory = (db: TDbClient) => {
           `${TableName.SecretTag}.id`
         )
         .where("projectId", projectId)
-        .whereIn("secretsId", secretIds)
+        .whereIn(`${TableName.Secret}Id`, secretIds)
         .delete()
         .returning("*");
       return tags;

backend-pg/src/services/secret/secret-service.ts

@@ -51,6 +51,7 @@ import {
   TUpdateSecretRawDTO
 } from "./secret-types";
 import { TSecretVersionDALFactory } from "./secret-version-dal";
+import { TSecretVersionTagDALFactory } from "./secret-version-tag-dal";
 
 type TSecretServiceFactoryDep = {
   secretDAL: TSecretDALFactory;
@@ -66,6 +67,7 @@ type TSecretServiceFactoryDep = {
   secretQueueService: Pick<TSecretQueueFactory, "syncSecrets">;
   projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
   secretImportDAL: Pick<TSecretImportDALFactory, "find">;
+  secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
 };
 
 export type TSecretServiceFactory = ReturnType<typeof secretServiceFactory>;
@@ -79,7 +81,8 @@ export const secretServiceFactory = ({
   snapshotService,
   secretQueueService,
   projectBotService,
-  secretImportDAL
+  secretImportDAL,
+  secretVersionTagDAL
 }: TSecretServiceFactoryDep) => {
   // utility function to get secret blind index data
   const interalGenSecBlindIndexByName = async (projectId: string, secretName: string) => {
@@ -116,10 +119,7 @@ export const secretServiceFactory = ({
         [`${TableName.Secret}Id` as const]: newSecretGroupByBlindIndex[secretBlindIndex][0].id
       }))
     );
-    if (newSecretTags.length) {
+    const secretVersions = await secretVersionDAL.insertMany(
-      await secretTagDAL.saveTagsToSecret(newSecretTags, tx);
-    }
-    await secretVersionDAL.insertMany(
       inputSecrets.map(({ tags, ...el }) => ({
         ...el,
         folderId,
@@ -127,6 +127,15 @@ export const secretServiceFactory = ({
       })),
       tx
     );
+    if (newSecretTags.length) {
+      const secTags = await secretTagDAL.saveTagsToSecret(newSecretTags, tx);
+      const secVersionsGroupBySecId = groupBy(secretVersions, (i) => i.secretId);
+      const newSecretVersionTags = secTags.flatMap(({ secretsId, secret_tagsId }) => ({
+        [`${TableName.SecretVersion}Id` as const]: secVersionsGroupBySecId[secretsId][0].id,
+        [`${TableName.SecretTag}Id` as const]: secret_tagsId
+      }));
+      await secretVersionTagDAL.insertMany(newSecretVersionTags, tx);
+    }
 
     return newSecrets;
   };
@@ -144,13 +153,20 @@ export const secretServiceFactory = ({
       })),
       tx
     );
+    const secretVersions = await secretVersionDAL.insertMany(
+      newSecrets.map(({ id, createdAt, updatedAt, ...el }) => ({
+        ...el,
+        secretId: id
+      })),
+      tx
+    );
     const secsUpdatedTag = inputSecrets.flatMap(({ data: { tags } }, i) =>
-      tags?.length ? { tags, secretId: newSecrets[i].id } : []
+      tags !== undefined ? { tags, secretId: newSecrets[i].id } : []
     );
     if (secsUpdatedTag.length) {
       await secretTagDAL.deleteTagsManySecret(
         projectId,
-        secsUpdatedTag.flatMap(({ tags }) => tags),
+        secsUpdatedTag.map(({ secretId }) => secretId),
         tx
       );
       const newSecretTags = secsUpdatedTag.flatMap(({ tags: secretTags = [], secretId }) =>
@@ -159,15 +175,16 @@ export const secretServiceFactory = ({
           [`${TableName.Secret}Id` as const]: secretId
         }))
       );
-      await secretTagDAL.saveTagsToSecret(newSecretTags, tx);
+      if (newSecretTags.length) {
+        const secTags = await secretTagDAL.saveTagsToSecret(newSecretTags, tx);
+        const secVersionsGroupBySecId = groupBy(secretVersions, (i) => i.secretId);
+        const newSecretVersionTags = secTags.flatMap(({ secretsId, secret_tagsId }) => ({
+          [`${TableName.SecretVersion}Id` as const]: secVersionsGroupBySecId[secretsId][0].id,
+          [`${TableName.SecretTag}Id` as const]: secret_tagsId
+        }));
+        await secretVersionTagDAL.insertMany(newSecretVersionTags, tx);
+      }
     }
-    await secretVersionDAL.insertMany(
-      newSecrets.map(({ id, createdAt, updatedAt, ...el }) => ({
-        ...el,
-        secretId: id
-      })),
-      tx
-    );
 
     return newSecrets;
   };

backend-pg/src/services/secret/secret-version-tag-dal.ts

@@ -0,0 +1,10 @@
+import { TDbClient } from "@app/db";
+import { TableName } from "@app/db/schemas";
+import { ormify } from "@app/lib/knex";
+
+export type TSecretVersionTagDALFactory = ReturnType<typeof secretVersionTagDALFactory>;
+
+export const secretVersionTagDALFactory = (db: TDbClient) => {
+  const secretVersionTagDAL = ormify(db, TableName.SecretVersionTag);
+  return secretVersionTagDAL;
+};