diff --git a/backend-pg/scripts/create-backend-file.ts b/backend-pg/scripts/create-backend-file.ts index d2b391051..b821aba21 100644 --- a/backend-pg/scripts/create-backend-file.ts +++ b/backend-pg/scripts/create-backend-file.ts @@ -27,8 +27,8 @@ if (componentType === 1) { .split("-") .map((el, index) => (index === 0 ? el : `${el[0].toUpperCase()}${el.slice(1)}`)) .join(""); - const dalTypeName = `T${pascalCase}DalFactory`; - const dalName = `${camelCase}DalFactory`; + const dalTypeName = `T${pascalCase}DALFactory`; + const dalName = `${camelCase}DALFactory`; const serviceTypeName = `T${pascalCase}ServiceFactory`; const serviceName = `${camelCase}ServiceFactory`; @@ -53,12 +53,12 @@ export const ${dalName} = (db: TDbClient) => { `import { ${dalTypeName} } from "./${componentName}-dal"; type ${serviceTypeName}Dep = { - ${camelCase}Dal: ${dalTypeName}; + ${camelCase}DAL: ${dalTypeName}; }; export type ${serviceTypeName} = ReturnType; -export const ${serviceName} = ({ ${camelCase}Dal }: ${serviceTypeName}Dep) => { +export const ${serviceName} = ({ ${camelCase}DAL }: ${serviceTypeName}Dep) => { return {}; }; ` @@ -75,8 +75,8 @@ export const ${serviceName} = ({ ${camelCase}Dal }: ${serviceTypeName}Dep) => { .split("-") .map((el, index) => (index === 0 ? el : `${el[0].toUpperCase()}${el.slice(1)}`)) .join(""); - const dalTypeName = `T${pascalCase}DalFactory`; - const dalName = `${camelCase}DalFactory`; + const dalTypeName = `T${pascalCase}DALFactory`; + const dalName = `${camelCase}DALFactory`; writeFileSync( path.join(__dirname, "../src/services", componentPath, `${componentName}-dal.ts`), diff --git a/backend-pg/src/@types/knex.d.ts b/backend-pg/src/@types/knex.d.ts index bd2391c4f..56ef68760 100644 --- a/backend-pg/src/@types/knex.d.ts +++ b/backend-pg/src/@types/knex.d.ts @@ -143,6 +143,9 @@ import { TSecretVersions, TSecretVersionsInsert, TSecretVersionsUpdate, + TSecretVersionTagJunction, + TSecretVersionTagJunctionInsert, + TSecretVersionTagJunctionUpdate, TServiceTokens, TServiceTokensInsert, TServiceTokensUpdate, @@ -163,8 +166,7 @@ import { TUsersUpdate, TWebhooks, TWebhooksInsert, - TWebhooksUpdate -} from "@app/db/schemas"; + TWebhooksUpdate} from "@app/db/schemas"; declare module "knex/types/tables" { interface Tables { @@ -403,5 +405,10 @@ declare module "knex/types/tables" { TSecretTagJunctionInsert, TSecretTagJunctionUpdate >; + [TableName.SecretVersionTag]: Knex.CompositeTableType< + TSecretVersionTagJunction, + TSecretVersionTagJunctionInsert, + TSecretVersionTagJunctionUpdate + >; } } diff --git a/backend-pg/src/db/migrations/20231220052508_secret-version.ts b/backend-pg/src/db/migrations/20231220052508_secret-version.ts index 4b515b8e1..6e3a8ae93 100644 --- a/backend-pg/src/db/migrations/20231220052508_secret-version.ts +++ b/backend-pg/src/db/migrations/20231220052508_secret-version.ts @@ -40,14 +40,14 @@ export async function up(knex: Knex): Promise { // many to many relation between tags await createJunctionTable( knex, - TableName.JnSecretVersionTag, + TableName.SecretVersionTag, TableName.SecretVersion, TableName.SecretTag ); } export async function down(knex: Knex): Promise { - await knex.schema.dropTableIfExists(TableName.JnSecretVersionTag); + await knex.schema.dropTableIfExists(TableName.SecretVersionTag); await knex.schema.dropTableIfExists(TableName.SecretVersion); await dropOnUpdateTrigger(knex, TableName.SecretVersion); } diff --git a/backend-pg/src/db/schemas/index.ts b/backend-pg/src/db/schemas/index.ts index ad29feaa7..62b01ebd2 100644 --- a/backend-pg/src/db/schemas/index.ts +++ b/backend-pg/src/db/schemas/index.ts @@ -44,6 +44,7 @@ export * from "./secret-snapshot-secrets"; export * from "./secret-snapshots"; export * from "./secret-tag-junction"; export * from "./secret-tags"; +export * from "./secret-version-tag-junction"; export * from "./secret-versions"; export * from "./secrets"; export * from "./service-tokens"; diff --git a/backend-pg/src/db/schemas/models.ts b/backend-pg/src/db/schemas/models.ts index 3dd0acbbb..f4a98a3ca 100644 --- a/backend-pg/src/db/schemas/models.ts +++ b/backend-pg/src/db/schemas/models.ts @@ -54,9 +54,9 @@ export enum TableName { GitAppOrg = "git_app_org", SecretScanningGitRisk = "secret_scanning_git_risks", TrustedIps = "trusted_ips", - // junction tables + // junction tables with tags JnSecretTag = "secret_tag_junction", - JnSecretVersionTag = "secret_version_tag_junction" + SecretVersionTag = "secret_version_tag_junction" } export type TImmutableDBKeys = "id" | "createdAt" | "updatedAt"; diff --git a/backend-pg/src/ee/routes/v1/snapshot-router.ts b/backend-pg/src/ee/routes/v1/snapshot-router.ts index f78f515e7..0bf119506 100644 --- a/backend-pg/src/ee/routes/v1/snapshot-router.ts +++ b/backend-pg/src/ee/routes/v1/snapshot-router.ts @@ -1,6 +1,6 @@ import { z } from "zod"; -import { SecretSnapshotsSchema, SecretVersionsSchema } from "@app/db/schemas"; +import { SecretSnapshotsSchema, SecretTagsSchema, SecretVersionsSchema } from "@app/db/schemas"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; import { AuthMode } from "@app/services/auth/auth-type"; @@ -22,7 +22,18 @@ export const registerSnapshotRouter = async (server: FastifyZodProvider) => { slug: z.string(), name: z.string() }), - secretVersions: SecretVersionsSchema.omit({ secretBlindIndex: true }).array(), + secretVersions: SecretVersionsSchema.omit({ secretBlindIndex: true }) + .merge( + z.object({ + tags: SecretTagsSchema.pick({ + id: true, + slug: true, + name: true, + color: true + }).array() + }) + ) + .array(), folderVersion: z.object({ id: z.string(), name: z.string() }).array(), createdAt: z.date(), updatedAt: z.date() diff --git a/backend-pg/src/ee/services/secret-snapshot/snapshot-dal.ts b/backend-pg/src/ee/services/secret-snapshot/snapshot-dal.ts index 370a70fbb..2174d0410 100644 --- a/backend-pg/src/ee/services/secret-snapshot/snapshot-dal.ts +++ b/backend-pg/src/ee/services/secret-snapshot/snapshot-dal.ts @@ -67,6 +67,16 @@ export const snapshotDALFactory = (db: TDbClient) => { `${TableName.SnapshotSecret}.secretVersionId`, `${TableName.SecretVersion}.id` ) + .leftJoin( + TableName.SecretVersionTag, + `${TableName.SecretVersionTag}.${TableName.SecretVersion}Id`, + `${TableName.SecretVersion}.id` + ) + .leftJoin( + TableName.SecretTag, + `${TableName.SecretVersionTag}.${TableName.SecretTag}Id`, + `${TableName.SecretTag}.id` + ) .leftJoin( TableName.SnapshotFolder, `${TableName.SnapshotFolder}.snapshotId`, @@ -87,7 +97,12 @@ export const snapshotDALFactory = (db: TDbClient) => { db.ref("slug").withSchema(TableName.Environment).as("envSlug"), db.ref("projectId").withSchema(TableName.Environment), db.ref("name").withSchema(TableName.SecretFolderVersion).as("folderVerName"), - db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("folderVerId") + db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("folderVerId"), + db.ref("id").withSchema(TableName.SecretTag).as("tagId"), + db.ref("id").withSchema(TableName.SecretVersionTag).as("tagVersionId"), + db.ref("color").withSchema(TableName.SecretTag).as("tagColor"), + db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug"), + db.ref("name").withSchema(TableName.SecretTag).as("tagName") ); return sqlNestRelationships({ data, @@ -111,7 +126,20 @@ export const snapshotDALFactory = (db: TDbClient) => { { key: "id", label: "secretVersions" as const, - mapper: (el) => SecretVersionsSchema.parse(el) + mapper: (el) => SecretVersionsSchema.parse(el), + childrenMapper: [ + { + key: "tagVersionId", + label: "tags" as const, + mapper: ({ + tagId: id, + tagName: name, + tagSlug: slug, + tagColor: color, + tagVersionId: vId + }) => ({ id, name, slug, color, vId }) + } + ] }, { key: "folderVerId", diff --git a/backend-pg/src/lib/knex/join.ts b/backend-pg/src/lib/knex/join.ts index 21b24880e..4e1ab09a3 100644 --- a/backend-pg/src/lib/knex/join.ts +++ b/backend-pg/src/lib/knex/join.ts @@ -45,10 +45,57 @@ export type TChildMapper R; + childrenMapper?: TChildMapper[]; }; type MappedRecord> = { - [K in T["label"]]: Exclude["mapper"]>, null | undefined>[]; + [K in T["label"]]: Array< + Exclude["mapper"]>, null | undefined> & + (Extract["childrenMapper"] extends Array + ? TChild extends TChildMapper + ? MappedRecord + : {} + : {}) + >; +}; + +const sqlChildMapper = < + T extends Record = {}, + P extends Record = {}, + C extends TChildMapper[] = TChildMapper[] +>( + doc: T, + docsByPk: P, + lookupTable: Set, + pk: keyof P, + prefix: string, + childrenMapper: C +) => { + if (!docsByPk) return; + type Cm = MappedRecord<(typeof childrenMapper)[number]>; + childrenMapper.forEach(({ label, mapper, key: childPk, childrenMapper: nestedMappers }) => { + // eslint-disable-next-line + if (!docsByPk?.[pk as keyof P]?.[label]) docsByPk[pk as keyof P][label as keyof Cm] = [] as any; + + if (doc?.[childPk] !== null && typeof doc?.[childPk] !== "undefined") { + const ck = `${prefix}-${label}-${doc[childPk]}`; + if (!lookupTable.has(ck)) { + const val = mapper(doc); + if (typeof val !== "undefined" && val !== null) docsByPk[pk as keyof P][label].push(val); + lookupTable.add(ck); + } + if (nestedMappers) { + sqlChildMapper( + doc, + docsByPk[pk][label as keyof P], + lookupTable, + docsByPk[pk][label as keyof P].length - 1, + ck, + nestedMappers + ); + } + } + }); }; export const sqlNestRelationships = < @@ -61,31 +108,21 @@ export const sqlNestRelationships = < parentMapper, childrenMapper }: TSqlPackRelationships) => { - const parentLookup = new Set(); - const childLookUp = new Set(); + const lookupTable = new Set(); const recordsOrder: string[] = []; type Cm = MappedRecord<(typeof childrenMapper)[number]>; const recordsGroupedByPk: Record = {}; - data.forEach((el) => { - const pk = el[key]; - if (!parentLookup.has(pk)) { - recordsGroupedByPk[pk] = parentMapper(el) as P & Cm; + data.forEach((doc) => { + const pk = doc[key]; + if (!lookupTable.has(pk)) { + recordsGroupedByPk[pk] = parentMapper(doc) as P & Cm; recordsOrder.push(pk); - parentLookup.add(pk); + lookupTable.add(pk); } - childrenMapper.forEach(({ label, mapper, key: cKey }) => { - if (!recordsGroupedByPk[pk][label]) recordsGroupedByPk[pk][label as keyof Cm] = [] as any; - if (el[cKey] !== null && typeof el[cKey] !== "undefined") { - const ck = `${pk}-${label}-${el[cKey]}`; - if (!childLookUp.has(ck)) { - const val = mapper(el); - if (typeof val !== "undefined" && val !== null) recordsGroupedByPk[pk][label].push(val); - childLookUp.add(ck); - } - } - }); + + sqlChildMapper(doc, recordsGroupedByPk, lookupTable, pk, "", childrenMapper); }); return recordsOrder.map((pkId) => recordsGroupedByPk[pkId]); }; diff --git a/backend-pg/src/server/routes/index.ts b/backend-pg/src/server/routes/index.ts index a0f43dcb1..f13409ac4 100644 --- a/backend-pg/src/server/routes/index.ts +++ b/backend-pg/src/server/routes/index.ts @@ -79,6 +79,7 @@ import { secretDALFactory } from "@app/services/secret/secret-dal"; import { secretQueueFactory } from "@app/services/secret/secret-queue"; import { secretServiceFactory } from "@app/services/secret/secret-service"; import { secretVersionDALFactory } from "@app/services/secret/secret-version-dal"; +import { secretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal"; import { secretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal"; import { secretFolderServiceFactory } from "@app/services/secret-folder/secret-folder-service"; import { secretFolderVersionDALFactory } from "@app/services/secret-folder/secret-folder-version-dal"; @@ -138,6 +139,7 @@ export const registerRoutes = async ( const folderVersionDAL = secretFolderVersionDALFactory(db); const secretImportDAL = secretImportDALFactory(db); const secretVersionDAL = secretVersionDALFactory(db); + const secretVersionTagDAL = secretVersionTagDALFactory(db); const secretBlindIndexDAL = secretBlindIndexDALFactory(db); const integrationDAL = integrationDALFactory(db); @@ -351,6 +353,7 @@ export const registerRoutes = async ( const secretService = secretServiceFactory({ folderDAL, secretVersionDAL, + secretVersionTagDAL, secretBlindIndexDAL, permissionService, secretDAL, diff --git a/backend-pg/src/services/secret-tag/secret-tag-dal.ts b/backend-pg/src/services/secret-tag/secret-tag-dal.ts index b1bc4108e..2e6b38c02 100644 --- a/backend-pg/src/services/secret-tag/secret-tag-dal.ts +++ b/backend-pg/src/services/secret-tag/secret-tag-dal.ts @@ -29,7 +29,7 @@ export const secretTagDALFactory = (db: TDbClient) => { `${TableName.SecretTag}.id` ) .where("projectId", projectId) - .whereIn("secretsId", secretIds) + .whereIn(`${TableName.Secret}Id`, secretIds) .delete() .returning("*"); return tags; diff --git a/backend-pg/src/services/secret/secret-service.ts b/backend-pg/src/services/secret/secret-service.ts index dbaab2ed6..eaf40e13b 100644 --- a/backend-pg/src/services/secret/secret-service.ts +++ b/backend-pg/src/services/secret/secret-service.ts @@ -51,6 +51,7 @@ import { TUpdateSecretRawDTO } from "./secret-types"; import { TSecretVersionDALFactory } from "./secret-version-dal"; +import { TSecretVersionTagDALFactory } from "./secret-version-tag-dal"; type TSecretServiceFactoryDep = { secretDAL: TSecretDALFactory; @@ -66,6 +67,7 @@ type TSecretServiceFactoryDep = { secretQueueService: Pick; projectBotService: Pick; secretImportDAL: Pick; + secretVersionTagDAL: Pick; }; export type TSecretServiceFactory = ReturnType; @@ -79,7 +81,8 @@ export const secretServiceFactory = ({ snapshotService, secretQueueService, projectBotService, - secretImportDAL + secretImportDAL, + secretVersionTagDAL }: TSecretServiceFactoryDep) => { // utility function to get secret blind index data const interalGenSecBlindIndexByName = async (projectId: string, secretName: string) => { @@ -116,10 +119,7 @@ export const secretServiceFactory = ({ [`${TableName.Secret}Id` as const]: newSecretGroupByBlindIndex[secretBlindIndex][0].id })) ); - if (newSecretTags.length) { - await secretTagDAL.saveTagsToSecret(newSecretTags, tx); - } - await secretVersionDAL.insertMany( + const secretVersions = await secretVersionDAL.insertMany( inputSecrets.map(({ tags, ...el }) => ({ ...el, folderId, @@ -127,6 +127,15 @@ export const secretServiceFactory = ({ })), tx ); + if (newSecretTags.length) { + const secTags = await secretTagDAL.saveTagsToSecret(newSecretTags, tx); + const secVersionsGroupBySecId = groupBy(secretVersions, (i) => i.secretId); + const newSecretVersionTags = secTags.flatMap(({ secretsId, secret_tagsId }) => ({ + [`${TableName.SecretVersion}Id` as const]: secVersionsGroupBySecId[secretsId][0].id, + [`${TableName.SecretTag}Id` as const]: secret_tagsId + })); + await secretVersionTagDAL.insertMany(newSecretVersionTags, tx); + } return newSecrets; }; @@ -144,13 +153,20 @@ export const secretServiceFactory = ({ })), tx ); + const secretVersions = await secretVersionDAL.insertMany( + newSecrets.map(({ id, createdAt, updatedAt, ...el }) => ({ + ...el, + secretId: id + })), + tx + ); const secsUpdatedTag = inputSecrets.flatMap(({ data: { tags } }, i) => - tags?.length ? { tags, secretId: newSecrets[i].id } : [] + tags !== undefined ? { tags, secretId: newSecrets[i].id } : [] ); if (secsUpdatedTag.length) { await secretTagDAL.deleteTagsManySecret( projectId, - secsUpdatedTag.flatMap(({ tags }) => tags), + secsUpdatedTag.map(({ secretId }) => secretId), tx ); const newSecretTags = secsUpdatedTag.flatMap(({ tags: secretTags = [], secretId }) => @@ -159,15 +175,16 @@ export const secretServiceFactory = ({ [`${TableName.Secret}Id` as const]: secretId })) ); - await secretTagDAL.saveTagsToSecret(newSecretTags, tx); + if (newSecretTags.length) { + const secTags = await secretTagDAL.saveTagsToSecret(newSecretTags, tx); + const secVersionsGroupBySecId = groupBy(secretVersions, (i) => i.secretId); + const newSecretVersionTags = secTags.flatMap(({ secretsId, secret_tagsId }) => ({ + [`${TableName.SecretVersion}Id` as const]: secVersionsGroupBySecId[secretsId][0].id, + [`${TableName.SecretTag}Id` as const]: secret_tagsId + })); + await secretVersionTagDAL.insertMany(newSecretVersionTags, tx); + } } - await secretVersionDAL.insertMany( - newSecrets.map(({ id, createdAt, updatedAt, ...el }) => ({ - ...el, - secretId: id - })), - tx - ); return newSecrets; }; diff --git a/backend-pg/src/services/secret/secret-version-tag-dal.ts b/backend-pg/src/services/secret/secret-version-tag-dal.ts new file mode 100644 index 000000000..e4563ff00 --- /dev/null +++ b/backend-pg/src/services/secret/secret-version-tag-dal.ts @@ -0,0 +1,10 @@ +import { TDbClient } from "@app/db"; +import { TableName } from "@app/db/schemas"; +import { ormify } from "@app/lib/knex"; + +export type TSecretVersionTagDALFactory = ReturnType; + +export const secretVersionTagDALFactory = (db: TDbClient) => { + const secretVersionTagDAL = ormify(db, TableName.SecretVersionTag); + return secretVersionTagDAL; +};