diff --git a/backend/e2e-test/vitest-environment-knex.ts b/backend/e2e-test/vitest-environment-knex.ts index 09ab05443..cc1f9afc2 100644 --- a/backend/e2e-test/vitest-environment-knex.ts +++ b/backend/e2e-test/vitest-environment-knex.ts @@ -3,7 +3,6 @@ import "ts-node/register"; import dotenv from "dotenv"; import jwt from "jsonwebtoken"; -import knex from "knex"; import path from "path"; import { seedData1 } from "@app/db/seed-data"; @@ -15,6 +14,7 @@ import { AuthMethod, AuthTokenType } from "@app/services/auth/auth-type"; import { mockQueue } from "./mocks/queue"; import { mockSmtpServer } from "./mocks/smtp"; import { mockKeyStore } from "./mocks/keystore"; +import { initDbConnection } from "@app/db"; dotenv.config({ path: path.join(__dirname, "../../.env.test"), debug: true }); export default { @@ -23,23 +23,21 @@ export default { async setup() { const logger = await initLogger(); const cfg = initEnvConfig(logger); - const db = knex({ - client: "pg", - connection: cfg.DB_CONNECTION_URI, - migrations: { - directory: path.join(__dirname, "../src/db/migrations"), - extension: "ts", - tableName: "infisical_migrations" - }, - seeds: { - directory: path.join(__dirname, "../src/db/seeds"), - extension: "ts" - } + const db = initDbConnection({ + dbConnectionUri: cfg.DB_CONNECTION_URI, + dbRootCert: cfg.DB_ROOT_CERT }); try { - await db.migrate.latest(); - await db.seed.run(); + await db.migrate.latest({ + directory: path.join(__dirname, "../src/db/migrations"), + extension: "ts", + tableName: "infisical_migrations" + }); + await db.seed.run({ + directory: path.join(__dirname, "../src/db/seeds"), + extension: "ts" + }); const smtp = mockSmtpServer(); const queue = mockQueue(); const keyStore = mockKeyStore(); @@ -74,7 +72,14 @@ export default { // @ts-expect-error type delete globalThis.jwtToken; // called after all tests with this env have been run - await db.migrate.rollback({}, true); + await db.migrate.rollback( + { + directory: path.join(__dirname, "../src/db/migrations"), + extension: "ts", + tableName: "infisical_migrations" + }, + true + ); await db.destroy(); } }; diff --git a/backend/src/@types/knex.d.ts b/backend/src/@types/knex.d.ts index 4fdfda70c..82ca89742 100644 --- a/backend/src/@types/knex.d.ts +++ b/backend/src/@types/knex.d.ts @@ -1,4 +1,4 @@ -import { Knex } from "knex"; +import { Knex as KnexOriginal } from "knex"; import { TableName, @@ -280,318 +280,371 @@ import { TWebhooksUpdate } from "@app/db/schemas"; +declare module "knex" { + namespace Knex { + interface QueryInterface { + primaryNode(): KnexOriginal; + replicaNode(): KnexOriginal; + } + } +} + declare module "knex/types/tables" { interface Tables { - [TableName.Users]: Knex.CompositeTableType; - [TableName.Groups]: Knex.CompositeTableType; - [TableName.CertificateAuthority]: Knex.CompositeTableType< + [TableName.Users]: KnexOriginal.CompositeTableType; + [TableName.Groups]: KnexOriginal.CompositeTableType; + [TableName.CertificateAuthority]: KnexOriginal.CompositeTableType< TCertificateAuthorities, TCertificateAuthoritiesInsert, TCertificateAuthoritiesUpdate >; - [TableName.CertificateAuthorityCert]: Knex.CompositeTableType< + [TableName.CertificateAuthorityCert]: KnexOriginal.CompositeTableType< TCertificateAuthorityCerts, TCertificateAuthorityCertsInsert, TCertificateAuthorityCertsUpdate >; - [TableName.CertificateAuthoritySecret]: Knex.CompositeTableType< + [TableName.CertificateAuthoritySecret]: KnexOriginal.CompositeTableType< TCertificateAuthoritySecret, TCertificateAuthoritySecretInsert, TCertificateAuthoritySecretUpdate >; - [TableName.CertificateAuthorityCrl]: Knex.CompositeTableType< + [TableName.CertificateAuthorityCrl]: KnexOriginal.CompositeTableType< TCertificateAuthorityCrl, TCertificateAuthorityCrlInsert, TCertificateAuthorityCrlUpdate >; - [TableName.Certificate]: Knex.CompositeTableType; - [TableName.CertificateBody]: Knex.CompositeTableType< + [TableName.Certificate]: KnexOriginal.CompositeTableType; + [TableName.CertificateBody]: KnexOriginal.CompositeTableType< TCertificateBodies, TCertificateBodiesInsert, TCertificateBodiesUpdate >; - [TableName.CertificateSecret]: Knex.CompositeTableType< + [TableName.CertificateSecret]: KnexOriginal.CompositeTableType< TCertificateSecrets, TCertificateSecretsInsert, TCertificateSecretsUpdate >; - [TableName.UserGroupMembership]: Knex.CompositeTableType< + [TableName.UserGroupMembership]: KnexOriginal.CompositeTableType< TUserGroupMembership, TUserGroupMembershipInsert, TUserGroupMembershipUpdate >; - [TableName.GroupProjectMembership]: Knex.CompositeTableType< + [TableName.GroupProjectMembership]: KnexOriginal.CompositeTableType< TGroupProjectMemberships, TGroupProjectMembershipsInsert, TGroupProjectMembershipsUpdate >; - [TableName.GroupProjectMembershipRole]: Knex.CompositeTableType< + [TableName.GroupProjectMembershipRole]: KnexOriginal.CompositeTableType< TGroupProjectMembershipRoles, TGroupProjectMembershipRolesInsert, TGroupProjectMembershipRolesUpdate >; - [TableName.UserAliases]: Knex.CompositeTableType; - [TableName.UserEncryptionKey]: Knex.CompositeTableType< + [TableName.UserAliases]: KnexOriginal.CompositeTableType; + [TableName.UserEncryptionKey]: KnexOriginal.CompositeTableType< TUserEncryptionKeys, TUserEncryptionKeysInsert, TUserEncryptionKeysUpdate >; - [TableName.AuthTokens]: Knex.CompositeTableType; - [TableName.AuthTokenSession]: Knex.CompositeTableType< + [TableName.AuthTokens]: KnexOriginal.CompositeTableType; + [TableName.AuthTokenSession]: KnexOriginal.CompositeTableType< TAuthTokenSessions, TAuthTokenSessionsInsert, TAuthTokenSessionsUpdate >; - [TableName.BackupPrivateKey]: Knex.CompositeTableType< + [TableName.BackupPrivateKey]: KnexOriginal.CompositeTableType< TBackupPrivateKey, TBackupPrivateKeyInsert, TBackupPrivateKeyUpdate >; - [TableName.Organization]: Knex.CompositeTableType; - [TableName.OrgMembership]: Knex.CompositeTableType; - [TableName.OrgRoles]: Knex.CompositeTableType; - [TableName.IncidentContact]: Knex.CompositeTableType< + [TableName.Organization]: KnexOriginal.CompositeTableType< + TOrganizations, + TOrganizationsInsert, + TOrganizationsUpdate + >; + [TableName.OrgMembership]: KnexOriginal.CompositeTableType< + TOrgMemberships, + TOrgMembershipsInsert, + TOrgMembershipsUpdate + >; + [TableName.OrgRoles]: KnexOriginal.CompositeTableType; + [TableName.IncidentContact]: KnexOriginal.CompositeTableType< TIncidentContacts, TIncidentContactsInsert, TIncidentContactsUpdate >; - [TableName.UserAction]: Knex.CompositeTableType; - [TableName.SuperAdmin]: Knex.CompositeTableType; - [TableName.ApiKey]: Knex.CompositeTableType; - [TableName.Project]: Knex.CompositeTableType; - [TableName.ProjectMembership]: Knex.CompositeTableType< + [TableName.UserAction]: KnexOriginal.CompositeTableType; + [TableName.SuperAdmin]: KnexOriginal.CompositeTableType; + [TableName.ApiKey]: KnexOriginal.CompositeTableType; + [TableName.Project]: KnexOriginal.CompositeTableType; + [TableName.ProjectMembership]: KnexOriginal.CompositeTableType< TProjectMemberships, TProjectMembershipsInsert, TProjectMembershipsUpdate >; - [TableName.Environment]: Knex.CompositeTableType< + [TableName.Environment]: KnexOriginal.CompositeTableType< TProjectEnvironments, TProjectEnvironmentsInsert, TProjectEnvironmentsUpdate >; - [TableName.ProjectBot]: Knex.CompositeTableType; - [TableName.ProjectUserMembershipRole]: Knex.CompositeTableType< + [TableName.ProjectBot]: KnexOriginal.CompositeTableType; + [TableName.ProjectUserMembershipRole]: KnexOriginal.CompositeTableType< TProjectUserMembershipRoles, TProjectUserMembershipRolesInsert, TProjectUserMembershipRolesUpdate >; - [TableName.ProjectRoles]: Knex.CompositeTableType; - [TableName.ProjectUserAdditionalPrivilege]: Knex.CompositeTableType< + [TableName.ProjectRoles]: KnexOriginal.CompositeTableType; + [TableName.ProjectUserAdditionalPrivilege]: KnexOriginal.CompositeTableType< TProjectUserAdditionalPrivilege, TProjectUserAdditionalPrivilegeInsert, TProjectUserAdditionalPrivilegeUpdate >; - [TableName.ProjectKeys]: Knex.CompositeTableType; - [TableName.Secret]: Knex.CompositeTableType; - [TableName.SecretReference]: Knex.CompositeTableType< + [TableName.ProjectKeys]: KnexOriginal.CompositeTableType; + [TableName.Secret]: KnexOriginal.CompositeTableType; + [TableName.SecretReference]: KnexOriginal.CompositeTableType< TSecretReferences, TSecretReferencesInsert, TSecretReferencesUpdate >; - [TableName.SecretBlindIndex]: Knex.CompositeTableType< + [TableName.SecretBlindIndex]: KnexOriginal.CompositeTableType< TSecretBlindIndexes, TSecretBlindIndexesInsert, TSecretBlindIndexesUpdate >; - [TableName.SecretVersion]: Knex.CompositeTableType; - [TableName.SecretFolder]: Knex.CompositeTableType; - [TableName.SecretFolderVersion]: Knex.CompositeTableType< + [TableName.SecretVersion]: KnexOriginal.CompositeTableType< + TSecretVersions, + TSecretVersionsInsert, + TSecretVersionsUpdate + >; + [TableName.SecretFolder]: KnexOriginal.CompositeTableType< + TSecretFolders, + TSecretFoldersInsert, + TSecretFoldersUpdate + >; + [TableName.SecretFolderVersion]: KnexOriginal.CompositeTableType< TSecretFolderVersions, TSecretFolderVersionsInsert, TSecretFolderVersionsUpdate >; - [TableName.SecretSharing]: Knex.CompositeTableType; - [TableName.RateLimit]: Knex.CompositeTableType; - [TableName.SecretTag]: Knex.CompositeTableType; - [TableName.SecretImport]: Knex.CompositeTableType; - [TableName.Integration]: Knex.CompositeTableType; - [TableName.Webhook]: Knex.CompositeTableType; - [TableName.ServiceToken]: Knex.CompositeTableType; - [TableName.IntegrationAuth]: Knex.CompositeTableType< + [TableName.SecretSharing]: KnexOriginal.CompositeTableType< + TSecretSharing, + TSecretSharingInsert, + TSecretSharingUpdate + >; + [TableName.RateLimit]: KnexOriginal.CompositeTableType; + [TableName.SecretTag]: KnexOriginal.CompositeTableType; + [TableName.SecretImport]: KnexOriginal.CompositeTableType< + TSecretImports, + TSecretImportsInsert, + TSecretImportsUpdate + >; + [TableName.Integration]: KnexOriginal.CompositeTableType; + [TableName.Webhook]: KnexOriginal.CompositeTableType; + [TableName.ServiceToken]: KnexOriginal.CompositeTableType< + TServiceTokens, + TServiceTokensInsert, + TServiceTokensUpdate + >; + [TableName.IntegrationAuth]: KnexOriginal.CompositeTableType< TIntegrationAuths, TIntegrationAuthsInsert, TIntegrationAuthsUpdate >; - [TableName.Identity]: Knex.CompositeTableType; - [TableName.IdentityUniversalAuth]: Knex.CompositeTableType< + [TableName.Identity]: KnexOriginal.CompositeTableType; + [TableName.IdentityUniversalAuth]: KnexOriginal.CompositeTableType< TIdentityUniversalAuths, TIdentityUniversalAuthsInsert, TIdentityUniversalAuthsUpdate >; - [TableName.IdentityKubernetesAuth]: Knex.CompositeTableType< + [TableName.IdentityKubernetesAuth]: KnexOriginal.CompositeTableType< TIdentityKubernetesAuths, TIdentityKubernetesAuthsInsert, TIdentityKubernetesAuthsUpdate >; - [TableName.IdentityGcpAuth]: Knex.CompositeTableType< + [TableName.IdentityGcpAuth]: KnexOriginal.CompositeTableType< TIdentityGcpAuths, TIdentityGcpAuthsInsert, TIdentityGcpAuthsUpdate >; - [TableName.IdentityAwsAuth]: Knex.CompositeTableType< + [TableName.IdentityAwsAuth]: KnexOriginal.CompositeTableType< TIdentityAwsAuths, TIdentityAwsAuthsInsert, TIdentityAwsAuthsUpdate >; - [TableName.IdentityAzureAuth]: Knex.CompositeTableType< + [TableName.IdentityAzureAuth]: KnexOriginal.CompositeTableType< TIdentityAzureAuths, TIdentityAzureAuthsInsert, TIdentityAzureAuthsUpdate >; - [TableName.IdentityUaClientSecret]: Knex.CompositeTableType< + [TableName.IdentityUaClientSecret]: KnexOriginal.CompositeTableType< TIdentityUaClientSecrets, TIdentityUaClientSecretsInsert, TIdentityUaClientSecretsUpdate >; - [TableName.IdentityAccessToken]: Knex.CompositeTableType< + [TableName.IdentityAccessToken]: KnexOriginal.CompositeTableType< TIdentityAccessTokens, TIdentityAccessTokensInsert, TIdentityAccessTokensUpdate >; - [TableName.IdentityOrgMembership]: Knex.CompositeTableType< + [TableName.IdentityOrgMembership]: KnexOriginal.CompositeTableType< TIdentityOrgMemberships, TIdentityOrgMembershipsInsert, TIdentityOrgMembershipsUpdate >; - [TableName.IdentityProjectMembership]: Knex.CompositeTableType< + [TableName.IdentityProjectMembership]: KnexOriginal.CompositeTableType< TIdentityProjectMemberships, TIdentityProjectMembershipsInsert, TIdentityProjectMembershipsUpdate >; - [TableName.IdentityProjectMembershipRole]: Knex.CompositeTableType< + [TableName.IdentityProjectMembershipRole]: KnexOriginal.CompositeTableType< TIdentityProjectMembershipRole, TIdentityProjectMembershipRoleInsert, TIdentityProjectMembershipRoleUpdate >; - [TableName.IdentityProjectAdditionalPrivilege]: Knex.CompositeTableType< + [TableName.IdentityProjectAdditionalPrivilege]: KnexOriginal.CompositeTableType< TIdentityProjectAdditionalPrivilege, TIdentityProjectAdditionalPrivilegeInsert, TIdentityProjectAdditionalPrivilegeUpdate >; - [TableName.AccessApprovalPolicy]: Knex.CompositeTableType< + [TableName.AccessApprovalPolicy]: KnexOriginal.CompositeTableType< TAccessApprovalPolicies, TAccessApprovalPoliciesInsert, TAccessApprovalPoliciesUpdate >; - [TableName.AccessApprovalPolicyApprover]: Knex.CompositeTableType< + [TableName.AccessApprovalPolicyApprover]: KnexOriginal.CompositeTableType< TAccessApprovalPoliciesApprovers, TAccessApprovalPoliciesApproversInsert, TAccessApprovalPoliciesApproversUpdate >; - [TableName.AccessApprovalRequest]: Knex.CompositeTableType< + [TableName.AccessApprovalRequest]: KnexOriginal.CompositeTableType< TAccessApprovalRequests, TAccessApprovalRequestsInsert, TAccessApprovalRequestsUpdate >; - [TableName.AccessApprovalRequestReviewer]: Knex.CompositeTableType< + [TableName.AccessApprovalRequestReviewer]: KnexOriginal.CompositeTableType< TAccessApprovalRequestsReviewers, TAccessApprovalRequestsReviewersInsert, TAccessApprovalRequestsReviewersUpdate >; - [TableName.ScimToken]: Knex.CompositeTableType; - [TableName.SecretApprovalPolicy]: Knex.CompositeTableType< + [TableName.ScimToken]: KnexOriginal.CompositeTableType; + [TableName.SecretApprovalPolicy]: KnexOriginal.CompositeTableType< TSecretApprovalPolicies, TSecretApprovalPoliciesInsert, TSecretApprovalPoliciesUpdate >; - [TableName.SecretApprovalPolicyApprover]: Knex.CompositeTableType< + [TableName.SecretApprovalPolicyApprover]: KnexOriginal.CompositeTableType< TSecretApprovalPoliciesApprovers, TSecretApprovalPoliciesApproversInsert, TSecretApprovalPoliciesApproversUpdate >; - [TableName.SecretApprovalRequest]: Knex.CompositeTableType< + [TableName.SecretApprovalRequest]: KnexOriginal.CompositeTableType< TSecretApprovalRequests, TSecretApprovalRequestsInsert, TSecretApprovalRequestsUpdate >; - [TableName.SecretApprovalRequestReviewer]: Knex.CompositeTableType< + [TableName.SecretApprovalRequestReviewer]: KnexOriginal.CompositeTableType< TSecretApprovalRequestsReviewers, TSecretApprovalRequestsReviewersInsert, TSecretApprovalRequestsReviewersUpdate >; - [TableName.SecretApprovalRequestSecret]: Knex.CompositeTableType< + [TableName.SecretApprovalRequestSecret]: KnexOriginal.CompositeTableType< TSecretApprovalRequestsSecrets, TSecretApprovalRequestsSecretsInsert, TSecretApprovalRequestsSecretsUpdate >; - [TableName.SecretApprovalRequestSecretTag]: Knex.CompositeTableType< + [TableName.SecretApprovalRequestSecretTag]: KnexOriginal.CompositeTableType< TSecretApprovalRequestSecretTags, TSecretApprovalRequestSecretTagsInsert, TSecretApprovalRequestSecretTagsUpdate >; - [TableName.SecretRotation]: Knex.CompositeTableType< + [TableName.SecretRotation]: KnexOriginal.CompositeTableType< TSecretRotations, TSecretRotationsInsert, TSecretRotationsUpdate >; - [TableName.SecretRotationOutput]: Knex.CompositeTableType< + [TableName.SecretRotationOutput]: KnexOriginal.CompositeTableType< TSecretRotationOutputs, TSecretRotationOutputsInsert, TSecretRotationOutputsUpdate >; - [TableName.Snapshot]: Knex.CompositeTableType; - [TableName.SnapshotSecret]: Knex.CompositeTableType< + [TableName.Snapshot]: KnexOriginal.CompositeTableType< + TSecretSnapshots, + TSecretSnapshotsInsert, + TSecretSnapshotsUpdate + >; + [TableName.SnapshotSecret]: KnexOriginal.CompositeTableType< TSecretSnapshotSecrets, TSecretSnapshotSecretsInsert, TSecretSnapshotSecretsUpdate >; - [TableName.SnapshotFolder]: Knex.CompositeTableType< + [TableName.SnapshotFolder]: KnexOriginal.CompositeTableType< TSecretSnapshotFolders, TSecretSnapshotFoldersInsert, TSecretSnapshotFoldersUpdate >; - [TableName.DynamicSecret]: Knex.CompositeTableType; - [TableName.DynamicSecretLease]: Knex.CompositeTableType< + [TableName.DynamicSecret]: KnexOriginal.CompositeTableType< + TDynamicSecrets, + TDynamicSecretsInsert, + TDynamicSecretsUpdate + >; + [TableName.DynamicSecretLease]: KnexOriginal.CompositeTableType< TDynamicSecretLeases, TDynamicSecretLeasesInsert, TDynamicSecretLeasesUpdate >; - [TableName.SamlConfig]: Knex.CompositeTableType; - [TableName.OidcConfig]: Knex.CompositeTableType; - [TableName.LdapConfig]: Knex.CompositeTableType; - [TableName.LdapGroupMap]: Knex.CompositeTableType; - [TableName.OrgBot]: Knex.CompositeTableType; - [TableName.AuditLog]: Knex.CompositeTableType; - [TableName.AuditLogStream]: Knex.CompositeTableType< + [TableName.SamlConfig]: KnexOriginal.CompositeTableType; + [TableName.OidcConfig]: KnexOriginal.CompositeTableType; + [TableName.LdapConfig]: KnexOriginal.CompositeTableType; + [TableName.LdapGroupMap]: KnexOriginal.CompositeTableType< + TLdapGroupMaps, + TLdapGroupMapsInsert, + TLdapGroupMapsUpdate + >; + [TableName.OrgBot]: KnexOriginal.CompositeTableType; + [TableName.AuditLog]: KnexOriginal.CompositeTableType; + [TableName.AuditLogStream]: KnexOriginal.CompositeTableType< TAuditLogStreams, TAuditLogStreamsInsert, TAuditLogStreamsUpdate >; - [TableName.GitAppInstallSession]: Knex.CompositeTableType< + [TableName.GitAppInstallSession]: KnexOriginal.CompositeTableType< TGitAppInstallSessions, TGitAppInstallSessionsInsert, TGitAppInstallSessionsUpdate >; - [TableName.GitAppOrg]: Knex.CompositeTableType; - [TableName.SecretScanningGitRisk]: Knex.CompositeTableType< + [TableName.GitAppOrg]: KnexOriginal.CompositeTableType; + [TableName.SecretScanningGitRisk]: KnexOriginal.CompositeTableType< TSecretScanningGitRisks, TSecretScanningGitRisksInsert, TSecretScanningGitRisksUpdate >; - [TableName.TrustedIps]: Knex.CompositeTableType; + [TableName.TrustedIps]: KnexOriginal.CompositeTableType; // Junction tables - [TableName.JnSecretTag]: Knex.CompositeTableType< + [TableName.JnSecretTag]: KnexOriginal.CompositeTableType< TSecretTagJunction, TSecretTagJunctionInsert, TSecretTagJunctionUpdate >; - [TableName.SecretVersionTag]: Knex.CompositeTableType< + [TableName.SecretVersionTag]: KnexOriginal.CompositeTableType< TSecretVersionTagJunction, TSecretVersionTagJunctionInsert, TSecretVersionTagJunctionUpdate >; // KMS service - [TableName.KmsServerRootConfig]: Knex.CompositeTableType< + [TableName.KmsServerRootConfig]: KnexOriginal.CompositeTableType< TKmsRootConfig, TKmsRootConfigInsert, TKmsRootConfigUpdate >; - [TableName.KmsKey]: Knex.CompositeTableType; - [TableName.KmsKeyVersion]: Knex.CompositeTableType; + [TableName.KmsKey]: KnexOriginal.CompositeTableType; + [TableName.KmsKeyVersion]: KnexOriginal.CompositeTableType< + TKmsKeyVersions, + TKmsKeyVersionsInsert, + TKmsKeyVersionsUpdate + >; } } diff --git a/backend/src/db/instance.ts b/backend/src/db/instance.ts index bd4ce99c1..f6162ad9c 100644 --- a/backend/src/db/instance.ts +++ b/backend/src/db/instance.ts @@ -1,8 +1,38 @@ -import knex from "knex"; +import knex, { Knex } from "knex"; export type TDbClient = ReturnType; -export const initDbConnection = ({ dbConnectionUri, dbRootCert }: { dbConnectionUri: string; dbRootCert?: string }) => { - const db = knex({ +export const initDbConnection = ({ + dbConnectionUri, + dbRootCert, + readReplicas = [] +}: { + dbConnectionUri: string; + dbRootCert?: string; + readReplicas?: { + dbConnectionUri: string; + dbRootCert?: string; + }[]; +}) => { + // akhilmhdh: the default Knex is knex.Knex. but when assigned with knex({}) the value is knex.Knex + // this was causing issue with files like `snapshot-dal` `findRecursivelySnapshots` this i am explicitly putting the any and unknown[] + // eslint-disable-next-line + let db: Knex; + // eslint-disable-next-line + let readReplicaDbs: Knex[]; + // @ts-expect-error the querybuilder type is expected but our intension is to return a knex instance + knex.QueryBuilder.extend("primaryNode", () => { + return db; + }); + + // @ts-expect-error the querybuilder type is expected but our intension is to return a knex instance + knex.QueryBuilder.extend("replicaNode", () => { + if (!readReplicaDbs.length) return db; + + const selectedReplica = readReplicaDbs[Math.floor(Math.random() * readReplicaDbs.length)]; + return selectedReplica; + }); + + db = knex({ client: "pg", connection: { connectionString: dbConnectionUri, @@ -22,5 +52,21 @@ export const initDbConnection = ({ dbConnectionUri, dbRootCert }: { dbConnection } }); + readReplicaDbs = readReplicas.map((el) => { + const replicaDbCertificate = el.dbRootCert || dbRootCert; + return knex({ + client: "pg", + connection: { + connectionString: el.dbConnectionUri, + ssl: replicaDbCertificate + ? { + rejectUnauthorized: true, + ca: Buffer.from(replicaDbCertificate, "base64").toString("ascii") + } + : false + } + }); + }); + return db; }; diff --git a/backend/src/ee/services/access-approval-policy/access-approval-policy-dal.ts b/backend/src/ee/services/access-approval-policy/access-approval-policy-dal.ts index 88e288832..77ae430c6 100644 --- a/backend/src/ee/services/access-approval-policy/access-approval-policy-dal.ts +++ b/backend/src/ee/services/access-approval-policy/access-approval-policy-dal.ts @@ -32,7 +32,7 @@ export const accessApprovalPolicyDALFactory = (db: TDbClient) => { const findById = async (id: string, tx?: Knex) => { try { - const doc = await accessApprovalPolicyFindQuery(tx || db, { + const doc = await accessApprovalPolicyFindQuery(tx || db.replicaNode(), { [`${TableName.AccessApprovalPolicy}.id` as "id"]: id }); const formatedDoc = mergeOneToManyRelation( @@ -54,7 +54,7 @@ export const accessApprovalPolicyDALFactory = (db: TDbClient) => { const find = async (filter: TFindFilter, tx?: Knex) => { try { - const docs = await accessApprovalPolicyFindQuery(tx || db, filter); + const docs = await accessApprovalPolicyFindQuery(tx || db.replicaNode(), filter); const formatedDoc = mergeOneToManyRelation( docs, "id", diff --git a/backend/src/ee/services/access-approval-request/access-approval-request-dal.ts b/backend/src/ee/services/access-approval-request/access-approval-request-dal.ts index c3f4c72a6..c3c0d24d0 100644 --- a/backend/src/ee/services/access-approval-request/access-approval-request-dal.ts +++ b/backend/src/ee/services/access-approval-request/access-approval-request-dal.ts @@ -14,7 +14,8 @@ export const accessApprovalRequestDALFactory = (db: TDbClient) => { const findRequestsWithPrivilegeByPolicyIds = async (policyIds: string[]) => { try { - const docs = await db(TableName.AccessApprovalRequest) + const docs = await db + .replicaNode()(TableName.AccessApprovalRequest) .whereIn(`${TableName.AccessApprovalRequest}.policyId`, policyIds) .leftJoin( @@ -170,7 +171,7 @@ export const accessApprovalRequestDALFactory = (db: TDbClient) => { const findById = async (id: string, tx?: Knex) => { try { - const sql = findQuery({ [`${TableName.AccessApprovalRequest}.id` as "id"]: id }, tx || db); + const sql = findQuery({ [`${TableName.AccessApprovalRequest}.id` as "id"]: id }, tx || db.replicaNode()); const docs = await sql; const formatedDoc = sqlNestRelationships({ data: docs, @@ -207,7 +208,8 @@ export const accessApprovalRequestDALFactory = (db: TDbClient) => { const getCount = async ({ projectId }: { projectId: string }) => { try { - const accessRequests = await db(TableName.AccessApprovalRequest) + const accessRequests = await db + .replicaNode()(TableName.AccessApprovalRequest) .leftJoin( TableName.AccessApprovalPolicy, `${TableName.AccessApprovalRequest}.policyId`, diff --git a/backend/src/ee/services/audit-log/audit-log-dal.ts b/backend/src/ee/services/audit-log/audit-log-dal.ts index b3ad8c2b6..ffb7de4f3 100644 --- a/backend/src/ee/services/audit-log/audit-log-dal.ts +++ b/backend/src/ee/services/audit-log/audit-log-dal.ts @@ -27,7 +27,7 @@ export const auditLogDALFactory = (db: TDbClient) => { tx?: Knex ) => { try { - const sqlQuery = (tx || db)(TableName.AuditLog) + const sqlQuery = (tx || db.replicaNode())(TableName.AuditLog) .where( stripUndefinedInWhere({ projectId, diff --git a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-dal.ts b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-dal.ts index 810628030..339b2d626 100644 --- a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-dal.ts +++ b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-dal.ts @@ -12,7 +12,10 @@ export const dynamicSecretLeaseDALFactory = (db: TDbClient) => { const countLeasesForDynamicSecret = async (dynamicSecretId: string, tx?: Knex) => { try { - const doc = await (tx || db)(TableName.DynamicSecretLease).count("*").where({ dynamicSecretId }).first(); + const doc = await (tx || db.replicaNode())(TableName.DynamicSecretLease) + .count("*") + .where({ dynamicSecretId }) + .first(); return parseInt(doc || "0", 10); } catch (error) { throw new DatabaseError({ error, name: "DynamicSecretCountLeases" }); @@ -21,7 +24,7 @@ export const dynamicSecretLeaseDALFactory = (db: TDbClient) => { const findById = async (id: string, tx?: Knex) => { try { - const doc = await (tx || db)(TableName.DynamicSecretLease) + const doc = await (tx || db.replicaNode())(TableName.DynamicSecretLease) .where({ [`${TableName.DynamicSecretLease}.id` as "id"]: id }) .first() .join( diff --git a/backend/src/ee/services/group/group-dal.ts b/backend/src/ee/services/group/group-dal.ts index 3da1f242c..4f8ffa664 100644 --- a/backend/src/ee/services/group/group-dal.ts +++ b/backend/src/ee/services/group/group-dal.ts @@ -12,7 +12,7 @@ export const groupDALFactory = (db: TDbClient) => { const findGroups = async (filter: TFindFilter, { offset, limit, sort, tx }: TFindOpt = {}) => { try { - const query = (tx || db)(TableName.Groups) + const query = (tx || db.replicaNode())(TableName.Groups) // eslint-disable-next-line .where(buildFindFilter(filter)) .select(selectAllTableCols(TableName.Groups)); @@ -32,7 +32,7 @@ export const groupDALFactory = (db: TDbClient) => { const findByOrgId = async (orgId: string, tx?: Knex) => { try { - const docs = await (tx || db)(TableName.Groups) + const docs = await (tx || db.replicaNode())(TableName.Groups) .where(`${TableName.Groups}.orgId`, orgId) .leftJoin(TableName.OrgRoles, `${TableName.Groups}.roleId`, `${TableName.OrgRoles}.id`) .select(selectAllTableCols(TableName.Groups)) @@ -74,11 +74,12 @@ export const groupDALFactory = (db: TDbClient) => { username?: string; }) => { try { - let query = db(TableName.OrgMembership) + let query = db + .replicaNode()(TableName.OrgMembership) .where(`${TableName.OrgMembership}.orgId`, orgId) .join(TableName.Users, `${TableName.OrgMembership}.userId`, `${TableName.Users}.id`) - .leftJoin(TableName.UserGroupMembership, function () { - this.on(`${TableName.UserGroupMembership}.userId`, "=", `${TableName.Users}.id`).andOn( + .leftJoin(TableName.UserGroupMembership, (bd) => { + bd.on(`${TableName.UserGroupMembership}.userId`, "=", `${TableName.Users}.id`).andOn( `${TableName.UserGroupMembership}.groupId`, "=", db.raw("?", [groupId]) diff --git a/backend/src/ee/services/group/user-group-membership-dal.ts b/backend/src/ee/services/group/user-group-membership-dal.ts index 1ab1839c5..e20cf317b 100644 --- a/backend/src/ee/services/group/user-group-membership-dal.ts +++ b/backend/src/ee/services/group/user-group-membership-dal.ts @@ -18,7 +18,7 @@ export const userGroupMembershipDALFactory = (db: TDbClient) => { */ const filterProjectsByUserMembership = async (userId: string, groupId: string, projectIds: string[], tx?: Knex) => { try { - const userProjectMemberships: string[] = await (tx || db)(TableName.ProjectMembership) + const userProjectMemberships: string[] = await (tx || db.replicaNode())(TableName.ProjectMembership) .where(`${TableName.ProjectMembership}.userId`, userId) .whereIn(`${TableName.ProjectMembership}.projectId`, projectIds) .pluck(`${TableName.ProjectMembership}.projectId`); @@ -43,7 +43,8 @@ export const userGroupMembershipDALFactory = (db: TDbClient) => { // special query const findUserGroupMembershipsInProject = async (usernames: string[], projectId: string) => { try { - const usernameDocs: string[] = await db(TableName.UserGroupMembership) + const usernameDocs: string[] = await db + .replicaNode()(TableName.UserGroupMembership) .join( TableName.GroupProjectMembership, `${TableName.UserGroupMembership}.groupId`, @@ -73,7 +74,7 @@ export const userGroupMembershipDALFactory = (db: TDbClient) => { try { // get list of groups in the project with id [projectId] // that that are not the group with id [groupId] - const groups: string[] = await (tx || db)(TableName.GroupProjectMembership) + const groups: string[] = await (tx || db.replicaNode())(TableName.GroupProjectMembership) .where(`${TableName.GroupProjectMembership}.projectId`, projectId) .whereNot(`${TableName.GroupProjectMembership}.groupId`, groupId) .pluck(`${TableName.GroupProjectMembership}.groupId`); @@ -83,8 +84,8 @@ export const userGroupMembershipDALFactory = (db: TDbClient) => { .where(`${TableName.UserGroupMembership}.groupId`, groupId) .where(`${TableName.UserGroupMembership}.isPending`, false) .join(TableName.Users, `${TableName.UserGroupMembership}.userId`, `${TableName.Users}.id`) - .leftJoin(TableName.ProjectMembership, function () { - this.on(`${TableName.Users}.id`, "=", `${TableName.ProjectMembership}.userId`).andOn( + .leftJoin(TableName.ProjectMembership, (bd) => { + bd.on(`${TableName.Users}.id`, "=", `${TableName.ProjectMembership}.userId`).andOn( `${TableName.ProjectMembership}.projectId`, "=", db.raw("?", [projectId]) @@ -107,9 +108,9 @@ export const userGroupMembershipDALFactory = (db: TDbClient) => { db.ref("publicKey").withSchema(TableName.UserEncryptionKey) ) .where({ isGhost: false }) // MAKE SURE USER IS NOT A GHOST USER - .whereNotIn(`${TableName.UserGroupMembership}.userId`, function () { + .whereNotIn(`${TableName.UserGroupMembership}.userId`, (bd) => { // eslint-disable-next-line @typescript-eslint/no-floating-promises - this.select(`${TableName.UserGroupMembership}.userId`) + bd.select(`${TableName.UserGroupMembership}.userId`) .from(TableName.UserGroupMembership) .whereIn(`${TableName.UserGroupMembership}.groupId`, groups); }); diff --git a/backend/src/ee/services/ldap-config/ldap-group-map-dal.ts b/backend/src/ee/services/ldap-config/ldap-group-map-dal.ts index 2264efa75..a08522e8d 100644 --- a/backend/src/ee/services/ldap-config/ldap-group-map-dal.ts +++ b/backend/src/ee/services/ldap-config/ldap-group-map-dal.ts @@ -10,7 +10,8 @@ export const ldapGroupMapDALFactory = (db: TDbClient) => { const findLdapGroupMapsByLdapConfigId = async (ldapConfigId: string) => { try { - const docs = await db(TableName.LdapGroupMap) + const docs = await db + .replicaNode()(TableName.LdapGroupMap) .where(`${TableName.LdapGroupMap}.ldapConfigId`, ldapConfigId) .join(TableName.Groups, `${TableName.LdapGroupMap}.groupId`, `${TableName.Groups}.id`) .select(selectAllTableCols(TableName.LdapGroupMap)) diff --git a/backend/src/ee/services/license/license-dal.ts b/backend/src/ee/services/license/license-dal.ts index 5cbfca1d6..cab428e86 100644 --- a/backend/src/ee/services/license/license-dal.ts +++ b/backend/src/ee/services/license/license-dal.ts @@ -9,7 +9,7 @@ export type TLicenseDALFactory = ReturnType; export const licenseDALFactory = (db: TDbClient) => { const countOfOrgMembers = async (orgId: string | null, tx?: Knex) => { try { - const doc = await (tx || db)(TableName.OrgMembership) + const doc = await (tx || db.replicaNode())(TableName.OrgMembership) .where({ status: OrgMembershipStatus.Accepted }) .andWhere((bd) => { if (orgId) { diff --git a/backend/src/ee/services/permission/permission-dal.ts b/backend/src/ee/services/permission/permission-dal.ts index d8114388e..d228ae109 100644 --- a/backend/src/ee/services/permission/permission-dal.ts +++ b/backend/src/ee/services/permission/permission-dal.ts @@ -10,7 +10,8 @@ export type TPermissionDALFactory = ReturnType; export const permissionDALFactory = (db: TDbClient) => { const getOrgPermission = async (userId: string, orgId: string) => { try { - const membership = await db(TableName.OrgMembership) + const membership = await db + .replicaNode()(TableName.OrgMembership) .leftJoin(TableName.OrgRoles, `${TableName.OrgMembership}.roleId`, `${TableName.OrgRoles}.id`) .join(TableName.Organization, `${TableName.OrgMembership}.orgId`, `${TableName.Organization}.id`) .where("userId", userId) @@ -28,7 +29,8 @@ export const permissionDALFactory = (db: TDbClient) => { const getOrgIdentityPermission = async (identityId: string, orgId: string) => { try { - const membership = await db(TableName.IdentityOrgMembership) + const membership = await db + .replicaNode()(TableName.IdentityOrgMembership) .leftJoin(TableName.OrgRoles, `${TableName.IdentityOrgMembership}.roleId`, `${TableName.OrgRoles}.id`) .join(TableName.Organization, `${TableName.IdentityOrgMembership}.orgId`, `${TableName.Organization}.id`) .where("identityId", identityId) @@ -45,11 +47,13 @@ export const permissionDALFactory = (db: TDbClient) => { const getProjectPermission = async (userId: string, projectId: string) => { try { - const groups: string[] = await db(TableName.GroupProjectMembership) + const groups: string[] = await db + .replicaNode()(TableName.GroupProjectMembership) .where(`${TableName.GroupProjectMembership}.projectId`, projectId) .pluck(`${TableName.GroupProjectMembership}.groupId`); - const groupDocs = await db(TableName.UserGroupMembership) + const groupDocs = await db + .replicaNode()(TableName.UserGroupMembership) .where(`${TableName.UserGroupMembership}.userId`, userId) .whereIn(`${TableName.UserGroupMembership}.groupId`, groups) .join( @@ -231,7 +235,8 @@ export const permissionDALFactory = (db: TDbClient) => { const getProjectIdentityPermission = async (identityId: string, projectId: string) => { try { - const docs = await db(TableName.IdentityProjectMembership) + const docs = await db + .replicaNode()(TableName.IdentityProjectMembership) .join( TableName.IdentityProjectMembershipRole, `${TableName.IdentityProjectMembershipRole}.projectMembershipId`, diff --git a/backend/src/ee/services/saml-config/saml-config-dal.ts b/backend/src/ee/services/saml-config/saml-config-dal.ts index 1e7b9e47e..aff42230f 100644 --- a/backend/src/ee/services/saml-config/saml-config-dal.ts +++ b/backend/src/ee/services/saml-config/saml-config-dal.ts @@ -10,7 +10,8 @@ export const samlConfigDALFactory = (db: TDbClient) => { const findEnforceableSamlCfg = async (orgId: string) => { try { - const samlCfg = await db(TableName.SamlConfig) + const samlCfg = await db + .replicaNode()(TableName.SamlConfig) .where({ orgId, isActive: true diff --git a/backend/src/ee/services/secret-approval-policy/secret-approval-policy-dal.ts b/backend/src/ee/services/secret-approval-policy/secret-approval-policy-dal.ts index eec3d9a1d..883e63747 100644 --- a/backend/src/ee/services/secret-approval-policy/secret-approval-policy-dal.ts +++ b/backend/src/ee/services/secret-approval-policy/secret-approval-policy-dal.ts @@ -30,7 +30,7 @@ export const secretApprovalPolicyDALFactory = (db: TDbClient) => { const findById = async (id: string, tx?: Knex) => { try { - const doc = await sapFindQuery(tx || db, { + const doc = await sapFindQuery(tx || db.replicaNode(), { [`${TableName.SecretApprovalPolicy}.id` as "id"]: id }); const formatedDoc = mergeOneToManyRelation( @@ -52,7 +52,7 @@ export const secretApprovalPolicyDALFactory = (db: TDbClient) => { const find = async (filter: TFindFilter, tx?: Knex) => { try { - const docs = await sapFindQuery(tx || db, filter); + const docs = await sapFindQuery(tx || db.replicaNode(), filter); const formatedDoc = mergeOneToManyRelation( docs, "id", diff --git a/backend/src/ee/services/secret-approval-request/secret-approval-request-dal.ts b/backend/src/ee/services/secret-approval-request/secret-approval-request-dal.ts index 05fe1b8f8..4eda64f8f 100644 --- a/backend/src/ee/services/secret-approval-request/secret-approval-request-dal.ts +++ b/backend/src/ee/services/secret-approval-request/secret-approval-request-dal.ts @@ -62,7 +62,7 @@ export const secretApprovalRequestDALFactory = (db: TDbClient) => { const findById = async (id: string, tx?: Knex) => { try { - const sql = findQuery({ [`${TableName.SecretApprovalRequest}.id` as "id"]: id }, tx || db); + const sql = findQuery({ [`${TableName.SecretApprovalRequest}.id` as "id"]: id }, tx || db.replicaNode()); const docs = await sql; const formatedDoc = sqlNestRelationships({ data: docs, @@ -102,7 +102,7 @@ export const secretApprovalRequestDALFactory = (db: TDbClient) => { const docs = await (tx || db) .with( "temp", - (tx || db)(TableName.SecretApprovalRequest) + (tx || db.replicaNode())(TableName.SecretApprovalRequest) .join(TableName.SecretFolder, `${TableName.SecretApprovalRequest}.folderId`, `${TableName.SecretFolder}.id`) .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) .join( @@ -148,7 +148,7 @@ export const secretApprovalRequestDALFactory = (db: TDbClient) => { try { // akhilmhdh: If ever u wanted a 1 to so many relationship connected with pagination // this is the place u wanna look at. - const query = (tx || db)(TableName.SecretApprovalRequest) + const query = (tx || db.replicaNode())(TableName.SecretApprovalRequest) .join(TableName.SecretFolder, `${TableName.SecretApprovalRequest}.folderId`, `${TableName.SecretFolder}.id`) .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) .join( diff --git a/backend/src/ee/services/secret-approval-request/secret-approval-request-secret-dal.ts b/backend/src/ee/services/secret-approval-request/secret-approval-request-secret-dal.ts index 736cd253e..8dc06aaf5 100644 --- a/backend/src/ee/services/secret-approval-request/secret-approval-request-secret-dal.ts +++ b/backend/src/ee/services/secret-approval-request/secret-approval-request-secret-dal.ts @@ -47,7 +47,7 @@ export const secretApprovalRequestSecretDALFactory = (db: TDbClient) => { const findByRequestId = async (requestId: string, tx?: Knex) => { try { - const doc = await (tx || db)({ + const doc = await (tx || db.replicaNode())({ secVerTag: TableName.SecretTag }) .from(TableName.SecretApprovalRequestSecret) diff --git a/backend/src/ee/services/secret-rotation/secret-rotation-dal.ts b/backend/src/ee/services/secret-rotation/secret-rotation-dal.ts index 7feafdc6b..57d86ff04 100644 --- a/backend/src/ee/services/secret-rotation/secret-rotation-dal.ts +++ b/backend/src/ee/services/secret-rotation/secret-rotation-dal.ts @@ -41,7 +41,7 @@ export const secretRotationDALFactory = (db: TDbClient) => { const find = async (filter: TFindFilter, tx?: Knex) => { try { - const data = await findQuery(filter, tx || db); + const data = await findQuery(filter, tx || db.replicaNode()); return sqlNestRelationships({ data, key: "id", @@ -93,7 +93,7 @@ export const secretRotationDALFactory = (db: TDbClient) => { const findById = async (id: string, tx?: Knex) => { try { - const doc = await (tx || db)(TableName.SecretRotation) + const doc = await (tx || db.replicaNode())(TableName.SecretRotation) .join(TableName.Environment, `${TableName.SecretRotation}.envId`, `${TableName.Environment}.id`) .where({ [`${TableName.SecretRotation}.id` as "id"]: id }) .select(selectAllTableCols(TableName.SecretRotation)) diff --git a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts index 4092bf356..a16b4548d 100644 --- a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts +++ b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts @@ -21,7 +21,7 @@ export const snapshotDALFactory = (db: TDbClient) => { const findById = async (id: string, tx?: Knex) => { try { - const data = await (tx || db)(TableName.Snapshot) + const data = await (tx || db.replicaNode())(TableName.Snapshot) .where(`${TableName.Snapshot}.id`, id) .join(TableName.Environment, `${TableName.Snapshot}.envId`, `${TableName.Environment}.id`) .select(selectAllTableCols(TableName.Snapshot)) @@ -43,7 +43,7 @@ export const snapshotDALFactory = (db: TDbClient) => { const countOfSnapshotsByFolderId = async (folderId: string, tx?: Knex) => { try { - const doc = await (tx || db)(TableName.Snapshot) + const doc = await (tx || db.replicaNode())(TableName.Snapshot) .where({ folderId }) .groupBy(["folderId"]) .count("folderId") @@ -56,7 +56,7 @@ export const snapshotDALFactory = (db: TDbClient) => { const findSecretSnapshotDataById = async (snapshotId: string, tx?: Knex) => { try { - const data = await (tx || db)(TableName.Snapshot) + const data = await (tx || db.replicaNode())(TableName.Snapshot) .where(`${TableName.Snapshot}.id`, snapshotId) .join(TableName.Environment, `${TableName.Snapshot}.envId`, `${TableName.Environment}.id`) .leftJoin(TableName.SnapshotSecret, `${TableName.Snapshot}.id`, `${TableName.SnapshotSecret}.snapshotId`) @@ -309,7 +309,7 @@ export const snapshotDALFactory = (db: TDbClient) => { // when we need to rollback we will pull from these snapshots const findLatestSnapshotByFolderId = async (folderId: string, tx?: Knex) => { try { - const docs = await (tx || db)(TableName.Snapshot) + const docs = await (tx || db.replicaNode())(TableName.Snapshot) .where(`${TableName.Snapshot}.folderId`, folderId) .join( (tx || db)(TableName.Snapshot).groupBy("folderId").max("createdAt").select("folderId").as("latestVersion"), diff --git a/backend/src/lib/config/env.ts b/backend/src/lib/config/env.ts index f4da71293..e6e8a42e2 100644 --- a/backend/src/lib/config/env.ts +++ b/backend/src/lib/config/env.ts @@ -10,6 +10,14 @@ const zodStrBool = z .optional() .transform((val) => val === "true"); +const databaseReadReplicaSchema = z + .object({ + DB_CONNECTION_URI: z.string().describe("Postgres read replica database connection string"), + DB_ROOT_CERT: zpStr(z.string().optional().describe("Postgres read replica database certificate string")) + }) + .array() + .optional(); + const envSchema = z .object({ PORT: z.coerce.number().default(4000), @@ -29,6 +37,7 @@ const envSchema = z DB_USER: zpStr(z.string().describe("Postgres database username").optional()), DB_PASSWORD: zpStr(z.string().describe("Postgres database password").optional()), DB_NAME: zpStr(z.string().describe("Postgres database name").optional()), + DB_READ_REPLICAS: zpStr(z.string().describe("Postgres read replicas").optional()), BCRYPT_SALT_ROUND: z.number().default(12), NODE_ENV: z.enum(["development", "test", "production"]).default("production"), SALT_ROUNDS: z.coerce.number().default(10), @@ -127,6 +136,9 @@ const envSchema = z }) .transform((data) => ({ ...data, + DB_READ_REPLICAS: data.DB_READ_REPLICAS + ? databaseReadReplicaSchema.parse(JSON.parse(data.DB_READ_REPLICAS)) + : undefined, isCloud: Boolean(data.LICENSE_SERVER_KEY), isSmtpConfigured: Boolean(data.SMTP_HOST), isRedisConfigured: Boolean(data.REDIS_URL), diff --git a/backend/src/lib/knex/index.ts b/backend/src/lib/knex/index.ts index 0faeba290..bd103af9d 100644 --- a/backend/src/lib/knex/index.ts +++ b/backend/src/lib/knex/index.ts @@ -50,7 +50,7 @@ export const ormify = (db: Kne }), findById: async (id: string, tx?: Knex) => { try { - const result = await (tx || db)(tableName) + const result = await (tx || db.replicaNode())(tableName) .where({ id } as never) .first("*"); return result; @@ -60,7 +60,7 @@ export const ormify = (db: Kne }, findOne: async (filter: Partial, tx?: Knex) => { try { - const res = await (tx || db)(tableName).where(filter).first("*"); + const res = await (tx || db.replicaNode())(tableName).where(filter).first("*"); return res; } catch (error) { throw new DatabaseError({ error, name: "Find one" }); @@ -71,7 +71,7 @@ export const ormify = (db: Kne { offset, limit, sort, tx }: TFindOpt = {} ) => { try { - const query = (tx || db)(tableName).where(buildFindFilter(filter)); + const query = (tx || db.replicaNode())(tableName).where(buildFindFilter(filter)); if (limit) void query.limit(limit); if (offset) void query.offset(offset); if (sort) { diff --git a/backend/src/main.ts b/backend/src/main.ts index 86681ef33..dac189b87 100644 --- a/backend/src/main.ts +++ b/backend/src/main.ts @@ -15,7 +15,11 @@ const run = async () => { const appCfg = initEnvConfig(logger); const db = initDbConnection({ dbConnectionUri: appCfg.DB_CONNECTION_URI, - dbRootCert: appCfg.DB_ROOT_CERT + dbRootCert: appCfg.DB_ROOT_CERT, + readReplicas: appCfg.DB_READ_REPLICAS?.map((el) => ({ + dbRootCert: el.DB_ROOT_CERT, + dbConnectionUri: el.DB_CONNECTION_URI + })) }); const smtp = smtpServiceFactory(formatSmtpConfig()); diff --git a/backend/src/services/auth-token/auth-token-dal.ts b/backend/src/services/auth-token/auth-token-dal.ts index 075ae7384..c058c13e8 100644 --- a/backend/src/services/auth-token/auth-token-dal.ts +++ b/backend/src/services/auth-token/auth-token-dal.ts @@ -14,7 +14,7 @@ export const tokenDALFactory = (db: TDbClient) => { const findOneTokenSession = async (filter: Partial): Promise => { try { - const doc = await db(TableName.AuthTokenSession).where(filter).first(); + const doc = await db.replicaNode()(TableName.AuthTokenSession).where(filter).first(); return doc; } catch (error) { throw new DatabaseError({ error, name: "FindOneTokenSession" }); @@ -44,7 +44,7 @@ export const tokenDALFactory = (db: TDbClient) => { const findTokenSessions = async (filter: Partial, tx?: Knex) => { try { - const sessions = await (tx || db)(TableName.AuthTokenSession).where(filter); + const sessions = await (tx || db.replicaNode())(TableName.AuthTokenSession).where(filter); return sessions; } catch (error) { throw new DatabaseError({ name: "Find all token session", error }); diff --git a/backend/src/services/certificate-authority/certificate-authority-dal.ts b/backend/src/services/certificate-authority/certificate-authority-dal.ts index 1b4b30e73..837bbcf37 100644 --- a/backend/src/services/certificate-authority/certificate-authority-dal.ts +++ b/backend/src/services/certificate-authority/certificate-authority-dal.ts @@ -16,6 +16,7 @@ export const certificateAuthorityDALFactory = (db: TDbClient) => { parentCaId?: string; encryptedCertificate: Buffer; }[] = await db + .replicaNode() .withRecursive("cte", (cte) => { void cte .select("ca.id as caId", "ca.parentCaId", "cert.encryptedCertificate") diff --git a/backend/src/services/certificate/certificate-dal.ts b/backend/src/services/certificate/certificate-dal.ts index 415bfabf9..67dca3aca 100644 --- a/backend/src/services/certificate/certificate-dal.ts +++ b/backend/src/services/certificate/certificate-dal.ts @@ -14,7 +14,8 @@ export const certificateDALFactory = (db: TDbClient) => { count: string; } - const count = await db(TableName.Certificate) + const count = await db + .replicaNode()(TableName.Certificate) .join(TableName.CertificateAuthority, `${TableName.Certificate}.caId`, `${TableName.CertificateAuthority}.id`) .join(TableName.Project, `${TableName.CertificateAuthority}.projectId`, `${TableName.Project}.id`) .where(`${TableName.Project}.id`, projectId) diff --git a/backend/src/services/group-project/group-project-dal.ts b/backend/src/services/group-project/group-project-dal.ts index 3b0523dde..a1d276376 100644 --- a/backend/src/services/group-project/group-project-dal.ts +++ b/backend/src/services/group-project/group-project-dal.ts @@ -12,7 +12,7 @@ export const groupProjectDALFactory = (db: TDbClient) => { const findByProjectId = async (projectId: string, tx?: Knex) => { try { - const docs = await (tx || db)(TableName.GroupProjectMembership) + const docs = await (tx || db.replicaNode())(TableName.GroupProjectMembership) .where(`${TableName.GroupProjectMembership}.projectId`, projectId) .join(TableName.Groups, `${TableName.GroupProjectMembership}.groupId`, `${TableName.Groups}.id`) .join( diff --git a/backend/src/services/identity-access-token/identity-access-token-dal.ts b/backend/src/services/identity-access-token/identity-access-token-dal.ts index a0f9fbc27..4f04ef0ac 100644 --- a/backend/src/services/identity-access-token/identity-access-token-dal.ts +++ b/backend/src/services/identity-access-token/identity-access-token-dal.ts @@ -12,7 +12,7 @@ export const identityAccessTokenDALFactory = (db: TDbClient) => { const findOne = async (filter: Partial, tx?: Knex) => { try { - const doc = await (tx || db)(TableName.IdentityAccessToken) + const doc = await (tx || db.replicaNode())(TableName.IdentityAccessToken) .where(filter) .join(TableName.Identity, `${TableName.Identity}.id`, `${TableName.IdentityAccessToken}.identityId`) .leftJoin(TableName.IdentityUaClientSecret, (qb) => { diff --git a/backend/src/services/identity-project/identity-project-dal.ts b/backend/src/services/identity-project/identity-project-dal.ts index c1cfe79cc..497d05c3c 100644 --- a/backend/src/services/identity-project/identity-project-dal.ts +++ b/backend/src/services/identity-project/identity-project-dal.ts @@ -12,7 +12,7 @@ export const identityProjectDALFactory = (db: TDbClient) => { const findByProjectId = async (projectId: string, filter: { identityId?: string } = {}, tx?: Knex) => { try { - const docs = await (tx || db)(TableName.IdentityProjectMembership) + const docs = await (tx || db.replicaNode())(TableName.IdentityProjectMembership) .where(`${TableName.IdentityProjectMembership}.projectId`, projectId) .join(TableName.Identity, `${TableName.IdentityProjectMembership}.identityId`, `${TableName.Identity}.id`) .where((qb) => { diff --git a/backend/src/services/identity/identity-org-dal.ts b/backend/src/services/identity/identity-org-dal.ts index 0d3199725..104b917e7 100644 --- a/backend/src/services/identity/identity-org-dal.ts +++ b/backend/src/services/identity/identity-org-dal.ts @@ -12,7 +12,7 @@ export const identityOrgDALFactory = (db: TDbClient) => { const findOne = async (filter: Partial, tx?: Knex) => { try { - const [data] = await (tx || db)(TableName.IdentityOrgMembership) + const [data] = await (tx || db.replicaNode())(TableName.IdentityOrgMembership) .where(filter) .join(TableName.Identity, `${TableName.IdentityOrgMembership}.identityId`, `${TableName.Identity}.id`) .select(selectAllTableCols(TableName.IdentityOrgMembership)) @@ -29,7 +29,7 @@ export const identityOrgDALFactory = (db: TDbClient) => { const find = async (filter: Partial, tx?: Knex) => { try { - const docs = await (tx || db)(TableName.IdentityOrgMembership) + const docs = await (tx || db.replicaNode())(TableName.IdentityOrgMembership) .where(filter) .join(TableName.Identity, `${TableName.IdentityOrgMembership}.identityId`, `${TableName.Identity}.id`) .leftJoin(TableName.OrgRoles, `${TableName.IdentityOrgMembership}.roleId`, `${TableName.OrgRoles}.id`) diff --git a/backend/src/services/integration/integration-dal.ts b/backend/src/services/integration/integration-dal.ts index bada253c5..8fc460d6f 100644 --- a/backend/src/services/integration/integration-dal.ts +++ b/backend/src/services/integration/integration-dal.ts @@ -22,7 +22,7 @@ export const integrationDALFactory = (db: TDbClient) => { const find = async (filter: Partial, tx?: Knex) => { try { - const docs = await integrationFindQuery(tx || db, filter); + const docs = await integrationFindQuery(tx || db.replicaNode(), filter); return docs.map(({ envId, envSlug, envName, ...el }) => ({ ...el, environment: { @@ -38,7 +38,7 @@ export const integrationDALFactory = (db: TDbClient) => { const findOne = async (filter: Partial, tx?: Knex) => { try { - const doc = await integrationFindQuery(tx || db, filter).first(); + const doc = await integrationFindQuery(tx || db.replicaNode(), filter).first(); if (!doc) return; const { envName: name, envSlug: slug, envId: id, ...el } = doc; @@ -50,7 +50,7 @@ export const integrationDALFactory = (db: TDbClient) => { const findById = async (id: string, tx?: Knex) => { try { - const doc = await integrationFindQuery(tx || db, { + const doc = await integrationFindQuery(tx || db.replicaNode(), { [`${TableName.Integration}.id` as "id"]: id }).first(); if (!doc) return; @@ -64,7 +64,7 @@ export const integrationDALFactory = (db: TDbClient) => { const findByProjectId = async (projectId: string, tx?: Knex) => { try { - const integrations = await (tx || db)(TableName.Integration) + const integrations = await (tx || db.replicaNode())(TableName.Integration) .where(`${TableName.Environment}.projectId`, projectId) .join(TableName.Environment, `${TableName.Integration}.envId`, `${TableName.Environment}.id`) .select(db.ref("name").withSchema(TableName.Environment).as("envName")) @@ -90,7 +90,7 @@ export const integrationDALFactory = (db: TDbClient) => { // used for syncing secrets // this will populate integration auth also const findByProjectIdV2 = async (projectId: string, environment: string, tx?: Knex) => { - const docs = await (tx || db)(TableName.Integration) + const docs = await (tx || db.replicaNode())(TableName.Integration) .where(`${TableName.Environment}.projectId`, projectId) .where("isActive", true) .where(`${TableName.Environment}.slug`, environment) diff --git a/backend/src/services/org/incident-contacts-dal.ts b/backend/src/services/org/incident-contacts-dal.ts index 1979a9c3e..9db87b517 100644 --- a/backend/src/services/org/incident-contacts-dal.ts +++ b/backend/src/services/org/incident-contacts-dal.ts @@ -16,7 +16,7 @@ export const incidentContactDALFactory = (db: TDbClient) => { const findByOrgId = async (orgId: string) => { try { - const incidentContacts = await db(TableName.IncidentContact).where({ orgId }); + const incidentContacts = await db.replicaNode()(TableName.IncidentContact).where({ orgId }); return incidentContacts; } catch (error) { throw new DatabaseError({ name: "Incident contact list", error }); @@ -25,7 +25,8 @@ export const incidentContactDALFactory = (db: TDbClient) => { const findOne = async (orgId: string, data: Partial) => { try { - const incidentContacts = await db(TableName.IncidentContact) + const incidentContacts = await db + .replicaNode()(TableName.IncidentContact) .where({ orgId, ...data }) .first(); return incidentContacts; diff --git a/backend/src/services/org/org-dal.ts b/backend/src/services/org/org-dal.ts index 1e52053b2..d518a698a 100644 --- a/backend/src/services/org/org-dal.ts +++ b/backend/src/services/org/org-dal.ts @@ -20,7 +20,7 @@ export const orgDALFactory = (db: TDbClient) => { const findOrgById = async (orgId: string) => { try { - const org = await db(TableName.Organization).where({ id: orgId }).first(); + const org = await db.replicaNode()(TableName.Organization).where({ id: orgId }).first(); return org; } catch (error) { throw new DatabaseError({ error, name: "Find org by id" }); @@ -30,7 +30,8 @@ export const orgDALFactory = (db: TDbClient) => { // special query const findAllOrgsByUserId = async (userId: string): Promise => { try { - const org = await db(TableName.OrgMembership) + const org = await db + .replicaNode()(TableName.OrgMembership) .where({ userId }) .join(TableName.Organization, `${TableName.OrgMembership}.orgId`, `${TableName.Organization}.id`) .select(selectAllTableCols(TableName.Organization)); @@ -42,7 +43,8 @@ export const orgDALFactory = (db: TDbClient) => { const findOrgByProjectId = async (projectId: string): Promise => { try { - const [org] = await db(TableName.Project) + const [org] = await db + .replicaNode()(TableName.Project) .where({ [`${TableName.Project}.id` as "id"]: projectId }) .join(TableName.Organization, `${TableName.Project}.orgId`, `${TableName.Organization}.id`) .select(selectAllTableCols(TableName.Organization)); @@ -56,7 +58,8 @@ export const orgDALFactory = (db: TDbClient) => { // special query const findAllOrgMembers = async (orgId: string) => { try { - const members = await db(TableName.OrgMembership) + const members = await db + .replicaNode()(TableName.OrgMembership) .where(`${TableName.OrgMembership}.orgId`, orgId) .join(TableName.Users, `${TableName.OrgMembership}.userId`, `${TableName.Users}.id`) .leftJoin( @@ -95,7 +98,8 @@ export const orgDALFactory = (db: TDbClient) => { count: string; } - const count = await db(TableName.OrgMembership) + const count = await db + .replicaNode()(TableName.OrgMembership) .where(`${TableName.OrgMembership}.orgId`, orgId) .count("*") .join(TableName.Users, `${TableName.OrgMembership}.userId`, `${TableName.Users}.id`) @@ -110,7 +114,8 @@ export const orgDALFactory = (db: TDbClient) => { const findOrgMembersByUsername = async (orgId: string, usernames: string[]) => { try { - const members = await db(TableName.OrgMembership) + const members = await db + .replicaNode()(TableName.OrgMembership) .where(`${TableName.OrgMembership}.orgId`, orgId) .join(TableName.Users, `${TableName.OrgMembership}.userId`, `${TableName.Users}.id`) .leftJoin( @@ -145,7 +150,8 @@ export const orgDALFactory = (db: TDbClient) => { const findOrgGhostUser = async (orgId: string) => { try { - const member = await db(TableName.OrgMembership) + const member = await db + .replicaNode()(TableName.OrgMembership) .where({ orgId }) .join(TableName.Users, `${TableName.OrgMembership}.userId`, `${TableName.Users}.id`) .leftJoin(TableName.UserEncryptionKey, `${TableName.UserEncryptionKey}.userId`, `${TableName.Users}.id`) @@ -169,7 +175,8 @@ export const orgDALFactory = (db: TDbClient) => { const ghostUserExists = async (orgId: string) => { try { - const member = await db(TableName.OrgMembership) + const member = await db + .replicaNode()(TableName.OrgMembership) .where({ orgId }) .join(TableName.Users, `${TableName.OrgMembership}.userId`, `${TableName.Users}.id`) .leftJoin(TableName.UserEncryptionKey, `${TableName.UserEncryptionKey}.userId`, `${TableName.Users}.id`) @@ -257,7 +264,7 @@ export const orgDALFactory = (db: TDbClient) => { { offset, limit, sort, tx }: TFindOpt = {} ) => { try { - const query = (tx || db)(TableName.OrgMembership) + const query = (tx || db.replicaNode())(TableName.OrgMembership) // eslint-disable-next-line .where(buildFindFilter(filter)) .join(TableName.Users, `${TableName.Users}.id`, `${TableName.OrgMembership}.userId`) diff --git a/backend/src/services/project-bot/project-bot-dal.ts b/backend/src/services/project-bot/project-bot-dal.ts index 74abf8f21..e25ebbd09 100644 --- a/backend/src/services/project-bot/project-bot-dal.ts +++ b/backend/src/services/project-bot/project-bot-dal.ts @@ -12,7 +12,7 @@ export const projectBotDALFactory = (db: TDbClient) => { const findOne = async (filter: Partial, tx?: Knex) => { try { - const bot = await (tx || db)(TableName.ProjectBot) + const bot = await (tx || db.replicaNode())(TableName.ProjectBot) .where(filter) .leftJoin(TableName.Users, `${TableName.ProjectBot}.senderId`, `${TableName.Users}.id`) .leftJoin(TableName.UserEncryptionKey, `${TableName.UserEncryptionKey}.userId`, `${TableName.Users}.id`) diff --git a/backend/src/services/project-env/project-env-dal.ts b/backend/src/services/project-env/project-env-dal.ts index 42a234298..d6f4429d0 100644 --- a/backend/src/services/project-env/project-env-dal.ts +++ b/backend/src/services/project-env/project-env-dal.ts @@ -12,7 +12,9 @@ export const projectEnvDALFactory = (db: TDbClient) => { const findBySlugs = async (projectId: string, env: string[], tx?: Knex) => { try { - const envs = await (tx || db)(TableName.Environment).where("projectId", projectId).whereIn("slug", env); + const envs = await (tx || db.replicaNode())(TableName.Environment) + .where("projectId", projectId) + .whereIn("slug", env); return envs; } catch (error) { throw new DatabaseError({ error, name: "Find by slugs" }); diff --git a/backend/src/services/project-key/project-key-dal.ts b/backend/src/services/project-key/project-key-dal.ts index d1b4053d0..ea4ed813c 100644 --- a/backend/src/services/project-key/project-key-dal.ts +++ b/backend/src/services/project-key/project-key-dal.ts @@ -16,7 +16,7 @@ export const projectKeyDALFactory = (db: TDbClient) => { tx?: Knex ): Promise<(TProjectKeys & { sender: { publicKey: string } }) | undefined> => { try { - const projectKey = await (tx || db)(TableName.ProjectKeys) + const projectKey = await (tx || db.replicaNode())(TableName.ProjectKeys) .join(TableName.Users, `${TableName.ProjectKeys}.senderId`, `${TableName.Users}.id`) .join(TableName.UserEncryptionKey, `${TableName.UserEncryptionKey}.userId`, `${TableName.Users}.id`) .where({ projectId, receiverId: userId }) @@ -34,7 +34,7 @@ export const projectKeyDALFactory = (db: TDbClient) => { const findAllProjectUserPubKeys = async (projectId: string, tx?: Knex) => { try { - const pubKeys = await (tx || db)(TableName.ProjectMembership) + const pubKeys = await (tx || db.replicaNode())(TableName.ProjectMembership) .where({ projectId }) .join(TableName.Users, `${TableName.ProjectMembership}.userId`, `${TableName.Users}.id`) .join(TableName.UserEncryptionKey, `${TableName.Users}.id`, `${TableName.UserEncryptionKey}.userId`) diff --git a/backend/src/services/project-membership/project-membership-dal.ts b/backend/src/services/project-membership/project-membership-dal.ts index 590c26ecc..93ec6597e 100644 --- a/backend/src/services/project-membership/project-membership-dal.ts +++ b/backend/src/services/project-membership/project-membership-dal.ts @@ -13,7 +13,8 @@ export const projectMembershipDALFactory = (db: TDbClient) => { // special query const findAllProjectMembers = async (projectId: string, filter: { usernames?: string[]; username?: string } = {}) => { try { - const docs = await db(TableName.ProjectMembership) + const docs = await db + .replicaNode()(TableName.ProjectMembership) .where({ [`${TableName.ProjectMembership}.projectId` as "projectId"]: projectId }) .join(TableName.Users, `${TableName.ProjectMembership}.userId`, `${TableName.Users}.id`) .where((qb) => { @@ -108,7 +109,7 @@ export const projectMembershipDALFactory = (db: TDbClient) => { const findProjectGhostUser = async (projectId: string, tx?: Knex) => { try { - const ghostUser = await (tx || db)(TableName.ProjectMembership) + const ghostUser = await (tx || db.replicaNode())(TableName.ProjectMembership) .where({ projectId }) .join(TableName.Users, `${TableName.ProjectMembership}.userId`, `${TableName.Users}.id`) .select(selectAllTableCols(TableName.Users)) @@ -123,7 +124,8 @@ export const projectMembershipDALFactory = (db: TDbClient) => { const findMembershipsByUsername = async (projectId: string, usernames: string[]) => { try { - const members = await db(TableName.ProjectMembership) + const members = await db + .replicaNode()(TableName.ProjectMembership) .where({ projectId }) .join(TableName.Users, `${TableName.ProjectMembership}.userId`, `${TableName.Users}.id`) .join( @@ -149,7 +151,8 @@ export const projectMembershipDALFactory = (db: TDbClient) => { const findProjectMembershipsByUserId = async (orgId: string, userId: string) => { try { - const memberships = await db(TableName.ProjectMembership) + const memberships = await db + .replicaNode()(TableName.ProjectMembership) .where({ userId }) .join(TableName.Project, `${TableName.ProjectMembership}.projectId`, `${TableName.Project}.id`) .where({ [`${TableName.Project}.orgId` as "orgId"]: orgId }) diff --git a/backend/src/services/project/project-dal.ts b/backend/src/services/project/project-dal.ts index a4ec99157..ce7f6324e 100644 --- a/backend/src/services/project/project-dal.ts +++ b/backend/src/services/project/project-dal.ts @@ -14,7 +14,8 @@ export const projectDALFactory = (db: TDbClient) => { const findAllProjects = async (userId: string) => { try { - const workspaces = await db(TableName.ProjectMembership) + const workspaces = await db + .replicaNode()(TableName.ProjectMembership) .where({ userId }) .join(TableName.Project, `${TableName.ProjectMembership}.projectId`, `${TableName.Project}.id`) .leftJoin(TableName.Environment, `${TableName.Environment}.projectId`, `${TableName.Project}.id`) @@ -83,7 +84,7 @@ export const projectDALFactory = (db: TDbClient) => { const findProjectGhostUser = async (projectId: string, tx?: Knex) => { try { - const ghostUser = await (tx || db)(TableName.ProjectMembership) + const ghostUser = await (tx || db.replicaNode())(TableName.ProjectMembership) .where({ projectId }) .join(TableName.Users, `${TableName.ProjectMembership}.userId`, `${TableName.Users}.id`) .select(selectAllTableCols(TableName.Users)) @@ -109,7 +110,8 @@ export const projectDALFactory = (db: TDbClient) => { const findAllProjectsByIdentity = async (identityId: string) => { try { - const workspaces = await db(TableName.IdentityProjectMembership) + const workspaces = await db + .replicaNode()(TableName.IdentityProjectMembership) .where({ identityId }) .join(TableName.Project, `${TableName.IdentityProjectMembership}.projectId`, `${TableName.Project}.id`) .leftJoin(TableName.Environment, `${TableName.Environment}.projectId`, `${TableName.Project}.id`) @@ -151,7 +153,8 @@ export const projectDALFactory = (db: TDbClient) => { const findProjectById = async (id: string) => { try { - const workspaces = await db(TableName.Project) + const workspaces = await db + .replicaNode()(TableName.Project) .where(`${TableName.Project}.id`, id) .leftJoin(TableName.Environment, `${TableName.Environment}.projectId`, `${TableName.Project}.id`) .select( @@ -198,7 +201,8 @@ export const projectDALFactory = (db: TDbClient) => { throw new BadRequestError({ message: "Organization ID is required when querying with slugs" }); } - const projects = await db(TableName.Project) + const projects = await db + .replicaNode()(TableName.Project) .where(`${TableName.Project}.slug`, slug) .where(`${TableName.Project}.orgId`, orgId) .leftJoin(TableName.Environment, `${TableName.Environment}.projectId`, `${TableName.Project}.id`) diff --git a/backend/src/services/secret-blind-index/secret-blind-index-dal.ts b/backend/src/services/secret-blind-index/secret-blind-index-dal.ts index 825dea3a7..e26495ce3 100644 --- a/backend/src/services/secret-blind-index/secret-blind-index-dal.ts +++ b/backend/src/services/secret-blind-index/secret-blind-index-dal.ts @@ -12,7 +12,7 @@ export const secretBlindIndexDALFactory = (db: TDbClient) => { const countOfSecretsWithNullSecretBlindIndex = async (projectId: string, tx?: Knex) => { try { - const doc = await (tx || db)(TableName.Secret) + const doc = await (tx || db.replicaNode())(TableName.Secret) .leftJoin(TableName.SecretFolder, `${TableName.SecretFolder}.id`, `${TableName.Secret}.folderId`) .leftJoin(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolder}.envId`) .where({ projectId }) @@ -26,7 +26,7 @@ export const secretBlindIndexDALFactory = (db: TDbClient) => { const findAllSecretsByProjectId = async (projectId: string, tx?: Knex) => { try { - const docs = await (tx || db)(TableName.Secret) + const docs = await (tx || db.replicaNode())(TableName.Secret) .leftJoin(TableName.SecretFolder, `${TableName.SecretFolder}.id`, `${TableName.Secret}.folderId`) .leftJoin(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolder}.envId`) .where({ projectId }) @@ -43,7 +43,7 @@ export const secretBlindIndexDALFactory = (db: TDbClient) => { const findSecretsByProjectId = async (projectId: string, secretIds: string[], tx?: Knex) => { try { - const docs = await (tx || db)(TableName.Secret) + const docs = await (tx || db.replicaNode())(TableName.Secret) .leftJoin(TableName.SecretFolder, `${TableName.SecretFolder}.id`, `${TableName.Secret}.folderId`) .leftJoin(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolder}.envId`) .where({ projectId }) diff --git a/backend/src/services/secret-folder/secret-folder-dal.ts b/backend/src/services/secret-folder/secret-folder-dal.ts index 0e896d0c6..283f60c7c 100644 --- a/backend/src/services/secret-folder/secret-folder-dal.ts +++ b/backend/src/services/secret-folder/secret-folder-dal.ts @@ -211,7 +211,12 @@ export const secretFolderDALFactory = (db: TDbClient) => { const findBySecretPath = async (projectId: string, environment: string, path: string, tx?: Knex) => { try { - const folder = await sqlFindFolderByPathQuery(tx || db, projectId, environment, removeTrailingSlash(path)) + const folder = await sqlFindFolderByPathQuery( + tx || db.replicaNode(), + projectId, + environment, + removeTrailingSlash(path) + ) .orderBy("depth", "desc") .first(); if (folder && folder.path !== removeTrailingSlash(path)) { @@ -230,7 +235,12 @@ export const secretFolderDALFactory = (db: TDbClient) => { // it will stop automatically at /path2 const findClosestFolder = async (projectId: string, environment: string, path: string, tx?: Knex) => { try { - const folder = await sqlFindFolderByPathQuery(tx || db, projectId, environment, removeTrailingSlash(path)) + const folder = await sqlFindFolderByPathQuery( + tx || db.replicaNode(), + projectId, + environment, + removeTrailingSlash(path) + ) .orderBy("depth", "desc") .first(); if (!folder) return; @@ -247,7 +257,7 @@ export const secretFolderDALFactory = (db: TDbClient) => { envId, secretPath: removeTrailingSlash(secretPath) })); - const folders = await sqlFindMultipleFolderByEnvPathQuery(tx || db, formatedQuery); + const folders = await sqlFindMultipleFolderByEnvPathQuery(tx || db.replicaNode(), formatedQuery); return formatedQuery.map(({ envId, secretPath }) => folders.find(({ path: targetPath, envId: targetEnvId }) => targetPath === secretPath && targetEnvId === envId) ); @@ -260,7 +270,7 @@ export const secretFolderDALFactory = (db: TDbClient) => { // that is instances in which for a given folderid find the secret path const findSecretPathByFolderIds = async (projectId: string, folderIds: string[], tx?: Knex) => { try { - const folders = await sqlFindSecretPathByFolderId(tx || db, projectId, folderIds); + const folders = await sqlFindSecretPathByFolderId(tx || db.replicaNode(), projectId, folderIds); // travelling all the way from leaf node to root contains real path const rootFolders = groupBy( @@ -299,7 +309,7 @@ export const secretFolderDALFactory = (db: TDbClient) => { const findById = async (id: string, tx?: Knex) => { try { - const folder = await (tx || db)(TableName.SecretFolder) + const folder = await (tx || db.replicaNode())(TableName.SecretFolder) .where({ [`${TableName.SecretFolder}.id` as "id"]: id }) .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) .select(selectAllTableCols(TableName.SecretFolder)) diff --git a/backend/src/services/secret-folder/secret-folder-version-dal.ts b/backend/src/services/secret-folder/secret-folder-version-dal.ts index fb68ce801..ba0c21f20 100644 --- a/backend/src/services/secret-folder/secret-folder-version-dal.ts +++ b/backend/src/services/secret-folder/secret-folder-version-dal.ts @@ -13,7 +13,7 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => { // This will fetch all latest secret versions from a folder const findLatestVersionByFolderId = async (folderId: string, tx?: Knex) => { try { - const docs = await (tx || db)(TableName.SecretFolderVersion) + const docs = await (tx || db.replicaNode())(TableName.SecretFolderVersion) .join(TableName.SecretFolder, `${TableName.SecretFolderVersion}.folderId`, `${TableName.SecretFolder}.id`) .where({ parentId: folderId, isReserved: false }) .join( @@ -38,7 +38,9 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => { const findLatestFolderVersions = async (folderIds: string[], tx?: Knex) => { try { - const docs: Array = await (tx || db)(TableName.SecretFolderVersion) + const docs: Array = await (tx || db.replicaNode())( + TableName.SecretFolderVersion + ) .whereIn("folderId", folderIds) .join( (tx || db)(TableName.SecretFolderVersion) diff --git a/backend/src/services/secret-import/secret-import-dal.ts b/backend/src/services/secret-import/secret-import-dal.ts index 0e73a8c23..9a7c7e4dc 100644 --- a/backend/src/services/secret-import/secret-import-dal.ts +++ b/backend/src/services/secret-import/secret-import-dal.ts @@ -51,7 +51,7 @@ export const secretImportDALFactory = (db: TDbClient) => { const find = async (filter: Partial, tx?: Knex) => { try { - const docs = await (tx || db)(TableName.SecretImport) + const docs = await (tx || db.replicaNode())(TableName.SecretImport) .where(filter) .join(TableName.Environment, `${TableName.SecretImport}.importEnv`, `${TableName.Environment}.id`) .select( @@ -72,7 +72,7 @@ export const secretImportDALFactory = (db: TDbClient) => { const findByFolderIds = async (folderIds: string[], tx?: Knex) => { try { - const docs = await (tx || db)(TableName.SecretImport) + const docs = await (tx || db.replicaNode())(TableName.SecretImport) .whereIn("folderId", folderIds) .where("isReplication", false) .join(TableName.Environment, `${TableName.SecretImport}.importEnv`, `${TableName.Environment}.id`) diff --git a/backend/src/services/secret-tag/secret-tag-dal.ts b/backend/src/services/secret-tag/secret-tag-dal.ts index f1ae2424a..98cd9af22 100644 --- a/backend/src/services/secret-tag/secret-tag-dal.ts +++ b/backend/src/services/secret-tag/secret-tag-dal.ts @@ -13,7 +13,7 @@ export const secretTagDALFactory = (db: TDbClient) => { const findManyTagsById = async (projectId: string, ids: string[], tx?: Knex) => { try { - const tags = await (tx || db)(TableName.SecretTag).where({ projectId }).whereIn("id", ids); + const tags = await (tx || db.replicaNode())(TableName.SecretTag).where({ projectId }).whereIn("id", ids); return tags; } catch (error) { throw new DatabaseError({ error, name: "Find all by ids" }); diff --git a/backend/src/services/secret/secret-dal.ts b/backend/src/services/secret/secret-dal.ts index 790b403dd..c26880e38 100644 --- a/backend/src/services/secret/secret-dal.ts +++ b/backend/src/services/secret/secret-dal.ts @@ -114,7 +114,7 @@ export const secretDALFactory = (db: TDbClient) => { userId = undefined; } - const secs = await (tx || db)(TableName.Secret) + const secs = await (tx || db.replicaNode())(TableName.Secret) .where({ folderId }) .where((bd) => { void bd.whereNull("userId").orWhere({ userId: userId || null }); @@ -152,7 +152,7 @@ export const secretDALFactory = (db: TDbClient) => { const getSecretTags = async (secretId: string, tx?: Knex) => { try { - const tags = await (tx || db)(TableName.JnSecretTag) + const tags = await (tx || db.replicaNode())(TableName.JnSecretTag) .join(TableName.SecretTag, `${TableName.JnSecretTag}.${TableName.SecretTag}Id`, `${TableName.SecretTag}.id`) .where({ [`${TableName.Secret}Id` as const]: secretId }) .select(db.ref("id").withSchema(TableName.SecretTag).as("tagId")) @@ -179,7 +179,7 @@ export const secretDALFactory = (db: TDbClient) => { userId = undefined; } - const secs = await (tx || db)(TableName.Secret) + const secs = await (tx || db.replicaNode())(TableName.Secret) .whereIn("folderId", folderIds) .where((bd) => { void bd.whereNull("userId").orWhere({ userId: userId || null }); @@ -223,7 +223,7 @@ export const secretDALFactory = (db: TDbClient) => { ) => { if (!blindIndexes.length) return []; try { - const secrets = await (tx || db)(TableName.Secret) + const secrets = await (tx || db.replicaNode())(TableName.Secret) .where({ folderId }) .where((bd) => { blindIndexes.forEach((el) => { @@ -278,7 +278,7 @@ export const secretDALFactory = (db: TDbClient) => { const findReferencedSecretReferences = async (projectId: string, envSlug: string, secretPath: string, tx?: Knex) => { try { - const docs = await (tx || db)(TableName.SecretReference) + const docs = await (tx || db.replicaNode())(TableName.SecretReference) .where({ secretPath, environment: envSlug @@ -298,7 +298,7 @@ export const secretDALFactory = (db: TDbClient) => { // special query to backfill secret value const findAllProjectSecretValues = async (projectId: string, tx?: Knex) => { try { - const docs = await (tx || db)(TableName.Secret) + const docs = await (tx || db.replicaNode())(TableName.Secret) .join(TableName.SecretFolder, `${TableName.Secret}.folderId`, `${TableName.SecretFolder}.id`) .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) .where("projectId", projectId) @@ -313,7 +313,7 @@ export const secretDALFactory = (db: TDbClient) => { const findOneWithTags = async (filter: Partial, tx?: Knex) => { try { - const rawDocs = await (tx || db)(TableName.Secret) + const rawDocs = await (tx || db.replicaNode())(TableName.Secret) .where(filter) .leftJoin(TableName.JnSecretTag, `${TableName.Secret}.id`, `${TableName.JnSecretTag}.${TableName.Secret}Id`) .leftJoin(TableName.SecretTag, `${TableName.JnSecretTag}.${TableName.SecretTag}Id`, `${TableName.SecretTag}.id`) diff --git a/backend/src/services/secret/secret-version-dal.ts b/backend/src/services/secret/secret-version-dal.ts index 4d641bb8d..39a5089b2 100644 --- a/backend/src/services/secret/secret-version-dal.ts +++ b/backend/src/services/secret/secret-version-dal.ts @@ -13,7 +13,7 @@ export const secretVersionDALFactory = (db: TDbClient) => { // This will fetch all latest secret versions from a folder const findLatestVersionByFolderId = async (folderId: string, tx?: Knex) => { try { - const docs = await (tx || db)(TableName.SecretVersion) + const docs = await (tx || db.replicaNode())(TableName.SecretVersion) .where(`${TableName.SecretVersion}.folderId`, folderId) .join(TableName.Secret, `${TableName.Secret}.id`, `${TableName.SecretVersion}.secretId`) .join( @@ -90,7 +90,7 @@ export const secretVersionDALFactory = (db: TDbClient) => { const findLatestVersionMany = async (folderId: string, secretIds: string[], tx?: Knex) => { try { if (!secretIds.length) return {}; - const docs: Array = await (tx || db)(TableName.SecretVersion) + const docs: Array = await (tx || db.replicaNode())(TableName.SecretVersion) .where("folderId", folderId) .whereIn(`${TableName.SecretVersion}.secretId`, secretIds) .join( diff --git a/backend/src/services/service-token/service-token-dal.ts b/backend/src/services/service-token/service-token-dal.ts index 5d3fcc5c8..ed9c5de7e 100644 --- a/backend/src/services/service-token/service-token-dal.ts +++ b/backend/src/services/service-token/service-token-dal.ts @@ -12,7 +12,7 @@ export const serviceTokenDALFactory = (db: TDbClient) => { const findById = async (id: string, tx?: Knex) => { try { - const doc = await (tx || db)(TableName.ServiceToken) + const doc = await (tx || db.replicaNode())(TableName.ServiceToken) .leftJoin( TableName.Users, `${TableName.Users}.id`, diff --git a/backend/src/services/user/user-dal.ts b/backend/src/services/user/user-dal.ts index f2da0df0e..e50d3cc68 100644 --- a/backend/src/services/user/user-dal.ts +++ b/backend/src/services/user/user-dal.ts @@ -22,7 +22,8 @@ export const userDALFactory = (db: TDbClient) => { // ------------------------- const findUserEncKeyByUsername = async ({ username }: { username: string }) => { try { - return await db(TableName.Users) + return await db + .replicaNode()(TableName.Users) .where({ username, isGhost: false @@ -36,7 +37,7 @@ export const userDALFactory = (db: TDbClient) => { const findUserEncKeyByUserIdsBatch = async ({ userIds }: { userIds: string[] }, tx?: Knex) => { try { - return await (tx || db)(TableName.Users) + return await (tx || db.replicaNode())(TableName.Users) .where({ isGhost: false }) @@ -49,7 +50,8 @@ export const userDALFactory = (db: TDbClient) => { const findUserEncKeyByUserId = async (userId: string) => { try { - const user = await db(TableName.Users) + const user = await db + .replicaNode()(TableName.Users) .where(`${TableName.Users}.id`, userId) .join(TableName.UserEncryptionKey, `${TableName.Users}.id`, `${TableName.UserEncryptionKey}.userId`) .first(); @@ -65,7 +67,8 @@ export const userDALFactory = (db: TDbClient) => { const findUserByProjectMembershipId = async (projectMembershipId: string) => { try { - return await db(TableName.ProjectMembership) + return await db + .replicaNode()(TableName.ProjectMembership) .where({ [`${TableName.ProjectMembership}.id` as "id"]: projectMembershipId }) .join(TableName.Users, `${TableName.ProjectMembership}.userId`, `${TableName.Users}.id`) .first(); @@ -76,7 +79,8 @@ export const userDALFactory = (db: TDbClient) => { const findUsersByProjectMembershipIds = async (projectMembershipIds: string[]) => { try { - return await db(TableName.ProjectMembership) + return await db + .replicaNode()(TableName.ProjectMembership) .whereIn(`${TableName.ProjectMembership}.id`, projectMembershipIds) .join(TableName.Users, `${TableName.ProjectMembership}.userId`, `${TableName.Users}.id`) .select("*"); @@ -128,7 +132,7 @@ export const userDALFactory = (db: TDbClient) => { // --------------------- const findOneUserAction = (filter: TUserActionsUpdate, tx?: Knex) => { try { - return (tx || db)(TableName.UserAction).where(filter).first("*"); + return (tx || db.replicaNode())(TableName.UserAction).where(filter).first("*"); } catch (error) { throw new DatabaseError({ error, name: "Find one user action" }); } diff --git a/backend/src/services/webhook/webhook-dal.ts b/backend/src/services/webhook/webhook-dal.ts index c33d79fdb..14d30a35e 100644 --- a/backend/src/services/webhook/webhook-dal.ts +++ b/backend/src/services/webhook/webhook-dal.ts @@ -22,7 +22,7 @@ export const webhookDALFactory = (db: TDbClient) => { const find = async (filter: Partial, tx?: Knex) => { try { - const docs = await webhookFindQuery(tx || db, filter); + const docs = await webhookFindQuery(tx || db.replicaNode(), filter); return docs.map(({ envId, envSlug, envName, ...el }) => ({ ...el, envId, @@ -39,7 +39,7 @@ export const webhookDALFactory = (db: TDbClient) => { const findOne = async (filter: Partial, tx?: Knex) => { try { - const doc = await webhookFindQuery(tx || db, filter).first(); + const doc = await webhookFindQuery(tx || db.replicaNode(), filter).first(); if (!doc) return; const { envName: name, envSlug: slug, envId: id, ...el } = doc; @@ -51,7 +51,7 @@ export const webhookDALFactory = (db: TDbClient) => { const findById = async (id: string, tx?: Knex) => { try { - const doc = await webhookFindQuery(tx || db, { + const doc = await webhookFindQuery(tx || db.replicaNode(), { [`${TableName.Webhook}.id` as "id"]: id }).first(); if (!doc) return; @@ -65,7 +65,7 @@ export const webhookDALFactory = (db: TDbClient) => { const findAllWebhooks = async (projectId: string, environment?: string, secretPath?: string, tx?: Knex) => { try { - const webhooks = await (tx || db)(TableName.Webhook) + const webhooks = await (tx || db.replicaNode())(TableName.Webhook) .where(`${TableName.Environment}.projectId`, projectId) .where((qb) => { if (environment) { diff --git a/docker-compose.dev-read-replica.yml b/docker-compose.dev-read-replica.yml new file mode 100644 index 000000000..7d1e6e7fe --- /dev/null +++ b/docker-compose.dev-read-replica.yml @@ -0,0 +1,191 @@ +version: "3.9" + +services: + nginx: + container_name: infisical-dev-nginx + image: nginx + restart: always + ports: + - 8080:80 + volumes: + - ./nginx/default.dev.conf:/etc/nginx/conf.d/default.conf:ro + depends_on: + - backend + - frontend + + db: + image: bitnami/postgresql:14 + ports: + - "5432:5432" + volumes: + - postgres-data:/var/lib/postgresql/data + environment: + POSTGRESQL_PASSWORD: infisical + POSTGRESQL_USERNAME: infisical + POSTGRESQL_DATABASE: infisical + POSTGRESQL_REPLICATION_MODE: master + POSTGRESQL_REPLICATION_USER: repl_user + POSTGRESQL_REPLICATION_PASSWORD: repl_password + POSTGRESQL_SYNCHRONOUS_COMMIT_MODE: on + POSTGRESQL_NUM_SYNCHRONOUS_REPLICAS: 1 + + db-slave: + image: bitnami/postgresql:14 + ports: + - "5433:5432" + volumes: + - postgres-data:/var/lib/postgresql/data + environment: + POSTGRESQL_PASSWORD: infisical + POSTGRESQL_USERNAME: infisical + POSTGRESQL_DATABASE: infisical + POSTGRESQL_REPLICATION_MODE: slave + POSTGRESQL_REPLICATION_USER: repl_user + POSTGRESQL_REPLICATION_PASSWORD: repl_password + POSTGRESQL_MASTER_HOST: db + POSTGRESQL_MASTER_PORT_NUMBER: 5432 + + + redis: + image: redis + container_name: infisical-dev-redis + environment: + - ALLOW_EMPTY_PASSWORD=yes + ports: + - 6379:6379 + volumes: + - redis_data:/data + + redis-commander: + container_name: infisical-dev-redis-commander + image: rediscommander/redis-commander + restart: always + depends_on: + - redis + environment: + - REDIS_HOSTS=local:redis:6379 + ports: + - "8085:8081" + + db-test: + profiles: ["test"] + image: postgres:14-alpine + ports: + - "5430:5432" + environment: + POSTGRES_PASSWORD: infisical + POSTGRES_USER: infisical + POSTGRES_DB: infisical-test + + db-migration: + container_name: infisical-db-migration + depends_on: + - db + build: + context: ./backend + dockerfile: Dockerfile.dev + env_file: .env + environment: + - DB_CONNECTION_URI=postgres://infisical:infisical@db/infisical?sslmode=disable + command: npm run migration:latest + volumes: + - ./backend/src:/app/src + + backend: + container_name: infisical-dev-api + build: + context: ./backend + dockerfile: Dockerfile.dev + depends_on: + db: + condition: service_started + redis: + condition: service_started + db-migration: + condition: service_completed_successfully + env_file: + - .env + ports: + - 4000:4000 + environment: + - NODE_ENV=development + - DB_CONNECTION_URI=postgres://infisical:infisical@db/infisical?sslmode=disable + - TELEMETRY_ENABLED=false + volumes: + - ./backend/src:/app/src + extra_hosts: + - "host.docker.internal:host-gateway" + + frontend: + container_name: infisical-dev-frontend + restart: unless-stopped + depends_on: + - backend + build: + context: ./frontend + dockerfile: Dockerfile.dev + volumes: + - ./frontend/src:/app/src/ # mounted whole src to avoid missing reload on new files + - ./frontend/public:/app/public + env_file: .env + environment: + - NEXT_PUBLIC_ENV=development + - INFISICAL_TELEMETRY_ENABLED=false + + pgadmin: + image: dpage/pgadmin4 + restart: always + environment: + PGADMIN_DEFAULT_EMAIL: admin@example.com + PGADMIN_DEFAULT_PASSWORD: pass + ports: + - 5050:80 + depends_on: + - db + + smtp-server: + container_name: infisical-dev-smtp-server + image: lytrax/mailhog:latest # https://github.com/mailhog/MailHog/issues/353#issuecomment-821137362 + restart: always + logging: + driver: "none" # disable saving logs + ports: + - 1025:1025 # SMTP server + - 8025:8025 # Web UI + + openldap: # note: more advanced configuration is available + image: osixia/openldap:1.5.0 + restart: always + environment: + LDAP_ORGANISATION: Acme + LDAP_DOMAIN: acme.com + LDAP_ADMIN_PASSWORD: admin + ports: + - 389:389 + - 636:636 + volumes: + - ldap_data:/var/lib/ldap + - ldap_config:/etc/ldap/slapd.d + profiles: [ldap] + + phpldapadmin: # username: cn=admin,dc=acme,dc=com, pass is admin + image: osixia/phpldapadmin:latest + restart: always + environment: + - PHPLDAPADMIN_LDAP_HOSTS=openldap + - PHPLDAPADMIN_HTTPS=false + ports: + - 6433:80 + depends_on: + - openldap + profiles: [ldap] + +volumes: + postgres-data: + driver: local + postgres-slave-data: + driver: local + redis_data: + driver: local + ldap_data: + ldap_config: diff --git a/docs/self-hosting/configuration/envars.mdx b/docs/self-hosting/configuration/envars.mdx index 754409dfe..fcb98d40e 100644 --- a/docs/self-hosting/configuration/envars.mdx +++ b/docs/self-hosting/configuration/envars.mdx @@ -47,6 +47,25 @@ The platform utilizes Postgres to persist all of its data and Redis for caching Redis connection string. + + Postgres database read replica connection strings. It accepts a JSON string. +``` +DB_READ_REPLICAS=[{"DB_CONNECTION_URI":""}] +``` + + + Postgres read replica connection string. + + + Configure the SSL certificate for securing a Postgres replica connection by first encoding it in base64. + Use the command below to encode your certificate: + `echo "" | base64` + + If not provided it will use master SSL certificate. + + + + ## Email service Without email configuration, Infisical's core functions like sign-up/login and secret operations work, but this disables multi-factor authentication, email invites for projects, alerts for suspicious logins, and all other email-dependent features.