Merge pull request #2019 from akhilmhdh/feat/read-replica

Postgres read replica support
This commit is contained in:
Maidul Islam
2024-06-27 19:36:44 -04:00
committed by GitHub
48 changed files with 605 additions and 226 deletions

View File

@@ -3,7 +3,6 @@ import "ts-node/register";
import dotenv from "dotenv";
import jwt from "jsonwebtoken";
import knex from "knex";
import path from "path";
import { seedData1 } from "@app/db/seed-data";
@@ -15,6 +14,7 @@ import { AuthMethod, AuthTokenType } from "@app/services/auth/auth-type";
import { mockQueue } from "./mocks/queue";
import { mockSmtpServer } from "./mocks/smtp";
import { mockKeyStore } from "./mocks/keystore";
import { initDbConnection } from "@app/db";
dotenv.config({ path: path.join(__dirname, "../../.env.test"), debug: true });
export default {
@@ -23,23 +23,21 @@ export default {
async setup() {
const logger = await initLogger();
const cfg = initEnvConfig(logger);
const db = knex({
client: "pg",
connection: cfg.DB_CONNECTION_URI,
migrations: {
directory: path.join(__dirname, "../src/db/migrations"),
extension: "ts",
tableName: "infisical_migrations"
},
seeds: {
directory: path.join(__dirname, "../src/db/seeds"),
extension: "ts"
}
const db = initDbConnection({
dbConnectionUri: cfg.DB_CONNECTION_URI,
dbRootCert: cfg.DB_ROOT_CERT
});
try {
await db.migrate.latest();
await db.seed.run();
await db.migrate.latest({
directory: path.join(__dirname, "../src/db/migrations"),
extension: "ts",
tableName: "infisical_migrations"
});
await db.seed.run({
directory: path.join(__dirname, "../src/db/seeds"),
extension: "ts"
});
const smtp = mockSmtpServer();
const queue = mockQueue();
const keyStore = mockKeyStore();
@@ -74,7 +72,14 @@ export default {
// @ts-expect-error type
delete globalThis.jwtToken;
// called after all tests with this env have been run
await db.migrate.rollback({}, true);
await db.migrate.rollback(
{
directory: path.join(__dirname, "../src/db/migrations"),
extension: "ts",
tableName: "infisical_migrations"
},
true
);
await db.destroy();
}
};

View File

@@ -1,4 +1,4 @@
import { Knex } from "knex";
import { Knex as KnexOriginal } from "knex";
import {
TableName,
@@ -280,318 +280,371 @@ import {
TWebhooksUpdate
} from "@app/db/schemas";
declare module "knex" {
namespace Knex {
interface QueryInterface {
primaryNode(): KnexOriginal;
replicaNode(): KnexOriginal;
}
}
}
declare module "knex/types/tables" {
interface Tables {
[TableName.Users]: Knex.CompositeTableType<TUsers, TUsersInsert, TUsersUpdate>;
[TableName.Groups]: Knex.CompositeTableType<TGroups, TGroupsInsert, TGroupsUpdate>;
[TableName.CertificateAuthority]: Knex.CompositeTableType<
[TableName.Users]: KnexOriginal.CompositeTableType<TUsers, TUsersInsert, TUsersUpdate>;
[TableName.Groups]: KnexOriginal.CompositeTableType<TGroups, TGroupsInsert, TGroupsUpdate>;
[TableName.CertificateAuthority]: KnexOriginal.CompositeTableType<
TCertificateAuthorities,
TCertificateAuthoritiesInsert,
TCertificateAuthoritiesUpdate
>;
[TableName.CertificateAuthorityCert]: Knex.CompositeTableType<
[TableName.CertificateAuthorityCert]: KnexOriginal.CompositeTableType<
TCertificateAuthorityCerts,
TCertificateAuthorityCertsInsert,
TCertificateAuthorityCertsUpdate
>;
[TableName.CertificateAuthoritySecret]: Knex.CompositeTableType<
[TableName.CertificateAuthoritySecret]: KnexOriginal.CompositeTableType<
TCertificateAuthoritySecret,
TCertificateAuthoritySecretInsert,
TCertificateAuthoritySecretUpdate
>;
[TableName.CertificateAuthorityCrl]: Knex.CompositeTableType<
[TableName.CertificateAuthorityCrl]: KnexOriginal.CompositeTableType<
TCertificateAuthorityCrl,
TCertificateAuthorityCrlInsert,
TCertificateAuthorityCrlUpdate
>;
[TableName.Certificate]: Knex.CompositeTableType<TCertificates, TCertificatesInsert, TCertificatesUpdate>;
[TableName.CertificateBody]: Knex.CompositeTableType<
[TableName.Certificate]: KnexOriginal.CompositeTableType<TCertificates, TCertificatesInsert, TCertificatesUpdate>;
[TableName.CertificateBody]: KnexOriginal.CompositeTableType<
TCertificateBodies,
TCertificateBodiesInsert,
TCertificateBodiesUpdate
>;
[TableName.CertificateSecret]: Knex.CompositeTableType<
[TableName.CertificateSecret]: KnexOriginal.CompositeTableType<
TCertificateSecrets,
TCertificateSecretsInsert,
TCertificateSecretsUpdate
>;
[TableName.UserGroupMembership]: Knex.CompositeTableType<
[TableName.UserGroupMembership]: KnexOriginal.CompositeTableType<
TUserGroupMembership,
TUserGroupMembershipInsert,
TUserGroupMembershipUpdate
>;
[TableName.GroupProjectMembership]: Knex.CompositeTableType<
[TableName.GroupProjectMembership]: KnexOriginal.CompositeTableType<
TGroupProjectMemberships,
TGroupProjectMembershipsInsert,
TGroupProjectMembershipsUpdate
>;
[TableName.GroupProjectMembershipRole]: Knex.CompositeTableType<
[TableName.GroupProjectMembershipRole]: KnexOriginal.CompositeTableType<
TGroupProjectMembershipRoles,
TGroupProjectMembershipRolesInsert,
TGroupProjectMembershipRolesUpdate
>;
[TableName.UserAliases]: Knex.CompositeTableType<TUserAliases, TUserAliasesInsert, TUserAliasesUpdate>;
[TableName.UserEncryptionKey]: Knex.CompositeTableType<
[TableName.UserAliases]: KnexOriginal.CompositeTableType<TUserAliases, TUserAliasesInsert, TUserAliasesUpdate>;
[TableName.UserEncryptionKey]: KnexOriginal.CompositeTableType<
TUserEncryptionKeys,
TUserEncryptionKeysInsert,
TUserEncryptionKeysUpdate
>;
[TableName.AuthTokens]: Knex.CompositeTableType<TAuthTokens, TAuthTokensInsert, TAuthTokensUpdate>;
[TableName.AuthTokenSession]: Knex.CompositeTableType<
[TableName.AuthTokens]: KnexOriginal.CompositeTableType<TAuthTokens, TAuthTokensInsert, TAuthTokensUpdate>;
[TableName.AuthTokenSession]: KnexOriginal.CompositeTableType<
TAuthTokenSessions,
TAuthTokenSessionsInsert,
TAuthTokenSessionsUpdate
>;
[TableName.BackupPrivateKey]: Knex.CompositeTableType<
[TableName.BackupPrivateKey]: KnexOriginal.CompositeTableType<
TBackupPrivateKey,
TBackupPrivateKeyInsert,
TBackupPrivateKeyUpdate
>;
[TableName.Organization]: Knex.CompositeTableType<TOrganizations, TOrganizationsInsert, TOrganizationsUpdate>;
[TableName.OrgMembership]: Knex.CompositeTableType<TOrgMemberships, TOrgMembershipsInsert, TOrgMembershipsUpdate>;
[TableName.OrgRoles]: Knex.CompositeTableType<TOrgRoles, TOrgRolesInsert, TOrgRolesUpdate>;
[TableName.IncidentContact]: Knex.CompositeTableType<
[TableName.Organization]: KnexOriginal.CompositeTableType<
TOrganizations,
TOrganizationsInsert,
TOrganizationsUpdate
>;
[TableName.OrgMembership]: KnexOriginal.CompositeTableType<
TOrgMemberships,
TOrgMembershipsInsert,
TOrgMembershipsUpdate
>;
[TableName.OrgRoles]: KnexOriginal.CompositeTableType<TOrgRoles, TOrgRolesInsert, TOrgRolesUpdate>;
[TableName.IncidentContact]: KnexOriginal.CompositeTableType<
TIncidentContacts,
TIncidentContactsInsert,
TIncidentContactsUpdate
>;
[TableName.UserAction]: Knex.CompositeTableType<TUserActions, TUserActionsInsert, TUserActionsUpdate>;
[TableName.SuperAdmin]: Knex.CompositeTableType<TSuperAdmin, TSuperAdminInsert, TSuperAdminUpdate>;
[TableName.ApiKey]: Knex.CompositeTableType<TApiKeys, TApiKeysInsert, TApiKeysUpdate>;
[TableName.Project]: Knex.CompositeTableType<TProjects, TProjectsInsert, TProjectsUpdate>;
[TableName.ProjectMembership]: Knex.CompositeTableType<
[TableName.UserAction]: KnexOriginal.CompositeTableType<TUserActions, TUserActionsInsert, TUserActionsUpdate>;
[TableName.SuperAdmin]: KnexOriginal.CompositeTableType<TSuperAdmin, TSuperAdminInsert, TSuperAdminUpdate>;
[TableName.ApiKey]: KnexOriginal.CompositeTableType<TApiKeys, TApiKeysInsert, TApiKeysUpdate>;
[TableName.Project]: KnexOriginal.CompositeTableType<TProjects, TProjectsInsert, TProjectsUpdate>;
[TableName.ProjectMembership]: KnexOriginal.CompositeTableType<
TProjectMemberships,
TProjectMembershipsInsert,
TProjectMembershipsUpdate
>;
[TableName.Environment]: Knex.CompositeTableType<
[TableName.Environment]: KnexOriginal.CompositeTableType<
TProjectEnvironments,
TProjectEnvironmentsInsert,
TProjectEnvironmentsUpdate
>;
[TableName.ProjectBot]: Knex.CompositeTableType<TProjectBots, TProjectBotsInsert, TProjectBotsUpdate>;
[TableName.ProjectUserMembershipRole]: Knex.CompositeTableType<
[TableName.ProjectBot]: KnexOriginal.CompositeTableType<TProjectBots, TProjectBotsInsert, TProjectBotsUpdate>;
[TableName.ProjectUserMembershipRole]: KnexOriginal.CompositeTableType<
TProjectUserMembershipRoles,
TProjectUserMembershipRolesInsert,
TProjectUserMembershipRolesUpdate
>;
[TableName.ProjectRoles]: Knex.CompositeTableType<TProjectRoles, TProjectRolesInsert, TProjectRolesUpdate>;
[TableName.ProjectUserAdditionalPrivilege]: Knex.CompositeTableType<
[TableName.ProjectRoles]: KnexOriginal.CompositeTableType<TProjectRoles, TProjectRolesInsert, TProjectRolesUpdate>;
[TableName.ProjectUserAdditionalPrivilege]: KnexOriginal.CompositeTableType<
TProjectUserAdditionalPrivilege,
TProjectUserAdditionalPrivilegeInsert,
TProjectUserAdditionalPrivilegeUpdate
>;
[TableName.ProjectKeys]: Knex.CompositeTableType<TProjectKeys, TProjectKeysInsert, TProjectKeysUpdate>;
[TableName.Secret]: Knex.CompositeTableType<TSecrets, TSecretsInsert, TSecretsUpdate>;
[TableName.SecretReference]: Knex.CompositeTableType<
[TableName.ProjectKeys]: KnexOriginal.CompositeTableType<TProjectKeys, TProjectKeysInsert, TProjectKeysUpdate>;
[TableName.Secret]: KnexOriginal.CompositeTableType<TSecrets, TSecretsInsert, TSecretsUpdate>;
[TableName.SecretReference]: KnexOriginal.CompositeTableType<
TSecretReferences,
TSecretReferencesInsert,
TSecretReferencesUpdate
>;
[TableName.SecretBlindIndex]: Knex.CompositeTableType<
[TableName.SecretBlindIndex]: KnexOriginal.CompositeTableType<
TSecretBlindIndexes,
TSecretBlindIndexesInsert,
TSecretBlindIndexesUpdate
>;
[TableName.SecretVersion]: Knex.CompositeTableType<TSecretVersions, TSecretVersionsInsert, TSecretVersionsUpdate>;
[TableName.SecretFolder]: Knex.CompositeTableType<TSecretFolders, TSecretFoldersInsert, TSecretFoldersUpdate>;
[TableName.SecretFolderVersion]: Knex.CompositeTableType<
[TableName.SecretVersion]: KnexOriginal.CompositeTableType<
TSecretVersions,
TSecretVersionsInsert,
TSecretVersionsUpdate
>;
[TableName.SecretFolder]: KnexOriginal.CompositeTableType<
TSecretFolders,
TSecretFoldersInsert,
TSecretFoldersUpdate
>;
[TableName.SecretFolderVersion]: KnexOriginal.CompositeTableType<
TSecretFolderVersions,
TSecretFolderVersionsInsert,
TSecretFolderVersionsUpdate
>;
[TableName.SecretSharing]: Knex.CompositeTableType<TSecretSharing, TSecretSharingInsert, TSecretSharingUpdate>;
[TableName.RateLimit]: Knex.CompositeTableType<TRateLimit, TRateLimitInsert, TRateLimitUpdate>;
[TableName.SecretTag]: Knex.CompositeTableType<TSecretTags, TSecretTagsInsert, TSecretTagsUpdate>;
[TableName.SecretImport]: Knex.CompositeTableType<TSecretImports, TSecretImportsInsert, TSecretImportsUpdate>;
[TableName.Integration]: Knex.CompositeTableType<TIntegrations, TIntegrationsInsert, TIntegrationsUpdate>;
[TableName.Webhook]: Knex.CompositeTableType<TWebhooks, TWebhooksInsert, TWebhooksUpdate>;
[TableName.ServiceToken]: Knex.CompositeTableType<TServiceTokens, TServiceTokensInsert, TServiceTokensUpdate>;
[TableName.IntegrationAuth]: Knex.CompositeTableType<
[TableName.SecretSharing]: KnexOriginal.CompositeTableType<
TSecretSharing,
TSecretSharingInsert,
TSecretSharingUpdate
>;
[TableName.RateLimit]: KnexOriginal.CompositeTableType<TRateLimit, TRateLimitInsert, TRateLimitUpdate>;
[TableName.SecretTag]: KnexOriginal.CompositeTableType<TSecretTags, TSecretTagsInsert, TSecretTagsUpdate>;
[TableName.SecretImport]: KnexOriginal.CompositeTableType<
TSecretImports,
TSecretImportsInsert,
TSecretImportsUpdate
>;
[TableName.Integration]: KnexOriginal.CompositeTableType<TIntegrations, TIntegrationsInsert, TIntegrationsUpdate>;
[TableName.Webhook]: KnexOriginal.CompositeTableType<TWebhooks, TWebhooksInsert, TWebhooksUpdate>;
[TableName.ServiceToken]: KnexOriginal.CompositeTableType<
TServiceTokens,
TServiceTokensInsert,
TServiceTokensUpdate
>;
[TableName.IntegrationAuth]: KnexOriginal.CompositeTableType<
TIntegrationAuths,
TIntegrationAuthsInsert,
TIntegrationAuthsUpdate
>;
[TableName.Identity]: Knex.CompositeTableType<TIdentities, TIdentitiesInsert, TIdentitiesUpdate>;
[TableName.IdentityUniversalAuth]: Knex.CompositeTableType<
[TableName.Identity]: KnexOriginal.CompositeTableType<TIdentities, TIdentitiesInsert, TIdentitiesUpdate>;
[TableName.IdentityUniversalAuth]: KnexOriginal.CompositeTableType<
TIdentityUniversalAuths,
TIdentityUniversalAuthsInsert,
TIdentityUniversalAuthsUpdate
>;
[TableName.IdentityKubernetesAuth]: Knex.CompositeTableType<
[TableName.IdentityKubernetesAuth]: KnexOriginal.CompositeTableType<
TIdentityKubernetesAuths,
TIdentityKubernetesAuthsInsert,
TIdentityKubernetesAuthsUpdate
>;
[TableName.IdentityGcpAuth]: Knex.CompositeTableType<
[TableName.IdentityGcpAuth]: KnexOriginal.CompositeTableType<
TIdentityGcpAuths,
TIdentityGcpAuthsInsert,
TIdentityGcpAuthsUpdate
>;
[TableName.IdentityAwsAuth]: Knex.CompositeTableType<
[TableName.IdentityAwsAuth]: KnexOriginal.CompositeTableType<
TIdentityAwsAuths,
TIdentityAwsAuthsInsert,
TIdentityAwsAuthsUpdate
>;
[TableName.IdentityAzureAuth]: Knex.CompositeTableType<
[TableName.IdentityAzureAuth]: KnexOriginal.CompositeTableType<
TIdentityAzureAuths,
TIdentityAzureAuthsInsert,
TIdentityAzureAuthsUpdate
>;
[TableName.IdentityUaClientSecret]: Knex.CompositeTableType<
[TableName.IdentityUaClientSecret]: KnexOriginal.CompositeTableType<
TIdentityUaClientSecrets,
TIdentityUaClientSecretsInsert,
TIdentityUaClientSecretsUpdate
>;
[TableName.IdentityAccessToken]: Knex.CompositeTableType<
[TableName.IdentityAccessToken]: KnexOriginal.CompositeTableType<
TIdentityAccessTokens,
TIdentityAccessTokensInsert,
TIdentityAccessTokensUpdate
>;
[TableName.IdentityOrgMembership]: Knex.CompositeTableType<
[TableName.IdentityOrgMembership]: KnexOriginal.CompositeTableType<
TIdentityOrgMemberships,
TIdentityOrgMembershipsInsert,
TIdentityOrgMembershipsUpdate
>;
[TableName.IdentityProjectMembership]: Knex.CompositeTableType<
[TableName.IdentityProjectMembership]: KnexOriginal.CompositeTableType<
TIdentityProjectMemberships,
TIdentityProjectMembershipsInsert,
TIdentityProjectMembershipsUpdate
>;
[TableName.IdentityProjectMembershipRole]: Knex.CompositeTableType<
[TableName.IdentityProjectMembershipRole]: KnexOriginal.CompositeTableType<
TIdentityProjectMembershipRole,
TIdentityProjectMembershipRoleInsert,
TIdentityProjectMembershipRoleUpdate
>;
[TableName.IdentityProjectAdditionalPrivilege]: Knex.CompositeTableType<
[TableName.IdentityProjectAdditionalPrivilege]: KnexOriginal.CompositeTableType<
TIdentityProjectAdditionalPrivilege,
TIdentityProjectAdditionalPrivilegeInsert,
TIdentityProjectAdditionalPrivilegeUpdate
>;
[TableName.AccessApprovalPolicy]: Knex.CompositeTableType<
[TableName.AccessApprovalPolicy]: KnexOriginal.CompositeTableType<
TAccessApprovalPolicies,
TAccessApprovalPoliciesInsert,
TAccessApprovalPoliciesUpdate
>;
[TableName.AccessApprovalPolicyApprover]: Knex.CompositeTableType<
[TableName.AccessApprovalPolicyApprover]: KnexOriginal.CompositeTableType<
TAccessApprovalPoliciesApprovers,
TAccessApprovalPoliciesApproversInsert,
TAccessApprovalPoliciesApproversUpdate
>;
[TableName.AccessApprovalRequest]: Knex.CompositeTableType<
[TableName.AccessApprovalRequest]: KnexOriginal.CompositeTableType<
TAccessApprovalRequests,
TAccessApprovalRequestsInsert,
TAccessApprovalRequestsUpdate
>;
[TableName.AccessApprovalRequestReviewer]: Knex.CompositeTableType<
[TableName.AccessApprovalRequestReviewer]: KnexOriginal.CompositeTableType<
TAccessApprovalRequestsReviewers,
TAccessApprovalRequestsReviewersInsert,
TAccessApprovalRequestsReviewersUpdate
>;
[TableName.ScimToken]: Knex.CompositeTableType<TScimTokens, TScimTokensInsert, TScimTokensUpdate>;
[TableName.SecretApprovalPolicy]: Knex.CompositeTableType<
[TableName.ScimToken]: KnexOriginal.CompositeTableType<TScimTokens, TScimTokensInsert, TScimTokensUpdate>;
[TableName.SecretApprovalPolicy]: KnexOriginal.CompositeTableType<
TSecretApprovalPolicies,
TSecretApprovalPoliciesInsert,
TSecretApprovalPoliciesUpdate
>;
[TableName.SecretApprovalPolicyApprover]: Knex.CompositeTableType<
[TableName.SecretApprovalPolicyApprover]: KnexOriginal.CompositeTableType<
TSecretApprovalPoliciesApprovers,
TSecretApprovalPoliciesApproversInsert,
TSecretApprovalPoliciesApproversUpdate
>;
[TableName.SecretApprovalRequest]: Knex.CompositeTableType<
[TableName.SecretApprovalRequest]: KnexOriginal.CompositeTableType<
TSecretApprovalRequests,
TSecretApprovalRequestsInsert,
TSecretApprovalRequestsUpdate
>;
[TableName.SecretApprovalRequestReviewer]: Knex.CompositeTableType<
[TableName.SecretApprovalRequestReviewer]: KnexOriginal.CompositeTableType<
TSecretApprovalRequestsReviewers,
TSecretApprovalRequestsReviewersInsert,
TSecretApprovalRequestsReviewersUpdate
>;
[TableName.SecretApprovalRequestSecret]: Knex.CompositeTableType<
[TableName.SecretApprovalRequestSecret]: KnexOriginal.CompositeTableType<
TSecretApprovalRequestsSecrets,
TSecretApprovalRequestsSecretsInsert,
TSecretApprovalRequestsSecretsUpdate
>;
[TableName.SecretApprovalRequestSecretTag]: Knex.CompositeTableType<
[TableName.SecretApprovalRequestSecretTag]: KnexOriginal.CompositeTableType<
TSecretApprovalRequestSecretTags,
TSecretApprovalRequestSecretTagsInsert,
TSecretApprovalRequestSecretTagsUpdate
>;
[TableName.SecretRotation]: Knex.CompositeTableType<
[TableName.SecretRotation]: KnexOriginal.CompositeTableType<
TSecretRotations,
TSecretRotationsInsert,
TSecretRotationsUpdate
>;
[TableName.SecretRotationOutput]: Knex.CompositeTableType<
[TableName.SecretRotationOutput]: KnexOriginal.CompositeTableType<
TSecretRotationOutputs,
TSecretRotationOutputsInsert,
TSecretRotationOutputsUpdate
>;
[TableName.Snapshot]: Knex.CompositeTableType<TSecretSnapshots, TSecretSnapshotsInsert, TSecretSnapshotsUpdate>;
[TableName.SnapshotSecret]: Knex.CompositeTableType<
[TableName.Snapshot]: KnexOriginal.CompositeTableType<
TSecretSnapshots,
TSecretSnapshotsInsert,
TSecretSnapshotsUpdate
>;
[TableName.SnapshotSecret]: KnexOriginal.CompositeTableType<
TSecretSnapshotSecrets,
TSecretSnapshotSecretsInsert,
TSecretSnapshotSecretsUpdate
>;
[TableName.SnapshotFolder]: Knex.CompositeTableType<
[TableName.SnapshotFolder]: KnexOriginal.CompositeTableType<
TSecretSnapshotFolders,
TSecretSnapshotFoldersInsert,
TSecretSnapshotFoldersUpdate
>;
[TableName.DynamicSecret]: Knex.CompositeTableType<TDynamicSecrets, TDynamicSecretsInsert, TDynamicSecretsUpdate>;
[TableName.DynamicSecretLease]: Knex.CompositeTableType<
[TableName.DynamicSecret]: KnexOriginal.CompositeTableType<
TDynamicSecrets,
TDynamicSecretsInsert,
TDynamicSecretsUpdate
>;
[TableName.DynamicSecretLease]: KnexOriginal.CompositeTableType<
TDynamicSecretLeases,
TDynamicSecretLeasesInsert,
TDynamicSecretLeasesUpdate
>;
[TableName.SamlConfig]: Knex.CompositeTableType<TSamlConfigs, TSamlConfigsInsert, TSamlConfigsUpdate>;
[TableName.OidcConfig]: Knex.CompositeTableType<TOidcConfigs, TOidcConfigsInsert, TOidcConfigsUpdate>;
[TableName.LdapConfig]: Knex.CompositeTableType<TLdapConfigs, TLdapConfigsInsert, TLdapConfigsUpdate>;
[TableName.LdapGroupMap]: Knex.CompositeTableType<TLdapGroupMaps, TLdapGroupMapsInsert, TLdapGroupMapsUpdate>;
[TableName.OrgBot]: Knex.CompositeTableType<TOrgBots, TOrgBotsInsert, TOrgBotsUpdate>;
[TableName.AuditLog]: Knex.CompositeTableType<TAuditLogs, TAuditLogsInsert, TAuditLogsUpdate>;
[TableName.AuditLogStream]: Knex.CompositeTableType<
[TableName.SamlConfig]: KnexOriginal.CompositeTableType<TSamlConfigs, TSamlConfigsInsert, TSamlConfigsUpdate>;
[TableName.OidcConfig]: KnexOriginal.CompositeTableType<TOidcConfigs, TOidcConfigsInsert, TOidcConfigsUpdate>;
[TableName.LdapConfig]: KnexOriginal.CompositeTableType<TLdapConfigs, TLdapConfigsInsert, TLdapConfigsUpdate>;
[TableName.LdapGroupMap]: KnexOriginal.CompositeTableType<
TLdapGroupMaps,
TLdapGroupMapsInsert,
TLdapGroupMapsUpdate
>;
[TableName.OrgBot]: KnexOriginal.CompositeTableType<TOrgBots, TOrgBotsInsert, TOrgBotsUpdate>;
[TableName.AuditLog]: KnexOriginal.CompositeTableType<TAuditLogs, TAuditLogsInsert, TAuditLogsUpdate>;
[TableName.AuditLogStream]: KnexOriginal.CompositeTableType<
TAuditLogStreams,
TAuditLogStreamsInsert,
TAuditLogStreamsUpdate
>;
[TableName.GitAppInstallSession]: Knex.CompositeTableType<
[TableName.GitAppInstallSession]: KnexOriginal.CompositeTableType<
TGitAppInstallSessions,
TGitAppInstallSessionsInsert,
TGitAppInstallSessionsUpdate
>;
[TableName.GitAppOrg]: Knex.CompositeTableType<TGitAppOrg, TGitAppOrgInsert, TGitAppOrgUpdate>;
[TableName.SecretScanningGitRisk]: Knex.CompositeTableType<
[TableName.GitAppOrg]: KnexOriginal.CompositeTableType<TGitAppOrg, TGitAppOrgInsert, TGitAppOrgUpdate>;
[TableName.SecretScanningGitRisk]: KnexOriginal.CompositeTableType<
TSecretScanningGitRisks,
TSecretScanningGitRisksInsert,
TSecretScanningGitRisksUpdate
>;
[TableName.TrustedIps]: Knex.CompositeTableType<TTrustedIps, TTrustedIpsInsert, TTrustedIpsUpdate>;
[TableName.TrustedIps]: KnexOriginal.CompositeTableType<TTrustedIps, TTrustedIpsInsert, TTrustedIpsUpdate>;
// Junction tables
[TableName.JnSecretTag]: Knex.CompositeTableType<
[TableName.JnSecretTag]: KnexOriginal.CompositeTableType<
TSecretTagJunction,
TSecretTagJunctionInsert,
TSecretTagJunctionUpdate
>;
[TableName.SecretVersionTag]: Knex.CompositeTableType<
[TableName.SecretVersionTag]: KnexOriginal.CompositeTableType<
TSecretVersionTagJunction,
TSecretVersionTagJunctionInsert,
TSecretVersionTagJunctionUpdate
>;
// KMS service
[TableName.KmsServerRootConfig]: Knex.CompositeTableType<
[TableName.KmsServerRootConfig]: KnexOriginal.CompositeTableType<
TKmsRootConfig,
TKmsRootConfigInsert,
TKmsRootConfigUpdate
>;
[TableName.KmsKey]: Knex.CompositeTableType<TKmsKeys, TKmsKeysInsert, TKmsKeysUpdate>;
[TableName.KmsKeyVersion]: Knex.CompositeTableType<TKmsKeyVersions, TKmsKeyVersionsInsert, TKmsKeyVersionsUpdate>;
[TableName.KmsKey]: KnexOriginal.CompositeTableType<TKmsKeys, TKmsKeysInsert, TKmsKeysUpdate>;
[TableName.KmsKeyVersion]: KnexOriginal.CompositeTableType<
TKmsKeyVersions,
TKmsKeyVersionsInsert,
TKmsKeyVersionsUpdate
>;
}
}

View File

@@ -1,8 +1,38 @@
import knex from "knex";
import knex, { Knex } from "knex";
export type TDbClient = ReturnType<typeof initDbConnection>;
export const initDbConnection = ({ dbConnectionUri, dbRootCert }: { dbConnectionUri: string; dbRootCert?: string }) => {
const db = knex({
export const initDbConnection = ({
dbConnectionUri,
dbRootCert,
readReplicas = []
}: {
dbConnectionUri: string;
dbRootCert?: string;
readReplicas?: {
dbConnectionUri: string;
dbRootCert?: string;
}[];
}) => {
// akhilmhdh: the default Knex is knex.Knex<any, any[]>. but when assigned with knex({<config>}) the value is knex.Knex<any, unknown[]>
// this was causing issue with files like `snapshot-dal` `findRecursivelySnapshots` this i am explicitly putting the any and unknown[]
// eslint-disable-next-line
let db: Knex<any, unknown[]>;
// eslint-disable-next-line
let readReplicaDbs: Knex<any, unknown[]>[];
// @ts-expect-error the querybuilder type is expected but our intension is to return a knex instance
knex.QueryBuilder.extend("primaryNode", () => {
return db;
});
// @ts-expect-error the querybuilder type is expected but our intension is to return a knex instance
knex.QueryBuilder.extend("replicaNode", () => {
if (!readReplicaDbs.length) return db;
const selectedReplica = readReplicaDbs[Math.floor(Math.random() * readReplicaDbs.length)];
return selectedReplica;
});
db = knex({
client: "pg",
connection: {
connectionString: dbConnectionUri,
@@ -22,5 +52,21 @@ export const initDbConnection = ({ dbConnectionUri, dbRootCert }: { dbConnection
}
});
readReplicaDbs = readReplicas.map((el) => {
const replicaDbCertificate = el.dbRootCert || dbRootCert;
return knex({
client: "pg",
connection: {
connectionString: el.dbConnectionUri,
ssl: replicaDbCertificate
? {
rejectUnauthorized: true,
ca: Buffer.from(replicaDbCertificate, "base64").toString("ascii")
}
: false
}
});
});
return db;
};

View File

@@ -32,7 +32,7 @@ export const accessApprovalPolicyDALFactory = (db: TDbClient) => {
const findById = async (id: string, tx?: Knex) => {
try {
const doc = await accessApprovalPolicyFindQuery(tx || db, {
const doc = await accessApprovalPolicyFindQuery(tx || db.replicaNode(), {
[`${TableName.AccessApprovalPolicy}.id` as "id"]: id
});
const formatedDoc = mergeOneToManyRelation(
@@ -54,7 +54,7 @@ export const accessApprovalPolicyDALFactory = (db: TDbClient) => {
const find = async (filter: TFindFilter<TAccessApprovalPolicies & { projectId: string }>, tx?: Knex) => {
try {
const docs = await accessApprovalPolicyFindQuery(tx || db, filter);
const docs = await accessApprovalPolicyFindQuery(tx || db.replicaNode(), filter);
const formatedDoc = mergeOneToManyRelation(
docs,
"id",

View File

@@ -14,7 +14,8 @@ export const accessApprovalRequestDALFactory = (db: TDbClient) => {
const findRequestsWithPrivilegeByPolicyIds = async (policyIds: string[]) => {
try {
const docs = await db(TableName.AccessApprovalRequest)
const docs = await db
.replicaNode()(TableName.AccessApprovalRequest)
.whereIn(`${TableName.AccessApprovalRequest}.policyId`, policyIds)
.leftJoin(
@@ -170,7 +171,7 @@ export const accessApprovalRequestDALFactory = (db: TDbClient) => {
const findById = async (id: string, tx?: Knex) => {
try {
const sql = findQuery({ [`${TableName.AccessApprovalRequest}.id` as "id"]: id }, tx || db);
const sql = findQuery({ [`${TableName.AccessApprovalRequest}.id` as "id"]: id }, tx || db.replicaNode());
const docs = await sql;
const formatedDoc = sqlNestRelationships({
data: docs,
@@ -207,7 +208,8 @@ export const accessApprovalRequestDALFactory = (db: TDbClient) => {
const getCount = async ({ projectId }: { projectId: string }) => {
try {
const accessRequests = await db(TableName.AccessApprovalRequest)
const accessRequests = await db
.replicaNode()(TableName.AccessApprovalRequest)
.leftJoin(
TableName.AccessApprovalPolicy,
`${TableName.AccessApprovalRequest}.policyId`,

View File

@@ -27,7 +27,7 @@ export const auditLogDALFactory = (db: TDbClient) => {
tx?: Knex
) => {
try {
const sqlQuery = (tx || db)(TableName.AuditLog)
const sqlQuery = (tx || db.replicaNode())(TableName.AuditLog)
.where(
stripUndefinedInWhere({
projectId,

View File

@@ -12,7 +12,10 @@ export const dynamicSecretLeaseDALFactory = (db: TDbClient) => {
const countLeasesForDynamicSecret = async (dynamicSecretId: string, tx?: Knex) => {
try {
const doc = await (tx || db)(TableName.DynamicSecretLease).count("*").where({ dynamicSecretId }).first();
const doc = await (tx || db.replicaNode())(TableName.DynamicSecretLease)
.count("*")
.where({ dynamicSecretId })
.first();
return parseInt(doc || "0", 10);
} catch (error) {
throw new DatabaseError({ error, name: "DynamicSecretCountLeases" });
@@ -21,7 +24,7 @@ export const dynamicSecretLeaseDALFactory = (db: TDbClient) => {
const findById = async (id: string, tx?: Knex) => {
try {
const doc = await (tx || db)(TableName.DynamicSecretLease)
const doc = await (tx || db.replicaNode())(TableName.DynamicSecretLease)
.where({ [`${TableName.DynamicSecretLease}.id` as "id"]: id })
.first()
.join(

View File

@@ -12,7 +12,7 @@ export const groupDALFactory = (db: TDbClient) => {
const findGroups = async (filter: TFindFilter<TGroups>, { offset, limit, sort, tx }: TFindOpt<TGroups> = {}) => {
try {
const query = (tx || db)(TableName.Groups)
const query = (tx || db.replicaNode())(TableName.Groups)
// eslint-disable-next-line
.where(buildFindFilter(filter))
.select(selectAllTableCols(TableName.Groups));
@@ -32,7 +32,7 @@ export const groupDALFactory = (db: TDbClient) => {
const findByOrgId = async (orgId: string, tx?: Knex) => {
try {
const docs = await (tx || db)(TableName.Groups)
const docs = await (tx || db.replicaNode())(TableName.Groups)
.where(`${TableName.Groups}.orgId`, orgId)
.leftJoin(TableName.OrgRoles, `${TableName.Groups}.roleId`, `${TableName.OrgRoles}.id`)
.select(selectAllTableCols(TableName.Groups))
@@ -74,11 +74,12 @@ export const groupDALFactory = (db: TDbClient) => {
username?: string;
}) => {
try {
let query = db(TableName.OrgMembership)
let query = db
.replicaNode()(TableName.OrgMembership)
.where(`${TableName.OrgMembership}.orgId`, orgId)
.join(TableName.Users, `${TableName.OrgMembership}.userId`, `${TableName.Users}.id`)
.leftJoin(TableName.UserGroupMembership, function () {
this.on(`${TableName.UserGroupMembership}.userId`, "=", `${TableName.Users}.id`).andOn(
.leftJoin(TableName.UserGroupMembership, (bd) => {
bd.on(`${TableName.UserGroupMembership}.userId`, "=", `${TableName.Users}.id`).andOn(
`${TableName.UserGroupMembership}.groupId`,
"=",
db.raw("?", [groupId])

View File

@@ -18,7 +18,7 @@ export const userGroupMembershipDALFactory = (db: TDbClient) => {
*/
const filterProjectsByUserMembership = async (userId: string, groupId: string, projectIds: string[], tx?: Knex) => {
try {
const userProjectMemberships: string[] = await (tx || db)(TableName.ProjectMembership)
const userProjectMemberships: string[] = await (tx || db.replicaNode())(TableName.ProjectMembership)
.where(`${TableName.ProjectMembership}.userId`, userId)
.whereIn(`${TableName.ProjectMembership}.projectId`, projectIds)
.pluck(`${TableName.ProjectMembership}.projectId`);
@@ -43,7 +43,8 @@ export const userGroupMembershipDALFactory = (db: TDbClient) => {
// special query
const findUserGroupMembershipsInProject = async (usernames: string[], projectId: string) => {
try {
const usernameDocs: string[] = await db(TableName.UserGroupMembership)
const usernameDocs: string[] = await db
.replicaNode()(TableName.UserGroupMembership)
.join(
TableName.GroupProjectMembership,
`${TableName.UserGroupMembership}.groupId`,
@@ -73,7 +74,7 @@ export const userGroupMembershipDALFactory = (db: TDbClient) => {
try {
// get list of groups in the project with id [projectId]
// that that are not the group with id [groupId]
const groups: string[] = await (tx || db)(TableName.GroupProjectMembership)
const groups: string[] = await (tx || db.replicaNode())(TableName.GroupProjectMembership)
.where(`${TableName.GroupProjectMembership}.projectId`, projectId)
.whereNot(`${TableName.GroupProjectMembership}.groupId`, groupId)
.pluck(`${TableName.GroupProjectMembership}.groupId`);
@@ -83,8 +84,8 @@ export const userGroupMembershipDALFactory = (db: TDbClient) => {
.where(`${TableName.UserGroupMembership}.groupId`, groupId)
.where(`${TableName.UserGroupMembership}.isPending`, false)
.join(TableName.Users, `${TableName.UserGroupMembership}.userId`, `${TableName.Users}.id`)
.leftJoin(TableName.ProjectMembership, function () {
this.on(`${TableName.Users}.id`, "=", `${TableName.ProjectMembership}.userId`).andOn(
.leftJoin(TableName.ProjectMembership, (bd) => {
bd.on(`${TableName.Users}.id`, "=", `${TableName.ProjectMembership}.userId`).andOn(
`${TableName.ProjectMembership}.projectId`,
"=",
db.raw("?", [projectId])
@@ -107,9 +108,9 @@ export const userGroupMembershipDALFactory = (db: TDbClient) => {
db.ref("publicKey").withSchema(TableName.UserEncryptionKey)
)
.where({ isGhost: false }) // MAKE SURE USER IS NOT A GHOST USER
.whereNotIn(`${TableName.UserGroupMembership}.userId`, function () {
.whereNotIn(`${TableName.UserGroupMembership}.userId`, (bd) => {
// eslint-disable-next-line @typescript-eslint/no-floating-promises
this.select(`${TableName.UserGroupMembership}.userId`)
bd.select(`${TableName.UserGroupMembership}.userId`)
.from(TableName.UserGroupMembership)
.whereIn(`${TableName.UserGroupMembership}.groupId`, groups);
});

View File

@@ -10,7 +10,8 @@ export const ldapGroupMapDALFactory = (db: TDbClient) => {
const findLdapGroupMapsByLdapConfigId = async (ldapConfigId: string) => {
try {
const docs = await db(TableName.LdapGroupMap)
const docs = await db
.replicaNode()(TableName.LdapGroupMap)
.where(`${TableName.LdapGroupMap}.ldapConfigId`, ldapConfigId)
.join(TableName.Groups, `${TableName.LdapGroupMap}.groupId`, `${TableName.Groups}.id`)
.select(selectAllTableCols(TableName.LdapGroupMap))

View File

@@ -9,7 +9,7 @@ export type TLicenseDALFactory = ReturnType<typeof licenseDALFactory>;
export const licenseDALFactory = (db: TDbClient) => {
const countOfOrgMembers = async (orgId: string | null, tx?: Knex) => {
try {
const doc = await (tx || db)(TableName.OrgMembership)
const doc = await (tx || db.replicaNode())(TableName.OrgMembership)
.where({ status: OrgMembershipStatus.Accepted })
.andWhere((bd) => {
if (orgId) {

View File

@@ -10,7 +10,8 @@ export type TPermissionDALFactory = ReturnType<typeof permissionDALFactory>;
export const permissionDALFactory = (db: TDbClient) => {
const getOrgPermission = async (userId: string, orgId: string) => {
try {
const membership = await db(TableName.OrgMembership)
const membership = await db
.replicaNode()(TableName.OrgMembership)
.leftJoin(TableName.OrgRoles, `${TableName.OrgMembership}.roleId`, `${TableName.OrgRoles}.id`)
.join(TableName.Organization, `${TableName.OrgMembership}.orgId`, `${TableName.Organization}.id`)
.where("userId", userId)
@@ -28,7 +29,8 @@ export const permissionDALFactory = (db: TDbClient) => {
const getOrgIdentityPermission = async (identityId: string, orgId: string) => {
try {
const membership = await db(TableName.IdentityOrgMembership)
const membership = await db
.replicaNode()(TableName.IdentityOrgMembership)
.leftJoin(TableName.OrgRoles, `${TableName.IdentityOrgMembership}.roleId`, `${TableName.OrgRoles}.id`)
.join(TableName.Organization, `${TableName.IdentityOrgMembership}.orgId`, `${TableName.Organization}.id`)
.where("identityId", identityId)
@@ -45,11 +47,13 @@ export const permissionDALFactory = (db: TDbClient) => {
const getProjectPermission = async (userId: string, projectId: string) => {
try {
const groups: string[] = await db(TableName.GroupProjectMembership)
const groups: string[] = await db
.replicaNode()(TableName.GroupProjectMembership)
.where(`${TableName.GroupProjectMembership}.projectId`, projectId)
.pluck(`${TableName.GroupProjectMembership}.groupId`);
const groupDocs = await db(TableName.UserGroupMembership)
const groupDocs = await db
.replicaNode()(TableName.UserGroupMembership)
.where(`${TableName.UserGroupMembership}.userId`, userId)
.whereIn(`${TableName.UserGroupMembership}.groupId`, groups)
.join(
@@ -231,7 +235,8 @@ export const permissionDALFactory = (db: TDbClient) => {
const getProjectIdentityPermission = async (identityId: string, projectId: string) => {
try {
const docs = await db(TableName.IdentityProjectMembership)
const docs = await db
.replicaNode()(TableName.IdentityProjectMembership)
.join(
TableName.IdentityProjectMembershipRole,
`${TableName.IdentityProjectMembershipRole}.projectMembershipId`,

View File

@@ -10,7 +10,8 @@ export const samlConfigDALFactory = (db: TDbClient) => {
const findEnforceableSamlCfg = async (orgId: string) => {
try {
const samlCfg = await db(TableName.SamlConfig)
const samlCfg = await db
.replicaNode()(TableName.SamlConfig)
.where({
orgId,
isActive: true

View File

@@ -30,7 +30,7 @@ export const secretApprovalPolicyDALFactory = (db: TDbClient) => {
const findById = async (id: string, tx?: Knex) => {
try {
const doc = await sapFindQuery(tx || db, {
const doc = await sapFindQuery(tx || db.replicaNode(), {
[`${TableName.SecretApprovalPolicy}.id` as "id"]: id
});
const formatedDoc = mergeOneToManyRelation(
@@ -52,7 +52,7 @@ export const secretApprovalPolicyDALFactory = (db: TDbClient) => {
const find = async (filter: TFindFilter<TSecretApprovalPolicies & { projectId: string }>, tx?: Knex) => {
try {
const docs = await sapFindQuery(tx || db, filter);
const docs = await sapFindQuery(tx || db.replicaNode(), filter);
const formatedDoc = mergeOneToManyRelation(
docs,
"id",

View File

@@ -62,7 +62,7 @@ export const secretApprovalRequestDALFactory = (db: TDbClient) => {
const findById = async (id: string, tx?: Knex) => {
try {
const sql = findQuery({ [`${TableName.SecretApprovalRequest}.id` as "id"]: id }, tx || db);
const sql = findQuery({ [`${TableName.SecretApprovalRequest}.id` as "id"]: id }, tx || db.replicaNode());
const docs = await sql;
const formatedDoc = sqlNestRelationships({
data: docs,
@@ -102,7 +102,7 @@ export const secretApprovalRequestDALFactory = (db: TDbClient) => {
const docs = await (tx || db)
.with(
"temp",
(tx || db)(TableName.SecretApprovalRequest)
(tx || db.replicaNode())(TableName.SecretApprovalRequest)
.join(TableName.SecretFolder, `${TableName.SecretApprovalRequest}.folderId`, `${TableName.SecretFolder}.id`)
.join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`)
.join(
@@ -148,7 +148,7 @@ export const secretApprovalRequestDALFactory = (db: TDbClient) => {
try {
// akhilmhdh: If ever u wanted a 1 to so many relationship connected with pagination
// this is the place u wanna look at.
const query = (tx || db)(TableName.SecretApprovalRequest)
const query = (tx || db.replicaNode())(TableName.SecretApprovalRequest)
.join(TableName.SecretFolder, `${TableName.SecretApprovalRequest}.folderId`, `${TableName.SecretFolder}.id`)
.join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`)
.join(

View File

@@ -47,7 +47,7 @@ export const secretApprovalRequestSecretDALFactory = (db: TDbClient) => {
const findByRequestId = async (requestId: string, tx?: Knex) => {
try {
const doc = await (tx || db)({
const doc = await (tx || db.replicaNode())({
secVerTag: TableName.SecretTag
})
.from(TableName.SecretApprovalRequestSecret)

View File

@@ -41,7 +41,7 @@ export const secretRotationDALFactory = (db: TDbClient) => {
const find = async (filter: TFindFilter<TSecretRotations & { projectId: string }>, tx?: Knex) => {
try {
const data = await findQuery(filter, tx || db);
const data = await findQuery(filter, tx || db.replicaNode());
return sqlNestRelationships({
data,
key: "id",
@@ -93,7 +93,7 @@ export const secretRotationDALFactory = (db: TDbClient) => {
const findById = async (id: string, tx?: Knex) => {
try {
const doc = await (tx || db)(TableName.SecretRotation)
const doc = await (tx || db.replicaNode())(TableName.SecretRotation)
.join(TableName.Environment, `${TableName.SecretRotation}.envId`, `${TableName.Environment}.id`)
.where({ [`${TableName.SecretRotation}.id` as "id"]: id })
.select(selectAllTableCols(TableName.SecretRotation))

View File

@@ -21,7 +21,7 @@ export const snapshotDALFactory = (db: TDbClient) => {
const findById = async (id: string, tx?: Knex) => {
try {
const data = await (tx || db)(TableName.Snapshot)
const data = await (tx || db.replicaNode())(TableName.Snapshot)
.where(`${TableName.Snapshot}.id`, id)
.join(TableName.Environment, `${TableName.Snapshot}.envId`, `${TableName.Environment}.id`)
.select(selectAllTableCols(TableName.Snapshot))
@@ -43,7 +43,7 @@ export const snapshotDALFactory = (db: TDbClient) => {
const countOfSnapshotsByFolderId = async (folderId: string, tx?: Knex) => {
try {
const doc = await (tx || db)(TableName.Snapshot)
const doc = await (tx || db.replicaNode())(TableName.Snapshot)
.where({ folderId })
.groupBy(["folderId"])
.count("folderId")
@@ -56,7 +56,7 @@ export const snapshotDALFactory = (db: TDbClient) => {
const findSecretSnapshotDataById = async (snapshotId: string, tx?: Knex) => {
try {
const data = await (tx || db)(TableName.Snapshot)
const data = await (tx || db.replicaNode())(TableName.Snapshot)
.where(`${TableName.Snapshot}.id`, snapshotId)
.join(TableName.Environment, `${TableName.Snapshot}.envId`, `${TableName.Environment}.id`)
.leftJoin(TableName.SnapshotSecret, `${TableName.Snapshot}.id`, `${TableName.SnapshotSecret}.snapshotId`)
@@ -309,7 +309,7 @@ export const snapshotDALFactory = (db: TDbClient) => {
// when we need to rollback we will pull from these snapshots
const findLatestSnapshotByFolderId = async (folderId: string, tx?: Knex) => {
try {
const docs = await (tx || db)(TableName.Snapshot)
const docs = await (tx || db.replicaNode())(TableName.Snapshot)
.where(`${TableName.Snapshot}.folderId`, folderId)
.join<TSecretSnapshots>(
(tx || db)(TableName.Snapshot).groupBy("folderId").max("createdAt").select("folderId").as("latestVersion"),

View File

@@ -10,6 +10,14 @@ const zodStrBool = z
.optional()
.transform((val) => val === "true");
const databaseReadReplicaSchema = z
.object({
DB_CONNECTION_URI: z.string().describe("Postgres read replica database connection string"),
DB_ROOT_CERT: zpStr(z.string().optional().describe("Postgres read replica database certificate string"))
})
.array()
.optional();
const envSchema = z
.object({
PORT: z.coerce.number().default(4000),
@@ -29,6 +37,7 @@ const envSchema = z
DB_USER: zpStr(z.string().describe("Postgres database username").optional()),
DB_PASSWORD: zpStr(z.string().describe("Postgres database password").optional()),
DB_NAME: zpStr(z.string().describe("Postgres database name").optional()),
DB_READ_REPLICAS: zpStr(z.string().describe("Postgres read replicas").optional()),
BCRYPT_SALT_ROUND: z.number().default(12),
NODE_ENV: z.enum(["development", "test", "production"]).default("production"),
SALT_ROUNDS: z.coerce.number().default(10),
@@ -127,6 +136,9 @@ const envSchema = z
})
.transform((data) => ({
...data,
DB_READ_REPLICAS: data.DB_READ_REPLICAS
? databaseReadReplicaSchema.parse(JSON.parse(data.DB_READ_REPLICAS))
: undefined,
isCloud: Boolean(data.LICENSE_SERVER_KEY),
isSmtpConfigured: Boolean(data.SMTP_HOST),
isRedisConfigured: Boolean(data.REDIS_URL),

View File

@@ -50,7 +50,7 @@ export const ormify = <DbOps extends object, Tname extends keyof Tables>(db: Kne
}),
findById: async (id: string, tx?: Knex) => {
try {
const result = await (tx || db)(tableName)
const result = await (tx || db.replicaNode())(tableName)
.where({ id } as never)
.first("*");
return result;
@@ -60,7 +60,7 @@ export const ormify = <DbOps extends object, Tname extends keyof Tables>(db: Kne
},
findOne: async (filter: Partial<Tables[Tname]["base"]>, tx?: Knex) => {
try {
const res = await (tx || db)(tableName).where(filter).first("*");
const res = await (tx || db.replicaNode())(tableName).where(filter).first("*");
return res;
} catch (error) {
throw new DatabaseError({ error, name: "Find one" });
@@ -71,7 +71,7 @@ export const ormify = <DbOps extends object, Tname extends keyof Tables>(db: Kne
{ offset, limit, sort, tx }: TFindOpt<Tables[Tname]["base"]> = {}
) => {
try {
const query = (tx || db)(tableName).where(buildFindFilter(filter));
const query = (tx || db.replicaNode())(tableName).where(buildFindFilter(filter));
if (limit) void query.limit(limit);
if (offset) void query.offset(offset);
if (sort) {

View File

@@ -15,7 +15,11 @@ const run = async () => {
const appCfg = initEnvConfig(logger);
const db = initDbConnection({
dbConnectionUri: appCfg.DB_CONNECTION_URI,
dbRootCert: appCfg.DB_ROOT_CERT
dbRootCert: appCfg.DB_ROOT_CERT,
readReplicas: appCfg.DB_READ_REPLICAS?.map((el) => ({
dbRootCert: el.DB_ROOT_CERT,
dbConnectionUri: el.DB_CONNECTION_URI
}))
});
const smtp = smtpServiceFactory(formatSmtpConfig());

View File

@@ -14,7 +14,7 @@ export const tokenDALFactory = (db: TDbClient) => {
const findOneTokenSession = async (filter: Partial<TAuthTokenSessions>): Promise<TAuthTokenSessions | undefined> => {
try {
const doc = await db(TableName.AuthTokenSession).where(filter).first();
const doc = await db.replicaNode()(TableName.AuthTokenSession).where(filter).first();
return doc;
} catch (error) {
throw new DatabaseError({ error, name: "FindOneTokenSession" });
@@ -44,7 +44,7 @@ export const tokenDALFactory = (db: TDbClient) => {
const findTokenSessions = async (filter: Partial<TAuthTokenSessions>, tx?: Knex) => {
try {
const sessions = await (tx || db)(TableName.AuthTokenSession).where(filter);
const sessions = await (tx || db.replicaNode())(TableName.AuthTokenSession).where(filter);
return sessions;
} catch (error) {
throw new DatabaseError({ name: "Find all token session", error });

View File

@@ -16,6 +16,7 @@ export const certificateAuthorityDALFactory = (db: TDbClient) => {
parentCaId?: string;
encryptedCertificate: Buffer;
}[] = await db
.replicaNode()
.withRecursive("cte", (cte) => {
void cte
.select("ca.id as caId", "ca.parentCaId", "cert.encryptedCertificate")

View File

@@ -14,7 +14,8 @@ export const certificateDALFactory = (db: TDbClient) => {
count: string;
}
const count = await db(TableName.Certificate)
const count = await db
.replicaNode()(TableName.Certificate)
.join(TableName.CertificateAuthority, `${TableName.Certificate}.caId`, `${TableName.CertificateAuthority}.id`)
.join(TableName.Project, `${TableName.CertificateAuthority}.projectId`, `${TableName.Project}.id`)
.where(`${TableName.Project}.id`, projectId)

View File

@@ -12,7 +12,7 @@ export const groupProjectDALFactory = (db: TDbClient) => {
const findByProjectId = async (projectId: string, tx?: Knex) => {
try {
const docs = await (tx || db)(TableName.GroupProjectMembership)
const docs = await (tx || db.replicaNode())(TableName.GroupProjectMembership)
.where(`${TableName.GroupProjectMembership}.projectId`, projectId)
.join(TableName.Groups, `${TableName.GroupProjectMembership}.groupId`, `${TableName.Groups}.id`)
.join(

View File

@@ -12,7 +12,7 @@ export const identityAccessTokenDALFactory = (db: TDbClient) => {
const findOne = async (filter: Partial<TIdentityAccessTokens>, tx?: Knex) => {
try {
const doc = await (tx || db)(TableName.IdentityAccessToken)
const doc = await (tx || db.replicaNode())(TableName.IdentityAccessToken)
.where(filter)
.join(TableName.Identity, `${TableName.Identity}.id`, `${TableName.IdentityAccessToken}.identityId`)
.leftJoin(TableName.IdentityUaClientSecret, (qb) => {

View File

@@ -12,7 +12,7 @@ export const identityProjectDALFactory = (db: TDbClient) => {
const findByProjectId = async (projectId: string, filter: { identityId?: string } = {}, tx?: Knex) => {
try {
const docs = await (tx || db)(TableName.IdentityProjectMembership)
const docs = await (tx || db.replicaNode())(TableName.IdentityProjectMembership)
.where(`${TableName.IdentityProjectMembership}.projectId`, projectId)
.join(TableName.Identity, `${TableName.IdentityProjectMembership}.identityId`, `${TableName.Identity}.id`)
.where((qb) => {

View File

@@ -12,7 +12,7 @@ export const identityOrgDALFactory = (db: TDbClient) => {
const findOne = async (filter: Partial<TIdentityOrgMemberships>, tx?: Knex) => {
try {
const [data] = await (tx || db)(TableName.IdentityOrgMembership)
const [data] = await (tx || db.replicaNode())(TableName.IdentityOrgMembership)
.where(filter)
.join(TableName.Identity, `${TableName.IdentityOrgMembership}.identityId`, `${TableName.Identity}.id`)
.select(selectAllTableCols(TableName.IdentityOrgMembership))
@@ -29,7 +29,7 @@ export const identityOrgDALFactory = (db: TDbClient) => {
const find = async (filter: Partial<TIdentityOrgMemberships>, tx?: Knex) => {
try {
const docs = await (tx || db)(TableName.IdentityOrgMembership)
const docs = await (tx || db.replicaNode())(TableName.IdentityOrgMembership)
.where(filter)
.join(TableName.Identity, `${TableName.IdentityOrgMembership}.identityId`, `${TableName.Identity}.id`)
.leftJoin(TableName.OrgRoles, `${TableName.IdentityOrgMembership}.roleId`, `${TableName.OrgRoles}.id`)

View File

@@ -22,7 +22,7 @@ export const integrationDALFactory = (db: TDbClient) => {
const find = async (filter: Partial<TIntegrations>, tx?: Knex) => {
try {
const docs = await integrationFindQuery(tx || db, filter);
const docs = await integrationFindQuery(tx || db.replicaNode(), filter);
return docs.map(({ envId, envSlug, envName, ...el }) => ({
...el,
environment: {
@@ -38,7 +38,7 @@ export const integrationDALFactory = (db: TDbClient) => {
const findOne = async (filter: Partial<TIntegrations>, tx?: Knex) => {
try {
const doc = await integrationFindQuery(tx || db, filter).first();
const doc = await integrationFindQuery(tx || db.replicaNode(), filter).first();
if (!doc) return;
const { envName: name, envSlug: slug, envId: id, ...el } = doc;
@@ -50,7 +50,7 @@ export const integrationDALFactory = (db: TDbClient) => {
const findById = async (id: string, tx?: Knex) => {
try {
const doc = await integrationFindQuery(tx || db, {
const doc = await integrationFindQuery(tx || db.replicaNode(), {
[`${TableName.Integration}.id` as "id"]: id
}).first();
if (!doc) return;
@@ -64,7 +64,7 @@ export const integrationDALFactory = (db: TDbClient) => {
const findByProjectId = async (projectId: string, tx?: Knex) => {
try {
const integrations = await (tx || db)(TableName.Integration)
const integrations = await (tx || db.replicaNode())(TableName.Integration)
.where(`${TableName.Environment}.projectId`, projectId)
.join(TableName.Environment, `${TableName.Integration}.envId`, `${TableName.Environment}.id`)
.select(db.ref("name").withSchema(TableName.Environment).as("envName"))
@@ -90,7 +90,7 @@ export const integrationDALFactory = (db: TDbClient) => {
// used for syncing secrets
// this will populate integration auth also
const findByProjectIdV2 = async (projectId: string, environment: string, tx?: Knex) => {
const docs = await (tx || db)(TableName.Integration)
const docs = await (tx || db.replicaNode())(TableName.Integration)
.where(`${TableName.Environment}.projectId`, projectId)
.where("isActive", true)
.where(`${TableName.Environment}.slug`, environment)

View File

@@ -16,7 +16,7 @@ export const incidentContactDALFactory = (db: TDbClient) => {
const findByOrgId = async (orgId: string) => {
try {
const incidentContacts = await db(TableName.IncidentContact).where({ orgId });
const incidentContacts = await db.replicaNode()(TableName.IncidentContact).where({ orgId });
return incidentContacts;
} catch (error) {
throw new DatabaseError({ name: "Incident contact list", error });
@@ -25,7 +25,8 @@ export const incidentContactDALFactory = (db: TDbClient) => {
const findOne = async (orgId: string, data: Partial<TIncidentContacts>) => {
try {
const incidentContacts = await db(TableName.IncidentContact)
const incidentContacts = await db
.replicaNode()(TableName.IncidentContact)
.where({ orgId, ...data })
.first();
return incidentContacts;

View File

@@ -20,7 +20,7 @@ export const orgDALFactory = (db: TDbClient) => {
const findOrgById = async (orgId: string) => {
try {
const org = await db(TableName.Organization).where({ id: orgId }).first();
const org = await db.replicaNode()(TableName.Organization).where({ id: orgId }).first();
return org;
} catch (error) {
throw new DatabaseError({ error, name: "Find org by id" });
@@ -30,7 +30,8 @@ export const orgDALFactory = (db: TDbClient) => {
// special query
const findAllOrgsByUserId = async (userId: string): Promise<TOrganizations[]> => {
try {
const org = await db(TableName.OrgMembership)
const org = await db
.replicaNode()(TableName.OrgMembership)
.where({ userId })
.join(TableName.Organization, `${TableName.OrgMembership}.orgId`, `${TableName.Organization}.id`)
.select(selectAllTableCols(TableName.Organization));
@@ -42,7 +43,8 @@ export const orgDALFactory = (db: TDbClient) => {
const findOrgByProjectId = async (projectId: string): Promise<TOrganizations> => {
try {
const [org] = await db(TableName.Project)
const [org] = await db
.replicaNode()(TableName.Project)
.where({ [`${TableName.Project}.id` as "id"]: projectId })
.join(TableName.Organization, `${TableName.Project}.orgId`, `${TableName.Organization}.id`)
.select(selectAllTableCols(TableName.Organization));
@@ -56,7 +58,8 @@ export const orgDALFactory = (db: TDbClient) => {
// special query
const findAllOrgMembers = async (orgId: string) => {
try {
const members = await db(TableName.OrgMembership)
const members = await db
.replicaNode()(TableName.OrgMembership)
.where(`${TableName.OrgMembership}.orgId`, orgId)
.join(TableName.Users, `${TableName.OrgMembership}.userId`, `${TableName.Users}.id`)
.leftJoin<TUserEncryptionKeys>(
@@ -95,7 +98,8 @@ export const orgDALFactory = (db: TDbClient) => {
count: string;
}
const count = await db(TableName.OrgMembership)
const count = await db
.replicaNode()(TableName.OrgMembership)
.where(`${TableName.OrgMembership}.orgId`, orgId)
.count("*")
.join(TableName.Users, `${TableName.OrgMembership}.userId`, `${TableName.Users}.id`)
@@ -110,7 +114,8 @@ export const orgDALFactory = (db: TDbClient) => {
const findOrgMembersByUsername = async (orgId: string, usernames: string[]) => {
try {
const members = await db(TableName.OrgMembership)
const members = await db
.replicaNode()(TableName.OrgMembership)
.where(`${TableName.OrgMembership}.orgId`, orgId)
.join(TableName.Users, `${TableName.OrgMembership}.userId`, `${TableName.Users}.id`)
.leftJoin<TUserEncryptionKeys>(
@@ -145,7 +150,8 @@ export const orgDALFactory = (db: TDbClient) => {
const findOrgGhostUser = async (orgId: string) => {
try {
const member = await db(TableName.OrgMembership)
const member = await db
.replicaNode()(TableName.OrgMembership)
.where({ orgId })
.join(TableName.Users, `${TableName.OrgMembership}.userId`, `${TableName.Users}.id`)
.leftJoin(TableName.UserEncryptionKey, `${TableName.UserEncryptionKey}.userId`, `${TableName.Users}.id`)
@@ -169,7 +175,8 @@ export const orgDALFactory = (db: TDbClient) => {
const ghostUserExists = async (orgId: string) => {
try {
const member = await db(TableName.OrgMembership)
const member = await db
.replicaNode()(TableName.OrgMembership)
.where({ orgId })
.join(TableName.Users, `${TableName.OrgMembership}.userId`, `${TableName.Users}.id`)
.leftJoin(TableName.UserEncryptionKey, `${TableName.UserEncryptionKey}.userId`, `${TableName.Users}.id`)
@@ -257,7 +264,7 @@ export const orgDALFactory = (db: TDbClient) => {
{ offset, limit, sort, tx }: TFindOpt<TOrgMemberships> = {}
) => {
try {
const query = (tx || db)(TableName.OrgMembership)
const query = (tx || db.replicaNode())(TableName.OrgMembership)
// eslint-disable-next-line
.where(buildFindFilter(filter))
.join(TableName.Users, `${TableName.Users}.id`, `${TableName.OrgMembership}.userId`)

View File

@@ -12,7 +12,7 @@ export const projectBotDALFactory = (db: TDbClient) => {
const findOne = async (filter: Partial<TProjectBots>, tx?: Knex) => {
try {
const bot = await (tx || db)(TableName.ProjectBot)
const bot = await (tx || db.replicaNode())(TableName.ProjectBot)
.where(filter)
.leftJoin(TableName.Users, `${TableName.ProjectBot}.senderId`, `${TableName.Users}.id`)
.leftJoin(TableName.UserEncryptionKey, `${TableName.UserEncryptionKey}.userId`, `${TableName.Users}.id`)

View File

@@ -12,7 +12,9 @@ export const projectEnvDALFactory = (db: TDbClient) => {
const findBySlugs = async (projectId: string, env: string[], tx?: Knex) => {
try {
const envs = await (tx || db)(TableName.Environment).where("projectId", projectId).whereIn("slug", env);
const envs = await (tx || db.replicaNode())(TableName.Environment)
.where("projectId", projectId)
.whereIn("slug", env);
return envs;
} catch (error) {
throw new DatabaseError({ error, name: "Find by slugs" });

View File

@@ -16,7 +16,7 @@ export const projectKeyDALFactory = (db: TDbClient) => {
tx?: Knex
): Promise<(TProjectKeys & { sender: { publicKey: string } }) | undefined> => {
try {
const projectKey = await (tx || db)(TableName.ProjectKeys)
const projectKey = await (tx || db.replicaNode())(TableName.ProjectKeys)
.join(TableName.Users, `${TableName.ProjectKeys}.senderId`, `${TableName.Users}.id`)
.join(TableName.UserEncryptionKey, `${TableName.UserEncryptionKey}.userId`, `${TableName.Users}.id`)
.where({ projectId, receiverId: userId })
@@ -34,7 +34,7 @@ export const projectKeyDALFactory = (db: TDbClient) => {
const findAllProjectUserPubKeys = async (projectId: string, tx?: Knex) => {
try {
const pubKeys = await (tx || db)(TableName.ProjectMembership)
const pubKeys = await (tx || db.replicaNode())(TableName.ProjectMembership)
.where({ projectId })
.join(TableName.Users, `${TableName.ProjectMembership}.userId`, `${TableName.Users}.id`)
.join(TableName.UserEncryptionKey, `${TableName.Users}.id`, `${TableName.UserEncryptionKey}.userId`)

View File

@@ -13,7 +13,8 @@ export const projectMembershipDALFactory = (db: TDbClient) => {
// special query
const findAllProjectMembers = async (projectId: string, filter: { usernames?: string[]; username?: string } = {}) => {
try {
const docs = await db(TableName.ProjectMembership)
const docs = await db
.replicaNode()(TableName.ProjectMembership)
.where({ [`${TableName.ProjectMembership}.projectId` as "projectId"]: projectId })
.join(TableName.Users, `${TableName.ProjectMembership}.userId`, `${TableName.Users}.id`)
.where((qb) => {
@@ -108,7 +109,7 @@ export const projectMembershipDALFactory = (db: TDbClient) => {
const findProjectGhostUser = async (projectId: string, tx?: Knex) => {
try {
const ghostUser = await (tx || db)(TableName.ProjectMembership)
const ghostUser = await (tx || db.replicaNode())(TableName.ProjectMembership)
.where({ projectId })
.join(TableName.Users, `${TableName.ProjectMembership}.userId`, `${TableName.Users}.id`)
.select(selectAllTableCols(TableName.Users))
@@ -123,7 +124,8 @@ export const projectMembershipDALFactory = (db: TDbClient) => {
const findMembershipsByUsername = async (projectId: string, usernames: string[]) => {
try {
const members = await db(TableName.ProjectMembership)
const members = await db
.replicaNode()(TableName.ProjectMembership)
.where({ projectId })
.join(TableName.Users, `${TableName.ProjectMembership}.userId`, `${TableName.Users}.id`)
.join<TUserEncryptionKeys>(
@@ -149,7 +151,8 @@ export const projectMembershipDALFactory = (db: TDbClient) => {
const findProjectMembershipsByUserId = async (orgId: string, userId: string) => {
try {
const memberships = await db(TableName.ProjectMembership)
const memberships = await db
.replicaNode()(TableName.ProjectMembership)
.where({ userId })
.join(TableName.Project, `${TableName.ProjectMembership}.projectId`, `${TableName.Project}.id`)
.where({ [`${TableName.Project}.orgId` as "orgId"]: orgId })

View File

@@ -14,7 +14,8 @@ export const projectDALFactory = (db: TDbClient) => {
const findAllProjects = async (userId: string) => {
try {
const workspaces = await db(TableName.ProjectMembership)
const workspaces = await db
.replicaNode()(TableName.ProjectMembership)
.where({ userId })
.join(TableName.Project, `${TableName.ProjectMembership}.projectId`, `${TableName.Project}.id`)
.leftJoin(TableName.Environment, `${TableName.Environment}.projectId`, `${TableName.Project}.id`)
@@ -83,7 +84,7 @@ export const projectDALFactory = (db: TDbClient) => {
const findProjectGhostUser = async (projectId: string, tx?: Knex) => {
try {
const ghostUser = await (tx || db)(TableName.ProjectMembership)
const ghostUser = await (tx || db.replicaNode())(TableName.ProjectMembership)
.where({ projectId })
.join(TableName.Users, `${TableName.ProjectMembership}.userId`, `${TableName.Users}.id`)
.select(selectAllTableCols(TableName.Users))
@@ -109,7 +110,8 @@ export const projectDALFactory = (db: TDbClient) => {
const findAllProjectsByIdentity = async (identityId: string) => {
try {
const workspaces = await db(TableName.IdentityProjectMembership)
const workspaces = await db
.replicaNode()(TableName.IdentityProjectMembership)
.where({ identityId })
.join(TableName.Project, `${TableName.IdentityProjectMembership}.projectId`, `${TableName.Project}.id`)
.leftJoin(TableName.Environment, `${TableName.Environment}.projectId`, `${TableName.Project}.id`)
@@ -151,7 +153,8 @@ export const projectDALFactory = (db: TDbClient) => {
const findProjectById = async (id: string) => {
try {
const workspaces = await db(TableName.Project)
const workspaces = await db
.replicaNode()(TableName.Project)
.where(`${TableName.Project}.id`, id)
.leftJoin(TableName.Environment, `${TableName.Environment}.projectId`, `${TableName.Project}.id`)
.select(
@@ -198,7 +201,8 @@ export const projectDALFactory = (db: TDbClient) => {
throw new BadRequestError({ message: "Organization ID is required when querying with slugs" });
}
const projects = await db(TableName.Project)
const projects = await db
.replicaNode()(TableName.Project)
.where(`${TableName.Project}.slug`, slug)
.where(`${TableName.Project}.orgId`, orgId)
.leftJoin(TableName.Environment, `${TableName.Environment}.projectId`, `${TableName.Project}.id`)

View File

@@ -12,7 +12,7 @@ export const secretBlindIndexDALFactory = (db: TDbClient) => {
const countOfSecretsWithNullSecretBlindIndex = async (projectId: string, tx?: Knex) => {
try {
const doc = await (tx || db)(TableName.Secret)
const doc = await (tx || db.replicaNode())(TableName.Secret)
.leftJoin(TableName.SecretFolder, `${TableName.SecretFolder}.id`, `${TableName.Secret}.folderId`)
.leftJoin(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolder}.envId`)
.where({ projectId })
@@ -26,7 +26,7 @@ export const secretBlindIndexDALFactory = (db: TDbClient) => {
const findAllSecretsByProjectId = async (projectId: string, tx?: Knex) => {
try {
const docs = await (tx || db)(TableName.Secret)
const docs = await (tx || db.replicaNode())(TableName.Secret)
.leftJoin(TableName.SecretFolder, `${TableName.SecretFolder}.id`, `${TableName.Secret}.folderId`)
.leftJoin(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolder}.envId`)
.where({ projectId })
@@ -43,7 +43,7 @@ export const secretBlindIndexDALFactory = (db: TDbClient) => {
const findSecretsByProjectId = async (projectId: string, secretIds: string[], tx?: Knex) => {
try {
const docs = await (tx || db)(TableName.Secret)
const docs = await (tx || db.replicaNode())(TableName.Secret)
.leftJoin(TableName.SecretFolder, `${TableName.SecretFolder}.id`, `${TableName.Secret}.folderId`)
.leftJoin(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolder}.envId`)
.where({ projectId })

View File

@@ -211,7 +211,12 @@ export const secretFolderDALFactory = (db: TDbClient) => {
const findBySecretPath = async (projectId: string, environment: string, path: string, tx?: Knex) => {
try {
const folder = await sqlFindFolderByPathQuery(tx || db, projectId, environment, removeTrailingSlash(path))
const folder = await sqlFindFolderByPathQuery(
tx || db.replicaNode(),
projectId,
environment,
removeTrailingSlash(path)
)
.orderBy("depth", "desc")
.first();
if (folder && folder.path !== removeTrailingSlash(path)) {
@@ -230,7 +235,12 @@ export const secretFolderDALFactory = (db: TDbClient) => {
// it will stop automatically at /path2
const findClosestFolder = async (projectId: string, environment: string, path: string, tx?: Knex) => {
try {
const folder = await sqlFindFolderByPathQuery(tx || db, projectId, environment, removeTrailingSlash(path))
const folder = await sqlFindFolderByPathQuery(
tx || db.replicaNode(),
projectId,
environment,
removeTrailingSlash(path)
)
.orderBy("depth", "desc")
.first();
if (!folder) return;
@@ -247,7 +257,7 @@ export const secretFolderDALFactory = (db: TDbClient) => {
envId,
secretPath: removeTrailingSlash(secretPath)
}));
const folders = await sqlFindMultipleFolderByEnvPathQuery(tx || db, formatedQuery);
const folders = await sqlFindMultipleFolderByEnvPathQuery(tx || db.replicaNode(), formatedQuery);
return formatedQuery.map(({ envId, secretPath }) =>
folders.find(({ path: targetPath, envId: targetEnvId }) => targetPath === secretPath && targetEnvId === envId)
);
@@ -260,7 +270,7 @@ export const secretFolderDALFactory = (db: TDbClient) => {
// that is instances in which for a given folderid find the secret path
const findSecretPathByFolderIds = async (projectId: string, folderIds: string[], tx?: Knex) => {
try {
const folders = await sqlFindSecretPathByFolderId(tx || db, projectId, folderIds);
const folders = await sqlFindSecretPathByFolderId(tx || db.replicaNode(), projectId, folderIds);
// travelling all the way from leaf node to root contains real path
const rootFolders = groupBy(
@@ -299,7 +309,7 @@ export const secretFolderDALFactory = (db: TDbClient) => {
const findById = async (id: string, tx?: Knex) => {
try {
const folder = await (tx || db)(TableName.SecretFolder)
const folder = await (tx || db.replicaNode())(TableName.SecretFolder)
.where({ [`${TableName.SecretFolder}.id` as "id"]: id })
.join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`)
.select(selectAllTableCols(TableName.SecretFolder))

View File

@@ -13,7 +13,7 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => {
// This will fetch all latest secret versions from a folder
const findLatestVersionByFolderId = async (folderId: string, tx?: Knex) => {
try {
const docs = await (tx || db)(TableName.SecretFolderVersion)
const docs = await (tx || db.replicaNode())(TableName.SecretFolderVersion)
.join(TableName.SecretFolder, `${TableName.SecretFolderVersion}.folderId`, `${TableName.SecretFolder}.id`)
.where({ parentId: folderId, isReserved: false })
.join<TSecretFolderVersions>(
@@ -38,7 +38,9 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => {
const findLatestFolderVersions = async (folderIds: string[], tx?: Knex) => {
try {
const docs: Array<TSecretFolderVersions & { max: number }> = await (tx || db)(TableName.SecretFolderVersion)
const docs: Array<TSecretFolderVersions & { max: number }> = await (tx || db.replicaNode())(
TableName.SecretFolderVersion
)
.whereIn("folderId", folderIds)
.join(
(tx || db)(TableName.SecretFolderVersion)

View File

@@ -51,7 +51,7 @@ export const secretImportDALFactory = (db: TDbClient) => {
const find = async (filter: Partial<TSecretImports & { projectId: string }>, tx?: Knex) => {
try {
const docs = await (tx || db)(TableName.SecretImport)
const docs = await (tx || db.replicaNode())(TableName.SecretImport)
.where(filter)
.join(TableName.Environment, `${TableName.SecretImport}.importEnv`, `${TableName.Environment}.id`)
.select(
@@ -72,7 +72,7 @@ export const secretImportDALFactory = (db: TDbClient) => {
const findByFolderIds = async (folderIds: string[], tx?: Knex) => {
try {
const docs = await (tx || db)(TableName.SecretImport)
const docs = await (tx || db.replicaNode())(TableName.SecretImport)
.whereIn("folderId", folderIds)
.where("isReplication", false)
.join(TableName.Environment, `${TableName.SecretImport}.importEnv`, `${TableName.Environment}.id`)

View File

@@ -13,7 +13,7 @@ export const secretTagDALFactory = (db: TDbClient) => {
const findManyTagsById = async (projectId: string, ids: string[], tx?: Knex) => {
try {
const tags = await (tx || db)(TableName.SecretTag).where({ projectId }).whereIn("id", ids);
const tags = await (tx || db.replicaNode())(TableName.SecretTag).where({ projectId }).whereIn("id", ids);
return tags;
} catch (error) {
throw new DatabaseError({ error, name: "Find all by ids" });

View File

@@ -114,7 +114,7 @@ export const secretDALFactory = (db: TDbClient) => {
userId = undefined;
}
const secs = await (tx || db)(TableName.Secret)
const secs = await (tx || db.replicaNode())(TableName.Secret)
.where({ folderId })
.where((bd) => {
void bd.whereNull("userId").orWhere({ userId: userId || null });
@@ -152,7 +152,7 @@ export const secretDALFactory = (db: TDbClient) => {
const getSecretTags = async (secretId: string, tx?: Knex) => {
try {
const tags = await (tx || db)(TableName.JnSecretTag)
const tags = await (tx || db.replicaNode())(TableName.JnSecretTag)
.join(TableName.SecretTag, `${TableName.JnSecretTag}.${TableName.SecretTag}Id`, `${TableName.SecretTag}.id`)
.where({ [`${TableName.Secret}Id` as const]: secretId })
.select(db.ref("id").withSchema(TableName.SecretTag).as("tagId"))
@@ -179,7 +179,7 @@ export const secretDALFactory = (db: TDbClient) => {
userId = undefined;
}
const secs = await (tx || db)(TableName.Secret)
const secs = await (tx || db.replicaNode())(TableName.Secret)
.whereIn("folderId", folderIds)
.where((bd) => {
void bd.whereNull("userId").orWhere({ userId: userId || null });
@@ -223,7 +223,7 @@ export const secretDALFactory = (db: TDbClient) => {
) => {
if (!blindIndexes.length) return [];
try {
const secrets = await (tx || db)(TableName.Secret)
const secrets = await (tx || db.replicaNode())(TableName.Secret)
.where({ folderId })
.where((bd) => {
blindIndexes.forEach((el) => {
@@ -278,7 +278,7 @@ export const secretDALFactory = (db: TDbClient) => {
const findReferencedSecretReferences = async (projectId: string, envSlug: string, secretPath: string, tx?: Knex) => {
try {
const docs = await (tx || db)(TableName.SecretReference)
const docs = await (tx || db.replicaNode())(TableName.SecretReference)
.where({
secretPath,
environment: envSlug
@@ -298,7 +298,7 @@ export const secretDALFactory = (db: TDbClient) => {
// special query to backfill secret value
const findAllProjectSecretValues = async (projectId: string, tx?: Knex) => {
try {
const docs = await (tx || db)(TableName.Secret)
const docs = await (tx || db.replicaNode())(TableName.Secret)
.join(TableName.SecretFolder, `${TableName.Secret}.folderId`, `${TableName.SecretFolder}.id`)
.join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`)
.where("projectId", projectId)
@@ -313,7 +313,7 @@ export const secretDALFactory = (db: TDbClient) => {
const findOneWithTags = async (filter: Partial<TSecrets>, tx?: Knex) => {
try {
const rawDocs = await (tx || db)(TableName.Secret)
const rawDocs = await (tx || db.replicaNode())(TableName.Secret)
.where(filter)
.leftJoin(TableName.JnSecretTag, `${TableName.Secret}.id`, `${TableName.JnSecretTag}.${TableName.Secret}Id`)
.leftJoin(TableName.SecretTag, `${TableName.JnSecretTag}.${TableName.SecretTag}Id`, `${TableName.SecretTag}.id`)

View File

@@ -13,7 +13,7 @@ export const secretVersionDALFactory = (db: TDbClient) => {
// This will fetch all latest secret versions from a folder
const findLatestVersionByFolderId = async (folderId: string, tx?: Knex) => {
try {
const docs = await (tx || db)(TableName.SecretVersion)
const docs = await (tx || db.replicaNode())(TableName.SecretVersion)
.where(`${TableName.SecretVersion}.folderId`, folderId)
.join(TableName.Secret, `${TableName.Secret}.id`, `${TableName.SecretVersion}.secretId`)
.join<TSecretVersions, TSecretVersions & { secretId: string; max: number }>(
@@ -90,7 +90,7 @@ export const secretVersionDALFactory = (db: TDbClient) => {
const findLatestVersionMany = async (folderId: string, secretIds: string[], tx?: Knex) => {
try {
if (!secretIds.length) return {};
const docs: Array<TSecretVersions & { max: number }> = await (tx || db)(TableName.SecretVersion)
const docs: Array<TSecretVersions & { max: number }> = await (tx || db.replicaNode())(TableName.SecretVersion)
.where("folderId", folderId)
.whereIn(`${TableName.SecretVersion}.secretId`, secretIds)
.join(

View File

@@ -12,7 +12,7 @@ export const serviceTokenDALFactory = (db: TDbClient) => {
const findById = async (id: string, tx?: Knex) => {
try {
const doc = await (tx || db)(TableName.ServiceToken)
const doc = await (tx || db.replicaNode())(TableName.ServiceToken)
.leftJoin<TUsers>(
TableName.Users,
`${TableName.Users}.id`,

View File

@@ -22,7 +22,8 @@ export const userDALFactory = (db: TDbClient) => {
// -------------------------
const findUserEncKeyByUsername = async ({ username }: { username: string }) => {
try {
return await db(TableName.Users)
return await db
.replicaNode()(TableName.Users)
.where({
username,
isGhost: false
@@ -36,7 +37,7 @@ export const userDALFactory = (db: TDbClient) => {
const findUserEncKeyByUserIdsBatch = async ({ userIds }: { userIds: string[] }, tx?: Knex) => {
try {
return await (tx || db)(TableName.Users)
return await (tx || db.replicaNode())(TableName.Users)
.where({
isGhost: false
})
@@ -49,7 +50,8 @@ export const userDALFactory = (db: TDbClient) => {
const findUserEncKeyByUserId = async (userId: string) => {
try {
const user = await db(TableName.Users)
const user = await db
.replicaNode()(TableName.Users)
.where(`${TableName.Users}.id`, userId)
.join(TableName.UserEncryptionKey, `${TableName.Users}.id`, `${TableName.UserEncryptionKey}.userId`)
.first();
@@ -65,7 +67,8 @@ export const userDALFactory = (db: TDbClient) => {
const findUserByProjectMembershipId = async (projectMembershipId: string) => {
try {
return await db(TableName.ProjectMembership)
return await db
.replicaNode()(TableName.ProjectMembership)
.where({ [`${TableName.ProjectMembership}.id` as "id"]: projectMembershipId })
.join(TableName.Users, `${TableName.ProjectMembership}.userId`, `${TableName.Users}.id`)
.first();
@@ -76,7 +79,8 @@ export const userDALFactory = (db: TDbClient) => {
const findUsersByProjectMembershipIds = async (projectMembershipIds: string[]) => {
try {
return await db(TableName.ProjectMembership)
return await db
.replicaNode()(TableName.ProjectMembership)
.whereIn(`${TableName.ProjectMembership}.id`, projectMembershipIds)
.join(TableName.Users, `${TableName.ProjectMembership}.userId`, `${TableName.Users}.id`)
.select("*");
@@ -128,7 +132,7 @@ export const userDALFactory = (db: TDbClient) => {
// ---------------------
const findOneUserAction = (filter: TUserActionsUpdate, tx?: Knex) => {
try {
return (tx || db)(TableName.UserAction).where(filter).first("*");
return (tx || db.replicaNode())(TableName.UserAction).where(filter).first("*");
} catch (error) {
throw new DatabaseError({ error, name: "Find one user action" });
}

View File

@@ -22,7 +22,7 @@ export const webhookDALFactory = (db: TDbClient) => {
const find = async (filter: Partial<TWebhooks>, tx?: Knex) => {
try {
const docs = await webhookFindQuery(tx || db, filter);
const docs = await webhookFindQuery(tx || db.replicaNode(), filter);
return docs.map(({ envId, envSlug, envName, ...el }) => ({
...el,
envId,
@@ -39,7 +39,7 @@ export const webhookDALFactory = (db: TDbClient) => {
const findOne = async (filter: Partial<TWebhooks>, tx?: Knex) => {
try {
const doc = await webhookFindQuery(tx || db, filter).first();
const doc = await webhookFindQuery(tx || db.replicaNode(), filter).first();
if (!doc) return;
const { envName: name, envSlug: slug, envId: id, ...el } = doc;
@@ -51,7 +51,7 @@ export const webhookDALFactory = (db: TDbClient) => {
const findById = async (id: string, tx?: Knex) => {
try {
const doc = await webhookFindQuery(tx || db, {
const doc = await webhookFindQuery(tx || db.replicaNode(), {
[`${TableName.Webhook}.id` as "id"]: id
}).first();
if (!doc) return;
@@ -65,7 +65,7 @@ export const webhookDALFactory = (db: TDbClient) => {
const findAllWebhooks = async (projectId: string, environment?: string, secretPath?: string, tx?: Knex) => {
try {
const webhooks = await (tx || db)(TableName.Webhook)
const webhooks = await (tx || db.replicaNode())(TableName.Webhook)
.where(`${TableName.Environment}.projectId`, projectId)
.where((qb) => {
if (environment) {

View File

@@ -0,0 +1,191 @@
version: "3.9"
services:
nginx:
container_name: infisical-dev-nginx
image: nginx
restart: always
ports:
- 8080:80
volumes:
- ./nginx/default.dev.conf:/etc/nginx/conf.d/default.conf:ro
depends_on:
- backend
- frontend
db:
image: bitnami/postgresql:14
ports:
- "5432:5432"
volumes:
- postgres-data:/var/lib/postgresql/data
environment:
POSTGRESQL_PASSWORD: infisical
POSTGRESQL_USERNAME: infisical
POSTGRESQL_DATABASE: infisical
POSTGRESQL_REPLICATION_MODE: master
POSTGRESQL_REPLICATION_USER: repl_user
POSTGRESQL_REPLICATION_PASSWORD: repl_password
POSTGRESQL_SYNCHRONOUS_COMMIT_MODE: on
POSTGRESQL_NUM_SYNCHRONOUS_REPLICAS: 1
db-slave:
image: bitnami/postgresql:14
ports:
- "5433:5432"
volumes:
- postgres-data:/var/lib/postgresql/data
environment:
POSTGRESQL_PASSWORD: infisical
POSTGRESQL_USERNAME: infisical
POSTGRESQL_DATABASE: infisical
POSTGRESQL_REPLICATION_MODE: slave
POSTGRESQL_REPLICATION_USER: repl_user
POSTGRESQL_REPLICATION_PASSWORD: repl_password
POSTGRESQL_MASTER_HOST: db
POSTGRESQL_MASTER_PORT_NUMBER: 5432
redis:
image: redis
container_name: infisical-dev-redis
environment:
- ALLOW_EMPTY_PASSWORD=yes
ports:
- 6379:6379
volumes:
- redis_data:/data
redis-commander:
container_name: infisical-dev-redis-commander
image: rediscommander/redis-commander
restart: always
depends_on:
- redis
environment:
- REDIS_HOSTS=local:redis:6379
ports:
- "8085:8081"
db-test:
profiles: ["test"]
image: postgres:14-alpine
ports:
- "5430:5432"
environment:
POSTGRES_PASSWORD: infisical
POSTGRES_USER: infisical
POSTGRES_DB: infisical-test
db-migration:
container_name: infisical-db-migration
depends_on:
- db
build:
context: ./backend
dockerfile: Dockerfile.dev
env_file: .env
environment:
- DB_CONNECTION_URI=postgres://infisical:infisical@db/infisical?sslmode=disable
command: npm run migration:latest
volumes:
- ./backend/src:/app/src
backend:
container_name: infisical-dev-api
build:
context: ./backend
dockerfile: Dockerfile.dev
depends_on:
db:
condition: service_started
redis:
condition: service_started
db-migration:
condition: service_completed_successfully
env_file:
- .env
ports:
- 4000:4000
environment:
- NODE_ENV=development
- DB_CONNECTION_URI=postgres://infisical:infisical@db/infisical?sslmode=disable
- TELEMETRY_ENABLED=false
volumes:
- ./backend/src:/app/src
extra_hosts:
- "host.docker.internal:host-gateway"
frontend:
container_name: infisical-dev-frontend
restart: unless-stopped
depends_on:
- backend
build:
context: ./frontend
dockerfile: Dockerfile.dev
volumes:
- ./frontend/src:/app/src/ # mounted whole src to avoid missing reload on new files
- ./frontend/public:/app/public
env_file: .env
environment:
- NEXT_PUBLIC_ENV=development
- INFISICAL_TELEMETRY_ENABLED=false
pgadmin:
image: dpage/pgadmin4
restart: always
environment:
PGADMIN_DEFAULT_EMAIL: admin@example.com
PGADMIN_DEFAULT_PASSWORD: pass
ports:
- 5050:80
depends_on:
- db
smtp-server:
container_name: infisical-dev-smtp-server
image: lytrax/mailhog:latest # https://github.com/mailhog/MailHog/issues/353#issuecomment-821137362
restart: always
logging:
driver: "none" # disable saving logs
ports:
- 1025:1025 # SMTP server
- 8025:8025 # Web UI
openldap: # note: more advanced configuration is available
image: osixia/openldap:1.5.0
restart: always
environment:
LDAP_ORGANISATION: Acme
LDAP_DOMAIN: acme.com
LDAP_ADMIN_PASSWORD: admin
ports:
- 389:389
- 636:636
volumes:
- ldap_data:/var/lib/ldap
- ldap_config:/etc/ldap/slapd.d
profiles: [ldap]
phpldapadmin: # username: cn=admin,dc=acme,dc=com, pass is admin
image: osixia/phpldapadmin:latest
restart: always
environment:
- PHPLDAPADMIN_LDAP_HOSTS=openldap
- PHPLDAPADMIN_HTTPS=false
ports:
- 6433:80
depends_on:
- openldap
profiles: [ldap]
volumes:
postgres-data:
driver: local
postgres-slave-data:
driver: local
redis_data:
driver: local
ldap_data:
ldap_config:

View File

@@ -47,6 +47,25 @@ The platform utilizes Postgres to persist all of its data and Redis for caching
Redis connection string.
</ParamField>
<ParamField query="DB_READ_REPLICAS" type="string" default="" optional>
Postgres database read replica connection strings. It accepts a JSON string.
```
DB_READ_REPLICAS=[{"DB_CONNECTION_URI":""}]
```
<Expandable title="Format">
<ParamField query="DB_CONNECTION_URI" type="string" default="" required>
Postgres read replica connection string.
</ParamField>
<ParamField query="DB_ROOT_CERT" type="string" default="" optional>
Configure the SSL certificate for securing a Postgres replica connection by first encoding it in base64.
Use the command below to encode your certificate:
`echo "<certificate>" | base64`
If not provided it will use master SSL certificate.
</ParamField>
</Expandable>
</ParamField>
## Email service
Without email configuration, Infisical's core functions like sign-up/login and secret operations work, but this disables multi-factor authentication, email invites for projects, alerts for suspicious logins, and all other email-dependent features.