awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-10-15 10:35:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into ENG-3639

Browse Source
This commit is contained in:
Carlos Monastyrski
2025-09-12 17:01:02 -03:00
parent 7f03f2cf1d c24b222ff0
commit 17cd03a00d
173 changed files with 6806 additions and 645 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/services/app-connection/app-connection-fns.ts
+4 -2
View File
@@ -6,6 +6,7 @@ import {
} from "@app/ee/services/app-connections/oci";
import { getOracleDBConnectionListItem, OracleDBConnectionMethod } from "@app/ee/services/app-connections/oracledb";
import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
import { TGatewayV2ServiceFactory } from "@app/ee/services/gateway-v2/gateway-v2-service";
import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
import { crypto } from "@app/lib/crypto/cryptography";
import { BadRequestError } from "@app/lib/errors";
@@ -219,7 +220,8 @@ export const decryptAppConnectionCredentials = async ({
export const validateAppConnectionCredentials = async (
appConnection: TAppConnectionConfig,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
gatewayV2Service: Pick<TGatewayV2ServiceFactory, "getPlatformConnectionDetailsByGatewayId">
): Promise<TAppConnection["credentials"]> => {
const VALIDATE_APP_CONNECTION_CREDENTIALS_MAP: Record<AppConnection, TAppConnectionCredentialsValidator> = {
[AppConnection.AWS]: validateAwsConnectionCredentials as TAppConnectionCredentialsValidator,
@@ -264,7 +266,7 @@ export const validateAppConnectionCredentials = async (
[AppConnection.Netlify]: validateNetlifyConnectionCredentials as TAppConnectionCredentialsValidator
};
return VALIDATE_APP_CONNECTION_CREDENTIALS_MAP[appConnection.app](appConnection, gatewayService);
return VALIDATE_APP_CONNECTION_CREDENTIALS_MAP[appConnection.app](appConnection, gatewayService, gatewayV2Service);
};
export const getAppConnectionMethodName = (method: TAppConnection["method"]) => {
backend/src/services/app-connection/app-connection-service.ts
+20 -8
View File
@@ -5,6 +5,8 @@ import { ociConnectionService } from "@app/ee/services/app-connections/oci/oci-c
import { ValidateOracleDBConnectionCredentialsSchema } from "@app/ee/services/app-connections/oracledb";
import { TGatewayDALFactory } from "@app/ee/services/gateway/gateway-dal";
import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
import { TGatewayV2DALFactory } from "@app/ee/services/gateway-v2/gateway-v2-dal";
import { TGatewayV2ServiceFactory } from "@app/ee/services/gateway-v2/gateway-v2-service";
import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
import {
OrgPermissionAppConnectionActions,
@@ -110,7 +112,9 @@ export type TAppConnectionServiceFactoryDep = {
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
licenseService: Pick<TLicenseServiceFactory, "getPlan">;
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">;
gatewayV2Service: Pick<TGatewayV2ServiceFactory, "getPlatformConnectionDetailsByGatewayId">;
gatewayDAL: Pick<TGatewayDALFactory, "find">;
gatewayV2DAL: Pick<TGatewayV2DALFactory, "find">;
};
export type TAppConnectionServiceFactory = ReturnType<typeof appConnectionServiceFactory>;
@@ -162,7 +166,9 @@ export const appConnectionServiceFactory = ({
kmsService,
licenseService,
gatewayService,
gatewayDAL
gatewayV2Service,
gatewayDAL,
gatewayV2DAL
}: TAppConnectionServiceFactoryDep) => {
const listAppConnectionsByOrg = async (actor: OrgServiceActor, app?: AppConnection) => {
const { permission } = await permissionService.getOrgPermission(
@@ -266,7 +272,8 @@ export const appConnectionServiceFactory = ({
);
const [gateway] = await gatewayDAL.find({ id: gatewayId, orgId: actor.orgId });
if (!gateway) {
const [gatewayV2] = await gatewayV2DAL.find({ id: gatewayId, orgId: actor.orgId });
if (!gateway && !gatewayV2) {
throw new NotFoundError({
message: `Gateway with ID ${gatewayId} not found for org`
});
@@ -288,7 +295,8 @@ export const appConnectionServiceFactory = ({
orgId: actor.orgId,
gatewayId
} as TAppConnectionConfig,
gatewayService
gatewayService,
gatewayV2Service
);
try {
@@ -321,7 +329,8 @@ export const appConnectionServiceFactory = ({
gatewayId
} as TAppConnectionConfig,
(platformCredentials) => createConnection(platformCredentials),
gatewayService
gatewayService,
gatewayV2Service
);
} else {
connection = await createConnection(validatedCredentials);
@@ -377,7 +386,8 @@ export const appConnectionServiceFactory = ({
if (gatewayId) {
const [gateway] = await gatewayDAL.find({ id: gatewayId, orgId: actor.orgId });
if (!gateway) {
const [gatewayV2] = await gatewayV2DAL.find({ id: gatewayId, orgId: actor.orgId });
if (!gateway && !gatewayV2) {
throw new NotFoundError({
message: `Gateway with ID ${gatewayId} not found for org`
});
@@ -417,7 +427,8 @@ export const appConnectionServiceFactory = ({
method,
gatewayId
} as TAppConnectionConfig,
gatewayService
gatewayService,
gatewayV2Service
);
if (!updatedCredentials)
@@ -458,7 +469,8 @@ export const appConnectionServiceFactory = ({
gatewayId
} as TAppConnectionConfig,
(platformCredentials) => updateConnection(platformCredentials),
gatewayService
gatewayService,
gatewayV2Service
);
} else {
updatedConnection = await updateConnection(updatedCredentials);
@@ -588,7 +600,7 @@ export const appConnectionServiceFactory = ({
deleteAppConnection,
connectAppConnectionById,
listAvailableAppConnectionsForUser,
github: githubConnectionService(connectAppConnectionById, gatewayService),
github: githubConnectionService(connectAppConnectionById, gatewayService, gatewayV2Service),
githubRadar: githubRadarConnectionService(connectAppConnectionById),
gcp: gcpConnectionService(connectAppConnectionById),
databricks: databricksConnectionService(connectAppConnectionById, appConnectionDAL, kmsService),
backend/src/services/app-connection/app-connection-types.ts
+5 -2
View File
@@ -10,6 +10,7 @@ import {
TValidateOracleDBConnectionCredentialsSchema
} from "@app/ee/services/app-connections/oracledb";
import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
import { TGatewayV2ServiceFactory } from "@app/ee/services/gateway-v2/gateway-v2-service";
import { TAppConnectionDALFactory } from "@app/services/app-connection/app-connection-dal";
import { TSqlConnectionConfig } from "@app/services/app-connection/shared/sql/sql-connection-types";
import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
@@ -411,13 +412,15 @@ export type TListAwsConnectionIamUsers = {
export type TAppConnectionCredentialsValidator = (
appConnection: TAppConnectionConfig,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
gatewayV2Service: Pick<TGatewayV2ServiceFactory, "getPlatformConnectionDetailsByGatewayId">
) => Promise<TAppConnection["credentials"]>;
export type TAppConnectionTransitionCredentialsToPlatform = (
appConnection: TAppConnectionConfig,
callback: (credentials: TAppConnection["credentials"]) => Promise<TAppConnectionRaw>,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
gatewayV2Service: Pick<TGatewayV2ServiceFactory, "getPlatformConnectionDetailsByGatewayId">
) => Promise<TAppConnectionRaw>;
export type TAppConnectionBaseConfig = {
backend/src/services/app-connection/github/github-connection-fns.ts
+101 -27
View File
@@ -4,11 +4,13 @@ import RE2 from "re2";
import { verifyHostInputValidity } from "@app/ee/services/dynamic-secret/dynamic-secret-fns";
import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
import { TGatewayV2ServiceFactory } from "@app/ee/services/gateway-v2/gateway-v2-service";
import { getConfig } from "@app/lib/config/env";
import { request as httpRequest } from "@app/lib/config/request";
import { crypto } from "@app/lib/crypto";
import { BadRequestError, ForbiddenRequestError, InternalServerError } from "@app/lib/errors";
import { GatewayProxyProtocol, withGatewayProxy } from "@app/lib/gateway";
import { withGatewayV2Proxy } from "@app/lib/gateway-v2/gateway-v2";
import { logger } from "@app/lib/logger";
import { blockLocalAndPrivateIpAddresses } from "@app/lib/validator";
import { getAppConnectionMethodName } from "@app/services/app-connection/app-connection-fns";
@@ -49,6 +51,7 @@ export const getGitHubInstanceApiUrl = async (config: {
export const requestWithGitHubGateway = async <T>(
appConnection: { gatewayId?: string | null },
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
gatewayV2Service: Pick<TGatewayV2ServiceFactory, "getPlatformConnectionDetailsByGatewayId">,
requestConfig: AxiosRequestConfig
): Promise<AxiosResponse<T>> => {
const { gatewayId } = appConnection;
@@ -63,6 +66,52 @@ export const requestWithGitHubGateway = async <T>(
await blockLocalAndPrivateIpAddresses(url.toString());
const [targetHost] = await verifyHostInputValidity(url.host, true);
const gatewayConnectionDetails = await gatewayV2Service.getPlatformConnectionDetailsByGatewayId({
gatewayId,
targetHost,
targetPort: 443
});
if (gatewayConnectionDetails) {
return withGatewayV2Proxy(
async (proxyPort) => {
const httpsAgent = new https.Agent({
servername: targetHost
});
url.protocol = "https:";
url.host = `localhost:${proxyPort}`;
const finalRequestConfig: AxiosRequestConfig = {
...requestConfig,
url: url.toString(),
httpsAgent,
headers: {
...requestConfig.headers,
Host: targetHost
}
};
try {
return await httpRequest.request(finalRequestConfig);
} catch (error) {
const axiosError = error as AxiosError;
logger.error(
{ message: axiosError.message, data: axiosError.response?.data },
"Error during GitHub gateway request:"
);
throw error;
}
},
{
protocol: GatewayProxyProtocol.Tcp,
relayHost: gatewayConnectionDetails.relayHost,
gateway: gatewayConnectionDetails.gateway,
relay: gatewayConnectionDetails.relay
}
);
}
const relayDetails = await gatewayService.fnGetGatewayClientTlsByGatewayId(gatewayId);
const [relayHost, relayPort] = relayDetails.relayAddress.split(":");
@@ -115,7 +164,8 @@ export const requestWithGitHubGateway = async <T>(
export const getGitHubAppAuthToken = async (
appConnection: TGitHubConnection,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
gatewayV2Service: Pick<TGatewayV2ServiceFactory, "getPlatformConnectionDetailsByGatewayId">
) => {
const appCfg = getConfig();
const appId = appCfg.INF_APP_CONNECTION_GITHUB_APP_ID;
@@ -151,6 +201,7 @@ export const getGitHubAppAuthToken = async (
const response = await requestWithGitHubGateway<{ token: string; expires_at: string }>(
appConnection,
gatewayService,
gatewayV2Service,
{
url: `https://${apiBaseUrl}/app/installations/${installationId}/access_tokens`,
method: "POST",
@@ -191,6 +242,7 @@ function extractNextPageUrl(linkHeader: string | undefined): string | null {
export const makePaginatedGitHubRequest = async <T, R = T[]>(
appConnection: TGitHubConnection,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
gatewayV2Service: Pick<TGatewayV2ServiceFactory, "getPlatformConnectionDetailsByGatewayId">,
path: string,
dataMapper?: (data: R) => T[]
): Promise<T[]> => {
@@ -199,7 +251,7 @@ export const makePaginatedGitHubRequest = async <T, R = T[]>(
const token =
method === GitHubConnectionMethod.OAuth
? credentials.accessToken
: await getGitHubAppAuthToken(appConnection, gatewayService);
: await getGitHubAppAuthToken(appConnection, gatewayService, gatewayV2Service);
const baseUrl = `https://${await getGitHubInstanceApiUrl(appConnection)}${path}`;
const initialUrlObj = new URL(baseUrl);
@@ -209,15 +261,20 @@ export const makePaginatedGitHubRequest = async <T, R = T[]>(
const maxIterations = 1000;
// Make initial request to get link header
const firstResponse: AxiosResponse<R> = await requestWithGitHubGateway<R>(appConnection, gatewayService, {
url: initialUrlObj.toString(),
method: "GET",
headers: {
Accept: "application/vnd.github+json",
Authorization: `Bearer ${token}`,
"X-GitHub-Api-Version": "2022-11-28"
const firstResponse: AxiosResponse<R> = await requestWithGitHubGateway<R>(
appConnection,
gatewayService,
gatewayV2Service,
{
url: initialUrlObj.toString(),
method: "GET",
headers: {
Accept: "application/vnd.github+json",
Authorization: `Bearer ${token}`,
"X-GitHub-Api-Version": "2022-11-28"
}
}
});
);
const firstPageItems = dataMapper ? dataMapper(firstResponse.data) : (firstResponse.data as unknown as T[]);
results = results.concat(firstPageItems);
@@ -237,7 +294,7 @@ export const makePaginatedGitHubRequest = async <T, R = T[]>(
pageUrlObj.searchParams.set("page", pageNum.toString());
pageRequests.push(
requestWithGitHubGateway<R>(appConnection, gatewayService, {
requestWithGitHubGateway<R>(appConnection, gatewayService, gatewayV2Service, {
url: pageUrlObj.toString(),
method: "GET",
headers: {
@@ -261,15 +318,20 @@ export const makePaginatedGitHubRequest = async <T, R = T[]>(
while (url && i < maxIterations) {
// eslint-disable-next-line no-await-in-loop
const response: AxiosResponse<R> = await requestWithGitHubGateway<R>(appConnection, gatewayService, {
url,
method: "GET",
headers: {
Accept: "application/vnd.github+json",
Authorization: `Bearer ${token}`,
"X-GitHub-Api-Version": "2022-11-28"
const response: AxiosResponse<R> = await requestWithGitHubGateway<R>(
appConnection,
gatewayService,
gatewayV2Service,
{
url,
method: "GET",
headers: {
Accept: "application/vnd.github+json",
Authorization: `Bearer ${token}`,
"X-GitHub-Api-Version": "2022-11-28"
}
}
});
);
const items = dataMapper ? dataMapper(response.data) : (response.data as unknown as T[]);
results = results.concat(items);
@@ -308,30 +370,39 @@ type GitHubEnvironment = {
export const getGitHubRepositories = async (
appConnection: TGitHubConnection,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
gatewayV2Service: Pick<TGatewayV2ServiceFactory, "getPlatformConnectionDetailsByGatewayId">
) => {
if (appConnection.method === GitHubConnectionMethod.App) {
return makePaginatedGitHubRequest<GitHubRepository, { repositories: GitHubRepository[] }>(
appConnection,
gatewayService,
gatewayV2Service,
"/installation/repositories",
(data) => data.repositories
);
}
const repos = await makePaginatedGitHubRequest<GitHubRepository>(appConnection, gatewayService, "/user/repos");
const repos = await makePaginatedGitHubRequest<GitHubRepository>(
appConnection,
gatewayService,
gatewayV2Service,
"/user/repos"
);
return repos.filter((repo) => repo.permissions?.admin);
};
export const getGitHubOrganizations = async (
appConnection: TGitHubConnection,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
gatewayV2Service: Pick<TGatewayV2ServiceFactory, "getPlatformConnectionDetailsByGatewayId">
) => {
if (appConnection.method === GitHubConnectionMethod.App) {
const installationRepositories = await makePaginatedGitHubRequest<
GitHubRepository,
{ repositories: GitHubRepository[] }
>(appConnection, gatewayService, "/installation/repositories", (data) => data.repositories);
>(appConnection, gatewayService, gatewayV2Service, "/installation/repositories", (data) => data.repositories);
const organizationMap: Record<string, GitHubOrganization> = {};
installationRepositories.forEach((repo) => {
@@ -343,12 +414,13 @@ export const getGitHubOrganizations = async (
return Object.values(organizationMap);
}
return makePaginatedGitHubRequest<GitHubOrganization>(appConnection, gatewayService, "/user/orgs");
return makePaginatedGitHubRequest<GitHubOrganization>(appConnection, gatewayService, gatewayV2Service, "/user/orgs");
};
export const getGitHubEnvironments = async (
appConnection: TGitHubConnection,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
gatewayV2Service: Pick<TGatewayV2ServiceFactory, "getPlatformConnectionDetailsByGatewayId">,
owner: string,
repo: string
) => {
@@ -356,6 +428,7 @@ export const getGitHubEnvironments = async (
return await makePaginatedGitHubRequest<GitHubEnvironment, { environments: GitHubEnvironment[] }>(
appConnection,
gatewayService,
gatewayV2Service,
`/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/environments`,
(data) => data.environments
);
@@ -383,7 +456,8 @@ export function isGithubErrorResponse(data: GithubTokenRespData): data is Github
export const validateGitHubConnectionCredentials = async (
config: TGitHubConnectionConfig,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
gatewayV2Service: Pick<TGatewayV2ServiceFactory, "getPlatformConnectionDetailsByGatewayId">
) => {
const { credentials, method } = config;
const {
@@ -419,7 +493,7 @@ export const validateGitHubConnectionCredentials = async (
const host = credentials.host || "github.com";
try {
tokenResp = await requestWithGitHubGateway<GithubTokenRespData>(config, gatewayService, {
tokenResp = await requestWithGitHubGateway<GithubTokenRespData>(config, gatewayService, gatewayV2Service, {
url: `https://${host}/login/oauth/access_token`,
method: "POST",
data: {
@@ -471,7 +545,7 @@ export const validateGitHubConnectionCredentials = async (
id: number;
};
}[];
}>(config, gatewayService, {
}>(config, gatewayService, gatewayV2Service, {
url: `https://${await getGitHubInstanceApiUrl(config)}/user/installations`,
headers: {
Accept: "application/json",
backend/src/services/app-connection/github/github-connection-service.ts
+6 -4
View File
@@ -1,4 +1,5 @@
import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
import { TGatewayV2ServiceFactory } from "@app/ee/services/gateway-v2/gateway-v2-service";
import { OrgServiceActor } from "@app/lib/types";
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
import {
@@ -22,12 +23,13 @@ type TListGitHubEnvironmentsDTO = {
export const githubConnectionService = (
getAppConnection: TGetAppConnectionFunc,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
gatewayV2Service: Pick<TGatewayV2ServiceFactory, "getPlatformConnectionDetailsByGatewayId">
) => {
const listRepositories = async (connectionId: string, actor: OrgServiceActor) => {
const appConnection = await getAppConnection(AppConnection.GitHub, connectionId, actor);
const repositories = await getGitHubRepositories(appConnection, gatewayService);
const repositories = await getGitHubRepositories(appConnection, gatewayService, gatewayV2Service);
return repositories;
};
@@ -35,7 +37,7 @@ export const githubConnectionService = (
const listOrganizations = async (connectionId: string, actor: OrgServiceActor) => {
const appConnection = await getAppConnection(AppConnection.GitHub, connectionId, actor);
const organizations = await getGitHubOrganizations(appConnection, gatewayService);
const organizations = await getGitHubOrganizations(appConnection, gatewayService, gatewayV2Service);
return organizations;
};
@@ -46,7 +48,7 @@ export const githubConnectionService = (
) => {
const appConnection = await getAppConnection(AppConnection.GitHub, connectionId, actor);
const environments = await getGitHubEnvironments(appConnection, gatewayService, owner, repo);
const environments = await getGitHubEnvironments(appConnection, gatewayService, gatewayV2Service, owner, repo);
return environments;
};
backend/src/services/app-connection/hc-vault/hc-vault-connection-fns.ts
+4 -1
View File
@@ -3,6 +3,7 @@ import https from "https";
import { verifyHostInputValidity } from "@app/ee/services/dynamic-secret/dynamic-secret-fns";
import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
import { TGatewayV2ServiceFactory } from "@app/ee/services/gateway-v2/gateway-v2-service";
import { request } from "@app/lib/config/request";
import { BadRequestError } from "@app/lib/errors";
import { removeTrailingSlash } from "@app/lib/fn";
@@ -144,7 +145,9 @@ export const getHCVaultAccessToken = async (
export const validateHCVaultConnectionCredentials = async (
connection: THCVaultConnection,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
// eslint-disable-next-line @typescript-eslint/no-unused-vars
_gatewayV2Service: Pick<TGatewayV2ServiceFactory, "getPlatformConnectionDetailsByGatewayId">
) => {
const instanceUrl = await getHCVaultInstanceUrl(connection);
backend/src/services/app-connection/shared/sql/sql-connection-fns.ts
+46 -5
View File
@@ -2,12 +2,14 @@ import knex, { Knex } from "knex";
import { verifyHostInputValidity } from "@app/ee/services/dynamic-secret/dynamic-secret-fns";
import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
import { TGatewayV2ServiceFactory } from "@app/ee/services/gateway-v2/gateway-v2-service";
import {
TSqlCredentialsRotationGeneratedCredentials,
TSqlCredentialsRotationWithConnection
} from "@app/ee/services/secret-rotation-v2/shared/sql-credentials/sql-credentials-rotation-types";
import { BadRequestError, DatabaseError } from "@app/lib/errors";
import { GatewayProxyProtocol, withGatewayProxy } from "@app/lib/gateway";
import { withGatewayV2Proxy } from "@app/lib/gateway-v2/gateway-v2";
import { alphaNumericNanoId } from "@app/lib/nanoid";
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
import { TAppConnectionRaw, TSqlConnection } from "@app/services/app-connection/app-connection-types";
@@ -104,12 +106,49 @@ export const getSqlConnectionClient = async (appConnection: Pick<TSqlConnection,
export const executeWithPotentialGateway = async <T>(
config: TSqlConnectionConfig,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
gatewayV2Service: Pick<TGatewayV2ServiceFactory, "getPlatformConnectionDetailsByGatewayId">,
operation: (client: Knex) => Promise<T>
): Promise<T> => {
const { credentials, app, gatewayId } = config;
if (gatewayId && gatewayService) {
if (gatewayId && gatewayService && gatewayV2Service) {
const [targetHost] = await verifyHostInputValidity(credentials.host, true);
const platformConnectionDetails = await gatewayV2Service.getPlatformConnectionDetailsByGatewayId({
gatewayId,
targetHost,
targetPort: credentials.port
});
if (platformConnectionDetails) {
return withGatewayV2Proxy(
async (proxyPort) => {
const client = knex({
client: SQL_CONNECTION_CLIENT_MAP[app],
connection: {
database: credentials.database,
port: proxyPort,
host: "localhost",
user: credentials.username,
password: credentials.password,
connectionTimeoutMillis: EXTERNAL_REQUEST_TIMEOUT,
...getConnectionConfig({ app, credentials })
}
});
try {
return await operation(client);
} finally {
await client.destroy();
}
},
{
protocol: GatewayProxyProtocol.Tcp,
relayHost: platformConnectionDetails.relayHost,
gateway: platformConnectionDetails.gateway,
relay: platformConnectionDetails.relay
}
);
}
const relayDetails = await gatewayService.fnGetGatewayClientTlsByGatewayId(gatewayId);
const [relayHost, relayPort] = relayDetails.relayAddress.split(":");
@@ -161,10 +200,11 @@ export const executeWithPotentialGateway = async <T>(
export const validateSqlConnectionCredentials = async (
config: TSqlConnectionConfig,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
gatewayV2Service: Pick<TGatewayV2ServiceFactory, "getPlatformConnectionDetailsByGatewayId">
) => {
try {
await executeWithPotentialGateway(config, gatewayService, async (client) => {
await executeWithPotentialGateway(config, gatewayService, gatewayV2Service, async (client) => {
await client.raw(config.app === AppConnection.OracleDB ? `SELECT 1 FROM DUAL` : `Select 1`);
});
return config.credentials;
@@ -191,14 +231,15 @@ export const SQL_CONNECTION_ALTER_LOGIN_STATEMENT: Record<
export const transferSqlConnectionCredentialsToPlatform = async (
config: TSqlConnectionConfig,
callback: (credentials: TSqlConnectionConfig["credentials"]) => Promise<TAppConnectionRaw>,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
gatewayV2Service: Pick<TGatewayV2ServiceFactory, "getPlatformConnectionDetailsByGatewayId">
) => {
const { credentials, app } = config;
const newPassword = alphaNumericNanoId(32);
try {
return await executeWithPotentialGateway(config, gatewayService, (client) => {
return await executeWithPotentialGateway(config, gatewayService, gatewayV2Service, (client) => {
return client.transaction(async (tx) => {
await tx.raw(
...SQL_CONNECTION_ALTER_LOGIN_STATEMENT[app]({ username: credentials.username, password: newPassword })
backend/src/services/auth-token/auth-token-service.ts
+9 -2
View File
@@ -36,6 +36,12 @@ export const getTokenConfig = (tokenType: TokenType) => {
const expiresAt = new Date(new Date().getTime() + 86400000);
return { token, triesLeft, expiresAt };
}
case TokenType.TOKEN_EMAIL_CHANGE_OTP: {
const token = String(crypto.randomInt(10 ** 5, 10 ** 6 - 1));
const triesLeft = 1;
const expiresAt = new Date(new Date().getTime() + 600000);
return { token, triesLeft, expiresAt };
}
case TokenType.TOKEN_EMAIL_MFA: {
// generate random 6-digit code
const token = String(crypto.randomInt(10 ** 5, 10 ** 6 - 1));
@@ -75,7 +81,7 @@ export const getTokenConfig = (tokenType: TokenType) => {
};
export const tokenServiceFactory = ({ tokenDAL, userDAL, orgMembershipDAL }: TAuthTokenServiceFactoryDep) => {
const createTokenForUser = async ({ type, userId, orgId, aliasId }: TCreateTokenForUserDTO) => {
const createTokenForUser = async ({ type, userId, orgId, aliasId, payload }: TCreateTokenForUserDTO) => {
const { token, ...tkCfg } = getTokenConfig(type);
const appCfg = getConfig();
const tokenHash = await crypto.hashing().createHash(token, appCfg.SALT_ROUNDS);
@@ -89,7 +95,8 @@ export const tokenServiceFactory = ({ tokenDAL, userDAL, orgMembershipDAL }: TAu
userId,
orgId,
triesLeft: tkCfg?.triesLeft,
aliasId
aliasId,
payload
},
tx
);
backend/src/services/auth-token/auth-token-types.ts
+2
View File
@@ -3,6 +3,7 @@ import { ProjectMembershipRole } from "@app/db/schemas";
export enum TokenType {
TOKEN_EMAIL_CONFIRMATION = "emailConfirmation",
TOKEN_EMAIL_VERIFICATION = "emailVerification", // unverified -> verified
TOKEN_EMAIL_CHANGE_OTP = "emailChangeOtp",
TOKEN_EMAIL_MFA = "emailMfa",
TOKEN_EMAIL_ORG_INVITATION = "organizationInvitation",
TOKEN_EMAIL_PASSWORD_RESET = "passwordReset",
@@ -15,6 +16,7 @@ export type TCreateTokenForUserDTO = {
userId: string;
orgId?: string;
aliasId?: string;
payload?: string;
};
export type TCreateOrgInviteTokenDTO = {
backend/src/services/certificate/certificate-fns.ts
+3
View File
@@ -52,6 +52,9 @@ export const constructPemChainFromCerts = (certificates: x509.X509Certificate[])
.join("\n")
.trim();
export const prependCertToPemChain = (cert: x509.X509Certificate, pemChain: string) =>
`${cert.toString("pem")}\n${pemChain}`;
export const splitPemChain = (pemText: string) => {
const re2Pattern = new RE2("-----BEGIN CERTIFICATE-----[^-]+-----END CERTIFICATE-----", "g");
backend/src/services/folder-commit/folder-commit-service.test.ts
+1 -1
View File
@@ -661,7 +661,7 @@ describe("folderCommitServiceFactory", () => {
// Assert
expect(mockFolderCommitDAL.create).toHaveBeenCalled();
expect(mockSecretV2BridgeDAL.invalidateSecretCacheByProjectId).toHaveBeenCalledWith(projectId);
expect(mockSecretV2BridgeDAL.invalidateSecretCacheByProjectId).toHaveBeenCalledWith(projectId, {});
// Check that we got the right counts
expect(result.totalChanges).toEqual(2);
backend/src/services/folder-commit/folder-commit-service.ts
+1 -1
View File
@@ -1386,7 +1386,7 @@ export const folderCommitServiceFactory = ({
);
// Invalidate cache to reflect the changes
await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId);
await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId, tx);
return {
secretChangesCount: secretChanges.length,
backend/src/services/identity-kubernetes-auth/identity-kubernetes-auth-service.ts
+89 -18
View File
@@ -6,6 +6,8 @@ import RE2 from "re2";
import { IdentityAuthMethod, TIdentityKubernetesAuthsUpdate } from "@app/db/schemas";
import { TGatewayDALFactory } from "@app/ee/services/gateway/gateway-dal";
import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
import { TGatewayV2DALFactory } from "@app/ee/services/gateway-v2/gateway-v2-dal";
import { TGatewayV2ServiceFactory } from "@app/ee/services/gateway-v2/gateway-v2-service";
import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
import {
OrgPermissionGatewayActions,
@@ -21,6 +23,7 @@ import { getConfig } from "@app/lib/config/env";
import { crypto } from "@app/lib/crypto";
import { BadRequestError, NotFoundError, PermissionBoundaryError, UnauthorizedError } from "@app/lib/errors";
import { GatewayHttpProxyActions, GatewayProxyProtocol, withGatewayProxy } from "@app/lib/gateway";
import { withGatewayV2Proxy } from "@app/lib/gateway-v2/gateway-v2";
import { extractIPDetails, isValidIpOrCidr } from "@app/lib/ip";
import { logger } from "@app/lib/logger";
@@ -54,11 +57,15 @@ type TIdentityKubernetesAuthServiceFactoryDep = {
licenseService: Pick<TLicenseServiceFactory, "getPlan">;
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
gatewayService: TGatewayServiceFactory;
gatewayV2Service: TGatewayV2ServiceFactory;
gatewayDAL: Pick<TGatewayDALFactory, "find">;
gatewayV2DAL: Pick<TGatewayV2DALFactory, "find">;
};
export type TIdentityKubernetesAuthServiceFactory = ReturnType<typeof identityKubernetesAuthServiceFactory>;
const GATEWAY_AUTH_DEFAULT_HOST = "https://kubernetes.default.svc.cluster.local";
export const identityKubernetesAuthServiceFactory = ({
identityKubernetesAuthDAL,
identityOrgMembershipDAL,
@@ -66,7 +73,9 @@ export const identityKubernetesAuthServiceFactory = ({
permissionService,
licenseService,
gatewayService,
gatewayV2Service,
gatewayDAL,
gatewayV2DAL,
kmsService
}: TIdentityKubernetesAuthServiceFactoryDep) => {
const $gatewayProxyWrapper = async <T>(
@@ -79,6 +88,42 @@ export const identityKubernetesAuthServiceFactory = ({
},
gatewayCallback: (host: string, port: number, httpsAgent?: https.Agent) => Promise<T>
): Promise<T> => {
const gatewayV2ConnectionDetails = await gatewayV2Service.getPlatformConnectionDetailsByGatewayId({
gatewayId: inputs.gatewayId,
targetHost: inputs.targetHost ?? GATEWAY_AUTH_DEFAULT_HOST,
targetPort: inputs.targetPort ?? 443
});
if (gatewayV2ConnectionDetails) {
let httpsAgent: https.Agent | undefined;
if (!inputs.reviewTokenThroughGateway) {
httpsAgent = new https.Agent({
ca: inputs.caCert,
rejectUnauthorized: Boolean(inputs.caCert)
});
}
const callbackResult = await withGatewayV2Proxy(
async (port) => {
const res = await gatewayCallback(
inputs.reviewTokenThroughGateway ? "http://localhost" : "https://localhost",
port,
httpsAgent
);
return res;
},
{
protocol: inputs.reviewTokenThroughGateway ? GatewayProxyProtocol.Http : GatewayProxyProtocol.Tcp,
relayHost: gatewayV2ConnectionDetails.relayHost,
gateway: gatewayV2ConnectionDetails.gateway,
relay: gatewayV2ConnectionDetails.relay,
httpsAgent
}
);
return callbackResult;
}
const relayDetails = await gatewayService.fnGetGatewayClientTlsByGatewayId(inputs.gatewayId);
const [relayHost, relayPort] = relayDetails.relayAddress.split(":");
@@ -277,7 +322,7 @@ export const identityKubernetesAuthServiceFactory = ({
let data: TCreateTokenReviewResponse | undefined;
if (identityKubernetesAuth.tokenReviewMode === IdentityKubernetesAuthTokenReviewMode.Gateway) {
if (!identityKubernetesAuth.gatewayId) {
if (!identityKubernetesAuth.gatewayId && !identityKubernetesAuth.gatewayV2Id) {
throw new BadRequestError({
message: "Gateway ID is required when token review mode is set to Gateway"
});
@@ -285,7 +330,7 @@ export const identityKubernetesAuthServiceFactory = ({
data = await $gatewayProxyWrapper(
{
gatewayId: identityKubernetesAuth.gatewayId,
gatewayId: (identityKubernetesAuth.gatewayV2Id ?? identityKubernetesAuth.gatewayId) as string,
reviewTokenThroughGateway: true
},
tokenReviewCallbackThroughGateway
@@ -304,17 +349,18 @@ export const identityKubernetesAuthServiceFactory = ({
const [k8sHost, k8sPort] = kubernetesHost.split(":");
data = identityKubernetesAuth.gatewayId
? await $gatewayProxyWrapper(
{
gatewayId: identityKubernetesAuth.gatewayId,
targetHost: k8sHost,
targetPort: k8sPort ? Number(k8sPort) : 443,
reviewTokenThroughGateway: false
},
tokenReviewCallbackRaw
)
: await tokenReviewCallbackRaw();
data =
identityKubernetesAuth.gatewayId || identityKubernetesAuth.gatewayV2Id
? await $gatewayProxyWrapper(
{
gatewayId: (identityKubernetesAuth.gatewayV2Id ?? identityKubernetesAuth.gatewayId) as string,
targetHost: k8sHost,
targetPort: k8sPort ? Number(k8sPort) : 443,
reviewTokenThroughGateway: false
},
tokenReviewCallbackRaw
)
: await tokenReviewCallbackRaw();
} else {
throw new BadRequestError({
message: `Invalid token review mode: ${identityKubernetesAuth.tokenReviewMode}`
@@ -490,14 +536,20 @@ export const identityKubernetesAuthServiceFactory = ({
return extractIPDetails(accessTokenTrustedIp.ipAddress);
});
let isGatewayV1 = true;
if (gatewayId) {
const [gateway] = await gatewayDAL.find({ id: gatewayId, orgId: identityMembershipOrg.orgId });
if (!gateway) {
const [gatewayV2] = await gatewayV2DAL.find({ id: gatewayId, orgId: identityMembershipOrg.orgId });
if (!gateway && !gatewayV2) {
throw new NotFoundError({
message: `Gateway with ID ${gatewayId} not found`
});
}
if (!gateway) {
isGatewayV1 = false;
}
const { permission: orgPermission } = await permissionService.getOrgPermission(
actor,
actorId,
@@ -528,7 +580,8 @@ export const identityKubernetesAuthServiceFactory = ({
accessTokenMaxTTL,
accessTokenTTL,
accessTokenNumUsesLimit,
gatewayId,
gatewayId: isGatewayV1 ? gatewayId : null,
gatewayV2Id: isGatewayV1 ? null : gatewayId,
accessTokenTrustedIps: JSON.stringify(reformattedAccessTokenTrustedIps),
encryptedKubernetesTokenReviewerJwt: tokenReviewerJwt
? encryptor({ plainText: Buffer.from(tokenReviewerJwt) }).cipherTextBlob
@@ -608,14 +661,21 @@ export const identityKubernetesAuthServiceFactory = ({
return extractIPDetails(accessTokenTrustedIp.ipAddress);
});
let isGatewayV1 = true;
if (gatewayId) {
const [gateway] = await gatewayDAL.find({ id: gatewayId, orgId: identityMembershipOrg.orgId });
if (!gateway) {
const [gatewayV2] = await gatewayV2DAL.find({ id: gatewayId, orgId: identityMembershipOrg.orgId });
if (!gateway && !gatewayV2) {
throw new NotFoundError({
message: `Gateway with ID ${gatewayId} not found`
});
}
if (!gateway) {
isGatewayV1 = false;
}
const { permission: orgPermission } = await permissionService.getOrgPermission(
actor,
actorId,
@@ -629,13 +689,18 @@ export const identityKubernetesAuthServiceFactory = ({
);
}
const shouldUpdateGatewayId = Boolean(gatewayId);
const gatewayIdValue = isGatewayV1 ? gatewayId : null;
const gatewayV2IdValue = isGatewayV1 ? null : gatewayId;
const updateQuery: TIdentityKubernetesAuthsUpdate = {
kubernetesHost,
tokenReviewMode,
allowedNamespaces,
allowedNames,
allowedAudience,
gatewayId,
gatewayId: shouldUpdateGatewayId ? gatewayIdValue : undefined,
gatewayV2Id: shouldUpdateGatewayId ? gatewayV2IdValue : undefined,
accessTokenMaxTTL,
accessTokenTTL,
accessTokenNumUsesLimit,
@@ -730,7 +795,13 @@ export const identityKubernetesAuthServiceFactory = ({
}).toString();
}
return { ...identityKubernetesAuth, caCert, tokenReviewerJwt, orgId: identityMembershipOrg.orgId };
return {
...identityKubernetesAuth,
caCert,
tokenReviewerJwt,
orgId: identityMembershipOrg.orgId,
gatewayId: identityKubernetesAuth.gatewayId ?? identityKubernetesAuth.gatewayV2Id
};
};
const revokeIdentityKubernetesAuth = async ({
backend/src/services/identity/identity-dal.ts
+1 -1
View File
@@ -25,7 +25,7 @@ export const identityDALFactory = (db: TDbClient) => {
} as const;
const tableName = authMethodToTableName[authMethod];
if (!tableName) return;
const data = await db(tableName).where({ identityId }).first();
const data = await db.replicaNode()(tableName).where({ identityId }).first();
if (!data) return;
return data.accessTokenTrustedIps;
};
backend/src/services/integration-auth/integration-auth-dal.ts
+1 -1
View File
@@ -30,7 +30,7 @@ export const integrationAuthDALFactory = (db: TDbClient) => {
const getByOrg = async (orgId: string, tx?: Knex) => {
try {
const integrationAuths = await (tx || db)(TableName.IntegrationAuth)
const integrationAuths = await (tx || db.replicaNode())(TableName.IntegrationAuth)
.join(TableName.Project, `${TableName.Project}.id`, `${TableName.IntegrationAuth}.projectId`)
.join(TableName.Organization, `${TableName.Organization}.id`, `${TableName.Project}.orgId`)
.where(`${TableName.Organization}.id`, "=", orgId)
backend/src/services/kms/kms-root-config-dal.ts
+1 -1
View File
@@ -12,7 +12,7 @@ export const kmsRootConfigDALFactory = (db: TDbClient) => {
const findById = async (id: string, tx?: Knex) => {
try {
const result = await (tx || db)(TableName.KmsServerRootConfig)
const result = await (tx || db?.replicaNode?.() || db)(TableName.KmsServerRootConfig)
.where({ id } as never)
.first("*");
return result;
backend/src/services/org/org-service.ts
+38 -3
View File
@@ -9,11 +9,14 @@ import {
ProjectMembershipRole,
ProjectVersion,
TableName,
TOidcConfigs,
TProjectMemberships,
TProjectUserMembershipRolesInsert,
TSamlConfigs,
TUsers
} from "@app/db/schemas";
import { TGroupDALFactory } from "@app/ee/services/group/group-dal";
import { TLdapConfigDALFactory } from "@app/ee/services/ldap-config/ldap-config-dal";
import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
import { TOidcConfigDALFactory } from "@app/ee/services/oidc/oidc-config-dal";
import {
@@ -125,6 +128,7 @@ type TOrgServiceFactoryDep = {
incidentContactDAL: TIncidentContactsDALFactory;
samlConfigDAL: Pick<TSamlConfigDALFactory, "findOne">;
oidcConfigDAL: Pick<TOidcConfigDALFactory, "findOne">;
ldapConfigDAL: Pick<TLdapConfigDALFactory, "findOne">;
smtpService: TSmtpService;
tokenService: TAuthTokenServiceFactory;
permissionService: TPermissionServiceFactory;
@@ -165,6 +169,7 @@ export const orgServiceFactory = ({
projectRoleDAL,
samlConfigDAL,
oidcConfigDAL,
ldapConfigDAL,
projectUserMembershipRoleDAL,
identityMetadataDAL,
projectBotService,
@@ -446,16 +451,20 @@ export const orgServiceFactory = ({
});
}
if (authEnforced) {
const samlCfg = await samlConfigDAL.findOne({
let samlCfg: TSamlConfigs | undefined;
let oidcCfg: TOidcConfigs | undefined;
if (authEnforced || googleSsoAuthEnforced) {
samlCfg = await samlConfigDAL.findOne({
orgId,
isActive: true
});
const oidcCfg = await oidcConfigDAL.findOne({
oidcCfg = await oidcConfigDAL.findOne({
orgId,
isActive: true
});
}
if (authEnforced) {
if (!samlCfg && !oidcCfg)
throw new NotFoundError({
message: `SAML or OIDC configuration for organization with ID '${orgId}' not found`
@@ -483,6 +492,32 @@ export const orgServiceFactory = ({
});
}
if (samlCfg) {
throw new BadRequestError({
message:
"Cannot enable Google OAuth enforcement while SAML SSO is configured. Disable SAML SSO to enforce Google OAuth."
});
}
if (oidcCfg) {
throw new BadRequestError({
message:
"Cannot enable Google OAuth enforcement while OIDC SSO is configured. Disable OIDC SSO to enforce Google OAuth."
});
}
const ldapCfg = await ldapConfigDAL.findOne({
orgId,
isActive: true
});
if (ldapCfg) {
throw new BadRequestError({
message:
"Cannot enable Google OAuth enforcement while LDAP SSO is configured. Disable LDAP SSO to enforce Google OAuth."
});
}
if (!currentOrg.googleSsoAuthLastUsed) {
throw new BadRequestError({
message:
backend/src/services/reminder/reminder-dal.ts
+4 -4
View File
@@ -39,7 +39,7 @@ export const reminderDALFactory = (db: TDbClient) => {
const findSecretDailyReminders = async (tx?: Knex) => {
const { startOfDay, endOfDay } = getTodayDateRange();
const rawReminders = await (tx || db)(TableName.Reminder)
const rawReminders = await (tx || db.replicaNode())(TableName.Reminder)
.whereBetween("nextReminderDate", [startOfDay, endOfDay])
.leftJoin(TableName.ReminderRecipient, `${TableName.Reminder}.id`, `${TableName.ReminderRecipient}.reminderId`)
.leftJoin<TUsers>(TableName.Users, `${TableName.ReminderRecipient}.userId`, `${TableName.Users}.id`)
@@ -90,7 +90,7 @@ export const reminderDALFactory = (db: TDbClient) => {
const futureDate = new Date(startOfDay);
futureDate.setDate(futureDate.getDate() + daysAhead);
const reminders = await (tx || db)(TableName.Reminder)
const reminders = await (tx || db.replicaNode())(TableName.Reminder)
.where("nextReminderDate", ">=", startOfDay)
.where("nextReminderDate", "<=", futureDate)
.orderBy("nextReminderDate", "asc")
@@ -101,7 +101,7 @@ export const reminderDALFactory = (db: TDbClient) => {
};
const findSecretReminder = async (secretId: string, tx?: Knex) => {
const rawReminders = await (tx || db)(TableName.Reminder)
const rawReminders = await (tx || db.replicaNode())(TableName.Reminder)
.where(`${TableName.Reminder}.secretId`, secretId)
.leftJoin(TableName.ReminderRecipient, `${TableName.Reminder}.id`, `${TableName.ReminderRecipient}.reminderId`)
.select(selectAllTableCols(TableName.Reminder))
@@ -125,7 +125,7 @@ export const reminderDALFactory = (db: TDbClient) => {
};
const findSecretReminders = async (secretIds: string[], tx?: Knex) => {
const rawReminders = await (tx || db)(TableName.Reminder)
const rawReminders = await (tx || db.replicaNode())(TableName.Reminder)
.whereIn(`${TableName.Reminder}.secretId`, secretIds)
.leftJoin(TableName.ReminderRecipient, `${TableName.Reminder}.id`, `${TableName.ReminderRecipient}.reminderId`)
.select(selectAllTableCols(TableName.Reminder))
backend/src/services/resource-cleanup/resource-cleanup-queue.ts
+9 -1
View File
@@ -1,5 +1,6 @@
import { TAuditLogDALFactory } from "@app/ee/services/audit-log/audit-log-dal";
import { TSnapshotDALFactory } from "@app/ee/services/secret-snapshot/snapshot-dal";
import { TKeyValueStoreDALFactory } from "@app/keystore/key-value-store-dal";
import { getConfig } from "@app/lib/config/env";
import { logger } from "@app/lib/logger";
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
@@ -27,6 +28,7 @@ type TDailyResourceCleanUpQueueServiceFactoryDep = {
queueService: TQueueServiceFactory;
orgService: TOrgServiceFactory;
userNotificationDAL: Pick<TUserNotificationDALFactory, "pruneNotifications">;
keyValueStoreDAL: Pick<TKeyValueStoreDALFactory, "pruneExpiredKeys">;
};
export type TDailyResourceCleanUpQueueServiceFactory = ReturnType<typeof dailyResourceCleanUpQueueServiceFactory>;
@@ -43,7 +45,8 @@ export const dailyResourceCleanUpQueueServiceFactory = ({
identityUniversalAuthClientSecretDAL,
serviceTokenService,
orgService,
userNotificationDAL
userNotificationDAL,
keyValueStoreDAL
}: TDailyResourceCleanUpQueueServiceFactoryDep) => {
const appCfg = getConfig();
@@ -52,6 +55,10 @@ export const dailyResourceCleanUpQueueServiceFactory = ({
}
const init = async () => {
if (appCfg.isSecondaryInstance) {
return;
}
await queueService.stopRepeatableJob(
QueueName.AuditLogPrune,
QueueJobs.AuditLogPrune,
@@ -82,6 +89,7 @@ export const dailyResourceCleanUpQueueServiceFactory = ({
await orgService.notifyInvitedUsers();
await auditLogDAL.pruneAuditLog();
await userNotificationDAL.pruneNotifications();
await keyValueStoreDAL.pruneExpiredKeys();
logger.info(`${QueueName.DailyResourceCleanUp}: queue task completed`);
} catch (error) {
logger.error(error, `${QueueName.DailyResourceCleanUp}: resource cleanup failed`);
backend/src/services/secret-folder/secret-folder-service.ts
+4 -1
View File
@@ -47,7 +47,7 @@ type TSecretFolderServiceFactoryDep = {
folderCommitService: Pick<TFolderCommitServiceFactory, "createCommit">;
projectDAL: Pick<TProjectDALFactory, "findProjectBySlug">;
secretApprovalPolicyService: Pick<TSecretApprovalPolicyServiceFactory, "getSecretApprovalPolicy">;
secretV2BridgeDAL: Pick<TSecretV2BridgeDALFactory, "findByFolderIds">;
secretV2BridgeDAL: Pick<TSecretV2BridgeDALFactory, "findByFolderIds" | "invalidateSecretCacheByProjectId">;
};
export type TSecretFolderServiceFactory = ReturnType<typeof secretFolderServiceFactory>;
@@ -398,6 +398,7 @@ export const secretFolderServiceFactory = ({
await Promise.all(result.map(async (res) => snapshotService.performSnapshot(res.newFolder.parentId as string)));
await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId);
return {
projectId,
newFolders: result.map((res) => res.newFolder),
@@ -522,6 +523,7 @@ export const secretFolderServiceFactory = ({
}
await snapshotService.performSnapshot(newFolder.parentId as string);
await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId);
return {
folder: { ...newFolder, path: newFolderWithFullPath.path },
old: { ...folder, path: folderWithFullPath.path }
@@ -724,6 +726,7 @@ export const secretFolderServiceFactory = ({
});
await snapshotService.performSnapshot(folder.parentId as string);
await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId);
return folder;
};
backend/src/services/secret-folder/secret-folder-version-dal.ts
+1 -1
View File
@@ -45,7 +45,7 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => {
)
.whereIn(`${TableName.SecretFolderVersion}.folderId`, folderIds)
.join(
(tx || db)(TableName.SecretFolderVersion)
(tx || db.replicaNode())(TableName.SecretFolderVersion)
.groupBy("folderId")
.max("version")
.select("folderId")
backend/src/services/secret-import/secret-import-dal.ts
+1 -1
View File
@@ -15,7 +15,7 @@ export const secretImportDALFactory = (db: TDbClient) => {
// we are using postion based sorting as its a small list
// this will return the last value of the position in a folder with secret imports
const findLastImportPosition = async (folderId: string, tx?: Knex) => {
const lastPos = await (tx || db)(TableName.SecretImport)
const lastPos = await (tx || db.replicaNode())(TableName.SecretImport)
.where({ folderId })
.max("position", { as: "position" })
.first();
backend/src/services/secret-sharing/secret-sharing-dal.ts
+1 -1
View File
@@ -119,7 +119,7 @@ export const secretSharingDALFactory = (db: TDbClient) => {
const findActiveSharedSecrets = async (filters: Partial<TSecretSharing>, tx?: Knex) => {
try {
const now = new Date();
return await (tx || db)(TableName.SecretSharing)
return await (tx || db.replicaNode())(TableName.SecretSharing)
.where(filters)
.andWhere("expiresAt", ">", now)
.andWhere("encryptedValue", "<>", "")
backend/src/services/secret-sync/github/github-sync-fns.ts
+22 -14
View File
@@ -1,6 +1,7 @@
import sodium from "libsodium-wrappers";
import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
import { TGatewayV2ServiceFactory } from "@app/ee/services/gateway-v2/gateway-v2-service";
import {
getGitHubAppAuthToken,
getGitHubInstanceApiUrl,
@@ -20,7 +21,8 @@ import { TGitHubPublicKey, TGitHubSecret, TGitHubSecretPayload, TGitHubSyncWithC
const getEncryptedSecrets = async (
secretSync: TGitHubSyncWithCredentials,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
gatewayV2Service: Pick<TGatewayV2ServiceFactory, "getPlatformConnectionDetailsByGatewayId">
) => {
const { destinationConfig, connection } = secretSync;
@@ -44,6 +46,7 @@ const getEncryptedSecrets = async (
return makePaginatedGitHubRequest<TGitHubSecret, { secrets: TGitHubSecret[] }>(
connection,
gatewayService,
gatewayV2Service,
path,
(data) => data.secrets
);
@@ -52,6 +55,7 @@ const getEncryptedSecrets = async (
const getPublicKey = async (
secretSync: TGitHubSyncWithCredentials,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
gatewayV2Service: Pick<TGatewayV2ServiceFactory, "getPlatformConnectionDetailsByGatewayId">,
token: string
) => {
const { destinationConfig, connection } = secretSync;
@@ -73,7 +77,7 @@ const getPublicKey = async (
}
}
const response = await requestWithGitHubGateway<TGitHubPublicKey>(connection, gatewayService, {
const response = await requestWithGitHubGateway<TGitHubPublicKey>(connection, gatewayService, gatewayV2Service, {
url: `https://${await getGitHubInstanceApiUrl(connection)}${path}`,
method: "GET",
headers: {
@@ -89,6 +93,7 @@ const getPublicKey = async (
const deleteSecret = async (
secretSync: TGitHubSyncWithCredentials,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
gatewayV2Service: Pick<TGatewayV2ServiceFactory, "getPlatformConnectionDetailsByGatewayId">,
token: string,
encryptedSecret: TGitHubSecret
) => {
@@ -111,7 +116,7 @@ const deleteSecret = async (
}
}
await requestWithGitHubGateway(connection, gatewayService, {
await requestWithGitHubGateway(connection, gatewayService, gatewayV2Service, {
url: `https://${await getGitHubInstanceApiUrl(connection)}${path}`,
method: "DELETE",
headers: {
@@ -125,6 +130,7 @@ const deleteSecret = async (
const putSecret = async (
secretSync: TGitHubSyncWithCredentials,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
gatewayV2Service: Pick<TGatewayV2ServiceFactory, "getPlatformConnectionDetailsByGatewayId">,
token: string,
payload: TGitHubSecretPayload
) => {
@@ -157,7 +163,7 @@ const putSecret = async (
}
}
await requestWithGitHubGateway(connection, gatewayService, {
await requestWithGitHubGateway(connection, gatewayService, gatewayV2Service, {
url: `https://${await getGitHubInstanceApiUrl(connection)}${path}`,
method: "PUT",
headers: {
@@ -173,7 +179,8 @@ export const GithubSyncFns = {
syncSecrets: async (
secretSync: TGitHubSyncWithCredentials,
ogSecretMap: TSecretMap,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
gatewayV2Service: Pick<TGatewayV2ServiceFactory, "getPlatformConnectionDetailsByGatewayId">
) => {
const secretMap = Object.fromEntries(Object.entries(ogSecretMap).map(([i, v]) => [i.toUpperCase(), v]));
@@ -207,10 +214,10 @@ export const GithubSyncFns = {
const token =
connection.method === GitHubConnectionMethod.OAuth
? connection.credentials.accessToken
: await getGitHubAppAuthToken(connection, gatewayService);
: await getGitHubAppAuthToken(connection, gatewayService, gatewayV2Service);
const encryptedSecrets = await getEncryptedSecrets(secretSync, gatewayService);
const publicKey = await getPublicKey(secretSync, gatewayService, token);
const encryptedSecrets = await getEncryptedSecrets(secretSync, gatewayService, gatewayV2Service);
const publicKey = await getPublicKey(secretSync, gatewayService, gatewayV2Service, token);
await sodium.ready;
for await (const key of Object.keys(secretMap)) {
@@ -225,7 +232,7 @@ export const GithubSyncFns = {
const encryptedSecretValue = sodium.to_base64(encryptedBytes, sodium.base64_variants.ORIGINAL);
try {
await putSecret(secretSync, gatewayService, token, {
await putSecret(secretSync, gatewayService, gatewayV2Service, token, {
secret_name: key,
encrypted_value: encryptedSecretValue,
key_id: publicKey.key_id
@@ -246,7 +253,7 @@ export const GithubSyncFns = {
continue;
if (!(encryptedSecret.name in secretMap)) {
await deleteSecret(secretSync, gatewayService, token, encryptedSecret);
await deleteSecret(secretSync, gatewayService, gatewayV2Service, token, encryptedSecret);
}
}
},
@@ -256,7 +263,8 @@ export const GithubSyncFns = {
removeSecrets: async (
secretSync: TGitHubSyncWithCredentials,
ogSecretMap: TSecretMap,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
gatewayV2Service: Pick<TGatewayV2ServiceFactory, "getPlatformConnectionDetailsByGatewayId">
) => {
const secretMap = Object.fromEntries(Object.entries(ogSecretMap).map(([i, v]) => [i.toUpperCase(), v]));
@@ -264,13 +272,13 @@ export const GithubSyncFns = {
const token =
connection.method === GitHubConnectionMethod.OAuth
? connection.credentials.accessToken
: await getGitHubAppAuthToken(connection, gatewayService);
: await getGitHubAppAuthToken(connection, gatewayService, gatewayV2Service);
const encryptedSecrets = await getEncryptedSecrets(secretSync, gatewayService);
const encryptedSecrets = await getEncryptedSecrets(secretSync, gatewayService, gatewayV2Service);
for await (const encryptedSecret of encryptedSecrets) {
if (encryptedSecret.name in secretMap) {
await deleteSecret(secretSync, gatewayService, token, encryptedSecret);
await deleteSecret(secretSync, gatewayService, gatewayV2Service, token, encryptedSecret);
}
}
}
backend/src/services/secret-sync/secret-sync-fns.ts
+6 -4
View File
@@ -2,6 +2,7 @@ import { AxiosError } from "axios";
import handlebars from "handlebars";
import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
import { TGatewayV2ServiceFactory } from "@app/ee/services/gateway-v2/gateway-v2-service";
import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
import { OCI_VAULT_SYNC_LIST_OPTION, OCIVaultSyncFns } from "@app/ee/services/secret-sync/oci-vault";
import { BadRequestError } from "@app/lib/errors";
@@ -101,6 +102,7 @@ type TSyncSecretDeps = {
appConnectionDAL: Pick<TAppConnectionDALFactory, "findById" | "update" | "updateById">;
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">;
gatewayV2Service: Pick<TGatewayV2ServiceFactory, "getPlatformConnectionDetailsByGatewayId">;
};
// Add schema to secret keys
@@ -195,7 +197,7 @@ export const SecretSyncFns = {
syncSecrets: (
secretSync: TSecretSyncWithCredentials,
secretMap: TSecretMap,
{ kmsService, appConnectionDAL, gatewayService }: TSyncSecretDeps
{ kmsService, appConnectionDAL, gatewayService, gatewayV2Service }: TSyncSecretDeps
): Promise<void> => {
const schemaSecretMap = addSchema(secretMap, secretSync.environment?.slug || "", secretSync.syncOptions.keySchema);
@@ -205,7 +207,7 @@ export const SecretSyncFns = {
case SecretSync.AWSSecretsManager:
return AwsSecretsManagerSyncFns.syncSecrets(secretSync, schemaSecretMap);
case SecretSync.GitHub:
return GithubSyncFns.syncSecrets(secretSync, schemaSecretMap, gatewayService);
return GithubSyncFns.syncSecrets(secretSync, schemaSecretMap, gatewayService, gatewayV2Service);
case SecretSync.GCPSecretManager:
return GcpSyncFns.syncSecrets(secretSync, schemaSecretMap);
case SecretSync.AzureKeyVault:
@@ -404,7 +406,7 @@ export const SecretSyncFns = {
removeSecrets: (
secretSync: TSecretSyncWithCredentials,
secretMap: TSecretMap,
{ kmsService, appConnectionDAL, gatewayService }: TSyncSecretDeps
{ kmsService, appConnectionDAL, gatewayService, gatewayV2Service }: TSyncSecretDeps
): Promise<void> => {
const schemaSecretMap = addSchema(secretMap, secretSync.environment?.slug || "", secretSync.syncOptions.keySchema);
@@ -414,7 +416,7 @@ export const SecretSyncFns = {
case SecretSync.AWSSecretsManager:
return AwsSecretsManagerSyncFns.removeSecrets(secretSync, schemaSecretMap);
case SecretSync.GitHub:
return GithubSyncFns.removeSecrets(secretSync, schemaSecretMap, gatewayService);
return GithubSyncFns.removeSecrets(secretSync, schemaSecretMap, gatewayService, gatewayV2Service);
case SecretSync.GCPSecretManager:
return GcpSyncFns.removeSecrets(secretSync, schemaSecretMap);
case SecretSync.AzureKeyVault:
backend/src/services/secret-sync/secret-sync-queue.ts
+10 -4
View File
@@ -5,6 +5,7 @@ import { Job } from "bullmq";
import { ProjectMembershipRole, SecretType } from "@app/db/schemas";
import { EventType, TAuditLogServiceFactory } from "@app/ee/services/audit-log/audit-log-types";
import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
import { TGatewayV2ServiceFactory } from "@app/ee/services/gateway-v2/gateway-v2-service";
import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore";
import { getConfig } from "@app/lib/config/env";
@@ -98,6 +99,7 @@ type TSecretSyncQueueFactoryDep = {
folderCommitService: Pick<TFolderCommitServiceFactory, "createCommit">;
licenseService: Pick<TLicenseServiceFactory, "getPlan">;
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">;
gatewayV2Service: Pick<TGatewayV2ServiceFactory, "getPlatformConnectionDetailsByGatewayId">;
};
type SecretSyncActionJob = Job<
@@ -139,7 +141,8 @@ export const secretSyncQueueFactory = ({
resourceMetadataDAL,
folderCommitService,
licenseService,
gatewayService
gatewayService,
gatewayV2Service
}: TSecretSyncQueueFactoryDep) => {
const appCfg = getConfig();
@@ -395,7 +398,8 @@ export const secretSyncQueueFactory = ({
const importedSecrets = await SecretSyncFns.getSecrets(secretSync, {
appConnectionDAL,
kmsService,
gatewayService
gatewayService,
gatewayV2Service
});
if (!Object.keys(importedSecrets).length) return {};
@@ -520,7 +524,8 @@ export const secretSyncQueueFactory = ({
await SecretSyncFns.syncSecrets(secretSyncWithCredentials, secretMap, {
appConnectionDAL,
kmsService,
gatewayService
gatewayService,
gatewayV2Service
});
isSynced = true;
@@ -762,7 +767,8 @@ export const secretSyncQueueFactory = ({
{
appConnectionDAL,
kmsService,
gatewayService
gatewayService,
gatewayV2Service
}
);
backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts
+9 -10
View File
@@ -50,20 +50,19 @@ interface TSecretV2DalArg {
}
export const SECRET_DAL_TTL = () => applyJitter(10 * 60, 2 * 60);
export const SECRET_DAL_VERSION_TTL = 15 * 60;
export const SECRET_DAL_VERSION_TTL = "15m";
export const MAX_SECRET_CACHE_BYTES = 25 * 1024 * 1024;
export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => {
const secretOrm = ormify(db, TableName.SecretV2);
const invalidateSecretCacheByProjectId = async (projectId: string) => {
const invalidateSecretCacheByProjectId = async (projectId: string, tx?: Knex) => {
const secretDalVersionKey = SecretServiceCacheKeys.getSecretDalVersion(projectId);
await keyStore.incrementBy(secretDalVersionKey, 1);
await keyStore.setExpiry(secretDalVersionKey, SECRET_DAL_VERSION_TTL);
await keyStore.pgIncrementBy(secretDalVersionKey, { incr: 1, tx, expiry: SECRET_DAL_VERSION_TTL });
};
const findOne = async (filter: Partial<TSecretsV2>, tx?: Knex) => {
try {
const docs = await (tx || db)(TableName.SecretV2)
const docs = await (tx || db.replicaNode())(TableName.SecretV2)
// eslint-disable-next-line @typescript-eslint/no-misused-promises
.where(buildFindFilter(filter, TableName.SecretV2))
.leftJoin(
@@ -144,7 +143,7 @@ export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => {
const find = async (filter: TFindFilter<TSecretsV2>, opts: TFindOpt<TSecretsV2> = {}) => {
const { offset, limit, sort, tx } = opts;
try {
const query = (tx || db)(TableName.SecretV2)
const query = (tx || db.replicaNode())(TableName.SecretV2)
// eslint-disable-next-line @typescript-eslint/no-misused-promises
.where(buildFindFilter(filter))
.leftJoin(
@@ -888,13 +887,13 @@ export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => {
const findSecretsWithReminderRecipients = async (ids: string[], limit: number, tx?: Knex) => {
try {
// Create a subquery to get limited secret IDs
const limitedSecretIds = (tx || db)(TableName.SecretV2)
const limitedSecretIds = (tx || db.replicaNode())(TableName.SecretV2)
.whereIn(`${TableName.SecretV2}.id`, ids)
.limit(limit)
.select("id");
// Join with all recipients for the limited secrets
const docs = await (tx || db)(TableName.SecretV2)
const docs = await (tx || db.replicaNode())(TableName.SecretV2)
.whereIn(`${TableName.SecretV2}.id`, limitedSecretIds)
.leftJoin(TableName.Reminder, `${TableName.SecretV2}.id`, `${TableName.Reminder}.secretId`)
.leftJoin(TableName.ReminderRecipient, `${TableName.Reminder}.id`, `${TableName.ReminderRecipient}.reminderId`)
@@ -926,13 +925,13 @@ export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => {
const findSecretsWithReminderRecipientsOld = async (ids: string[], limit: number, tx?: Knex) => {
try {
// Create a subquery to get limited secret IDs
const limitedSecretIds = (tx || db)(TableName.SecretV2)
const limitedSecretIds = (tx || db.replicaNode())(TableName.SecretV2)
.whereIn(`${TableName.SecretV2}.id`, ids)
.limit(limit)
.select("id");
// Join with all recipients for the limited secrets
const docs = await (tx || db)(TableName.SecretV2)
const docs = await (tx || db.replicaNode())(TableName.SecretV2)
.whereIn(`${TableName.SecretV2}.id`, limitedSecretIds)
.leftJoin(TableName.Reminder, `${TableName.SecretV2}.id`, `${TableName.Reminder}.secretId`)
.leftJoin(
backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts
+31 -18
View File
@@ -118,7 +118,7 @@ type TSecretV2BridgeServiceFactoryDep = {
>;
snapshotService: Pick<TSecretSnapshotServiceFactory, "performSnapshot">;
resourceMetadataDAL: Pick<TResourceMetadataDALFactory, "insertMany" | "delete">;
keyStore: Pick<TKeyStoreFactory, "getItem" | "setExpiry" | "setItemWithExpiry" | "deleteItem">;
keyStore: Pick<TKeyStoreFactory, "getItem" | "setExpiry" | "setItemWithExpiry" | "deleteItem" | "pgGetIntItem">;
reminderService: Pick<TReminderServiceFactory, "createReminder" | "getReminder">;
};
@@ -373,6 +373,7 @@ export const secretV2BridgeServiceFactory = ({
tx
});
await secretDAL.invalidateSecretCacheByProjectId(projectId, tx);
return createdSecret;
});
@@ -390,7 +391,6 @@ export const secretV2BridgeServiceFactory = ({
});
}
await secretDAL.invalidateSecretCacheByProjectId(projectId);
if (inputSecret.type === SecretType.Shared) {
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({
@@ -579,8 +579,16 @@ export const secretV2BridgeServiceFactory = ({
}
: {};
const updatedSecret = await secretDAL.transaction(async (tx) =>
fnSecretBulkUpdate({
if (secretValue) {
const { nestedReferences, localReferences } = getAllSecretReferences(secretValue);
const allSecretReferences = nestedReferences.concat(
localReferences.map((el) => ({ secretKey: el, secretPath, environment }))
);
await $validateSecretReferences(projectId, permission, allSecretReferences);
}
const updatedSecret = await secretDAL.transaction(async (tx) => {
const modifiedSecretsInDB = await fnSecretBulkUpdate({
folderId,
orgId: actorOrgId,
resourceMetadataDAL,
@@ -611,8 +619,11 @@ export const secretV2BridgeServiceFactory = ({
actorId
},
tx
})
);
});
await secretDAL.invalidateSecretCacheByProjectId(projectId, tx);
return modifiedSecretsInDB;
});
if (inputSecret.secretReminderRepeatDays) {
await reminderService.createReminder({
actor,
@@ -628,7 +639,6 @@ export const secretV2BridgeServiceFactory = ({
});
}
await secretDAL.invalidateSecretCacheByProjectId(projectId);
if (inputSecret.type === SecretType.Shared) {
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({
@@ -728,8 +738,8 @@ export const secretV2BridgeServiceFactory = ({
);
try {
const deletedSecret = await secretDAL.transaction(async (tx) =>
fnSecretBulkDelete({
const deletedSecret = await secretDAL.transaction(async (tx) => {
const modifiedSecretsInDB = await fnSecretBulkDelete({
projectId,
folderId,
actorId,
@@ -745,10 +755,11 @@ export const secretV2BridgeServiceFactory = ({
}
],
tx
})
);
});
await secretDAL.invalidateSecretCacheByProjectId(projectId, tx);
return modifiedSecretsInDB;
});
await secretDAL.invalidateSecretCacheByProjectId(projectId);
if (inputSecret.type === SecretType.Shared) {
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({
@@ -1040,7 +1051,7 @@ export const secretV2BridgeServiceFactory = ({
});
throwIfMissingSecretReadValueOrDescribePermission(permission, ProjectPermissionSecretActions.DescribeSecret);
const cachedSecretDalVersion = await keyStore.getItem(SecretServiceCacheKeys.getSecretDalVersion(projectId));
const cachedSecretDalVersion = await keyStore.pgGetIntItem(SecretServiceCacheKeys.getSecretDalVersion(projectId));
const secretDalVersion = Number(cachedSecretDalVersion || 0);
const cacheKey = SecretServiceCacheKeys.getSecretsOfServiceLayer(projectId, secretDalVersion, {
...dto,
@@ -1705,7 +1716,7 @@ export const secretV2BridgeServiceFactory = ({
await kmsService.createCipherPairWithDataKey({ type: KmsDataKey.SecretManager, projectId });
const executeBulkInsert = async (tx: Knex) => {
return fnSecretBulkInsert({
const modifiedSecretsInDB = await fnSecretBulkInsert({
inputSecrets: inputSecrets.map((el) => {
const references = secretReferencesGroupByInputSecretKey[el.secretKey]?.nestedReferences;
@@ -1741,13 +1752,14 @@ export const secretV2BridgeServiceFactory = ({
},
tx
});
await secretDAL.invalidateSecretCacheByProjectId(projectId, tx);
return modifiedSecretsInDB;
};
const newSecrets = providedTx
? await executeBulkInsert(providedTx)
: await secretDAL.transaction(executeBulkInsert);
await secretDAL.invalidateSecretCacheByProjectId(projectId);
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({
actor,
@@ -2112,6 +2124,7 @@ export const secretV2BridgeServiceFactory = ({
}
}
await secretDAL.invalidateSecretCacheByProjectId(projectId, tx);
return updatedSecrets;
};
@@ -2119,7 +2132,6 @@ export const secretV2BridgeServiceFactory = ({
? await executeBulkUpdate(providedTx)
: await secretDAL.transaction(executeBulkUpdate);
await secretDAL.invalidateSecretCacheByProjectId(projectId);
await Promise.allSettled(folders.map((el) => (el?.id ? snapshotService.performSnapshot(el.id) : undefined)));
await Promise.allSettled(
folders.map((el) =>
@@ -2246,7 +2258,7 @@ export const secretV2BridgeServiceFactory = ({
});
const executeBulkDelete = async (tx: Knex) => {
return fnSecretBulkDelete({
const modifiedSecretsInDB = await fnSecretBulkDelete({
secretDAL,
secretQueueService,
folderCommitService,
@@ -2262,6 +2274,8 @@ export const secretV2BridgeServiceFactory = ({
commitChanges,
tx
});
await secretDAL.invalidateSecretCacheByProjectId(projectId, tx);
return modifiedSecretsInDB;
};
try {
@@ -2269,7 +2283,6 @@ export const secretV2BridgeServiceFactory = ({
? await executeBulkDelete(providedTx)
: await secretDAL.transaction(executeBulkDelete);
await secretDAL.invalidateSecretCacheByProjectId(projectId);
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({
actor,
backend/src/services/secret-v2-bridge/secret-version-dal.ts
+3 -3
View File
@@ -72,7 +72,7 @@ export const secretVersionV2BridgeDALFactory = (db: TDbClient) => {
.where(`${TableName.SecretVersionV2}.folderId`, folderId)
.join(TableName.SecretV2, `${TableName.SecretV2}.id`, `${TableName.SecretVersionV2}.secretId`)
.join<TSecretVersionsV2, TSecretVersionsV2 & { secretId: string; max: number }>(
(tx || db)(TableName.SecretVersionV2)
(tx || db.replicaNode())(TableName.SecretVersionV2)
.where(`${TableName.SecretVersionV2}.folderId`, folderId)
.groupBy("secretId")
.max("version")
@@ -121,7 +121,7 @@ export const secretVersionV2BridgeDALFactory = (db: TDbClient) => {
.where("folderId", folderId)
.whereIn(`${TableName.SecretVersionV2}.secretId`, secretIds)
.join(
(tx || db)(TableName.SecretVersionV2)
(tx || db.replicaNode())(TableName.SecretVersionV2)
.groupBy("secretId")
.max("version")
.select("secretId")
@@ -189,7 +189,7 @@ export const secretVersionV2BridgeDALFactory = (db: TDbClient) => {
}) => {
try {
const { offset, limit, sort = [["createdAt", "desc"]] } = findOpt;
const query = (tx || db)(TableName.SecretVersionV2)
const query = (tx || db.replicaNode())(TableName.SecretVersionV2)
.leftJoin(TableName.Users, `${TableName.Users}.id`, `${TableName.SecretVersionV2}.userActorId`)
.leftJoin(
TableName.ProjectMembership,
backend/src/services/super-admin/super-admin-dal.ts
+1 -1
View File
@@ -11,7 +11,7 @@ export const superAdminDALFactory = (db: TDbClient) => {
const superAdminOrm = ormify(db, TableName.SuperAdmin);
const findById = async (id: string, tx?: Knex) => {
const config = await (tx || db)(TableName.SuperAdmin)
const config = await (tx || db.replicaNode())(TableName.SuperAdmin)
.where(`${TableName.SuperAdmin}.id`, id)
.leftJoin(TableName.Organization, `${TableName.SuperAdmin}.defaultAuthOrgId`, `${TableName.Organization}.id`)
.leftJoin(TableName.SamlConfig, (qb) => {
backend/src/services/user/user-dal.ts
+8 -4
View File
@@ -19,12 +19,16 @@ export type TUserDALFactory = ReturnType<typeof userDALFactory>;
export const userDALFactory = (db: TDbClient) => {
const userOrm = ormify(db, TableName.Users);
const findUserByUsername = async (username: string, tx?: Knex) =>
(tx || db)(TableName.Users).whereRaw('lower("username") = :username', { username: username.toLowerCase() });
(tx || db.replicaNode())(TableName.Users).whereRaw('lower("username") = :username', {
username: username.toLowerCase()
});
const findUserByEmail = async (email: string, tx?: Knex) =>
(tx || db)(TableName.Users).whereRaw('lower("email") = :email', { email: email.toLowerCase() }).where({
isEmailVerified: true
});
(tx || db.replicaNode())(TableName.Users)
.whereRaw('lower("email") = :email', { email: email.toLowerCase() })
.where({
isEmailVerified: true
});
const getUsersByFilter = async ({
limit,
backend/src/services/user/user-service.ts
+138 -3
View File
@@ -1,4 +1,5 @@
import { ForbiddenError } from "@casl/ability";
import { Knex } from "knex";
import { OrgPermissionActions, OrgPermissionSubjects } from "@app/ee/services/permission/org-permission";
import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service-types";
@@ -7,6 +8,7 @@ import { BadRequestError, ForbiddenRequestError, NotFoundError } from "@app/lib/
import { logger } from "@app/lib/logger";
import { TAuthTokenServiceFactory } from "@app/services/auth-token/auth-token-service";
import { TokenType } from "@app/services/auth-token/auth-token-types";
import { TOrgDALFactory } from "@app/services/org/org-dal";
import { TOrgMembershipDALFactory } from "@app/services/org-membership/org-membership-dal";
import { SmtpTemplates, TSmtpService } from "@app/services/smtp/smtp-service";
@@ -15,7 +17,7 @@ import { TGroupProjectDALFactory } from "../group-project/group-project-dal";
import { TProjectMembershipDALFactory } from "../project-membership/project-membership-dal";
import { TUserAliasDALFactory } from "../user-alias/user-alias-dal";
import { TUserDALFactory } from "./user-dal";
import { TListUserGroupsDTO, TUpdateUserMfaDTO } from "./user-types";
import { TListUserGroupsDTO, TUpdateUserEmailDTO, TUpdateUserMfaDTO } from "./user-types";
type TUserServiceFactoryDep = {
userDAL: Pick<
@@ -34,18 +36,20 @@ type TUserServiceFactoryDep = {
| "findAllMyAccounts"
>;
groupProjectDAL: Pick<TGroupProjectDALFactory, "findByUserId">;
orgDAL: Pick<TOrgDALFactory, "findById" | "find">;
orgMembershipDAL: Pick<TOrgMembershipDALFactory, "find" | "insertMany" | "findOne" | "updateById">;
tokenService: Pick<TAuthTokenServiceFactory, "createTokenForUser" | "validateTokenForUser">;
tokenService: Pick<TAuthTokenServiceFactory, "createTokenForUser" | "validateTokenForUser" | "revokeAllMySessions">;
projectMembershipDAL: Pick<TProjectMembershipDALFactory, "find">;
smtpService: Pick<TSmtpService, "sendMail">;
permissionService: TPermissionServiceFactory;
userAliasDAL: Pick<TUserAliasDALFactory, "findOne" | "find" | "updateById">;
userAliasDAL: Pick<TUserAliasDALFactory, "findOne" | "find" | "updateById" | "delete">;
};
export type TUserServiceFactory = ReturnType<typeof userServiceFactory>;
export const userServiceFactory = ({
userDAL,
orgDAL,
orgMembershipDAL,
projectMembershipDAL,
groupProjectDAL,
@@ -178,6 +182,135 @@ export const userServiceFactory = ({
return updatedUser;
};
const checkUserScimRestriction = async (userId: string, tx?: Knex) => {
const userOrgs = await orgMembershipDAL.find({ userId }, { tx });
if (userOrgs.length === 0) {
return false;
}
const orgIds = userOrgs.map((membership) => membership.orgId);
const organizations = await orgDAL.find({ $in: { id: orgIds } }, { tx });
return organizations.some((org) => org.scimEnabled);
};
const requestEmailChangeOTP = async ({ userId, newEmail }: TUpdateUserEmailDTO) => {
const startTime = new Date();
const changeEmailOTP = await userDAL.transaction(async (tx) => {
const user = await userDAL.findById(userId, tx);
if (!user)
throw new NotFoundError({ message: `User with ID '${userId}' not found`, name: "RequestEmailChangeOTP" });
if (user.authMethods?.includes(AuthMethod.LDAP)) {
throw new BadRequestError({ message: "Cannot update email for LDAP users", name: "RequestEmailChangeOTP" });
}
const hasScimRestriction = await checkUserScimRestriction(userId, tx);
if (hasScimRestriction) {
throw new BadRequestError({
message: "Email changes are disabled because SCIM is enabled for one or more of your organizations",
name: "RequestEmailChangeOTP"
});
}
// Silently check if another user already has this email - don't send OTP if email is taken
const existingUsers = await userDAL.findUserByUsername(newEmail.toLowerCase(), tx);
const existingUser = existingUsers?.find((u) => u.id !== userId);
if (!existingUser) {
// Generate 6-digit OTP
const otpCode = await tokenService.createTokenForUser({
type: TokenType.TOKEN_EMAIL_CHANGE_OTP,
userId,
payload: newEmail.toLowerCase()
});
// Send OTP to NEW email address
await smtpService.sendMail({
template: SmtpTemplates.EmailVerification,
subjectLine: "Infisical email change verification",
recipients: [newEmail.toLowerCase()],
substitutions: {
code: otpCode
}
});
}
return { success: true, message: "Verification code sent to new email address" };
});
// Force this function to have a minimum execution time of 2 seconds to avoid possible information disclosure about existing users
const endTime = new Date();
const timeDiff = endTime.getTime() - startTime.getTime();
if (timeDiff < 2000) {
await new Promise((resolve) => {
setTimeout(resolve, 2000 - timeDiff);
});
}
return changeEmailOTP;
};
const updateUserEmail = async ({ userId, newEmail, otpCode }: TUpdateUserEmailDTO & { otpCode: string }) => {
const changedUser = await userDAL.transaction(async (tx) => {
const user = await userDAL.findById(userId, tx);
if (!user) throw new NotFoundError({ message: `User with ID '${userId}' not found`, name: "UpdateUserEmail" });
if (user.authMethods?.includes(AuthMethod.LDAP)) {
throw new BadRequestError({ message: "Cannot update email for LDAP users", name: "UpdateUserEmail" });
}
const hasScimRestriction = await checkUserScimRestriction(userId, tx);
if (hasScimRestriction) {
throw new BadRequestError({
message: "You are part of an organization that has SCIM enabled, and email changes are not allowed",
name: "UpdateUserEmail"
});
}
// Validate OTP and get the new email from token aliasId field
let tokenData;
try {
tokenData = await tokenService.validateTokenForUser({
type: TokenType.TOKEN_EMAIL_CHANGE_OTP,
userId,
code: otpCode
});
} catch (error) {
throw new BadRequestError({ message: "Invalid verification code", name: "UpdateUserEmail" });
}
// Verify the new email matches what was stored in payload
const tokenNewEmail = tokenData?.payload;
if (!tokenNewEmail || tokenNewEmail !== newEmail.toLowerCase()) {
throw new BadRequestError({ message: "Invalid verification code", name: "UpdateUserEmail" });
}
// Final check if another user has this email
const existingUsers = await userDAL.findUserByUsername(newEmail.toLowerCase(), tx);
const existingUser = existingUsers?.find((u) => u.id !== userId);
if (existingUser) {
throw new BadRequestError({ message: "Email is no longer available", name: "UpdateUserEmail" });
}
// Delete all user aliases since the email is changing
await userAliasDAL.delete({ userId }, tx);
const updatedUser = await userDAL.updateById(
userId,
{
email: newEmail.toLowerCase(),
username: newEmail.toLowerCase()
},
tx
);
// Revoke all sessions to force re-login
await tokenService.revokeAllMySessions(userId);
return updatedUser;
});
return changedUser;
};
const getAllMyAccounts = async (email: string, userId: string) => {
const users = await userDAL.findAllMyAccounts(email);
return users?.map((el) => ({ ...el, isMyAccount: el.id === userId }));
@@ -313,6 +446,8 @@ export const userServiceFactory = ({
updateUserMfa,
updateUserName,
updateAuthMethods,
requestEmailChangeOTP,
updateUserEmail,
deleteUser,
getMe,
createUserAction,
backend/src/services/user/user-types.ts
+5
View File
@@ -16,3 +16,8 @@ export type TUpdateUserMfaDTO = {
isMfaEnabled?: boolean;
selectedMfaMethod?: MfaMethod;
};
export type TUpdateUserEmailDTO = {
userId: string;
newEmail: string;
};