mirror of
https://github.com/awatertrevi/infisical.git
synced 2026-10-14 10:29:52 +00:00
Merge branch 'main' into ENG-3009
This commit is contained in:
x032205
@@ -1,3 +1,4 @@
|
||||
import { logger } from "@app/lib/logger";
|
||||
import { OrgServiceActor } from "@app/lib/types";
|
||||
|
||||
import { AppConnection } from "../app-connection-enums";
|
||||
@@ -19,6 +20,7 @@ export const gcpConnectionService = (getAppConnection: TGetAppConnectionFunc) =>
|
||||
|
||||
return projects;
|
||||
} catch (error) {
|
||||
logger.error(error, "Error listing GCP secret manager projects");
|
||||
return [];
|
||||
}
|
||||
};
|
||||
|
||||
@@ -17,70 +17,11 @@ export const identityAccessTokenDALFactory = (db: TDbClient) => {
|
||||
const doc = await (tx || db.replicaNode())(TableName.IdentityAccessToken)
|
||||
.where(filter)
|
||||
.join(TableName.Identity, `${TableName.Identity}.id`, `${TableName.IdentityAccessToken}.identityId`)
|
||||
.leftJoin(
|
||||
TableName.IdentityUaClientSecret,
|
||||
`${TableName.IdentityAccessToken}.identityUAClientSecretId`,
|
||||
`${TableName.IdentityUaClientSecret}.id`
|
||||
)
|
||||
.leftJoin(
|
||||
TableName.IdentityUniversalAuth,
|
||||
`${TableName.IdentityUaClientSecret}.identityUAId`,
|
||||
`${TableName.IdentityUniversalAuth}.id`
|
||||
)
|
||||
.leftJoin(TableName.IdentityGcpAuth, `${TableName.Identity}.id`, `${TableName.IdentityGcpAuth}.identityId`)
|
||||
.leftJoin(
|
||||
TableName.IdentityAliCloudAuth,
|
||||
`${TableName.Identity}.id`,
|
||||
`${TableName.IdentityAliCloudAuth}.identityId`
|
||||
)
|
||||
.leftJoin(TableName.IdentityAwsAuth, `${TableName.Identity}.id`, `${TableName.IdentityAwsAuth}.identityId`)
|
||||
.leftJoin(TableName.IdentityAzureAuth, `${TableName.Identity}.id`, `${TableName.IdentityAzureAuth}.identityId`)
|
||||
.leftJoin(TableName.IdentityLdapAuth, `${TableName.Identity}.id`, `${TableName.IdentityLdapAuth}.identityId`)
|
||||
.leftJoin(
|
||||
TableName.IdentityKubernetesAuth,
|
||||
`${TableName.Identity}.id`,
|
||||
`${TableName.IdentityKubernetesAuth}.identityId`
|
||||
)
|
||||
.leftJoin(TableName.IdentityOciAuth, `${TableName.Identity}.id`, `${TableName.IdentityOciAuth}.identityId`)
|
||||
.leftJoin(TableName.IdentityOidcAuth, `${TableName.Identity}.id`, `${TableName.IdentityOidcAuth}.identityId`)
|
||||
.leftJoin(TableName.IdentityTokenAuth, `${TableName.Identity}.id`, `${TableName.IdentityTokenAuth}.identityId`)
|
||||
.leftJoin(TableName.IdentityJwtAuth, `${TableName.Identity}.id`, `${TableName.IdentityJwtAuth}.identityId`)
|
||||
.select(selectAllTableCols(TableName.IdentityAccessToken))
|
||||
.select(
|
||||
db.ref("accessTokenTrustedIps").withSchema(TableName.IdentityUniversalAuth).as("accessTokenTrustedIpsUa"),
|
||||
db.ref("accessTokenTrustedIps").withSchema(TableName.IdentityGcpAuth).as("accessTokenTrustedIpsGcp"),
|
||||
db
|
||||
.ref("accessTokenTrustedIps")
|
||||
.withSchema(TableName.IdentityAliCloudAuth)
|
||||
.as("accessTokenTrustedIpsAliCloud"),
|
||||
db.ref("accessTokenTrustedIps").withSchema(TableName.IdentityAwsAuth).as("accessTokenTrustedIpsAws"),
|
||||
db.ref("accessTokenTrustedIps").withSchema(TableName.IdentityAzureAuth).as("accessTokenTrustedIpsAzure"),
|
||||
db.ref("accessTokenTrustedIps").withSchema(TableName.IdentityKubernetesAuth).as("accessTokenTrustedIpsK8s"),
|
||||
db.ref("accessTokenTrustedIps").withSchema(TableName.IdentityOciAuth).as("accessTokenTrustedIpsOci"),
|
||||
db.ref("accessTokenTrustedIps").withSchema(TableName.IdentityOidcAuth).as("accessTokenTrustedIpsOidc"),
|
||||
db.ref("accessTokenTrustedIps").withSchema(TableName.IdentityTokenAuth).as("accessTokenTrustedIpsToken"),
|
||||
db.ref("accessTokenTrustedIps").withSchema(TableName.IdentityJwtAuth).as("accessTokenTrustedIpsJwt"),
|
||||
db.ref("accessTokenTrustedIps").withSchema(TableName.IdentityLdapAuth).as("accessTokenTrustedIpsLdap"),
|
||||
db.ref("name").withSchema(TableName.Identity)
|
||||
)
|
||||
.select(db.ref("name").withSchema(TableName.Identity))
|
||||
.first();
|
||||
|
||||
if (!doc) return;
|
||||
|
||||
return {
|
||||
...doc,
|
||||
trustedIpsUniversalAuth: doc.accessTokenTrustedIpsUa,
|
||||
trustedIpsGcpAuth: doc.accessTokenTrustedIpsGcp,
|
||||
trustedIpsAliCloudAuth: doc.accessTokenTrustedIpsAliCloud,
|
||||
trustedIpsAwsAuth: doc.accessTokenTrustedIpsAws,
|
||||
trustedIpsAzureAuth: doc.accessTokenTrustedIpsAzure,
|
||||
trustedIpsKubernetesAuth: doc.accessTokenTrustedIpsK8s,
|
||||
trustedIpsOciAuth: doc.accessTokenTrustedIpsOci,
|
||||
trustedIpsOidcAuth: doc.accessTokenTrustedIpsOidc,
|
||||
trustedIpsAccessTokenAuth: doc.accessTokenTrustedIpsToken,
|
||||
trustedIpsAccessJwtAuth: doc.accessTokenTrustedIpsJwt,
|
||||
trustedIpsAccessLdapAuth: doc.accessTokenTrustedIpsLdap
|
||||
};
|
||||
return doc;
|
||||
} catch (error) {
|
||||
throw new DatabaseError({ error, name: "IdAccessTokenFindOne" });
|
||||
}
|
||||
|
||||
@@ -7,12 +7,14 @@ import { checkIPAgainstBlocklist, TIp } from "@app/lib/ip";
|
||||
|
||||
import { TAccessTokenQueueServiceFactory } from "../access-token-queue/access-token-queue";
|
||||
import { AuthTokenType } from "../auth/auth-type";
|
||||
import { TIdentityDALFactory } from "../identity/identity-dal";
|
||||
import { TIdentityOrgDALFactory } from "../identity/identity-org-dal";
|
||||
import { TIdentityAccessTokenDALFactory } from "./identity-access-token-dal";
|
||||
import { TIdentityAccessTokenJwtPayload, TRenewAccessTokenDTO } from "./identity-access-token-types";
|
||||
|
||||
type TIdentityAccessTokenServiceFactoryDep = {
|
||||
identityAccessTokenDAL: TIdentityAccessTokenDALFactory;
|
||||
identityDAL: Pick<TIdentityDALFactory, "getTrustedIpsByAuthMethod">;
|
||||
identityOrgMembershipDAL: TIdentityOrgDALFactory;
|
||||
accessTokenQueue: Pick<
|
||||
TAccessTokenQueueServiceFactory,
|
||||
@@ -25,7 +27,8 @@ export type TIdentityAccessTokenServiceFactory = ReturnType<typeof identityAcces
|
||||
export const identityAccessTokenServiceFactory = ({
|
||||
identityAccessTokenDAL,
|
||||
identityOrgMembershipDAL,
|
||||
accessTokenQueue
|
||||
accessTokenQueue,
|
||||
identityDAL
|
||||
}: TIdentityAccessTokenServiceFactoryDep) => {
|
||||
const validateAccessTokenExp = async (identityAccessToken: TIdentityAccessTokens) => {
|
||||
const {
|
||||
@@ -190,23 +193,11 @@ export const identityAccessTokenServiceFactory = ({
|
||||
message: "Failed to authorize revoked access token, access token is revoked"
|
||||
});
|
||||
|
||||
const trustedIpsMap: Record<IdentityAuthMethod, unknown> = {
|
||||
[IdentityAuthMethod.UNIVERSAL_AUTH]: identityAccessToken.trustedIpsUniversalAuth,
|
||||
[IdentityAuthMethod.GCP_AUTH]: identityAccessToken.trustedIpsGcpAuth,
|
||||
[IdentityAuthMethod.ALICLOUD_AUTH]: identityAccessToken.trustedIpsAliCloudAuth,
|
||||
[IdentityAuthMethod.AWS_AUTH]: identityAccessToken.trustedIpsAwsAuth,
|
||||
[IdentityAuthMethod.OCI_AUTH]: identityAccessToken.trustedIpsOciAuth,
|
||||
[IdentityAuthMethod.AZURE_AUTH]: identityAccessToken.trustedIpsAzureAuth,
|
||||
[IdentityAuthMethod.KUBERNETES_AUTH]: identityAccessToken.trustedIpsKubernetesAuth,
|
||||
[IdentityAuthMethod.OIDC_AUTH]: identityAccessToken.trustedIpsOidcAuth,
|
||||
[IdentityAuthMethod.TOKEN_AUTH]: identityAccessToken.trustedIpsAccessTokenAuth,
|
||||
[IdentityAuthMethod.JWT_AUTH]: identityAccessToken.trustedIpsAccessJwtAuth,
|
||||
[IdentityAuthMethod.LDAP_AUTH]: identityAccessToken.trustedIpsAccessLdapAuth
|
||||
};
|
||||
|
||||
const trustedIps = trustedIpsMap[identityAccessToken.authMethod as IdentityAuthMethod];
|
||||
|
||||
if (ipAddress) {
|
||||
const trustedIps = await identityDAL.getTrustedIpsByAuthMethod(
|
||||
identityAccessToken.identityId,
|
||||
identityAccessToken.authMethod as IdentityAuthMethod
|
||||
);
|
||||
if (ipAddress && trustedIps) {
|
||||
checkIPAgainstBlocklist({
|
||||
ipAddress,
|
||||
trustedIps: trustedIps as TIp[]
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
import { TDbClient } from "@app/db";
|
||||
import { TableName } from "@app/db/schemas";
|
||||
import { ormify, TOrmify } from "@app/lib/knex";
|
||||
|
||||
export type TIdentityTlsCertAuthDALFactory = TOrmify<TableName.IdentityTlsCertAuth>;
|
||||
|
||||
export const identityTlsCertAuthDALFactory = (db: TDbClient) => {
|
||||
const orm = ormify(db, TableName.IdentityTlsCertAuth);
|
||||
return orm;
|
||||
};
|
||||
@@ -0,0 +1,423 @@
|
||||
import crypto from "node:crypto";
|
||||
|
||||
import { ForbiddenError } from "@casl/ability";
|
||||
import jwt from "jsonwebtoken";
|
||||
|
||||
import { IdentityAuthMethod } from "@app/db/schemas";
|
||||
import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
|
||||
import { OrgPermissionIdentityActions, OrgPermissionSubjects } from "@app/ee/services/permission/org-permission";
|
||||
import {
|
||||
constructPermissionErrorMessage,
|
||||
validatePrivilegeChangeOperation
|
||||
} from "@app/ee/services/permission/permission-fns";
|
||||
import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service-types";
|
||||
import { extractX509CertFromChain } from "@app/lib/certificates/extract-certificate";
|
||||
import { getConfig } from "@app/lib/config/env";
|
||||
import { BadRequestError, NotFoundError, PermissionBoundaryError, UnauthorizedError } from "@app/lib/errors";
|
||||
import { extractIPDetails, isValidIpOrCidr } from "@app/lib/ip";
|
||||
|
||||
import { ActorType, AuthTokenType } from "../auth/auth-type";
|
||||
import { TIdentityOrgDALFactory } from "../identity/identity-org-dal";
|
||||
import { TIdentityAccessTokenDALFactory } from "../identity-access-token/identity-access-token-dal";
|
||||
import { TIdentityAccessTokenJwtPayload } from "../identity-access-token/identity-access-token-types";
|
||||
import { TKmsServiceFactory } from "../kms/kms-service";
|
||||
import { KmsDataKey } from "../kms/kms-types";
|
||||
import { validateIdentityUpdateForSuperAdminPrivileges } from "../super-admin/super-admin-fns";
|
||||
import { TIdentityTlsCertAuthDALFactory } from "./identity-tls-cert-auth-dal";
|
||||
import { TIdentityTlsCertAuthServiceFactory } from "./identity-tls-cert-auth-types";
|
||||
|
||||
type TIdentityTlsCertAuthServiceFactoryDep = {
|
||||
identityAccessTokenDAL: Pick<TIdentityAccessTokenDALFactory, "create" | "delete">;
|
||||
identityTlsCertAuthDAL: Pick<
|
||||
TIdentityTlsCertAuthDALFactory,
|
||||
"findOne" | "transaction" | "create" | "updateById" | "delete"
|
||||
>;
|
||||
identityOrgMembershipDAL: Pick<TIdentityOrgDALFactory, "findOne">;
|
||||
licenseService: Pick<TLicenseServiceFactory, "getPlan">;
|
||||
permissionService: Pick<TPermissionServiceFactory, "getOrgPermission">;
|
||||
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
|
||||
};
|
||||
|
||||
const parseSubjectDetails = (data: string) => {
|
||||
const values: Record<string, string> = {};
|
||||
data.split("\n").forEach((el) => {
|
||||
const [key, value] = el.split("=");
|
||||
values[key.trim()] = value.trim();
|
||||
});
|
||||
return values;
|
||||
};
|
||||
|
||||
export const identityTlsCertAuthServiceFactory = ({
|
||||
identityAccessTokenDAL,
|
||||
identityTlsCertAuthDAL,
|
||||
identityOrgMembershipDAL,
|
||||
licenseService,
|
||||
permissionService,
|
||||
kmsService
|
||||
}: TIdentityTlsCertAuthServiceFactoryDep): TIdentityTlsCertAuthServiceFactory => {
|
||||
const login: TIdentityTlsCertAuthServiceFactory["login"] = async ({ identityId, clientCertificate }) => {
|
||||
const identityTlsCertAuth = await identityTlsCertAuthDAL.findOne({ identityId });
|
||||
if (!identityTlsCertAuth) {
|
||||
throw new NotFoundError({
|
||||
message: "TLS Certificate auth method not found for identity, did you configure TLS Certificate auth?"
|
||||
});
|
||||
}
|
||||
|
||||
const identityMembershipOrg = await identityOrgMembershipDAL.findOne({
|
||||
identityId: identityTlsCertAuth.identityId
|
||||
});
|
||||
|
||||
if (!identityMembershipOrg) {
|
||||
throw new NotFoundError({
|
||||
message: `Identity organization membership for identity with ID '${identityTlsCertAuth.identityId}' not found`
|
||||
});
|
||||
}
|
||||
|
||||
const { decryptor } = await kmsService.createCipherPairWithDataKey({
|
||||
type: KmsDataKey.Organization,
|
||||
orgId: identityMembershipOrg.orgId
|
||||
});
|
||||
|
||||
const caCertificate = decryptor({
|
||||
cipherTextBlob: identityTlsCertAuth.encryptedCaCertificate
|
||||
}).toString();
|
||||
|
||||
const leafCertificate = extractX509CertFromChain(decodeURIComponent(clientCertificate))?.[0];
|
||||
if (!leafCertificate) {
|
||||
throw new BadRequestError({ message: "Missing client certificate" });
|
||||
}
|
||||
|
||||
const clientCertificateX509 = new crypto.X509Certificate(leafCertificate);
|
||||
const caCertificateX509 = new crypto.X509Certificate(caCertificate);
|
||||
|
||||
const isValidCertificate = clientCertificateX509.verify(caCertificateX509.publicKey);
|
||||
if (!isValidCertificate)
|
||||
throw new UnauthorizedError({
|
||||
message: "Access denied: Certificate not issued by the provided CA."
|
||||
});
|
||||
|
||||
if (new Date(clientCertificateX509.validTo) < new Date()) {
|
||||
throw new UnauthorizedError({
|
||||
message: "Access denied: Certificate has expired."
|
||||
});
|
||||
}
|
||||
|
||||
if (new Date(clientCertificateX509.validFrom) > new Date()) {
|
||||
throw new UnauthorizedError({
|
||||
message: "Access denied: Certificate not yet valid."
|
||||
});
|
||||
}
|
||||
|
||||
const subjectDetails = parseSubjectDetails(clientCertificateX509.subject);
|
||||
if (identityTlsCertAuth.allowedCommonNames) {
|
||||
const isValidCommonName = identityTlsCertAuth.allowedCommonNames.split(",").includes(subjectDetails.CN);
|
||||
if (!isValidCommonName) {
|
||||
throw new UnauthorizedError({
|
||||
message: "Access denied: TLS Certificate Auth common name not allowed."
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Generate the token
|
||||
const identityAccessToken = await identityTlsCertAuthDAL.transaction(async (tx) => {
|
||||
const newToken = await identityAccessTokenDAL.create(
|
||||
{
|
||||
identityId: identityTlsCertAuth.identityId,
|
||||
isAccessTokenRevoked: false,
|
||||
accessTokenTTL: identityTlsCertAuth.accessTokenTTL,
|
||||
accessTokenMaxTTL: identityTlsCertAuth.accessTokenMaxTTL,
|
||||
accessTokenNumUses: 0,
|
||||
accessTokenNumUsesLimit: identityTlsCertAuth.accessTokenNumUsesLimit,
|
||||
authMethod: IdentityAuthMethod.TLS_CERT_AUTH
|
||||
},
|
||||
tx
|
||||
);
|
||||
return newToken;
|
||||
});
|
||||
|
||||
const appCfg = getConfig();
|
||||
const accessToken = jwt.sign(
|
||||
{
|
||||
identityId: identityTlsCertAuth.identityId,
|
||||
identityAccessTokenId: identityAccessToken.id,
|
||||
authTokenType: AuthTokenType.IDENTITY_ACCESS_TOKEN
|
||||
} as TIdentityAccessTokenJwtPayload,
|
||||
appCfg.AUTH_SECRET,
|
||||
Number(identityAccessToken.accessTokenTTL) === 0
|
||||
? undefined
|
||||
: {
|
||||
expiresIn: Number(identityAccessToken.accessTokenTTL)
|
||||
}
|
||||
);
|
||||
|
||||
return {
|
||||
identityTlsCertAuth,
|
||||
accessToken,
|
||||
identityAccessToken,
|
||||
identityMembershipOrg
|
||||
};
|
||||
};
|
||||
|
||||
const attachTlsCertAuth: TIdentityTlsCertAuthServiceFactory["attachTlsCertAuth"] = async ({
|
||||
identityId,
|
||||
accessTokenTTL,
|
||||
accessTokenMaxTTL,
|
||||
accessTokenNumUsesLimit,
|
||||
accessTokenTrustedIps,
|
||||
actorId,
|
||||
actorAuthMethod,
|
||||
actor,
|
||||
actorOrgId,
|
||||
isActorSuperAdmin,
|
||||
caCertificate,
|
||||
allowedCommonNames
|
||||
}) => {
|
||||
await validateIdentityUpdateForSuperAdminPrivileges(identityId, isActorSuperAdmin);
|
||||
|
||||
const identityMembershipOrg = await identityOrgMembershipDAL.findOne({ identityId });
|
||||
if (!identityMembershipOrg) throw new NotFoundError({ message: `Failed to find identity with ID ${identityId}` });
|
||||
|
||||
if (identityMembershipOrg.identity.authMethods.includes(IdentityAuthMethod.TLS_CERT_AUTH)) {
|
||||
throw new BadRequestError({
|
||||
message: "Failed to add TLS Certificate Auth to already configured identity"
|
||||
});
|
||||
}
|
||||
|
||||
if (accessTokenMaxTTL > 0 && accessTokenTTL > accessTokenMaxTTL) {
|
||||
throw new BadRequestError({ message: "Access token TTL cannot be greater than max TTL" });
|
||||
}
|
||||
|
||||
const { permission } = await permissionService.getOrgPermission(
|
||||
actor,
|
||||
actorId,
|
||||
identityMembershipOrg.orgId,
|
||||
actorAuthMethod,
|
||||
actorOrgId
|
||||
);
|
||||
ForbiddenError.from(permission).throwUnlessCan(OrgPermissionIdentityActions.Create, OrgPermissionSubjects.Identity);
|
||||
|
||||
const plan = await licenseService.getPlan(identityMembershipOrg.orgId);
|
||||
const reformattedAccessTokenTrustedIps = accessTokenTrustedIps.map((accessTokenTrustedIp) => {
|
||||
if (
|
||||
!plan.ipAllowlisting &&
|
||||
accessTokenTrustedIp.ipAddress !== "0.0.0.0/0" &&
|
||||
accessTokenTrustedIp.ipAddress !== "::/0"
|
||||
)
|
||||
throw new BadRequestError({
|
||||
message:
|
||||
"Failed to add IP access range to access token due to plan restriction. Upgrade plan to add IP access range."
|
||||
});
|
||||
if (!isValidIpOrCidr(accessTokenTrustedIp.ipAddress))
|
||||
throw new BadRequestError({
|
||||
message: "The IP is not a valid IPv4, IPv6, or CIDR block"
|
||||
});
|
||||
return extractIPDetails(accessTokenTrustedIp.ipAddress);
|
||||
});
|
||||
|
||||
const { encryptor } = await kmsService.createCipherPairWithDataKey({
|
||||
type: KmsDataKey.Organization,
|
||||
orgId: identityMembershipOrg.orgId
|
||||
});
|
||||
|
||||
const identityTlsCertAuth = await identityTlsCertAuthDAL.transaction(async (tx) => {
|
||||
const doc = await identityTlsCertAuthDAL.create(
|
||||
{
|
||||
identityId: identityMembershipOrg.identityId,
|
||||
accessTokenMaxTTL,
|
||||
allowedCommonNames,
|
||||
accessTokenTTL,
|
||||
encryptedCaCertificate: encryptor({ plainText: Buffer.from(caCertificate) }).cipherTextBlob,
|
||||
accessTokenNumUsesLimit,
|
||||
accessTokenTrustedIps: JSON.stringify(reformattedAccessTokenTrustedIps)
|
||||
},
|
||||
tx
|
||||
);
|
||||
return doc;
|
||||
});
|
||||
return { ...identityTlsCertAuth, orgId: identityMembershipOrg.orgId };
|
||||
};
|
||||
|
||||
const updateTlsCertAuth: TIdentityTlsCertAuthServiceFactory["updateTlsCertAuth"] = async ({
|
||||
identityId,
|
||||
caCertificate,
|
||||
allowedCommonNames,
|
||||
accessTokenTTL,
|
||||
accessTokenMaxTTL,
|
||||
accessTokenNumUsesLimit,
|
||||
accessTokenTrustedIps,
|
||||
actorId,
|
||||
actorAuthMethod,
|
||||
actor,
|
||||
actorOrgId
|
||||
}) => {
|
||||
const identityMembershipOrg = await identityOrgMembershipDAL.findOne({ identityId });
|
||||
if (!identityMembershipOrg) throw new NotFoundError({ message: `Failed to find identity with ID ${identityId}` });
|
||||
|
||||
if (!identityMembershipOrg.identity.authMethods.includes(IdentityAuthMethod.TLS_CERT_AUTH)) {
|
||||
throw new NotFoundError({
|
||||
message: "The identity does not have TLS Certificate Auth attached"
|
||||
});
|
||||
}
|
||||
|
||||
const identityTlsCertAuth = await identityTlsCertAuthDAL.findOne({ identityId });
|
||||
|
||||
if (
|
||||
(accessTokenMaxTTL || identityTlsCertAuth.accessTokenMaxTTL) > 0 &&
|
||||
(accessTokenTTL || identityTlsCertAuth.accessTokenTTL) >
|
||||
(accessTokenMaxTTL || identityTlsCertAuth.accessTokenMaxTTL)
|
||||
) {
|
||||
throw new BadRequestError({ message: "Access token TTL cannot be greater than max TTL" });
|
||||
}
|
||||
|
||||
const { permission } = await permissionService.getOrgPermission(
|
||||
actor,
|
||||
actorId,
|
||||
identityMembershipOrg.orgId,
|
||||
actorAuthMethod,
|
||||
actorOrgId
|
||||
);
|
||||
ForbiddenError.from(permission).throwUnlessCan(OrgPermissionIdentityActions.Edit, OrgPermissionSubjects.Identity);
|
||||
|
||||
const plan = await licenseService.getPlan(identityMembershipOrg.orgId);
|
||||
const reformattedAccessTokenTrustedIps = accessTokenTrustedIps?.map((accessTokenTrustedIp) => {
|
||||
if (
|
||||
!plan.ipAllowlisting &&
|
||||
accessTokenTrustedIp.ipAddress !== "0.0.0.0/0" &&
|
||||
accessTokenTrustedIp.ipAddress !== "::/0"
|
||||
)
|
||||
throw new BadRequestError({
|
||||
message:
|
||||
"Failed to add IP access range to access token due to plan restriction. Upgrade plan to add IP access range."
|
||||
});
|
||||
if (!isValidIpOrCidr(accessTokenTrustedIp.ipAddress))
|
||||
throw new BadRequestError({
|
||||
message: "The IP is not a valid IPv4, IPv6, or CIDR block"
|
||||
});
|
||||
return extractIPDetails(accessTokenTrustedIp.ipAddress);
|
||||
});
|
||||
const { encryptor } = await kmsService.createCipherPairWithDataKey({
|
||||
type: KmsDataKey.Organization,
|
||||
orgId: identityMembershipOrg.orgId
|
||||
});
|
||||
|
||||
const updatedTlsCertAuth = await identityTlsCertAuthDAL.updateById(identityTlsCertAuth.id, {
|
||||
allowedCommonNames,
|
||||
encryptedCaCertificate: caCertificate
|
||||
? encryptor({ plainText: Buffer.from(caCertificate) }).cipherTextBlob
|
||||
: undefined,
|
||||
accessTokenMaxTTL,
|
||||
accessTokenTTL,
|
||||
accessTokenNumUsesLimit,
|
||||
accessTokenTrustedIps: reformattedAccessTokenTrustedIps
|
||||
? JSON.stringify(reformattedAccessTokenTrustedIps)
|
||||
: undefined
|
||||
});
|
||||
|
||||
return { ...updatedTlsCertAuth, orgId: identityMembershipOrg.orgId };
|
||||
};
|
||||
|
||||
const getTlsCertAuth: TIdentityTlsCertAuthServiceFactory["getTlsCertAuth"] = async ({
|
||||
identityId,
|
||||
actorId,
|
||||
actor,
|
||||
actorAuthMethod,
|
||||
actorOrgId
|
||||
}) => {
|
||||
const identityMembershipOrg = await identityOrgMembershipDAL.findOne({ identityId });
|
||||
if (!identityMembershipOrg) throw new NotFoundError({ message: `Failed to find identity with ID ${identityId}` });
|
||||
|
||||
if (!identityMembershipOrg.identity.authMethods.includes(IdentityAuthMethod.TLS_CERT_AUTH)) {
|
||||
throw new BadRequestError({
|
||||
message: "The identity does not have TLS Certificate Auth attached"
|
||||
});
|
||||
}
|
||||
|
||||
const identityAuth = await identityTlsCertAuthDAL.findOne({ identityId });
|
||||
|
||||
const { permission } = await permissionService.getOrgPermission(
|
||||
actor,
|
||||
actorId,
|
||||
identityMembershipOrg.orgId,
|
||||
actorAuthMethod,
|
||||
actorOrgId
|
||||
);
|
||||
ForbiddenError.from(permission).throwUnlessCan(OrgPermissionIdentityActions.Read, OrgPermissionSubjects.Identity);
|
||||
const { decryptor } = await kmsService.createCipherPairWithDataKey({
|
||||
type: KmsDataKey.Organization,
|
||||
orgId: identityMembershipOrg.orgId
|
||||
});
|
||||
let caCertificate = "";
|
||||
if (identityAuth.encryptedCaCertificate) {
|
||||
caCertificate = decryptor({ cipherTextBlob: identityAuth.encryptedCaCertificate }).toString();
|
||||
}
|
||||
|
||||
return { ...identityAuth, caCertificate, orgId: identityMembershipOrg.orgId };
|
||||
};
|
||||
|
||||
const revokeTlsCertAuth: TIdentityTlsCertAuthServiceFactory["revokeTlsCertAuth"] = async ({
|
||||
identityId,
|
||||
actorId,
|
||||
actor,
|
||||
actorAuthMethod,
|
||||
actorOrgId
|
||||
}) => {
|
||||
const identityMembershipOrg = await identityOrgMembershipDAL.findOne({ identityId });
|
||||
if (!identityMembershipOrg) throw new NotFoundError({ message: `Failed to find identity with ID ${identityId}` });
|
||||
if (!identityMembershipOrg.identity.authMethods.includes(IdentityAuthMethod.TLS_CERT_AUTH)) {
|
||||
throw new BadRequestError({
|
||||
message: "The identity does not have TLS Certificate auth"
|
||||
});
|
||||
}
|
||||
const { permission, membership } = await permissionService.getOrgPermission(
|
||||
actor,
|
||||
actorId,
|
||||
identityMembershipOrg.orgId,
|
||||
actorAuthMethod,
|
||||
actorOrgId
|
||||
);
|
||||
ForbiddenError.from(permission).throwUnlessCan(OrgPermissionIdentityActions.Edit, OrgPermissionSubjects.Identity);
|
||||
|
||||
const { permission: rolePermission } = await permissionService.getOrgPermission(
|
||||
ActorType.IDENTITY,
|
||||
identityMembershipOrg.identityId,
|
||||
identityMembershipOrg.orgId,
|
||||
actorAuthMethod,
|
||||
actorOrgId
|
||||
);
|
||||
|
||||
const permissionBoundary = validatePrivilegeChangeOperation(
|
||||
membership.shouldUseNewPrivilegeSystem,
|
||||
OrgPermissionIdentityActions.RevokeAuth,
|
||||
OrgPermissionSubjects.Identity,
|
||||
permission,
|
||||
rolePermission
|
||||
);
|
||||
|
||||
if (!permissionBoundary.isValid)
|
||||
throw new PermissionBoundaryError({
|
||||
message: constructPermissionErrorMessage(
|
||||
"Failed to revoke TLS Certificate auth of identity with more privileged role",
|
||||
membership.shouldUseNewPrivilegeSystem,
|
||||
OrgPermissionIdentityActions.RevokeAuth,
|
||||
OrgPermissionSubjects.Identity
|
||||
),
|
||||
details: { missingPermissions: permissionBoundary.missingPermissions }
|
||||
});
|
||||
|
||||
const revokedIdentityTlsCertAuth = await identityTlsCertAuthDAL.transaction(async (tx) => {
|
||||
const deletedTlsCertAuth = await identityTlsCertAuthDAL.delete({ identityId }, tx);
|
||||
await identityAccessTokenDAL.delete({ identityId, authMethod: IdentityAuthMethod.TLS_CERT_AUTH }, tx);
|
||||
|
||||
return { ...deletedTlsCertAuth?.[0], orgId: identityMembershipOrg.orgId };
|
||||
});
|
||||
return revokedIdentityTlsCertAuth;
|
||||
};
|
||||
|
||||
return {
|
||||
login,
|
||||
attachTlsCertAuth,
|
||||
updateTlsCertAuth,
|
||||
getTlsCertAuth,
|
||||
revokeTlsCertAuth
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,49 @@
|
||||
import { TIdentityAccessTokens, TIdentityOrgMemberships, TIdentityTlsCertAuths } from "@app/db/schemas";
|
||||
import { TProjectPermission } from "@app/lib/types";
|
||||
|
||||
export type TLoginTlsCertAuthDTO = {
|
||||
identityId: string;
|
||||
clientCertificate: string;
|
||||
};
|
||||
|
||||
export type TAttachTlsCertAuthDTO = {
|
||||
identityId: string;
|
||||
caCertificate: string;
|
||||
allowedCommonNames?: string | null;
|
||||
accessTokenTTL: number;
|
||||
accessTokenMaxTTL: number;
|
||||
accessTokenNumUsesLimit: number;
|
||||
accessTokenTrustedIps: { ipAddress: string }[];
|
||||
isActorSuperAdmin?: boolean;
|
||||
} & Omit<TProjectPermission, "projectId">;
|
||||
|
||||
export type TUpdateTlsCertAuthDTO = {
|
||||
identityId: string;
|
||||
caCertificate?: string;
|
||||
allowedCommonNames?: string | null;
|
||||
accessTokenTTL?: number;
|
||||
accessTokenMaxTTL?: number;
|
||||
accessTokenNumUsesLimit?: number;
|
||||
accessTokenTrustedIps?: { ipAddress: string }[];
|
||||
} & Omit<TProjectPermission, "projectId">;
|
||||
|
||||
export type TGetTlsCertAuthDTO = {
|
||||
identityId: string;
|
||||
} & Omit<TProjectPermission, "projectId">;
|
||||
|
||||
export type TRevokeTlsCertAuthDTO = {
|
||||
identityId: string;
|
||||
} & Omit<TProjectPermission, "projectId">;
|
||||
|
||||
export type TIdentityTlsCertAuthServiceFactory = {
|
||||
login: (dto: TLoginTlsCertAuthDTO) => Promise<{
|
||||
identityTlsCertAuth: TIdentityTlsCertAuths;
|
||||
accessToken: string;
|
||||
identityAccessToken: TIdentityAccessTokens;
|
||||
identityMembershipOrg: TIdentityOrgMemberships;
|
||||
}>;
|
||||
attachTlsCertAuth: (dto: TAttachTlsCertAuthDTO) => Promise<TIdentityTlsCertAuths>;
|
||||
updateTlsCertAuth: (dto: TUpdateTlsCertAuthDTO) => Promise<TIdentityTlsCertAuths>;
|
||||
revokeTlsCertAuth: (dto: TRevokeTlsCertAuthDTO) => Promise<TIdentityTlsCertAuths>;
|
||||
getTlsCertAuth: (dto: TGetTlsCertAuthDTO) => Promise<TIdentityTlsCertAuths & { caCertificate: string }>;
|
||||
};
|
||||
@@ -1,5 +1,5 @@
|
||||
import { TDbClient } from "@app/db";
|
||||
import { TableName, TIdentities } from "@app/db/schemas";
|
||||
import { IdentityAuthMethod, TableName, TIdentities } from "@app/db/schemas";
|
||||
import { DatabaseError } from "@app/lib/errors";
|
||||
import { ormify, selectAllTableCols } from "@app/lib/knex";
|
||||
|
||||
@@ -8,6 +8,28 @@ export type TIdentityDALFactory = ReturnType<typeof identityDALFactory>;
|
||||
export const identityDALFactory = (db: TDbClient) => {
|
||||
const identityOrm = ormify(db, TableName.Identity);
|
||||
|
||||
const getTrustedIpsByAuthMethod = async (identityId: string, authMethod: IdentityAuthMethod) => {
|
||||
const authMethodToTableName = {
|
||||
[IdentityAuthMethod.TOKEN_AUTH]: TableName.IdentityTokenAuth,
|
||||
[IdentityAuthMethod.UNIVERSAL_AUTH]: TableName.IdentityUniversalAuth,
|
||||
[IdentityAuthMethod.KUBERNETES_AUTH]: TableName.IdentityKubernetesAuth,
|
||||
[IdentityAuthMethod.GCP_AUTH]: TableName.IdentityGcpAuth,
|
||||
[IdentityAuthMethod.ALICLOUD_AUTH]: TableName.IdentityAliCloudAuth,
|
||||
[IdentityAuthMethod.AWS_AUTH]: TableName.IdentityAwsAuth,
|
||||
[IdentityAuthMethod.AZURE_AUTH]: TableName.IdentityAzureAuth,
|
||||
[IdentityAuthMethod.TLS_CERT_AUTH]: TableName.IdentityTlsCertAuth,
|
||||
[IdentityAuthMethod.OCI_AUTH]: TableName.IdentityOciAuth,
|
||||
[IdentityAuthMethod.OIDC_AUTH]: TableName.IdentityOidcAuth,
|
||||
[IdentityAuthMethod.JWT_AUTH]: TableName.IdentityJwtAuth,
|
||||
[IdentityAuthMethod.LDAP_AUTH]: TableName.IdentityLdapAuth
|
||||
} as const;
|
||||
const tableName = authMethodToTableName[authMethod];
|
||||
if (!tableName) return;
|
||||
const data = await db(tableName).where({ identityId }).first();
|
||||
if (!data) return;
|
||||
return data.accessTokenTrustedIps;
|
||||
};
|
||||
|
||||
const getIdentitiesByFilter = async ({
|
||||
limit,
|
||||
offset,
|
||||
@@ -38,5 +60,5 @@ export const identityDALFactory = (db: TDbClient) => {
|
||||
}
|
||||
};
|
||||
|
||||
return { ...identityOrm, getIdentitiesByFilter };
|
||||
return { ...identityOrm, getTrustedIpsByAuthMethod, getIdentitiesByFilter };
|
||||
};
|
||||
|
||||
@@ -11,7 +11,8 @@ export const buildAuthMethods = ({
|
||||
azureId,
|
||||
tokenId,
|
||||
jwtId,
|
||||
ldapId
|
||||
ldapId,
|
||||
tlsCertId
|
||||
}: {
|
||||
uaId?: string;
|
||||
gcpId?: string;
|
||||
@@ -24,6 +25,7 @@ export const buildAuthMethods = ({
|
||||
tokenId?: string;
|
||||
jwtId?: string;
|
||||
ldapId?: string;
|
||||
tlsCertId?: string;
|
||||
}) => {
|
||||
return [
|
||||
...[uaId ? IdentityAuthMethod.UNIVERSAL_AUTH : null],
|
||||
@@ -36,6 +38,7 @@ export const buildAuthMethods = ({
|
||||
...[azureId ? IdentityAuthMethod.AZURE_AUTH : null],
|
||||
...[tokenId ? IdentityAuthMethod.TOKEN_AUTH : null],
|
||||
...[jwtId ? IdentityAuthMethod.JWT_AUTH : null],
|
||||
...[ldapId ? IdentityAuthMethod.LDAP_AUTH : null]
|
||||
...[ldapId ? IdentityAuthMethod.LDAP_AUTH : null],
|
||||
...[tlsCertId ? IdentityAuthMethod.TLS_CERT_AUTH : null]
|
||||
].filter((authMethod) => authMethod) as IdentityAuthMethod[];
|
||||
};
|
||||
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
TIdentityOciAuths,
|
||||
TIdentityOidcAuths,
|
||||
TIdentityOrgMemberships,
|
||||
TIdentityTlsCertAuths,
|
||||
TIdentityTokenAuths,
|
||||
TIdentityUniversalAuths,
|
||||
TOrgRoles
|
||||
@@ -99,7 +100,11 @@ export const identityOrgDALFactory = (db: TDbClient) => {
|
||||
`${TableName.IdentityOrgMembership}.identityId`,
|
||||
`${TableName.IdentityLdapAuth}.identityId`
|
||||
)
|
||||
|
||||
.leftJoin<TIdentityTlsCertAuths>(
|
||||
TableName.IdentityTlsCertAuth,
|
||||
`${TableName.IdentityOrgMembership}.identityId`,
|
||||
`${TableName.IdentityTlsCertAuth}.identityId`
|
||||
)
|
||||
.select(
|
||||
selectAllTableCols(TableName.IdentityOrgMembership),
|
||||
|
||||
@@ -114,6 +119,7 @@ export const identityOrgDALFactory = (db: TDbClient) => {
|
||||
db.ref("id").as("tokenId").withSchema(TableName.IdentityTokenAuth),
|
||||
db.ref("id").as("jwtId").withSchema(TableName.IdentityJwtAuth),
|
||||
db.ref("id").as("ldapId").withSchema(TableName.IdentityLdapAuth),
|
||||
db.ref("id").as("tlsCertId").withSchema(TableName.IdentityTlsCertAuth),
|
||||
db.ref("name").withSchema(TableName.Identity),
|
||||
db.ref("hasDeleteProtection").withSchema(TableName.Identity)
|
||||
);
|
||||
@@ -238,7 +244,11 @@ export const identityOrgDALFactory = (db: TDbClient) => {
|
||||
"paginatedIdentity.identityId",
|
||||
`${TableName.IdentityLdapAuth}.identityId`
|
||||
)
|
||||
|
||||
.leftJoin<TIdentityTlsCertAuths>(
|
||||
TableName.IdentityTlsCertAuth,
|
||||
"paginatedIdentity.identityId",
|
||||
`${TableName.IdentityTlsCertAuth}.identityId`
|
||||
)
|
||||
.select(
|
||||
db.ref("id").withSchema("paginatedIdentity"),
|
||||
db.ref("role").withSchema("paginatedIdentity"),
|
||||
@@ -260,7 +270,8 @@ export const identityOrgDALFactory = (db: TDbClient) => {
|
||||
db.ref("id").as("azureId").withSchema(TableName.IdentityAzureAuth),
|
||||
db.ref("id").as("tokenId").withSchema(TableName.IdentityTokenAuth),
|
||||
db.ref("id").as("jwtId").withSchema(TableName.IdentityJwtAuth),
|
||||
db.ref("id").as("ldapId").withSchema(TableName.IdentityLdapAuth)
|
||||
db.ref("id").as("ldapId").withSchema(TableName.IdentityLdapAuth),
|
||||
db.ref("id").as("tlsCertId").withSchema(TableName.IdentityTlsCertAuth)
|
||||
)
|
||||
// cr stands for custom role
|
||||
.select(db.ref("id").as("crId").withSchema(TableName.OrgRoles))
|
||||
@@ -306,6 +317,7 @@ export const identityOrgDALFactory = (db: TDbClient) => {
|
||||
azureId,
|
||||
tokenId,
|
||||
ldapId,
|
||||
tlsCertId,
|
||||
createdAt,
|
||||
updatedAt
|
||||
}) => ({
|
||||
@@ -313,7 +325,6 @@ export const identityOrgDALFactory = (db: TDbClient) => {
|
||||
roleId,
|
||||
identityId,
|
||||
id,
|
||||
|
||||
orgId,
|
||||
createdAt,
|
||||
updatedAt,
|
||||
@@ -341,7 +352,8 @@ export const identityOrgDALFactory = (db: TDbClient) => {
|
||||
azureId,
|
||||
tokenId,
|
||||
jwtId,
|
||||
ldapId
|
||||
ldapId,
|
||||
tlsCertId
|
||||
})
|
||||
}
|
||||
}),
|
||||
|
||||
@@ -7,13 +7,18 @@ import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
|
||||
|
||||
import { getServerCfg } from "../super-admin/super-admin-service";
|
||||
import { TTelemetryDALFactory } from "./telemetry-dal";
|
||||
import { TELEMETRY_SECRET_OPERATIONS_KEY, TELEMETRY_SECRET_PROCESSED_KEY } from "./telemetry-service";
|
||||
import {
|
||||
TELEMETRY_SECRET_OPERATIONS_KEY,
|
||||
TELEMETRY_SECRET_PROCESSED_KEY,
|
||||
TTelemetryServiceFactory
|
||||
} from "./telemetry-service";
|
||||
import { PostHogEventTypes } from "./telemetry-types";
|
||||
|
||||
type TTelemetryQueueServiceFactoryDep = {
|
||||
queueService: TQueueServiceFactory;
|
||||
keyStore: Pick<TKeyStoreFactory, "getItem" | "deleteItem">;
|
||||
telemetryDAL: TTelemetryDALFactory;
|
||||
telemetryService: TTelemetryServiceFactory;
|
||||
};
|
||||
|
||||
export type TTelemetryQueueServiceFactory = ReturnType<typeof telemetryQueueServiceFactory>;
|
||||
@@ -21,7 +26,8 @@ export type TTelemetryQueueServiceFactory = ReturnType<typeof telemetryQueueServ
|
||||
export const telemetryQueueServiceFactory = ({
|
||||
queueService,
|
||||
keyStore,
|
||||
telemetryDAL
|
||||
telemetryDAL,
|
||||
telemetryService
|
||||
}: TTelemetryQueueServiceFactoryDep) => {
|
||||
const appCfg = getConfig();
|
||||
const postHog =
|
||||
@@ -48,6 +54,10 @@ export const telemetryQueueServiceFactory = ({
|
||||
await keyStore.deleteItem(TELEMETRY_SECRET_OPERATIONS_KEY);
|
||||
});
|
||||
|
||||
queueService.start(QueueName.TelemetryAggregatedEvents, async () => {
|
||||
await telemetryService.processAggregatedEvents();
|
||||
});
|
||||
|
||||
// every day at midnight a telemetry job executes on self-hosted instances
|
||||
// this sends some telemetry information like instance id secrets operated etc
|
||||
const startTelemetryCheck = async () => {
|
||||
@@ -60,11 +70,26 @@ export const telemetryQueueServiceFactory = ({
|
||||
{ pattern: "0 0 * * *", utc: true },
|
||||
QueueName.TelemetryInstanceStats // just a job id
|
||||
);
|
||||
|
||||
// clear previous aggregated events job
|
||||
await queueService.stopRepeatableJob(
|
||||
QueueName.TelemetryAggregatedEvents,
|
||||
QueueJobs.TelemetryAggregatedEvents,
|
||||
{ pattern: "*/5 * * * *", utc: true },
|
||||
QueueName.TelemetryAggregatedEvents // just a job id
|
||||
);
|
||||
|
||||
if (postHog) {
|
||||
await queueService.queue(QueueName.TelemetryInstanceStats, QueueJobs.TelemetryInstanceStats, undefined, {
|
||||
jobId: QueueName.TelemetryInstanceStats,
|
||||
repeat: { pattern: "0 0 * * *", utc: true }
|
||||
});
|
||||
|
||||
// Start aggregated events job (runs every five minutes)
|
||||
await queueService.queue(QueueName.TelemetryAggregatedEvents, QueueJobs.TelemetryAggregatedEvents, undefined, {
|
||||
jobId: QueueName.TelemetryAggregatedEvents,
|
||||
repeat: { pattern: "*/5 * * * *", utc: true }
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
@@ -72,6 +97,10 @@ export const telemetryQueueServiceFactory = ({
|
||||
logger.error(err?.failedReason, `${QueueName.TelemetryInstanceStats}: failed`);
|
||||
});
|
||||
|
||||
queueService.listen(QueueName.TelemetryAggregatedEvents, "failed", (err) => {
|
||||
logger.error(err?.failedReason, `${QueueName.TelemetryAggregatedEvents}: failed`);
|
||||
});
|
||||
|
||||
return {
|
||||
startTelemetryCheck
|
||||
};
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { createHash, randomUUID } from "crypto";
|
||||
import { PostHog } from "posthog-node";
|
||||
|
||||
import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
|
||||
@@ -12,12 +13,49 @@ import { PostHogEventTypes, TPostHogEvent, TSecretModifiedEvent } from "./teleme
|
||||
export const TELEMETRY_SECRET_PROCESSED_KEY = "telemetry-secret-processed";
|
||||
export const TELEMETRY_SECRET_OPERATIONS_KEY = "telemetry-secret-operations";
|
||||
|
||||
export const POSTHOG_AGGREGATED_EVENTS = [PostHogEventTypes.SecretPulled];
|
||||
const TELEMETRY_AGGREGATED_KEY_EXP = 900; // 15mins
|
||||
|
||||
// Bucket configuration
|
||||
const TELEMETRY_BUCKET_COUNT = 30;
|
||||
const TELEMETRY_BUCKET_NAMES = Array.from(
|
||||
{ length: TELEMETRY_BUCKET_COUNT },
|
||||
(_, i) => `bucket-${i.toString().padStart(2, "0")}`
|
||||
);
|
||||
|
||||
type AggregatedEventData = Record<string, unknown>;
|
||||
type SingleEventData = {
|
||||
distinctId: string;
|
||||
event: string;
|
||||
properties: unknown;
|
||||
organizationId: string;
|
||||
};
|
||||
|
||||
export type TTelemetryServiceFactory = ReturnType<typeof telemetryServiceFactory>;
|
||||
export type TTelemetryServiceFactoryDep = {
|
||||
keyStore: Pick<TKeyStoreFactory, "getItem" | "incrementBy">;
|
||||
keyStore: Pick<
|
||||
TKeyStoreFactory,
|
||||
"incrementBy" | "deleteItemsByKeyIn" | "setItemWithExpiry" | "getKeysByPattern" | "getItems"
|
||||
>;
|
||||
licenseService: Pick<TLicenseServiceFactory, "getInstanceType">;
|
||||
};
|
||||
|
||||
const getBucketForDistinctId = (distinctId: string): string => {
|
||||
// Use SHA-256 hash for consistent distribution
|
||||
const hash = createHash("sha256").update(distinctId).digest("hex");
|
||||
|
||||
// Take first 8 characters and convert to number for better distribution
|
||||
const hashNumber = parseInt(hash.substring(0, 8), 16);
|
||||
const bucketIndex = hashNumber % TELEMETRY_BUCKET_COUNT;
|
||||
|
||||
return TELEMETRY_BUCKET_NAMES[bucketIndex];
|
||||
};
|
||||
|
||||
export const createTelemetryEventKey = (event: string, distinctId: string): string => {
|
||||
const bucketId = getBucketForDistinctId(distinctId);
|
||||
return `telemetry-event-${event}-${bucketId}-${distinctId}-${randomUUID()}`;
|
||||
};
|
||||
|
||||
export const telemetryServiceFactory = ({ keyStore, licenseService }: TTelemetryServiceFactoryDep) => {
|
||||
const appCfg = getConfig();
|
||||
|
||||
@@ -64,11 +102,33 @@ To opt into telemetry, you can set "TELEMETRY_ENABLED=true" within the environme
|
||||
const instanceType = licenseService.getInstanceType();
|
||||
// capture posthog only when its cloud or signup event happens in self-hosted
|
||||
if (instanceType === InstanceType.Cloud || event.event === PostHogEventTypes.UserSignedUp) {
|
||||
postHog.capture({
|
||||
event: event.event,
|
||||
distinctId: event.distinctId,
|
||||
properties: event.properties
|
||||
});
|
||||
if (event.organizationId) {
|
||||
try {
|
||||
postHog.groupIdentify({ groupType: "organization", groupKey: event.organizationId });
|
||||
} catch (error) {
|
||||
logger.error(error, "Failed to identify PostHog organization");
|
||||
}
|
||||
}
|
||||
if (POSTHOG_AGGREGATED_EVENTS.includes(event.event)) {
|
||||
const eventKey = createTelemetryEventKey(event.event, event.distinctId);
|
||||
await keyStore.setItemWithExpiry(
|
||||
eventKey,
|
||||
TELEMETRY_AGGREGATED_KEY_EXP,
|
||||
JSON.stringify({
|
||||
distinctId: event.distinctId,
|
||||
event: event.event,
|
||||
properties: event.properties,
|
||||
organizationId: event.organizationId
|
||||
})
|
||||
);
|
||||
} else {
|
||||
postHog.capture({
|
||||
event: event.event,
|
||||
distinctId: event.distinctId,
|
||||
properties: event.properties,
|
||||
...(event.organizationId ? { groups: { organization: event.organizationId } } : {})
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -89,6 +149,160 @@ To opt into telemetry, you can set "TELEMETRY_ENABLED=true" within the environme
|
||||
}
|
||||
};
|
||||
|
||||
const aggregateGroupProperties = (events: SingleEventData[]): AggregatedEventData => {
|
||||
const aggregatedData: AggregatedEventData = {};
|
||||
|
||||
// Set the total count
|
||||
aggregatedData.count = events.length;
|
||||
|
||||
events.forEach((event) => {
|
||||
if (!event.properties) return;
|
||||
|
||||
Object.entries(event.properties as Record<string, unknown>).forEach(([key, value]: [string, unknown]) => {
|
||||
if (Array.isArray(value)) {
|
||||
// For arrays, count occurrences of each item
|
||||
const existingCounts =
|
||||
aggregatedData[key] &&
|
||||
typeof aggregatedData[key] === "object" &&
|
||||
aggregatedData[key]?.constructor === Object
|
||||
? (aggregatedData[key] as Record<string, number>)
|
||||
: {};
|
||||
|
||||
value.forEach((item) => {
|
||||
const itemKey = typeof item === "object" ? JSON.stringify(item) : String(item);
|
||||
existingCounts[itemKey] = (existingCounts[itemKey] || 0) + 1;
|
||||
});
|
||||
|
||||
aggregatedData[key] = existingCounts;
|
||||
} else if (typeof value === "object" && value?.constructor === Object) {
|
||||
// For objects, count occurrences of each field value
|
||||
const existingCounts =
|
||||
aggregatedData[key] &&
|
||||
typeof aggregatedData[key] === "object" &&
|
||||
aggregatedData[key]?.constructor === Object
|
||||
? (aggregatedData[key] as Record<string, number>)
|
||||
: {};
|
||||
|
||||
if (value) {
|
||||
Object.values(value).forEach((fieldValue) => {
|
||||
const valueKey = typeof fieldValue === "object" ? JSON.stringify(fieldValue) : String(fieldValue);
|
||||
existingCounts[valueKey] = (existingCounts[valueKey] || 0) + 1;
|
||||
});
|
||||
}
|
||||
aggregatedData[key] = existingCounts;
|
||||
} else if (typeof value === "number") {
|
||||
// For numbers, add to existing sum
|
||||
aggregatedData[key] = ((aggregatedData[key] as number) || 0) + value;
|
||||
} else if (value !== undefined && value !== null) {
|
||||
// For other types (strings, booleans, etc.), count occurrences
|
||||
const stringValue = String(value);
|
||||
const existingValue = aggregatedData[key];
|
||||
|
||||
if (!existingValue) {
|
||||
aggregatedData[key] = { [stringValue]: 1 };
|
||||
} else if (existingValue && typeof existingValue === "object" && existingValue.constructor === Object) {
|
||||
const countObject = existingValue as Record<string, number>;
|
||||
countObject[stringValue] = (countObject[stringValue] || 0) + 1;
|
||||
} else {
|
||||
const oldValue = String(existingValue);
|
||||
aggregatedData[key] = {
|
||||
[oldValue]: 1,
|
||||
[stringValue]: 1
|
||||
};
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
return aggregatedData;
|
||||
};
|
||||
|
||||
const processBucketEvents = async (eventType: string, bucketId: string) => {
|
||||
if (!postHog) return 0;
|
||||
|
||||
try {
|
||||
const bucketPattern = `telemetry-event-${eventType}-${bucketId}-*`;
|
||||
const bucketKeys = await keyStore.getKeysByPattern(bucketPattern);
|
||||
|
||||
if (bucketKeys.length === 0) return 0;
|
||||
|
||||
const bucketEvents = await keyStore.getItems(bucketKeys);
|
||||
let bucketEventsParsed: SingleEventData[] = [];
|
||||
|
||||
try {
|
||||
bucketEventsParsed = bucketEvents
|
||||
.filter((event) => event !== null)
|
||||
.map((event) => JSON.parse(event as string) as SingleEventData);
|
||||
} catch (error) {
|
||||
logger.error(error, `Failed to parse bucket events for ${eventType} in ${bucketId}`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
const eventsGrouped = new Map<string, SingleEventData[]>();
|
||||
|
||||
bucketEventsParsed.forEach((event) => {
|
||||
const key = JSON.stringify({ id: event.distinctId, org: event.organizationId });
|
||||
if (!eventsGrouped.has(key)) {
|
||||
eventsGrouped.set(key, []);
|
||||
}
|
||||
eventsGrouped.get(key)!.push(event);
|
||||
});
|
||||
|
||||
if (eventsGrouped.size === 0) return 0;
|
||||
|
||||
for (const [eventsKey, events] of eventsGrouped) {
|
||||
const key = JSON.parse(eventsKey) as { id: string; org?: string };
|
||||
if (key.org) {
|
||||
try {
|
||||
postHog.groupIdentify({ groupType: "organization", groupKey: key.org });
|
||||
} catch (error) {
|
||||
logger.error(error, "Failed to identify PostHog organization");
|
||||
}
|
||||
}
|
||||
const properties = aggregateGroupProperties(events);
|
||||
|
||||
postHog.capture({
|
||||
event: `${eventType} aggregated`,
|
||||
distinctId: key.id,
|
||||
properties,
|
||||
...(key.org ? { groups: { organization: key.org } } : {})
|
||||
});
|
||||
}
|
||||
|
||||
// Clean up processed data for this bucket
|
||||
await keyStore.deleteItemsByKeyIn(bucketKeys);
|
||||
|
||||
logger.info(`Processed ${bucketEventsParsed.length} events from bucket ${bucketId} for ${eventType}`);
|
||||
return bucketEventsParsed.length;
|
||||
} catch (error) {
|
||||
logger.error(error, `Failed to process bucket ${bucketId} for ${eventType}`);
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
|
||||
const processAggregatedEvents = async () => {
|
||||
if (!postHog) return;
|
||||
|
||||
for (const eventType of POSTHOG_AGGREGATED_EVENTS) {
|
||||
let totalProcessed = 0;
|
||||
|
||||
logger.info(`Starting bucket processing for ${eventType}`);
|
||||
|
||||
// Process each bucket sequentially to control memory usage
|
||||
for (const bucketId of TELEMETRY_BUCKET_NAMES) {
|
||||
try {
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
const processed = await processBucketEvents(eventType, bucketId);
|
||||
totalProcessed += processed;
|
||||
} catch (error) {
|
||||
logger.error(error, `Failed to process bucket ${bucketId} for ${eventType}`);
|
||||
}
|
||||
}
|
||||
|
||||
logger.info(`Completed processing ${totalProcessed} total events for ${eventType}`);
|
||||
}
|
||||
};
|
||||
|
||||
const flushAll = async () => {
|
||||
if (postHog) {
|
||||
await postHog.shutdownAsync();
|
||||
@@ -98,6 +312,8 @@ To opt into telemetry, you can set "TELEMETRY_ENABLED=true" within the environme
|
||||
return {
|
||||
sendLoopsEvent,
|
||||
sendPostHogEvents,
|
||||
flushAll
|
||||
processAggregatedEvents,
|
||||
flushAll,
|
||||
getBucketForDistinctId
|
||||
};
|
||||
};
|
||||
|
||||
@@ -1,3 +1,13 @@
|
||||
import {
|
||||
IdentityActor,
|
||||
KmipClientActor,
|
||||
PlatformActor,
|
||||
ScimClientActor,
|
||||
ServiceActor,
|
||||
UnknownUserActor,
|
||||
UserActor
|
||||
} from "@app/ee/services/audit-log/audit-log-types";
|
||||
|
||||
export enum PostHogEventTypes {
|
||||
SecretPush = "secrets pushed",
|
||||
SecretPulled = "secrets pulled",
|
||||
@@ -40,6 +50,14 @@ export type TSecretModifiedEvent = {
|
||||
secretPath: string;
|
||||
channel?: string;
|
||||
userAgent?: string;
|
||||
actor?:
|
||||
| UserActor
|
||||
| IdentityActor
|
||||
| ServiceActor
|
||||
| ScimClientActor
|
||||
| PlatformActor
|
||||
| UnknownUserActor
|
||||
| KmipClientActor;
|
||||
};
|
||||
};
|
||||
|
||||
@@ -214,7 +232,7 @@ export type TInvalidateCacheEvent = {
|
||||
};
|
||||
};
|
||||
|
||||
export type TPostHogEvent = { distinctId: string } & (
|
||||
export type TPostHogEvent = { distinctId: string; organizationId?: string } & (
|
||||
| TSecretModifiedEvent
|
||||
| TAdminInitEvent
|
||||
| TUserSignedUpEvent
|
||||
|
||||
Reference in New Issue
Block a user