awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-10-11 06:29:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2429 from Infisical/daniel/integration-ui-improvements

Browse Source 
feat: integration details page with logging
This commit is contained in:
Daniel Hougaard
2024-09-17 14:29:26 +04:00
committed by GitHub
parent 10e7999334 09887a7405
commit 5740d2b4e4
33 changed files with 1248 additions and 191 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/ee/routes/v1/project-router.ts
+10 -6
View File
@@ -146,12 +146,16 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => {
actorId: req.permission.id,
actorOrgId: req.permission.orgId,
actorAuthMethod: req.permission.authMethod,
projectId: req.params.workspaceId,
...req.query,
endDate: req.query.endDate,
startDate: req.query.startDate || getLastMidnightDateISO(),
auditLogActor: req.query.actor,
actor: req.permission.type
actor: req.permission.type,
filter: {
...req.query,
projectId: req.params.workspaceId,
endDate: req.query.endDate,
startDate: req.query.startDate || getLastMidnightDateISO(),
auditLogActorId: req.query.actor,
eventType: req.query.eventType ? [req.query.eventType] : undefined
}
});
return { auditLogs };
}
backend/src/ee/services/audit-log/audit-log-dal.ts
+37 -4
View File
@@ -6,6 +6,9 @@ import { DatabaseError } from "@app/lib/errors";
import { ormify, selectAllTableCols, stripUndefinedInWhere } from "@app/lib/knex";
import { logger } from "@app/lib/logger";
import { QueueName } from "@app/queue";
import { ActorType } from "@app/services/auth/auth-type";
import { EventType } from "./audit-log-types";
export type TAuditLogDALFactory = ReturnType<typeof auditLogDALFactory>;
@@ -25,7 +28,24 @@ export const auditLogDALFactory = (db: TDbClient) => {
const auditLogOrm = ormify(db, TableName.AuditLog);
const find = async (
{ orgId, projectId, userAgentType, startDate, endDate, limit = 20, offset = 0, actor, eventType }: TFindQuery,
{
orgId,
projectId,
userAgentType,
startDate,
endDate,
limit = 20,
offset = 0,
actorId,
actorType,
eventType,
eventMetadata
}: Omit<TFindQuery, "actor" | "eventType"> & {
actorId?: string;
actorType?: ActorType;
eventType?: EventType[];
eventMetadata?: Record<string, string>;
},
tx?: Knex
) => {
try {
@@ -34,7 +54,6 @@ export const auditLogDALFactory = (db: TDbClient) => {
stripUndefinedInWhere({
projectId,
[`${TableName.AuditLog}.orgId`]: orgId,
eventType,
userAgentType
})
)
@@ -52,8 +71,22 @@ export const auditLogDALFactory = (db: TDbClient) => {
.offset(offset)
.orderBy(`${TableName.AuditLog}.createdAt`, "desc");
if (actor) {
void sqlQuery.whereRaw(`"actorMetadata"->>'userId' = ?`, [actor]);
if (actorId) {
void sqlQuery.whereRaw(`"actorMetadata"->>'userId' = ?`, [actorId]);
}
if (eventMetadata && Object.keys(eventMetadata).length) {
Object.entries(eventMetadata).forEach(([key, value]) => {
void sqlQuery.whereRaw(`"eventMetadata"->>'${key}' = ?`, [value]);
});
}
if (actorType) {
void sqlQuery.where("actor", actorType);
}
if (eventType?.length) {
void sqlQuery.whereIn("eventType", eventType);
}
if (startDate) {
backend/src/ee/services/audit-log/audit-log-service.ts
+13 -24
View File
@@ -23,25 +23,12 @@ export const auditLogServiceFactory = ({
auditLogQueue,
permissionService
}: TAuditLogServiceFactoryDep) => {
const listAuditLogs = async ({
userAgentType,
eventType,
offset,
limit,
endDate,
startDate,
actor,
actorId,
actorOrgId,
actorAuthMethod,
projectId,
auditLogActor
}: TListProjectAuditLogDTO) => {
if (projectId) {
const listAuditLogs = async ({ actorAuthMethod, actorId, actorOrgId, actor, filter }: TListProjectAuditLogDTO) => {
if (filter.projectId) {
const { permission } = await permissionService.getProjectPermission(
actor,
actorId,
projectId,
filter.projectId,
actorAuthMethod,
actorOrgId
);
@@ -65,14 +52,16 @@ export const auditLogServiceFactory = ({
// If project ID is not provided, then we need to return all the audit logs for the organization itself.
const auditLogs = await auditLogDAL.find({
startDate,
endDate,
limit,
offset,
eventType,
userAgentType,
actor: auditLogActor,
...(projectId ? { projectId } : { orgId: actorOrgId })
startDate: filter.startDate,
endDate: filter.endDate,
limit: filter.limit,
offset: filter.offset,
eventType: filter.eventType,
userAgentType: filter.userAgentType,
actorId: filter.auditLogActorId,
actorType: filter.actorType,
eventMetadata: filter.eventMetadata,
...(filter.projectId ? { projectId: filter.projectId } : { orgId: actorOrgId })
});
return auditLogs.map(({ eventType: logEventType, actor: eActor, actorMetadata, eventMetadata, ...el }) => ({
backend/src/ee/services/audit-log/audit-log-types.ts
+35 -12
View File
@@ -5,19 +5,23 @@ import { TIdentityTrustedIp } from "@app/services/identity/identity-types";
import { PkiItemType } from "@app/services/pki-collection/pki-collection-types";
export type TListProjectAuditLogDTO = {
auditLogActor?: string;
projectId?: string;
eventType?: string;
startDate?: string;
endDate?: string;
userAgentType?: string;
limit?: number;
offset?: number;
filter: {
userAgentType?: UserAgentType;
eventType?: EventType[];
offset?: number;
limit: number;
endDate?: string;
startDate?: string;
projectId?: string;
auditLogActorId?: string;
actorType?: ActorType;
eventMetadata?: Record<string, string>;
};
} & Omit<TProjectPermission, "projectId">;
export type TCreateAuditLogDTO = {
event: Event;
actor: UserActor | IdentityActor | ServiceActor | ScimClientActor;
actor: UserActor | IdentityActor | ServiceActor | ScimClientActor | PlatformActor;
orgId?: string;
projectId?: string;
} & BaseAuthData;
@@ -177,7 +181,8 @@ export enum EventType {
UPDATE_SLACK_INTEGRATION = "update-slack-integration",
DELETE_SLACK_INTEGRATION = "delete-slack-integration",
GET_PROJECT_SLACK_CONFIG = "get-project-slack-config",
UPDATE_PROJECT_SLACK_CONFIG = "update-project-slack-config"
UPDATE_PROJECT_SLACK_CONFIG = "update-project-slack-config",
INTEGRATION_SYNCED = "integration-synced"
}
interface UserActorMetadata {
@@ -198,6 +203,8 @@ interface IdentityActorMetadata {
interface ScimClientActorMetadata {}
interface PlatformActorMetadata {}
export interface UserActor {
type: ActorType.USER;
metadata: UserActorMetadata;
@@ -208,6 +215,11 @@ export interface ServiceActor {
metadata: ServiceActorMetadata;
}
export interface PlatformActor {
type: ActorType.PLATFORM;
metadata: PlatformActorMetadata;
}
export interface IdentityActor {
type: ActorType.IDENTITY;
metadata: IdentityActorMetadata;
@@ -218,7 +230,7 @@ export interface ScimClientActor {
metadata: ScimClientActorMetadata;
}
export type Actor = UserActor | ServiceActor | IdentityActor | ScimClientActor;
export type Actor = UserActor | ServiceActor | IdentityActor | ScimClientActor | PlatformActor;
interface GetSecretsEvent {
type: EventType.GET_SECRETS;
@@ -1518,6 +1530,16 @@ interface GetProjectSlackConfig {
id: string;
};
}
interface IntegrationSyncedEvent {
type: EventType.INTEGRATION_SYNCED;
metadata: {
integrationId: string;
lastSyncJobId: string;
lastUsed: Date;
syncMessage: string;
isSynced: boolean;
};
}
export type Event =
| GetSecretsEvent
@@ -1657,4 +1679,5 @@ export type Event =
| DeleteSlackIntegration
| GetSlackIntegration
| UpdateProjectSlackConfig
| GetProjectSlackConfig;
| GetProjectSlackConfig
| IntegrationSyncedEvent;
backend/src/queue/queue-service.ts
+2
View File
@@ -91,6 +91,8 @@ export type TQueueJobTypes = {
[QueueName.IntegrationSync]: {
name: QueueJobs.IntegrationSync;
payload: {
isManual?: boolean;
actorId?: string;
projectId: string;
environment: string;
secretPath: string;
backend/src/server/routes/index.ts
+2
View File
@@ -810,6 +810,8 @@ export const registerRoutes = async (
projectEnvDAL,
webhookDAL,
orgDAL,
auditLogService,
userDAL,
projectMembershipDAL,
smtpService,
projectDAL,
backend/src/server/routes/v1/integration-router.ts
+43 -1
View File
@@ -4,7 +4,7 @@ import { IntegrationsSchema } from "@app/db/schemas";
import { EventType } from "@app/ee/services/audit-log/audit-log-types";
import { INTEGRATION } from "@app/lib/api-docs";
import { removeTrailingSlash, shake } from "@app/lib/fn";
import { writeLimit } from "@app/server/config/rateLimiter";
import { readLimit, writeLimit } from "@app/server/config/rateLimiter";
import { getTelemetryDistinctId } from "@app/server/lib/telemetry";
import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
import { AuthMode } from "@app/services/auth/auth-type";
@@ -154,6 +154,48 @@ export const registerIntegrationRouter = async (server: FastifyZodProvider) => {
}
});
server.route({
method: "GET",
url: "/:integrationId",
config: {
rateLimit: readLimit
},
schema: {
description: "Get an integration by integration id",
security: [
{
bearerAuth: []
}
],
params: z.object({
integrationId: z.string().trim().describe(INTEGRATION.UPDATE.integrationId)
}),
response: {
200: z.object({
integration: IntegrationsSchema.extend({
environment: z.object({
slug: z.string().trim(),
name: z.string().trim(),
id: z.string().trim()
})
})
})
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const integration = await server.services.integration.getIntegration({
actorId: req.permission.id,
actor: req.permission.type,
actorAuthMethod: req.permission.authMethod,
actorOrgId: req.permission.orgId,
id: req.params.integrationId
});
return { integration };
}
});
server.route({
method: "DELETE",
url: "/:integrationId",
backend/src/server/routes/v1/organization-router.ts
+39 -6
View File
@@ -14,7 +14,7 @@ import { AUDIT_LOGS, ORGANIZATIONS } from "@app/lib/api-docs";
import { getLastMidnightDateISO } from "@app/lib/fn";
import { readLimit, writeLimit } from "@app/server/config/rateLimiter";
import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
import { AuthMode } from "@app/services/auth/auth-type";
import { ActorType, AuthMode } from "@app/services/auth/auth-type";
export const registerOrgRouter = async (server: FastifyZodProvider) => {
server.route({
@@ -74,8 +74,35 @@ export const registerOrgRouter = async (server: FastifyZodProvider) => {
schema: {
description: "Get all audit logs for an organization",
querystring: z.object({
eventType: z.nativeEnum(EventType).optional().describe(AUDIT_LOGS.EXPORT.eventType),
projectId: z.string().optional(),
actorType: z.nativeEnum(ActorType).optional(),
// eventType is split with , for multiple values, we need to transform it to array
eventType: z
.string()
.optional()
.transform((val) => (val ? val.split(",") : undefined)),
userAgentType: z.nativeEnum(UserAgentType).optional().describe(AUDIT_LOGS.EXPORT.userAgentType),
eventMetadata: z
.string()
.optional()
.transform((val) => {
if (!val) {
return undefined;
}
const pairs = val.split(",");
return pairs.reduce(
(acc, pair) => {
const [key, value] = pair.split("=");
if (key && value) {
acc[key] = value;
}
return acc;
},
{} as Record<string, string>
);
}),
startDate: z.string().datetime().optional().describe(AUDIT_LOGS.EXPORT.startDate),
endDate: z.string().datetime().optional().describe(AUDIT_LOGS.EXPORT.endDate),
offset: z.coerce.number().default(0).describe(AUDIT_LOGS.EXPORT.offset),
@@ -114,13 +141,19 @@ export const registerOrgRouter = async (server: FastifyZodProvider) => {
onRequest: verifyAuth([AuthMode.JWT]),
handler: async (req) => {
const auditLogs = await server.services.auditLog.listAuditLogs({
filter: {
...req.query,
endDate: req.query.endDate,
projectId: req.query.projectId,
startDate: req.query.startDate || getLastMidnightDateISO(),
auditLogActorId: req.query.actor,
actorType: req.query.actorType,
eventType: req.query.eventType as EventType[] | undefined
},
actorId: req.permission.id,
actorOrgId: req.permission.orgId,
actorAuthMethod: req.permission.authMethod,
...req.query,
endDate: req.query.endDate,
startDate: req.query.startDate || getLastMidnightDateISO(),
auditLogActor: req.query.actor,
actor: req.permission.type
});
return { auditLogs };
backend/src/services/auth/auth-type.ts
+1
View File
@@ -34,6 +34,7 @@ export enum AuthMode {
}
export enum ActorType { // would extend to AWS, Azure, ...
PLATFORM = "platform", // Useful for when we want to perform logging on automated actions such as integration syncs.
USER = "user", // userIdentity
SERVICE = "service",
IDENTITY = "identity",
backend/src/services/integration/integration-service.ts
+26 -1
View File
@@ -2,7 +2,7 @@ import { ForbiddenError, subject } from "@casl/ability";
import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission";
import { BadRequestError } from "@app/lib/errors";
import { BadRequestError, NotFoundError } from "@app/lib/errors";
import { TProjectPermission } from "@app/lib/types";
import { TIntegrationAuthDALFactory } from "../integration-auth/integration-auth-dal";
@@ -19,6 +19,7 @@ import { TIntegrationDALFactory } from "./integration-dal";
import {
TCreateIntegrationDTO,
TDeleteIntegrationDTO,
TGetIntegrationDTO,
TSyncIntegrationDTO,
TUpdateIntegrationDTO
} from "./integration-types";
@@ -180,6 +181,27 @@ export const integrationServiceFactory = ({
return updatedIntegration;
};
const getIntegration = async ({ id, actor, actorAuthMethod, actorId, actorOrgId }: TGetIntegrationDTO) => {
const integration = await integrationDAL.findById(id);
const { permission } = await permissionService.getProjectPermission(
actor,
actorId,
integration?.projectId || "",
actorAuthMethod,
actorOrgId
);
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Integrations);
if (!integration) {
throw new NotFoundError({
message: "Integration not found"
});
}
return { ...integration, envId: integration.environment.id };
};
const deleteIntegration = async ({
actorId,
id,
@@ -276,6 +298,8 @@ export const integrationServiceFactory = ({
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Integrations);
await secretQueueService.syncIntegrations({
isManual: true,
actorId,
environment: integration.environment.slug,
secretPath: integration.secretPath,
projectId: integration.projectId
@@ -289,6 +313,7 @@ export const integrationServiceFactory = ({
updateIntegration,
deleteIntegration,
listIntegrationByProject,
getIntegration,
syncIntegration
};
};
backend/src/services/integration/integration-types.ts
+4
View File
@@ -39,6 +39,10 @@ export type TCreateIntegrationDTO = {
};
} & Omit<TProjectPermission, "projectId">;
export type TGetIntegrationDTO = {
id: string;
} & Omit<TProjectPermission, "projectId">;
export type TUpdateIntegrationDTO = {
id: string;
app?: string;
backend/src/services/secret/secret-queue.ts
+68 -5
View File
@@ -2,6 +2,8 @@
import { AxiosError } from "axios";
import { ProjectUpgradeStatus, ProjectVersion, TSecretSnapshotSecretsV2, TSecretVersionsV2 } from "@app/db/schemas";
import { TAuditLogServiceFactory } from "@app/ee/services/audit-log/audit-log-service";
import { Actor, EventType } from "@app/ee/services/audit-log/audit-log-types";
import { TSecretApprovalRequestDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-dal";
import { TSecretRotationDALFactory } from "@app/ee/services/secret-rotation/secret-rotation-dal";
import { TSnapshotDALFactory } from "@app/ee/services/secret-snapshot/snapshot-dal";
@@ -21,6 +23,7 @@ import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version
import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal";
import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
import { ActorType } from "../auth/auth-type";
import { TIntegrationDALFactory } from "../integration/integration-dal";
import { TIntegrationAuthDALFactory } from "../integration-auth/integration-auth-dal";
import { TIntegrationAuthServiceFactory } from "../integration-auth/integration-auth-service";
@@ -40,6 +43,7 @@ import { expandSecretReferencesFactory, getAllNestedSecretReferences } from "../
import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-dal";
import { TSecretVersionV2TagDALFactory } from "../secret-v2-bridge/secret-version-tag-dal";
import { SmtpTemplates, TSmtpService } from "../smtp/smtp-service";
import { TUserDALFactory } from "../user/user-dal";
import { TWebhookDALFactory } from "../webhook/webhook-dal";
import { fnTriggerWebhook } from "../webhook/webhook-fns";
import { TSecretDALFactory } from "./secret-dal";
@@ -71,6 +75,7 @@ type TSecretQueueFactoryDep = {
secretVersionDAL: TSecretVersionDALFactory;
secretBlindIndexDAL: TSecretBlindIndexDALFactory;
secretTagDAL: TSecretTagDALFactory;
userDAL: Pick<TUserDALFactory, "findById">;
secretVersionTagDAL: TSecretVersionTagDALFactory;
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
secretV2BridgeDAL: TSecretV2BridgeDALFactory;
@@ -81,6 +86,7 @@ type TSecretQueueFactoryDep = {
snapshotDAL: Pick<TSnapshotDALFactory, "findNSecretV1SnapshotByFolderId" | "deleteSnapshotsAboveLimit">;
snapshotSecretV2BridgeDAL: Pick<TSnapshotSecretV2DALFactory, "insertMany" | "batchInsert">;
keyStore: Pick<TKeyStoreFactory, "acquireLock" | "setItemWithExpiry" | "getItem">;
auditLogService: Pick<TAuditLogServiceFactory, "createAuditLog">;
};
export type TGetSecrets = {
@@ -106,6 +112,7 @@ export const secretQueueFactory = ({
secretDAL,
secretImportDAL,
folderDAL,
userDAL,
webhookDAL,
projectEnvDAL,
orgDAL,
@@ -125,7 +132,8 @@ export const secretQueueFactory = ({
snapshotDAL,
snapshotSecretV2BridgeDAL,
secretApprovalRequestDAL,
keyStore
keyStore,
auditLogService
}: TSecretQueueFactoryDep) => {
const removeSecretReminder = async (dto: TRemoveSecretReminderDTO) => {
const appCfg = getConfig();
@@ -430,7 +438,9 @@ export const secretQueueFactory = ({
return content;
};
const syncIntegrations = async (dto: TGetSecrets & { deDupeQueue?: Record<string, boolean> }) => {
const syncIntegrations = async (
dto: TGetSecrets & { isManual?: boolean; actorId?: string; deDupeQueue?: Record<string, boolean> }
) => {
await queueService.queue(QueueName.IntegrationSync, QueueJobs.IntegrationSync, dto, {
attempts: 3,
delay: 1000,
@@ -528,7 +538,7 @@ export const secretQueueFactory = ({
}
}
);
await syncIntegrations({ secretPath, projectId, environment, deDupeQueue });
await syncIntegrations({ secretPath, projectId, environment, deDupeQueue, isManual: false });
if (!excludeReplication) {
await replicateSecrets({
_deDupeReplicationQueue: deDupeReplicationQueue,
@@ -544,7 +554,7 @@ export const secretQueueFactory = ({
});
queueService.start(QueueName.IntegrationSync, async (job) => {
const { environment, projectId, secretPath, depth = 1, deDupeQueue = {} } = job.data;
const { environment, actorId, isManual, projectId, secretPath, depth = 1, deDupeQueue = {} } = job.data;
if (depth > MAX_SYNC_SECRET_DEPTH) return;
const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
@@ -693,6 +703,30 @@ export const secretQueueFactory = ({
});
}
const generateActor = async (): Promise<Actor> => {
if (isManual && actorId) {
const user = await userDAL.findById(actorId);
if (!user) {
throw new Error("User not found");
}
return {
type: ActorType.USER,
metadata: {
email: user.email,
username: user.username,
userId: user.id
}
};
}
return {
type: ActorType.PLATFORM,
metadata: {}
};
};
// akhilmhdh: this try catch is for lock release
try {
const secrets = shouldUseSecretV2Bridge
@@ -778,6 +812,21 @@ export const secretQueueFactory = ({
}
});
await auditLogService.createAuditLog({
projectId,
actor: await generateActor(),
event: {
type: EventType.INTEGRATION_SYNCED,
metadata: {
integrationId: integration.id,
isSynced: response?.isSynced ?? true,
lastSyncJobId: job?.id ?? "",
lastUsed: new Date(),
syncMessage: response?.syncMessage ?? ""
}
}
});
await integrationDAL.updateById(integration.id, {
lastSyncJobId: job.id,
lastUsed: new Date(),
@@ -794,9 +843,23 @@ export const secretQueueFactory = ({
(err instanceof AxiosError ? JSON.stringify(err?.response?.data) : (err as Error)?.message) ||
"Unknown error occurred.";
await auditLogService.createAuditLog({
projectId,
actor: await generateActor(),
event: {
type: EventType.INTEGRATION_SYNCED,
metadata: {
integrationId: integration.id,
isSynced: false,
lastSyncJobId: job?.id ?? "",
lastUsed: new Date(),
syncMessage: message
}
}
});
await integrationDAL.updateById(integration.id, {
lastSyncJobId: job.id,
lastUsed: new Date(),
syncMessage: message,
isSynced: false
});