Merge remote-tracking branch 'origin/main' into feat/resource-metadata

This commit is contained in:
Sheen Capadngan
2024-12-20 14:23:14 +08:00
231 changed files with 11454 additions and 167 deletions
+32 -1
View File
@@ -1,8 +1,10 @@
import { ForbiddenError, PureAbility } from "@casl/ability";
import opentelemetry from "@opentelemetry/api";
import fastifyPlugin from "fastify-plugin";
import jwt from "jsonwebtoken";
import { ZodError } from "zod";
import { getConfig } from "@app/lib/config/env";
import {
BadRequestError,
DatabaseError,
@@ -35,8 +37,30 @@ enum HttpStatusCodes {
}
export const fastifyErrHandler = fastifyPlugin(async (server: FastifyZodProvider) => {
const appCfg = getConfig();
const apiMeter = opentelemetry.metrics.getMeter("API");
const errorHistogram = apiMeter.createHistogram("API_errors", {
description: "API errors by type, status code, and name",
unit: "1"
});
server.setErrorHandler((error, req, res) => {
req.log.error(error);
if (appCfg.OTEL_TELEMETRY_COLLECTION_ENABLED) {
const { method } = req;
const route = req.routerPath;
const errorType =
error instanceof jwt.JsonWebTokenError ? "TokenError" : error.constructor.name || "UnknownError";
errorHistogram.record(1, {
route,
method,
type: errorType,
name: error.name
});
}
if (error instanceof BadRequestError) {
void res
.status(HttpStatusCodes.BadRequest)
@@ -52,13 +76,20 @@ export const fastifyErrHandler = fastifyPlugin(async (server: FastifyZodProvider
message: error.message,
error: error.name
});
} else if (error instanceof DatabaseError || error instanceof InternalServerError) {
} else if (error instanceof DatabaseError) {
void res.status(HttpStatusCodes.InternalServerError).send({
reqId: req.id,
statusCode: HttpStatusCodes.InternalServerError,
message: "Something went wrong",
error: error.name
});
} else if (error instanceof InternalServerError) {
void res.status(HttpStatusCodes.InternalServerError).send({
reqId: req.id,
statusCode: HttpStatusCodes.InternalServerError,
message: error.message ?? "Something went wrong",
error: error.name
});
} else if (error instanceof GatewayTimeoutError) {
void res.status(HttpStatusCodes.GatewayTimeout).send({
reqId: req.id,
+46 -1
View File
@@ -75,6 +75,13 @@ import { snapshotDALFactory } from "@app/ee/services/secret-snapshot/snapshot-da
import { snapshotFolderDALFactory } from "@app/ee/services/secret-snapshot/snapshot-folder-dal";
import { snapshotSecretDALFactory } from "@app/ee/services/secret-snapshot/snapshot-secret-dal";
import { snapshotSecretV2DALFactory } from "@app/ee/services/secret-snapshot/snapshot-secret-v2-dal";
import { sshCertificateAuthorityDALFactory } from "@app/ee/services/ssh/ssh-certificate-authority-dal";
import { sshCertificateAuthoritySecretDALFactory } from "@app/ee/services/ssh/ssh-certificate-authority-secret-dal";
import { sshCertificateAuthorityServiceFactory } from "@app/ee/services/ssh/ssh-certificate-authority-service";
import { sshCertificateBodyDALFactory } from "@app/ee/services/ssh-certificate/ssh-certificate-body-dal";
import { sshCertificateDALFactory } from "@app/ee/services/ssh-certificate/ssh-certificate-dal";
import { sshCertificateTemplateDALFactory } from "@app/ee/services/ssh-certificate-template/ssh-certificate-template-dal";
import { sshCertificateTemplateServiceFactory } from "@app/ee/services/ssh-certificate-template/ssh-certificate-template-service";
import { trustedIpDALFactory } from "@app/ee/services/trusted-ip/trusted-ip-dal";
import { trustedIpServiceFactory } from "@app/ee/services/trusted-ip/trusted-ip-service";
import { TKeyStoreFactory } from "@app/keystore/keystore";
@@ -84,6 +91,8 @@ import { readLimit } from "@app/server/config/rateLimiter";
import { accessTokenQueueServiceFactory } from "@app/services/access-token-queue/access-token-queue";
import { apiKeyDALFactory } from "@app/services/api-key/api-key-dal";
import { apiKeyServiceFactory } from "@app/services/api-key/api-key-service";
import { appConnectionDALFactory } from "@app/services/app-connection/app-connection-dal";
import { appConnectionServiceFactory } from "@app/services/app-connection/app-connection-service";
import { authDALFactory } from "@app/services/auth/auth-dal";
import { authLoginServiceFactory } from "@app/services/auth/auth-login-service";
import { authPaswordServiceFactory } from "@app/services/auth/auth-password-service";
@@ -308,6 +317,7 @@ export const registerRoutes = async (
const auditLogStreamDAL = auditLogStreamDALFactory(db);
const trustedIpDAL = trustedIpDALFactory(db);
const telemetryDAL = telemetryDALFactory(db);
const appConnectionDAL = appConnectionDALFactory(db);
// ee db layer ops
const permissionDAL = permissionDALFactory(db);
@@ -346,6 +356,12 @@ export const registerRoutes = async (
const dynamicSecretDAL = dynamicSecretDALFactory(db);
const dynamicSecretLeaseDAL = dynamicSecretLeaseDALFactory(db);
const sshCertificateDAL = sshCertificateDALFactory(db);
const sshCertificateBodyDAL = sshCertificateBodyDALFactory(db);
const sshCertificateAuthorityDAL = sshCertificateAuthorityDALFactory(db);
const sshCertificateAuthoritySecretDAL = sshCertificateAuthoritySecretDALFactory(db);
const sshCertificateTemplateDAL = sshCertificateTemplateDALFactory(db);
const kmsDAL = kmskeyDALFactory(db);
const internalKmsDAL = internalKmsDALFactory(db);
const externalKmsDAL = externalKmsDALFactory(db);
@@ -714,6 +730,22 @@ export const registerRoutes = async (
queueService
});
const sshCertificateAuthorityService = sshCertificateAuthorityServiceFactory({
sshCertificateAuthorityDAL,
sshCertificateAuthoritySecretDAL,
sshCertificateTemplateDAL,
sshCertificateDAL,
sshCertificateBodyDAL,
kmsService,
permissionService
});
const sshCertificateTemplateService = sshCertificateTemplateServiceFactory({
sshCertificateTemplateDAL,
sshCertificateAuthorityDAL,
permissionService
});
const certificateAuthorityService = certificateAuthorityServiceFactory({
certificateAuthorityDAL,
certificateAuthorityCertDAL,
@@ -851,6 +883,9 @@ export const registerRoutes = async (
certificateDAL,
pkiAlertDAL,
pkiCollectionDAL,
sshCertificateAuthorityDAL,
sshCertificateDAL,
sshCertificateTemplateDAL,
projectUserMembershipRoleDAL,
identityProjectMembershipRoleDAL,
keyStore,
@@ -1328,6 +1363,13 @@ export const registerRoutes = async (
externalGroupOrgRoleMappingDAL
});
const appConnectionService = appConnectionServiceFactory({
appConnectionDAL,
permissionService,
kmsService,
licenseService
});
await superAdminService.initServerCfg();
// setup the communication with license key server
@@ -1396,6 +1438,8 @@ export const registerRoutes = async (
auditLog: auditLogService,
auditLogStream: auditLogStreamService,
certificate: certificateService,
sshCertificateAuthority: sshCertificateAuthorityService,
sshCertificateTemplate: sshCertificateTemplateService,
certificateAuthority: certificateAuthorityService,
certificateTemplate: certificateTemplateService,
certificateAuthorityCrl: certificateAuthorityCrlService,
@@ -1422,7 +1466,8 @@ export const registerRoutes = async (
migration: migrationService,
externalGroupOrgRoleMapping: externalGroupOrgRoleMappingService,
projectTemplate: projectTemplateService,
totp: totpService
totp: totpService,
appConnection: appConnectionService
});
const cronJobs: CronJob[] = [];
@@ -0,0 +1,74 @@
import { z } from "zod";
import { EventType } from "@app/ee/services/audit-log/audit-log-types";
import { readLimit } from "@app/server/config/rateLimiter";
import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
import { AwsConnectionListItemSchema, SanitizedAwsConnectionSchema } from "@app/services/app-connection/aws";
import { GitHubConnectionListItemSchema, SanitizedGitHubConnectionSchema } from "@app/services/app-connection/github";
import { AuthMode } from "@app/services/auth/auth-type";
// can't use discriminated due to multiple schemas for certain apps
const SanitizedAppConnectionSchema = z.union([
...SanitizedAwsConnectionSchema.options,
...SanitizedGitHubConnectionSchema.options
]);
const AppConnectionOptionsSchema = z.discriminatedUnion("app", [
AwsConnectionListItemSchema,
GitHubConnectionListItemSchema
]);
export const registerAppConnectionRouter = async (server: FastifyZodProvider) => {
server.route({
method: "GET",
url: "/options",
config: {
rateLimit: readLimit
},
schema: {
description: "List the available App Connection Options.",
response: {
200: z.object({
appConnectionOptions: AppConnectionOptionsSchema.array()
})
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: () => {
const appConnectionOptions = server.services.appConnection.listAppConnectionOptions();
return { appConnectionOptions };
}
});
server.route({
method: "GET",
url: "/",
config: {
rateLimit: readLimit
},
schema: {
description: "List all the App Connections for the current organization.",
response: {
200: z.object({ appConnections: SanitizedAppConnectionSchema.array() })
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const appConnections = await server.services.appConnection.listAppConnectionsByOrg(req.permission);
await server.services.auditLog.createAuditLog({
...req.auditLogInfo,
orgId: req.permission.orgId,
event: {
type: EventType.GET_APP_CONNECTIONS,
metadata: {
count: appConnections.length,
connectionIds: appConnections.map((connection) => connection.id)
}
}
});
return { appConnections };
}
});
};
@@ -0,0 +1,274 @@
import { z } from "zod";
import { EventType } from "@app/ee/services/audit-log/audit-log-types";
import { AppConnections } from "@app/lib/api-docs";
import { startsWithVowel } from "@app/lib/fn";
import { readLimit, writeLimit } from "@app/server/config/rateLimiter";
import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
import { APP_CONNECTION_NAME_MAP } from "@app/services/app-connection/app-connection-maps";
import { TAppConnection, TAppConnectionInput } from "@app/services/app-connection/app-connection-types";
import { AuthMode } from "@app/services/auth/auth-type";
export const registerAppConnectionEndpoints = <T extends TAppConnection, I extends TAppConnectionInput>({
server,
app,
createSchema,
updateSchema,
responseSchema
}: {
app: AppConnection;
server: FastifyZodProvider;
createSchema: z.ZodType<{
name: string;
method: I["method"];
credentials: I["credentials"];
description?: string | null;
}>;
updateSchema: z.ZodType<{ name?: string; credentials?: I["credentials"]; description?: string | null }>;
responseSchema: z.ZodTypeAny;
}) => {
const appName = APP_CONNECTION_NAME_MAP[app];
server.route({
method: "GET",
url: `/`,
config: {
rateLimit: readLimit
},
schema: {
description: `List the ${appName} Connections for the current organization.`,
response: {
200: z.object({ appConnections: responseSchema.array() })
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const appConnections = (await server.services.appConnection.listAppConnectionsByOrg(req.permission, app)) as T[];
await server.services.auditLog.createAuditLog({
...req.auditLogInfo,
orgId: req.permission.orgId,
event: {
type: EventType.GET_APP_CONNECTIONS,
metadata: {
app,
count: appConnections.length,
connectionIds: appConnections.map((connection) => connection.id)
}
}
});
return { appConnections };
}
});
server.route({
method: "GET",
url: "/:connectionId",
config: {
rateLimit: readLimit
},
schema: {
description: `Get the specified ${appName} Connection by ID.`,
params: z.object({
connectionId: z.string().uuid().describe(AppConnections.GET_BY_ID(app).connectionId)
}),
response: {
200: z.object({ appConnection: responseSchema })
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const { connectionId } = req.params;
const appConnection = (await server.services.appConnection.findAppConnectionById(
app,
connectionId,
req.permission
)) as T;
await server.services.auditLog.createAuditLog({
...req.auditLogInfo,
orgId: req.permission.orgId,
event: {
type: EventType.GET_APP_CONNECTION,
metadata: {
connectionId
}
}
});
return { appConnection };
}
});
server.route({
method: "GET",
url: `/name/:connectionName`,
config: {
rateLimit: readLimit
},
schema: {
description: `Get the specified ${appName} Connection by name.`,
params: z.object({
connectionName: z
.string()
.min(0, "Connection name required")
.describe(AppConnections.GET_BY_NAME(app).connectionName)
}),
response: {
200: z.object({ appConnection: responseSchema })
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const { connectionName } = req.params;
const appConnection = (await server.services.appConnection.findAppConnectionByName(
app,
connectionName,
req.permission
)) as T;
await server.services.auditLog.createAuditLog({
...req.auditLogInfo,
orgId: req.permission.orgId,
event: {
type: EventType.GET_APP_CONNECTION,
metadata: {
connectionId: appConnection.id
}
}
});
return { appConnection };
}
});
server.route({
method: "POST",
url: "/",
config: {
rateLimit: writeLimit
},
schema: {
description: `Create ${
startsWithVowel(appName) ? "an" : "a"
} ${appName} Connection for the current organization.`,
body: createSchema,
response: {
200: z.object({ appConnection: responseSchema })
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const { name, method, credentials, description } = req.body;
const appConnection = (await server.services.appConnection.createAppConnection(
{ name, method, app, credentials, description },
req.permission
)) as TAppConnection;
await server.services.auditLog.createAuditLog({
...req.auditLogInfo,
orgId: req.permission.orgId,
event: {
type: EventType.CREATE_APP_CONNECTION,
metadata: {
name,
method,
app,
connectionId: appConnection.id
}
}
});
return { appConnection };
}
});
server.route({
method: "PATCH",
url: "/:connectionId",
config: {
rateLimit: writeLimit
},
schema: {
description: `Update the specified ${appName} Connection.`,
params: z.object({
connectionId: z.string().uuid().describe(AppConnections.UPDATE(app).connectionId)
}),
body: updateSchema,
response: {
200: z.object({ appConnection: responseSchema })
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const { name, credentials, description } = req.body;
const { connectionId } = req.params;
const appConnection = (await server.services.appConnection.updateAppConnection(
{ name, credentials, connectionId, description },
req.permission
)) as T;
await server.services.auditLog.createAuditLog({
...req.auditLogInfo,
orgId: req.permission.orgId,
event: {
type: EventType.UPDATE_APP_CONNECTION,
metadata: {
name,
description,
credentialsUpdated: Boolean(credentials),
connectionId
}
}
});
return { appConnection };
}
});
server.route({
method: "DELETE",
url: `/:connectionId`,
config: {
rateLimit: writeLimit
},
schema: {
description: `Delete the specified ${appName} Connection.`,
params: z.object({
connectionId: z.string().uuid().describe(AppConnections.DELETE(app).connectionId)
}),
response: {
200: z.object({ appConnection: responseSchema })
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const { connectionId } = req.params;
const appConnection = (await server.services.appConnection.deleteAppConnection(
app,
connectionId,
req.permission
)) as T;
await server.services.auditLog.createAuditLog({
...req.auditLogInfo,
orgId: req.permission.orgId,
event: {
type: EventType.DELETE_APP_CONNECTION,
metadata: {
connectionId
}
}
});
return { appConnection };
}
});
};
@@ -0,0 +1,17 @@
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
import {
CreateAwsConnectionSchema,
SanitizedAwsConnectionSchema,
UpdateAwsConnectionSchema
} from "@app/services/app-connection/aws";
import { registerAppConnectionEndpoints } from "./app-connection-endpoints";
export const registerAwsConnectionRouter = async (server: FastifyZodProvider) =>
registerAppConnectionEndpoints({
app: AppConnection.AWS,
server,
responseSchema: SanitizedAwsConnectionSchema,
createSchema: CreateAwsConnectionSchema,
updateSchema: UpdateAwsConnectionSchema
});
@@ -0,0 +1,17 @@
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
import {
CreateGitHubConnectionSchema,
SanitizedGitHubConnectionSchema,
UpdateGitHubConnectionSchema
} from "@app/services/app-connection/github";
import { registerAppConnectionEndpoints } from "./app-connection-endpoints";
export const registerGitHubConnectionRouter = async (server: FastifyZodProvider) =>
registerAppConnectionEndpoints({
app: AppConnection.GitHub,
server,
responseSchema: SanitizedGitHubConnectionSchema,
createSchema: CreateGitHubConnectionSchema,
updateSchema: UpdateGitHubConnectionSchema
});
@@ -0,0 +1,8 @@
import { registerAwsConnectionRouter } from "@app/server/routes/v1/app-connection-routers/apps/aws-connection-router";
import { registerGitHubConnectionRouter } from "@app/server/routes/v1/app-connection-routers/apps/github-connection-router";
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
export const APP_CONNECTION_REGISTER_MAP: Record<AppConnection, (server: FastifyZodProvider) => Promise<void>> = {
[AppConnection.AWS]: registerAwsConnectionRouter,
[AppConnection.GitHub]: registerGitHubConnectionRouter
};
@@ -0,0 +1,2 @@
export * from "./app-connection-router";
export * from "./apps";
+11
View File
@@ -1,3 +1,4 @@
import { APP_CONNECTION_REGISTER_MAP, registerAppConnectionRouter } from "@app/server/routes/v1/app-connection-routers";
import { registerCmekRouter } from "@app/server/routes/v1/cmek-router";
import { registerDashboardRouter } from "@app/server/routes/v1/dashboard-router";
@@ -110,4 +111,14 @@ export const registerV1Routes = async (server: FastifyZodProvider) => {
await server.register(registerDashboardRouter, { prefix: "/dashboard" });
await server.register(registerCmekRouter, { prefix: "/kms" });
await server.register(registerExternalGroupOrgRoleMappingRouter, { prefix: "/external-group-mappings" });
await server.register(
async (appConnectionsRouter) => {
await appConnectionsRouter.register(registerAppConnectionRouter);
for await (const [app, router] of Object.entries(APP_CONNECTION_REGISTER_MAP)) {
await appConnectionsRouter.register(router, { prefix: `/${app}` });
}
},
{ prefix: "/app-connections" }
);
};
@@ -137,7 +137,9 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => {
.enum(["true", "false"])
.default("false")
.transform((value) => value === "true"),
type: z.enum([ProjectType.SecretManager, ProjectType.KMS, ProjectType.CertificateManager, "all"]).optional()
type: z
.enum([ProjectType.SecretManager, ProjectType.KMS, ProjectType.CertificateManager, ProjectType.SSH, "all"])
.optional()
}),
response: {
200: z.object({
@@ -10,6 +10,9 @@ import {
} from "@app/db/schemas";
import { EventType } from "@app/ee/services/audit-log/audit-log-types";
import { InfisicalProjectTemplate } from "@app/ee/services/project-template/project-template-types";
import { sanitizedSshCa } from "@app/ee/services/ssh/ssh-certificate-authority-schema";
import { sanitizedSshCertificate } from "@app/ee/services/ssh-certificate/ssh-certificate-schema";
import { sanitizedSshCertificateTemplate } from "@app/ee/services/ssh-certificate-template/ssh-certificate-template-schema";
import { PROJECTS } from "@app/lib/api-docs";
import { readLimit, writeLimit } from "@app/server/config/rateLimiter";
import { slugSchema } from "@app/server/lib/schemas";
@@ -500,4 +503,101 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => {
return { certificateTemplates };
}
});
server.route({
method: "GET",
url: "/:projectId/ssh-certificates",
config: {
rateLimit: readLimit
},
schema: {
params: z.object({
projectId: z.string().trim().describe(PROJECTS.LIST_SSH_CAS.projectId)
}),
querystring: z.object({
offset: z.coerce.number().default(0).describe(PROJECTS.LIST_SSH_CERTIFICATES.offset),
limit: z.coerce.number().default(25).describe(PROJECTS.LIST_SSH_CERTIFICATES.limit)
}),
response: {
200: z.object({
certificates: z.array(sanitizedSshCertificate),
totalCount: z.number()
})
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const { certificates, totalCount } = await server.services.project.listProjectSshCertificates({
actorId: req.permission.id,
actorOrgId: req.permission.orgId,
actorAuthMethod: req.permission.authMethod,
actor: req.permission.type,
projectId: req.params.projectId,
offset: req.query.offset,
limit: req.query.limit
});
return { certificates, totalCount };
}
});
server.route({
method: "GET",
url: "/:projectId/ssh-certificate-templates",
config: {
rateLimit: readLimit
},
schema: {
params: z.object({
projectId: z.string().trim().describe(PROJECTS.LIST_SSH_CERTIFICATE_TEMPLATES.projectId)
}),
response: {
200: z.object({
certificateTemplates: z.array(sanitizedSshCertificateTemplate)
})
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const { certificateTemplates } = await server.services.project.listProjectSshCertificateTemplates({
actorId: req.permission.id,
actorOrgId: req.permission.orgId,
actorAuthMethod: req.permission.authMethod,
actor: req.permission.type,
projectId: req.params.projectId
});
return { certificateTemplates };
}
});
server.route({
method: "GET",
url: "/:projectId/ssh-cas",
config: {
rateLimit: readLimit
},
schema: {
params: z.object({
projectId: z.string().trim().describe(PROJECTS.LIST_SSH_CAS.projectId)
}),
response: {
200: z.object({
cas: z.array(sanitizedSshCa)
})
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const cas = await server.services.project.listProjectSshCas({
actorId: req.permission.id,
actorOrgId: req.permission.orgId,
actorAuthMethod: req.permission.authMethod,
actor: req.permission.type,
projectId: req.params.projectId
});
return { cas };
}
});
};