awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-10-08 23:28:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin' into stv3-update

Browse Source
This commit is contained in:
Tuan Dang
2023-11-01 18:57:31 +02:00
parent 7d802b41a8 2350219cc9
commit 7bf2e96ad3
18 changed files with 1355 additions and 1824 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/package-lock.json
Generated
+867 -1060
View File
File diff suppressed because it is too large Load Diff
backend/package.json
+6 -4
View File
@@ -6,7 +6,7 @@
"@godaddy/terminus": "^4.12.0", "@godaddy/terminus": "^4.12.0",
"@node-saml/passport-saml": "^4.0.4", "@node-saml/passport-saml": "^4.0.4",
"@octokit/rest": "^19.0.5", "@octokit/rest": "^19.0.5",
"@sentry/node": "^7.49.0", "@sentry/node": "^7.77.0",
"@sentry/tracing": "^7.48.0", "@sentry/tracing": "^7.48.0",
"@types/crypto-js": "^4.1.1", "@types/crypto-js": "^4.1.1",
"@types/libsodium-wrappers": "^0.7.10", "@types/libsodium-wrappers": "^0.7.10",
@@ -42,6 +42,8 @@
"passport-github": "^1.1.0", "passport-github": "^1.1.0",
"passport-gitlab2": "^5.0.0", "passport-gitlab2": "^5.0.0",
"passport-google-oauth20": "^2.0.0", "passport-google-oauth20": "^2.0.0",
"pino": "^8.16.1",
"pino-http": "^8.5.1",
"posthog-node": "^2.6.0", "posthog-node": "^2.6.0",
"probot": "^12.3.1", "probot": "^12.3.1",
"query-string": "^7.1.3", "query-string": "^7.1.3",
@@ -52,8 +54,6 @@
"tweetnacl-util": "^0.15.1", "tweetnacl-util": "^0.15.1",
"typescript": "^4.9.3", "typescript": "^4.9.3",
"utility-types": "^3.10.0", "utility-types": "^3.10.0",
"winston": "^3.8.2",
"winston-loki": "^6.0.6",
"zod": "^3.22.3" "zod": "^3.22.3"
}, },
"overrides": { "overrides": {
@@ -66,7 +66,7 @@
"main": "src/index.js", "main": "src/index.js",
"scripts": { "scripts": {
"start": "node build/index.js", "start": "node build/index.js",
"dev": "nodemon", "dev": "nodemon index.js | pino-pretty --colorize",
"swagger-autogen": "node ./swagger/index.ts", "swagger-autogen": "node ./swagger/index.ts",
"build": "rimraf ./build && tsc && cp -R ./src/templates ./build && cp -R ./src/data ./build", "build": "rimraf ./build && tsc && cp -R ./src/templates ./build && cp -R ./src/data ./build",
"lint": "eslint . --ext .ts", "lint": "eslint . --ext .ts",
@@ -104,6 +104,7 @@
"@types/nodemailer": "^6.4.6", "@types/nodemailer": "^6.4.6",
"@types/passport": "^1.0.12", "@types/passport": "^1.0.12",
"@types/picomatch": "^2.3.0", "@types/picomatch": "^2.3.0",
"@types/pino": "^7.0.5",
"@types/supertest": "^2.0.12", "@types/supertest": "^2.0.12",
"@types/swagger-jsdoc": "^6.0.1", "@types/swagger-jsdoc": "^6.0.1",
"@types/swagger-ui-express": "^4.1.3", "@types/swagger-ui-express": "^4.1.3",
@@ -117,6 +118,7 @@
"jest-junit": "^15.0.0", "jest-junit": "^15.0.0",
"nodemon": "^2.0.19", "nodemon": "^2.0.19",
"npm": "^8.19.3", "npm": "^8.19.3",
"pino-pretty": "^10.2.3",
"smee-client": "^1.2.3", "smee-client": "^1.2.3",
"supertest": "^6.3.3", "supertest": "^6.3.3",
"swagger-autogen": "^2.23.5", "swagger-autogen": "^2.23.5",
backend/src/helpers/database.ts
+3 -3
View File
@@ -1,5 +1,5 @@
import mongoose from "mongoose"; import mongoose from "mongoose";
import { getLogger } from "../utils/logger"; import { logger } from "../utils/logging";
/** /**
* Initialize database connection * Initialize database connection
@@ -18,10 +18,10 @@ export const initDatabaseHelper = async ({
// allow empty strings to pass the required validator // allow empty strings to pass the required validator
mongoose.Schema.Types.String.checkRequired(v => typeof v === "string"); mongoose.Schema.Types.String.checkRequired(v => typeof v === "string");
(await getLogger("database")).info("Database connection established"); logger.info("Database connection established");
} catch (err) { } catch (err) {
(await getLogger("database")).error(`Unable to establish Database connection due to the error.\n${err}`); logger.error(err, "Unable to establish database connection");
} }
return mongoose.connection; return mongoose.connection;
backend/src/helpers/organization.ts
+204 -300
View File
@@ -1,4 +1,4 @@
import mongoose, { Types, mongo } from "mongoose"; import { Types } from "mongoose";
import { import {
Bot, Bot,
BotKey, BotKey,
@@ -55,7 +55,7 @@ import {
import { import {
createBotOrg createBotOrg
} from "./botOrg"; } from "./botOrg";
import { InternalServerError, ResourceNotFoundError } from "../utils/errors"; import { ResourceNotFoundError } from "../utils/errors";
/** /**
* Create an organization with name [name] * Create an organization with name [name]
@@ -111,311 +111,215 @@ export const createOrganization = async ({
* @returns * @returns
*/ */
export const deleteOrganization = async ({ export const deleteOrganization = async ({
organizationId, organizationId
existingSession
}: { }: {
organizationId: Types.ObjectId; organizationId: Types.ObjectId;
existingSession?: mongo.ClientSession;
}) => { }) => {
let session; const organization = await Organization.findByIdAndDelete(
organizationId
if (existingSession) { );
session = existingSession;
} else {
session = await mongoose.startSession();
session.startTransaction();
}
try { if (!organization) throw ResourceNotFoundError();
const organization = await Organization.findByIdAndDelete(
organizationId, await MembershipOrg.deleteMany({
{ organization: organization._id
session });
}
await BotOrg.deleteMany({
organization: organization._id
});
await SSOConfig.deleteMany({
organization: organization._id
});
await Role.deleteMany({
organization: organization._id
});
await IncidentContactOrg.deleteMany({
organization: organization._id
});
await GitRisks.deleteMany({
organization: organization._id
});
await GitAppInstallationSession.deleteMany({
organization: organization._id
});
await GitAppOrganizationInstallation.deleteMany({
organization: organization._id
});
const workspaceIds = await Workspace.distinct("_id", {
organization: organization._id
});
await Workspace.deleteMany({
organization: organization._id
});
await Membership.deleteMany({
workspace: {
$in: workspaceIds
}
});
await Key.deleteMany({
workspace: {
$in: workspaceIds
}
});
await Bot.deleteMany({
workspace: {
$in: workspaceIds
}
});
await BotKey.deleteMany({
workspace: {
$in: workspaceIds
}
});
await SecretBlindIndexData.deleteMany({
workspace: {
$in: workspaceIds
}
});
await Secret.deleteMany({
workspace: {
$in: workspaceIds
}
});
await SecretVersion.deleteMany({
workspace: {
$in: workspaceIds
}
});
await SecretSnapshot.deleteMany({
workspace: {
$in: workspaceIds
}
});
await SecretImport.deleteMany({
workspace: {
$in: workspaceIds
}
});
await Folder.deleteMany({
workspace: {
$in: workspaceIds
}
});
await FolderVersion.deleteMany({
workspace: {
$in: workspaceIds
}
});
await Webhook.deleteMany({
workspace: {
$in: workspaceIds
}
});
await TrustedIP.deleteMany({
workspace: {
$in: workspaceIds
}
});
await Tag.deleteMany({
workspace: {
$in: workspaceIds
}
});
await IntegrationAuth.deleteMany({
workspace: {
$in: workspaceIds
}
});
await Integration.deleteMany({
workspace: {
$in: workspaceIds
}
});
await ServiceToken.deleteMany({
workspace: {
$in: workspaceIds
}
});
await ServiceTokenData.deleteMany({
workspace: {
$in: workspaceIds
}
});
await ServiceTokenDataV3.deleteMany({
workspace: {
$in: workspaceIds
}
});
await ServiceTokenDataV3Key.deleteMany({
workspace: {
$in: workspaceIds
}
});
await AuditLog.deleteMany({
workspace: {
$in: workspaceIds
}
});
await Log.deleteMany({
workspace: {
$in: workspaceIds
}
});
await Action.deleteMany({
workspace: {
$in: workspaceIds
}
});
await SecretApprovalPolicy.deleteMany({
workspace: {
$in: workspaceIds
}
});
await SecretApprovalRequest.deleteMany({
workspace: {
$in: workspaceIds
}
});
if (organization.customerId) {
// delete from stripe here
await licenseServerKeyRequest.delete(
`${await getLicenseServerUrl()}/api/license-server/v1/customers/${organization.customerId}`
); );
if (!organization) throw ResourceNotFoundError();
await MembershipOrg.deleteMany({
organization: organization._id
}, {
session
});
await BotOrg.deleteMany({
organization: organization._id
}, {
session
});
await SSOConfig.deleteMany({
organization: organization._id
}, {
session
});
await Role.deleteMany({
organization: organization._id
}, {
session
});
await IncidentContactOrg.deleteMany({
organization: organization._id
}, {
session
});
await GitRisks.deleteMany({
organization: organization._id
}, {
session
});
await GitAppInstallationSession.deleteMany({
organization: organization._id
}, {
session
});
await GitAppOrganizationInstallation.deleteMany({
organization: organization._id
}, {
session
});
const workspaceIds = await Workspace.distinct("_id", {
organization: organization._id
});
await Workspace.deleteMany({
organization: organization._id
}, {
session
});
await Membership.deleteMany({
workspace: {
$in: workspaceIds
}
}, {
session
});
await Key.deleteMany({
workspace: {
$in: workspaceIds
}
}, {
session
});
await Bot.deleteMany({
workspace: {
$in: workspaceIds
}
}, {
session
});
await BotKey.deleteMany({
workspace: {
$in: workspaceIds
}
}, {
session
});
await SecretBlindIndexData.deleteMany({
workspace: {
$in: workspaceIds
}
}, {
session
});
await Secret.deleteMany({
workspace: {
$in: workspaceIds
}
}, {
session
});
await SecretVersion.deleteMany({
workspace: {
$in: workspaceIds
}
}, {
session
});
await SecretSnapshot.deleteMany({
workspace: {
$in: workspaceIds
}
}, {
session
});
await SecretImport.deleteMany({
workspace: {
$in: workspaceIds
}
}, {
session
});
await Folder.deleteMany({
workspace: {
$in: workspaceIds
}
}, {
session
});
await FolderVersion.deleteMany({
workspace: {
$in: workspaceIds
}
}, {
session
});
await Webhook.deleteMany({
workspace: {
$in: workspaceIds
}
}, {
session
});
await TrustedIP.deleteMany({
workspace: {
$in: workspaceIds
}
}, {
session
});
await Tag.deleteMany({
workspace: {
$in: workspaceIds
}
}, {
session
});
await IntegrationAuth.deleteMany({
workspace: {
$in: workspaceIds
}
}, {
session
});
await Integration.deleteMany({
workspace: {
$in: workspaceIds
}
}, {
session
});
await ServiceToken.deleteMany({
workspace: {
$in: workspaceIds
}
}, {
session
});
await ServiceTokenData.deleteMany({
workspace: {
$in: workspaceIds
}
}, {
session
});
await ServiceTokenDataV3.deleteMany({
workspace: {
$in: workspaceIds
}
}, {
session
});
await ServiceTokenDataV3Key.deleteMany({
workspace: {
$in: workspaceIds
}
}, {
session
});
await AuditLog.deleteMany({
workspace: {
$in: workspaceIds
}
}, {
session
});
await Log.deleteMany({
workspace: {
$in: workspaceIds
}
}, {
session
});
await Action.deleteMany({
workspace: {
$in: workspaceIds
}
}, {
session
});
await SecretApprovalPolicy.deleteMany({
workspace: {
$in: workspaceIds
}
}, {
session
});
await SecretApprovalRequest.deleteMany({
workspace: {
$in: workspaceIds
}
}, {
session
});
if (organization.customerId) {
// delete from stripe here
await licenseServerKeyRequest.delete(
`${await getLicenseServerUrl()}/api/license-server/v1/customers/${organization.customerId}`
);
}
return organization;
} catch (err) {
if (!existingSession) {
await session.abortTransaction();
}
throw InternalServerError({
message: "Failed to delete organization"
});
} finally {
if (!existingSession) {
await session.commitTransaction();
session.endSession();
}
} }
return organization;
} }
/** /**
backend/src/helpers/user.ts
+76 -125
View File
@@ -1,4 +1,4 @@
import mongoose, { Types, mongo } from "mongoose"; import { Types } from "mongoose";
import { import {
APIKeyData, APIKeyData,
BackupPrivateKey, BackupPrivateKey,
@@ -222,141 +222,92 @@ const checkDeleteUserConditions = async ({
* @returns {User} user - deleted user * @returns {User} user - deleted user
*/ */
export const deleteUser = async ({ export const deleteUser = async ({
userId, userId
existingSession
}: { }: {
userId: Types.ObjectId; userId: Types.ObjectId;
existingSession?: mongo.ClientSession;
}) => { }) => {
const user = await User.findByIdAndDelete(userId);
let session; if (!user) throw ResourceNotFoundError();
await checkDeleteUserConditions({
userId: user._id
});
if (existingSession) { await UserAction.deleteMany({
session = existingSession; user: user._id
} else { });
session = await mongoose.startSession();
session.startTransaction(); await BackupPrivateKey.deleteMany({
user: user._id
});
await APIKeyData.deleteMany({
user: user._id
});
await Action.deleteMany({
user: user._id
});
await Log.deleteMany({
user: user._id
});
await TokenVersion.deleteMany({
user: user._id
});
await Key.deleteMany({
receiver: user._id
});
const membershipOrgs = await MembershipOrg.find({
user: userId
});
// delete organizations where user is only member
for await (const membershipOrg of membershipOrgs) {
const memberCount = await MembershipOrg.countDocuments({
organization: membershipOrg.organization
});
if (memberCount === 1) {
// organization only has 1 member (the current user)
await deleteOrganization({
organizationId: membershipOrg.organization
});
}
} }
try { const memberships = await Membership.find({
const user = await User.findByIdAndDelete(userId, { user: userId
session });
// delete workspaces where user is only member
for await (const membership of memberships) {
const memberCount = await Membership.countDocuments({
workspace: membership.workspace
}); });
if (!user) throw ResourceNotFoundError(); if (memberCount === 1) {
// workspace only has 1 member (the current user) -> delete workspace
await checkDeleteUserConditions({ await deleteWorkspace({
userId: user._id workspaceId: membership.workspace
});
await UserAction.deleteMany({
user: user._id
}, {
session
});
await BackupPrivateKey.deleteMany({
user: user._id
}, {
session
});
await APIKeyData.deleteMany({
user: user._id
}, {
session
});
await Action.deleteMany({
user: user._id
}, {
session
});
await Log.deleteMany({
user: user._id
}, {
session
});
await TokenVersion.deleteMany({
user: user._id
});
await Key.deleteMany({
receiver: user._id
}, {
session
});
const membershipOrgs = await MembershipOrg.find({
user: userId
}, null, {
session
});
// delete organizations where user is only member
for await (const membershipOrg of membershipOrgs) {
const memberCount = await MembershipOrg.countDocuments({
organization: membershipOrg.organization
}); });
if (memberCount === 1) {
// organization only has 1 member (the current user)
await deleteOrganization({
organizationId: membershipOrg.organization,
existingSession: session
});
}
}
const memberships = await Membership.find({
user: userId
}, null, {
session
});
// delete workspaces where user is only member
for await (const membership of memberships) {
const memberCount = await Membership.countDocuments({
workspace: membership.workspace
});
if (memberCount === 1) {
// workspace only has 1 member (the current user) -> delete workspace
await deleteWorkspace({
workspaceId: membership.workspace,
existingSession: session
});
}
}
await MembershipOrg.deleteMany({
user: userId
}, {
session
});
await Membership.deleteMany({
user: userId
}, {
session
});
return user;
} catch (err) {
if (!existingSession) {
await session.abortTransaction();
}
throw InternalServerError({
message: "Failed to delete account"
})
} finally {
if (!existingSession) {
await session.commitTransaction();
session.endSession();
} }
} }
await MembershipOrg.deleteMany({
user: userId
});
await Membership.deleteMany({
user: userId
});
return user;
} }
backend/src/helpers/workspace.ts
+86 -163
View File
@@ -1,4 +1,4 @@
import mongoose, { Types, mongo } from "mongoose"; import { Types } from "mongoose";
import { import {
Bot, Bot,
BotKey, BotKey,
@@ -33,8 +33,7 @@ import {
import { createBot } from "../helpers/bot"; import { createBot } from "../helpers/bot";
import { EELicenseService } from "../ee/services"; import { EELicenseService } from "../ee/services";
import { SecretService } from "../services"; import { SecretService } from "../services";
import { import {
InternalServerError,
ResourceNotFoundError ResourceNotFoundError
} from "../utils/errors"; } from "../utils/errors";
@@ -102,189 +101,113 @@ export const createWorkspace = async ({
* @param {String} obj.id - id of workspace to delete * @param {String} obj.id - id of workspace to delete
*/ */
export const deleteWorkspace = async ({ export const deleteWorkspace = async ({
workspaceId, workspaceId
existingSession
}: { }: {
workspaceId: Types.ObjectId; workspaceId: Types.ObjectId;
existingSession?: mongo.ClientSession;
}) => { }) => {
const workspace = await Workspace.findByIdAndDelete(workspaceId);
let session;
if (existingSession) {
session = existingSession;
} else {
session = await mongoose.startSession();
session.startTransaction();
}
try { if (!workspace) throw ResourceNotFoundError();
const workspace = await Workspace.findByIdAndDelete(workspaceId, { session });
await Membership.deleteMany({
if (!workspace) throw ResourceNotFoundError(); workspace: workspace._id
});
await Membership.deleteMany({
workspace: workspace._id await Key.deleteMany({
}, { workspace: workspace._id
session });
});
await Bot.deleteMany({
await Key.deleteMany({ workspace: workspace._id
workspace: workspace._id });
}, {
session
});
await Bot.deleteMany({
workspace: workspace._id
}, {
session
});
await BotKey.deleteMany({ await BotKey.deleteMany({
workspace: workspace._id workspace: workspace._id
}, { });
session
});
await SecretBlindIndexData.deleteMany({ await SecretBlindIndexData.deleteMany({
workspace: workspace._id workspace: workspace._id
}, { });
session
});
await Secret.deleteMany({ await Secret.deleteMany({
workspace: workspace._id workspace: workspace._id
}, { });
session
}); await SecretVersion.deleteMany({
workspace: workspace._id
await SecretVersion.deleteMany({ });
workspace: workspace._id
}, {
session
});
await SecretSnapshot.deleteMany({ await SecretSnapshot.deleteMany({
workspace: workspace._id workspace: workspace._id
}, { });
session
});
await SecretImport.deleteMany({ await SecretImport.deleteMany({
workspace: workspace._id workspace: workspace._id
}, { });
session
});
await Folder.deleteMany({ await Folder.deleteMany({
workspace: workspace._id workspace: workspace._id
}, { });
session
});
await FolderVersion.deleteMany({ await FolderVersion.deleteMany({
workspace: workspace._id workspace: workspace._id
}, { });
session
});
await Webhook.deleteMany({ await Webhook.deleteMany({
workspace: workspace._id workspace: workspace._id
}, { });
session
});
await TrustedIP.deleteMany({ await TrustedIP.deleteMany({
workspace: workspace._id workspace: workspace._id
}, { });
session
});
await Tag.deleteMany({ await Tag.deleteMany({
workspace: workspace._id workspace: workspace._id
}, { });
session
});
await IntegrationAuth.deleteMany({ await IntegrationAuth.deleteMany({
workspace: workspace._id workspace: workspace._id
}, { });
session
});
await Integration.deleteMany({ await Integration.deleteMany({
workspace: workspace._id workspace: workspace._id
}, { });
session
});
await ServiceToken.deleteMany({ await ServiceToken.deleteMany({
workspace: workspace._id workspace: workspace._id
}, { });
session
});
await ServiceTokenData.deleteMany({ await ServiceTokenData.deleteMany({
workspace: workspace._id workspace: workspace._id
}, { });
session
});
await ServiceTokenDataV3.deleteMany({ await ServiceTokenDataV3.deleteMany({
workspace: workspace._id workspace: workspace._id
}, { });
session
});
await ServiceTokenDataV3Key.deleteMany({ await ServiceTokenDataV3Key.deleteMany({
workspace: workspace._id workspace: workspace._id
}, { });
session
});
await AuditLog.deleteMany({ await AuditLog.deleteMany({
workspace: workspace._id workspace: workspace._id
}, { });
session
});
await Log.deleteMany({ await Log.deleteMany({
workspace: workspace._id workspace: workspace._id
}, { });
session
});
await Action.deleteMany({ await Action.deleteMany({
workspace: workspace._id workspace: workspace._id
}, { });
session
});
await SecretApprovalPolicy.deleteMany({ await SecretApprovalPolicy.deleteMany({
workspace: workspace._id workspace: workspace._id
}, { });
session
});
await SecretApprovalRequest.deleteMany({ await SecretApprovalRequest.deleteMany({
workspace: workspace._id workspace: workspace._id
}, { });
session
}); return workspace;
return workspace;
} catch (err) {
if (!existingSession) {
await session.abortTransaction();
}
throw InternalServerError({
message: "Failed to delete organization"
});
} finally {
if (!existingSession) {
await session.commitTransaction();
session.endSession();
}
}
}; };
backend/src/index.ts
+14 -4
View File
@@ -5,6 +5,8 @@ import express from "express";
require("express-async-errors"); require("express-async-errors");
import helmet from "helmet"; import helmet from "helmet";
import cors from "cors"; import cors from "cors";
import { logger } from "./utils/logging";
import httpLogger from "pino-http";
import { DatabaseService } from "./services"; import { DatabaseService } from "./services";
import { EELicenseService, GithubSecretScanningService } from "./ee/services"; import { EELicenseService, GithubSecretScanningService } from "./ee/services";
import { setUpHealthEndpoint } from "./services/health"; import { setUpHealthEndpoint } from "./services/health";
@@ -73,7 +75,7 @@ import {
workspaces as v3WorkspacesRouter workspaces as v3WorkspacesRouter
} from "./routes/v3"; } from "./routes/v3";
import { healthCheck } from "./routes/status"; import { healthCheck } from "./routes/status";
import { getLogger } from "./utils/logger"; // import { getLogger } from "./utils/logger";
import { RouteNotFoundError } from "./utils/errors"; import { RouteNotFoundError } from "./utils/errors";
import { requestErrorHandler } from "./middleware/requestErrorHandler"; import { requestErrorHandler } from "./middleware/requestErrorHandler";
import { import {
@@ -94,12 +96,20 @@ import path from "path";
let handler: null | any = null; let handler: null | any = null;
const main = async () => { const main = async () => {
const port = await getPort();
await setup(); await setup();
await EELicenseService.initGlobalFeatureSet(); await EELicenseService.initGlobalFeatureSet();
const app = express(); const app = express();
app.enable("trust proxy"); app.enable("trust proxy");
app.use(httpLogger({
logger,
autoLogging: false
}));
app.use(express.json()); app.use(express.json());
app.use(express.urlencoded({ extended: false })); app.use(express.urlencoded({ extended: false }));
app.use(cookieParser()); app.use(cookieParser());
@@ -164,7 +174,7 @@ const main = async () => {
const nextApp = new NextServer({ const nextApp = new NextServer({
dev: false, dev: false,
dir: nextJsBuildPath, dir: nextJsBuildPath,
port: await getPort(), port,
conf, conf,
hostname: "local", hostname: "local",
customServer: false customServer: false
@@ -255,8 +265,8 @@ const main = async () => {
app.use(requestErrorHandler); app.use(requestErrorHandler);
const server = app.listen(await getPort(), async () => { const server = app.listen(port, async () => {
(await getLogger("backend-main")).info(`Server started listening at port ${await getPort()}`); logger.info(`Server started listening at port ${port}`);
}); });
// await createTestUserForDevelopment(); // await createTestUserForDevelopment();
backend/src/middleware/requestErrorHandler.ts
+27 -31
View File
@@ -2,49 +2,45 @@ import * as Sentry from "@sentry/node";
import { ErrorRequestHandler } from "express"; import { ErrorRequestHandler } from "express";
import { TokenExpiredError } from "jsonwebtoken"; import { TokenExpiredError } from "jsonwebtoken";
import { InternalServerError, UnauthorizedRequestError } from "../utils/errors"; import { InternalServerError, UnauthorizedRequestError } from "../utils/errors";
import { getLogger } from "../utils/logger"; import { logger } from "../utils/logging";
import RequestError from "../utils/requestError"; import RequestError, { mapToPinoLogLevel } from "../utils/requestError";
import { ForbiddenError } from "@casl/ability"; import { ForbiddenError } from "@casl/ability";
export const requestErrorHandler: ErrorRequestHandler = async ( export const requestErrorHandler: ErrorRequestHandler = async (
error: RequestError | Error, err: RequestError | Error,
req, req,
res, res,
next next
) => { ) => {
if (res.headersSent) return next(); if (res.headersSent) return next();
const logAndCaptureException = async (error: RequestError) => { let error: RequestError;
(await getLogger("backend-main")).log(
(<RequestError>error).levelName.toLowerCase(), switch (true) {
`${error.stack}\n${error.message}` case err instanceof TokenExpiredError:
); error = UnauthorizedRequestError({ stack: err.stack, message: "Token expired" });
break;
//* Set Sentry user identification if req.user is populated case err instanceof ForbiddenError:
if (req.user !== undefined && req.user !== null) { error = UnauthorizedRequestError({ context: { exception: err.message }, stack: err.stack })
Sentry.setUser({ email: (req.user as any).email }); break;
} case err instanceof RequestError:
error = err as RequestError;
Sentry.captureException(error); break;
}; default:
error = InternalServerError({ context: { exception: err.message }, stack: err.stack });
if (error instanceof RequestError) { break;
if (error instanceof TokenExpiredError) {
error = UnauthorizedRequestError({ stack: error.stack, message: "Token expired" });
}
await logAndCaptureException((<RequestError>error));
} else {
if (error instanceof ForbiddenError) {
error = UnauthorizedRequestError({ context: { exception: error.message }, stack: error.stack })
} else {
error = InternalServerError({ context: { exception: error.message }, stack: error.stack });
}
await logAndCaptureException((<RequestError>error));
} }
logger[mapToPinoLogLevel(error.level)](error);
if (req.user) {
Sentry.setUser({ email: (req.user as any).email });
}
Sentry.captureException(error);
delete (<any>error).stacktrace // remove stack trace from being sent to client delete (<any>error).stacktrace // remove stack trace from being sent to client
res.status((<RequestError>error).statusCode).json(error); res.status((<RequestError>error).statusCode).json(error); // revise json part here
next(); next();
}; };
backend/src/services/TelemetryService.ts
+4 -4
View File
@@ -1,5 +1,5 @@
import { PostHog } from "posthog-node"; import { PostHog } from "posthog-node";
import { getLogger } from "../utils/logger"; import { logger } from "../utils/logging";
import { AuthData } from "../interfaces/middleware"; import { AuthData } from "../interfaces/middleware";
import { import {
getNodeEnv, getNodeEnv,
@@ -22,13 +22,13 @@ class Telemetry {
* Logs telemetry enable/disable notice. * Logs telemetry enable/disable notice.
*/ */
static logTelemetryMessage = async () => { static logTelemetryMessage = async () => {
if(!(await getTelemetryEnabled())){ if(!(await getTelemetryEnabled())){
(await getLogger("backend-main")).info([ [
"",
"To improve, Infisical collects telemetry data about general usage.", "To improve, Infisical collects telemetry data about general usage.",
"This helps us understand how the product is doing and guide our product development to create the best possible platform; it also helps us demonstrate growth as we support Infisical as open-source software.", "This helps us understand how the product is doing and guide our product development to create the best possible platform; it also helps us demonstrate growth as we support Infisical as open-source software.",
"To opt into telemetry, you can set `TELEMETRY_ENABLED=true` within the environment variables.", "To opt into telemetry, you can set `TELEMETRY_ENABLED=true` within the environment variables.",
].join("\n")) ].forEach(line => logger.info(line));
} }
} }
backend/src/services/health.ts
+2 -2
View File
@@ -1,10 +1,10 @@
import mongoose from "mongoose"; import mongoose from "mongoose";
import { createTerminus } from "@godaddy/terminus"; import { createTerminus } from "@godaddy/terminus";
import { getLogger } from "../utils/logger"; import { logger } from "../utils/logging";
export const setUpHealthEndpoint = <T>(server: T) => { export const setUpHealthEndpoint = <T>(server: T) => {
const onSignal = async () => { const onSignal = async () => {
(await getLogger("backend-main")).info("Server is starting clean-up"); logger.info("Server is starting clean-up");
return Promise.all([ return Promise.all([
new Promise((resolve) => { new Promise((resolve) => {
if (mongoose.connection && mongoose.connection.readyState == 1) { if (mongoose.connection && mongoose.connection.readyState == 1) {
backend/src/services/smtp.ts
+3 -4
View File
@@ -16,7 +16,7 @@ import {
getSmtpSecure, getSmtpSecure,
getSmtpUsername, getSmtpUsername,
} from "../config"; } from "../config";
import { getLogger } from "../utils/logger"; import { logger } from "../utils/logging";
export const initSmtp = async () => { export const initSmtp = async () => {
const mailOpts: SMTPConnection.Options = { const mailOpts: SMTPConnection.Options = {
@@ -84,15 +84,14 @@ export const initSmtp = async () => {
.then(async () => { .then(async () => {
Sentry.setUser(null); Sentry.setUser(null);
Sentry.captureMessage("SMTP - Successfully connected"); Sentry.captureMessage("SMTP - Successfully connected");
(await getLogger("backend-main")).info( logger.info("SMTP - Successfully connected");
"SMTP - Successfully connected"
);
}) })
.catch(async (err) => { .catch(async (err) => {
Sentry.setUser(null); Sentry.setUser(null);
Sentry.captureException( Sentry.captureException(
`SMTP - Failed to connect to ${await getSmtpHost()}:${await getSmtpPort()} \n\t${err}` `SMTP - Failed to connect to ${await getSmtpHost()}:${await getSmtpPort()} \n\t${err}`
); );
logger.error(err, `SMTP - Failed to connect to ${await getSmtpHost()}:${await getSmtpPort()}`);
}); });
return transporter; return transporter;
backend/src/utils/errors.ts
+1 -1
View File
@@ -29,7 +29,7 @@ export const UnauthorizedRequestError = (error?: Partial<RequestErrorContext>) =
}); });
export const ForbiddenRequestError = (error?: Partial<RequestErrorContext>) => new RequestError({ export const ForbiddenRequestError = (error?: Partial<RequestErrorContext>) => new RequestError({
logLevel: error?.logLevel ?? LogLevel.INFO, logLevel: error?.logLevel ?? LogLevel.WARN,
statusCode: error?.statusCode ?? 403, statusCode: error?.statusCode ?? 403,
type: error?.type ?? "forbidden", type: error?.type ?? "forbidden",
message: error?.message ?? "You are not allowed to access this resource", message: error?.message ?? "You are not allowed to access this resource",
backend/src/utils/logger.ts
-67
View File
@@ -1,67 +0,0 @@
/* eslint-disable no-console */
import { createLogger, format, transports } from "winston";
import LokiTransport from "winston-loki";
import { getLokiHost, getNodeEnv } from "../config";
const { combine, colorize, label, printf, splat, timestamp } = format;
const logFormat = (prefix: string) => combine(
timestamp(),
splat(),
label({ label: prefix }),
printf((info) => `${info.timestamp} ${info.label} ${info.level}: ${info.message}`)
);
const createLoggerWithLabel = async (level: string, label: string) => {
const _level = level.toLowerCase() || "info"
//* Always add Console output to transports
const _transports: any[] = [
new transports.Console({
format: combine(
colorize(),
logFormat(label),
// format.json()
),
}),
]
//* Add LokiTransport if it's enabled
if((await getLokiHost()) !== undefined){
_transports.push(
new LokiTransport({
host: await getLokiHost(),
handleExceptions: true,
handleRejections: true,
batching: true,
level: _level,
timeout: 30000,
format: format.combine(
format.json()
),
labels: {
app: process.env.npm_package_name,
version: process.env.npm_package_version,
environment: await getNodeEnv(),
},
onConnectionError: (err: Error)=> console.error("Connection error while connecting to Loki Server.\n", err),
})
)
}
return createLogger({
level: _level,
transports: _transports,
format: format.combine(
logFormat(label),
format.metadata({ fillExcept: ["message", "level", "timestamp", "label"] })
),
});
}
export const getLogger = async (loggerName: "backend-main" | "database") => {
const logger = {
"backend-main": await createLoggerWithLabel("info", "[IFSC:backend-main]"),
"database": await createLoggerWithLabel("info", "[IFSC:database]"),
}
return logger[loggerName]
}
backend/src/utils/logging/index.ts
+1
View File
@@ -0,0 +1 @@
export { logger } from "./logger";
backend/src/utils/logging/logger.ts
+15
View File
@@ -0,0 +1,15 @@
import pino from "pino";
export const logger = pino({
level: process.env.PINO_LOG_LEVEL || "trace",
timestamp: pino.stdTimeFunctions.isoTime,
formatters: {
bindings: (bindings) => {
return {
pid: bindings.pid,
hostname: bindings.hostname
// node_version: process.version
};
},
}
});
backend/src/utils/requestError.ts
+24 -23
View File
@@ -2,34 +2,30 @@ import { Request } from "express"
import { getVerboseErrorOutput } from "../config"; import { getVerboseErrorOutput } from "../config";
export enum LogLevel { export enum LogLevel {
DEBUG = 100, TRACE = 10,
INFO = 200, DEBUG = 20,
NOTICE = 250, INFO = 30,
WARNING = 300, WARN = 40,
ERROR = 400, ERROR = 50,
CRITICAL = 500, FATAL = 60
ALERT = 550,
EMERGENCY = 600,
} }
export const mapToWinstonLogLevel = (customLogLevel: LogLevel): string => { type PinoLogLevel = "trace" | "debug" | "info" | "warn" | "error" | "fatal";
export const mapToPinoLogLevel = (customLogLevel: LogLevel): PinoLogLevel => {
switch (customLogLevel) { switch (customLogLevel) {
case LogLevel.TRACE:
return "trace";
case LogLevel.DEBUG: case LogLevel.DEBUG:
return "debug"; return "debug";
case LogLevel.INFO: case LogLevel.INFO:
return "info"; return "info";
case LogLevel.NOTICE: case LogLevel.WARN:
return "notice";
case LogLevel.WARNING:
return "warn"; return "warn";
case LogLevel.ERROR: case LogLevel.ERROR:
return "error"; return "error";
case LogLevel.CRITICAL: case LogLevel.FATAL:
return "crit"; return "fatal";
case LogLevel.ALERT:
return "alert";
case LogLevel.EMERGENCY:
return "emerg";
} }
} }
@@ -42,10 +38,10 @@ export type RequestErrorContext = {
stack?: string|undefined stack?: string|undefined
} }
export default class RequestError extends Error{ export default class RequestError extends Error {
private _logLevel: LogLevel private _logLevel: LogLevel
private _logName: string private _logName: string;
statusCode: number statusCode: number
type: string type: string
context: Record<string, unknown> context: Record<string, unknown>
@@ -55,9 +51,10 @@ export default class RequestError extends Error{
constructor( constructor(
{logLevel, statusCode, type, message, context, stack} : RequestErrorContext {logLevel, statusCode, type, message, context, stack} : RequestErrorContext
){ ){
super(message) super(message)
this._logLevel = logLevel || LogLevel.INFO this._logLevel = logLevel || LogLevel.INFO
this._logName = LogLevel[this._logLevel] this._logName = LogLevel[this._logLevel];
this.statusCode = statusCode this.statusCode = statusCode
this.type = type this.type = type
this.context = context || {} this.context = context || {}
@@ -83,8 +80,12 @@ export default class RequestError extends Error{
}) })
} }
get level(){ return this._logLevel } get level(){
get levelName(){ return this._logName } return this._logLevel
}
get levelName(){
return this._logName
}
withTags(...tags: string[]|number[]){ withTags(...tags: string[]|number[]){
this.context["tags"] = Object.assign(tags, this.context["tags"]) this.context["tags"] = Object.assign(tags, this.context["tags"])
backend/src/utils/setup/backfillData.ts
+13 -13
View File
@@ -1,4 +1,3 @@
/* eslint-disable no-console */
import crypto from "crypto"; import crypto from "crypto";
import { Types } from "mongoose"; import { Types } from "mongoose";
import { encryptSymmetric128BitHexKeyUTF8 } from "../crypto"; import { encryptSymmetric128BitHexKeyUTF8 } from "../crypto";
@@ -47,6 +46,7 @@ import {
ProjectPermissionSub, ProjectPermissionSub,
memberProjectPermissions memberProjectPermissions
} from "../../ee/services/ProjectRoleService"; } from "../../ee/services/ProjectRoleService";
import { logger } from "../logging";
/** /**
* Backfill secrets to ensure that they're all versioned and have * Backfill secrets to ensure that they're all versioned and have
@@ -88,7 +88,7 @@ export const backfillSecretVersions = async () => {
) )
}); });
} }
console.log("Migration: Secret version migration v1 complete"); logger.info("Migration: Secret version migration v1 complete");
}; };
/** /**
@@ -518,7 +518,7 @@ export const backfillSecretFolders = async () => {
.limit(50); .limit(50);
} }
console.log("Migration: Folder migration v1 complete"); logger.info("Migration: Folder migration v1 complete");
}; };
export const backfillServiceToken = async () => { export const backfillServiceToken = async () => {
@@ -534,7 +534,7 @@ export const backfillServiceToken = async () => {
} }
} }
); );
console.log("Migration: Service token migration v1 complete"); logger.info("Migration: Service token migration v1 complete");
}; };
export const backfillIntegration = async () => { export const backfillIntegration = async () => {
@@ -550,7 +550,7 @@ export const backfillIntegration = async () => {
} }
} }
); );
console.log("Migration: Integration migration v1 complete"); logger.info("Migration: Integration migration v1 complete");
}; };
export const backfillServiceTokenMultiScope = async () => { export const backfillServiceTokenMultiScope = async () => {
@@ -575,7 +575,7 @@ export const backfillServiceTokenMultiScope = async () => {
} }
} }
console.log("Migration: Service token migration v2 complete"); logger.info("Migration: Service token migration v2 complete");
}; };
/** /**
@@ -650,7 +650,7 @@ export const backfillTrustedIps = async () => {
}); });
await TrustedIP.bulkWrite(operations); await TrustedIP.bulkWrite(operations);
console.log("Backfill: Trusted IPs complete"); logger.info("Backfill: Trusted IPs complete");
} }
}; };
@@ -698,7 +698,7 @@ export const backfillPermission = async () => {
if (lock) { if (lock) {
try { try {
console.info("Lock acquired for script [backfillPermission]"); logger.info("Lock acquired for script [backfillPermission]");
const memberships = await Membership.find({ const memberships = await Membership.find({
deniedPermissions: { deniedPermissions: {
@@ -801,7 +801,7 @@ export const backfillPermission = async () => {
} }
} }
console.info("Backfill: Finished converting old denied permission in workspace to viewers"); logger.info("Backfill: Finished converting old denied permission in workspace to viewers");
await MembershipOrg.updateMany( await MembershipOrg.updateMany(
{ {
@@ -814,14 +814,14 @@ export const backfillPermission = async () => {
} }
); );
console.info("Backfill: Finished converting owner role to member"); logger.info("Backfill: Finished converting owner role to member");
} catch (error) { } catch (error) {
console.error("An error occurred when running script [backfillPermission]:", error); logger.error(error, "An error occurred when running script [backfillPermission]");
} }
} else { } else {
console.info("Could not acquire lock for script [backfillPermission], skipping"); logger.info("Could not acquire lock for script [backfillPermission], skipping");
} }
}; };
@@ -837,5 +837,5 @@ export const migrateRoleFromOwnerToAdmin = async () => {
} }
); );
console.info("Backfill: Finished converting owner role to member"); logger.info("Backfill: Finished converting owner role to member");
} }
docs/self-hosting/configuration/envars.mdx
+9 -20
View File
@@ -153,29 +153,18 @@ Other environment variables are listed below to increase the functionality of yo
JWT token lifetime expressed in seconds or a string describing a time span JWT token lifetime expressed in seconds or a string describing a time span
</ParamField> </ParamField>
{" "} #### Logging
<ParamField
query="MONGO_USERNAME"
type="string"
default="none"
optional
></ParamField>
{" "}
<ParamField
query="MONGO_PASSWORD"
type="string"
default="none"
optional
></ParamField>
#### Error logging
Infisical uses Sentry to report error logs Infisical uses Sentry to report error logs
{" "} <ParamField
query="PINO_LOG_LEVEL"
type="string"
default="info"
optional
>
The minimum log level for application logging; can be one of `trace`, `debug`, `info`, `warn`, `error`, or `fatal`.
</ParamField>
<ParamField <ParamField
query="SENTRY_DSN" query="SENTRY_DSN"