diff --git a/backend/src/db/knexfile.ts b/backend/src/db/knexfile.ts index 0e51e94e2..82a8adea0 100644 --- a/backend/src/db/knexfile.ts +++ b/backend/src/db/knexfile.ts @@ -4,6 +4,7 @@ import "ts-node/register"; import dotenv from "dotenv"; import type { Knex } from "knex"; import path from "path"; +import { initLogger } from "@app/lib/logger"; // Update with your config settings. . dotenv.config({ @@ -13,6 +14,8 @@ dotenv.config({ path: path.join(__dirname, "../../../.env") }); +initLogger(); + export default { development: { client: "postgres", diff --git a/backend/src/db/migrations/20250710022434_add-index-for-access-token.ts b/backend/src/db/migrations/20250710022434_add-index-for-access-token.ts new file mode 100644 index 000000000..a0215891d --- /dev/null +++ b/backend/src/db/migrations/20250710022434_add-index-for-access-token.ts @@ -0,0 +1,48 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; + +const MIGRATION_TIMEOUT = 30 * 60 * 1000; // 30 minutes + +export async function up(knex: Knex): Promise { + const result = await knex.raw("SHOW statement_timeout"); + const originalTimeout = result.rows[0].statement_timeout; + + try { + await knex.raw(`SET statement_timeout = ${MIGRATION_TIMEOUT}`); + + // iat means IdentityAccessToken + await knex.raw(` + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_iat_identity_id + ON ${TableName.IdentityAccessToken} ("identityId") + `); + + await knex.raw(` + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_iat_ua_client_secret_id + ON ${TableName.IdentityAccessToken} ("identityUAClientSecretId") + `); + } finally { + await knex.raw(`SET statement_timeout = '${originalTimeout}'`); + } +} + +export async function down(knex: Knex): Promise { + const result = await knex.raw("SHOW statement_timeout"); + const originalTimeout = result.rows[0].statement_timeout; + + try { + await knex.raw(`SET statement_timeout = ${MIGRATION_TIMEOUT}`); + + await knex.raw(` + DROP INDEX IF EXISTS idx_iat_identity_id + `); + + await knex.raw(` + DROP INDEX IF EXISTS idx_iat_ua_client_secret_id + `); + } finally { + await knex.raw(`SET statement_timeout = '${originalTimeout}'`); + } +} + +export const config = { transaction: false }; diff --git a/backend/src/services/org-membership/org-membership-dal.ts b/backend/src/services/org-membership/org-membership-dal.ts index ebf1700d0..ed4867025 100644 --- a/backend/src/services/org-membership/org-membership-dal.ts +++ b/backend/src/services/org-membership/org-membership-dal.ts @@ -108,16 +108,16 @@ export const orgMembershipDALFactory = (db: TDbClient) => { const now = new Date(); const oneWeekAgo = new Date(now.getTime() - 7 * 24 * 60 * 60 * 1000); const oneMonthAgo = new Date(now.getTime() - 30 * 24 * 60 * 60 * 1000); - const threeMonthsAgo = new Date(now.getTime() - 90 * 24 * 60 * 60 * 1000); + const twelveMonthsAgo = new Date(now.getTime() - 360 * 24 * 60 * 60 * 1000); const memberships = await db .replicaNode()(TableName.OrgMembership) .where("status", "invited") .where((qb) => { - // lastInvitedAt is null AND createdAt is between 1 week and 3 months ago + // lastInvitedAt is null AND createdAt is between 1 week and 12 months ago void qb .whereNull(`${TableName.OrgMembership}.lastInvitedAt`) - .whereBetween(`${TableName.OrgMembership}.createdAt`, [threeMonthsAgo, oneWeekAgo]); + .whereBetween(`${TableName.OrgMembership}.createdAt`, [twelveMonthsAgo, oneWeekAgo]); }) .orWhere((qb) => { // lastInvitedAt is older than 1 week ago AND createdAt is younger than 1 month ago diff --git a/backend/src/services/org/org-service.ts b/backend/src/services/org/org-service.ts index 5941eb013..8917a4fc1 100644 --- a/backend/src/services/org/org-service.ts +++ b/backend/src/services/org/org-service.ts @@ -36,6 +36,8 @@ import { getConfig } from "@app/lib/config/env"; import { generateAsymmetricKeyPair } from "@app/lib/crypto"; import { generateSymmetricKey, infisicalSymmetricDecrypt, infisicalSymmetricEncypt } from "@app/lib/crypto/encryption"; import { generateUserSrpKeys } from "@app/lib/crypto/srp"; +import { applyJitter } from "@app/lib/dates"; +import { delay as delayMs } from "@app/lib/delay"; import { BadRequestError, ForbiddenRequestError, @@ -44,9 +46,10 @@ import { UnauthorizedError } from "@app/lib/errors"; import { groupBy } from "@app/lib/fn"; +import { logger } from "@app/lib/logger"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { isDisposableEmail } from "@app/lib/validator"; -import { TQueueServiceFactory } from "@app/queue"; +import { QueueName, TQueueServiceFactory } from "@app/queue"; import { getDefaultOrgMembershipRoleForUpdateOrg } from "@app/services/org/org-role-fns"; import { TOrgMembershipDALFactory } from "@app/services/org-membership/org-membership-dal"; import { TUserAliasDALFactory } from "@app/services/user-alias/user-alias-dal"; @@ -1438,6 +1441,8 @@ export const orgServiceFactory = ({ * Re-send emails to users who haven't accepted an invite yet */ const notifyInvitedUsers = async () => { + logger.info(`${QueueName.DailyResourceCleanUp}: notify invited users started`); + const invitedUsers = await orgMembershipDAL.findRecentInvitedMemberships(); const appCfg = getConfig(); @@ -1461,24 +1466,32 @@ export const orgServiceFactory = ({ }); if (invitedUser.inviteEmail) { - await smtpService.sendMail({ - template: SmtpTemplates.OrgInvite, - subjectLine: `Reminder: You have been invited to ${org.name} on Infisical`, - recipients: [invitedUser.inviteEmail], - substitutions: { - organizationName: org.name, - email: invitedUser.inviteEmail, - organizationId: org.id.toString(), - token, - callback_url: `${appCfg.SITE_URL}/signupinvite` - } - }); - notifiedUsers.push(invitedUser.id); + await delayMs(Math.max(0, applyJitter(0, 2000))); + + try { + await smtpService.sendMail({ + template: SmtpTemplates.OrgInvite, + subjectLine: `Reminder: You have been invited to ${org.name} on Infisical`, + recipients: [invitedUser.inviteEmail], + substitutions: { + organizationName: org.name, + email: invitedUser.inviteEmail, + organizationId: org.id.toString(), + token, + callback_url: `${appCfg.SITE_URL}/signupinvite` + } + }); + notifiedUsers.push(invitedUser.id); + } catch (err) { + logger.error(err, `${QueueName.DailyResourceCleanUp}: notify invited users failed to send email`); + } } }) ); await orgMembershipDAL.updateLastInvitedAtByIds(notifiedUsers); + + logger.info(`${QueueName.DailyResourceCleanUp}: notify invited users completed`); }; return { diff --git a/docs/Dockerfile b/docs/Dockerfile new file mode 100644 index 000000000..34730d01e --- /dev/null +++ b/docs/Dockerfile @@ -0,0 +1,6 @@ +FROM node:20-alpine +WORKDIR /app +RUN npm install -g mint +COPY . . +EXPOSE 3000 +CMD ["mint", "dev"] diff --git a/frontend/src/components/utilities/parseSecrets.ts b/frontend/src/components/utilities/parseSecrets.ts index d1df7cd68..3e5a57edf 100644 --- a/frontend/src/components/utilities/parseSecrets.ts +++ b/frontend/src/components/utilities/parseSecrets.ts @@ -77,3 +77,91 @@ export const parseJson = (src: ArrayBuffer | string) => { }); return env; }; + +/** + * Parses simple flat YAML with support for multiline strings using |, |-, and >. + * @param {ArrayBuffer | string} src + * @returns {Record} + */ +export function parseYaml(src: ArrayBuffer | string) { + const result: Record = {}; + + const content = src.toString().replace(/\r\n?/g, "\n"); + const lines = content.split("\n"); + + let i = 0; + let comments: string[] = []; + + while (i < lines.length) { + const line = lines[i].trim(); + + // Collect comment + if (line.startsWith("#")) { + comments.push(line.slice(1).trim()); + i += 1; // move to next line + } else { + // Match key: value or key: |, key: >, etc. + const keyMatch = lines[i].match(/^([\w.-]+)\s*:\s*(.*)$/); + if (keyMatch) { + const [, key, rawValue] = keyMatch; + let value = rawValue.trim(); + + // Multiline string handling + if (value === "|-" || value === "|" || value === ">") { + const isFolded = value === ">"; + + const baseIndent = lines[i + 1]?.match(/^(\s*)/)?.[1]?.length ?? 0; + const collectedLines: string[] = []; + + i += 1; // move to first content line + + while (i < lines.length) { + const current = lines[i]; + const currentIndent = current.match(/^(\s*)/)?.[1]?.length ?? 0; + + if (current.trim() === "" || currentIndent >= baseIndent) { + collectedLines.push(current.slice(baseIndent)); + i += 1; // move to next line + } else { + break; + } + } + + if (isFolded) { + // Join lines with space for `>` folded style + value = collectedLines.map((l) => l.trim()).join(" "); + } else { + // Keep lines with newlines for `|` and `|-` + value = collectedLines.join("\n"); + } + } else { + // Inline value — strip quotes and inline comment + const commentIndex = value.indexOf(" #"); + if (commentIndex !== -1) { + value = value.slice(0, commentIndex).trim(); + } + + // Remove surrounding quotes + if ( + (value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'")) + ) { + value = value.slice(1, -1); + } + + i += 1; // advance to next line + } + + result[key] = { + value, + comments: [...comments] + }; + comments = []; // reset + } else { + i += 1; // skip unknown line + } + } + } + + return result; +} diff --git a/frontend/src/pages/secret-manager/SecretDashboardPage/components/SecretDropzone/SecretDropzone.tsx b/frontend/src/pages/secret-manager/SecretDashboardPage/components/SecretDropzone/SecretDropzone.tsx index 017604fc4..b0f61dc89 100644 --- a/frontend/src/pages/secret-manager/SecretDashboardPage/components/SecretDropzone/SecretDropzone.tsx +++ b/frontend/src/pages/secret-manager/SecretDashboardPage/components/SecretDropzone/SecretDropzone.tsx @@ -9,7 +9,7 @@ import { twMerge } from "tailwind-merge"; import { createNotification } from "@app/components/notifications"; import { ProjectPermissionCan } from "@app/components/permissions"; // TODO:(akhilmhdh) convert all the util functions like this into a lib folder grouped by functionality -import { parseDotEnv, parseJson } from "@app/components/utilities/parseSecrets"; +import { parseDotEnv, parseJson, parseYaml } from "@app/components/utilities/parseSecrets"; import { Button, Lottie, Modal, ModalContent } from "@app/components/v2"; import { ProjectPermissionActions, ProjectPermissionSub } from "@app/context"; import { usePopUp, useToggle } from "@app/hooks"; @@ -127,7 +127,7 @@ export const SecretDropzone = ({ } }; - const parseFile = (file?: File, isJson?: boolean) => { + const parseFile = (file?: File) => { const reader = new FileReader(); if (!file) { createNotification({ @@ -140,10 +140,25 @@ export const SecretDropzone = ({ setIsLoading.on(); reader.onload = (event) => { if (!event?.target?.result) return; - // parse function's argument looks like to be ArrayBuffer - const env = isJson - ? parseJson(event.target.result as ArrayBuffer) - : parseDotEnv(event.target.result as ArrayBuffer); + + let env: TParsedEnv; + + const src = event.target.result as ArrayBuffer; + + switch (file.type) { + case "application/json": + env = parseJson(src); + break; + case "text/yaml": + case "application/x-yaml": + case "application/yaml": + env = parseYaml(src); + break; + + default: + env = parseDotEnv(src); + break; + } setIsLoading.off(); handleParsedEnv(env); }; @@ -165,12 +180,12 @@ export const SecretDropzone = ({ e.dataTransfer.dropEffect = "copy"; setDragActive.off(); - parseFile(e.dataTransfer.files[0], e.dataTransfer.files[0].type === "application/json"); + parseFile(e.dataTransfer.files[0]); }; const handleFileUpload = (e: ChangeEvent) => { e.preventDefault(); - parseFile(e.target?.files?.[0], e.target?.files?.[0]?.type === "application/json"); + parseFile(e.target?.files?.[0]); }; const handleSaveSecrets = async () => { diff --git a/helm-charts/secrets-operator/Chart.yaml b/helm-charts/secrets-operator/Chart.yaml index 68159eca4..a11238e94 100644 --- a/helm-charts/secrets-operator/Chart.yaml +++ b/helm-charts/secrets-operator/Chart.yaml @@ -13,9 +13,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: v0.9.3 +version: v0.9.4 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "v0.9.3" +appVersion: "v0.9.4" diff --git a/helm-charts/secrets-operator/values.yaml b/helm-charts/secrets-operator/values.yaml index 898850299..54b64d4ca 100644 --- a/helm-charts/secrets-operator/values.yaml +++ b/helm-charts/secrets-operator/values.yaml @@ -32,7 +32,7 @@ controllerManager: - ALL image: repository: infisical/kubernetes-operator - tag: v0.9.3 + tag: v0.9.4 resources: limits: cpu: 500m diff --git a/k8-operator/go.mod b/k8-operator/go.mod index c9b868b00..b5de6b278 100644 --- a/k8-operator/go.mod +++ b/k8-operator/go.mod @@ -5,7 +5,7 @@ go 1.21 require ( github.com/Masterminds/sprig/v3 v3.3.0 github.com/aws/smithy-go v1.20.3 - github.com/infisical/go-sdk v0.4.4 + github.com/infisical/go-sdk v0.5.97 github.com/lestrrat-go/jwx/v2 v2.1.4 github.com/onsi/ginkgo/v2 v2.6.0 github.com/onsi/gomega v1.24.1 @@ -43,6 +43,7 @@ require ( github.com/google/s2a-go v0.1.7 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.12.5 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/huandu/xstrings v1.5.0 // indirect github.com/lestrrat-go/blackmagic v1.0.2 // indirect github.com/lestrrat-go/httpcc v1.0.1 // indirect @@ -105,7 +106,7 @@ require ( go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.24.0 // indirect golang.org/x/crypto v0.32.0 - golang.org/x/net v0.27.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/oauth2 v0.21.0 // indirect golang.org/x/sys v0.29.0 // indirect golang.org/x/term v0.28.0 // indirect diff --git a/k8-operator/go.sum b/k8-operator/go.sum index 2e151b0a4..a082da41c 100644 --- a/k8-operator/go.sum +++ b/k8-operator/go.sum @@ -228,13 +228,15 @@ github.com/googleapis/gax-go/v2 v2.12.5 h1:8gw9KZK8TiVKB6q3zHY3SBzLnrGp6HQjyfYBY github.com/googleapis/gax-go/v2 v2.12.5/go.mod h1:BUDKcWo+RaKq5SC9vVYL0wLADa3VcfswbOMMRmB9H3E= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI= github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/infisical/go-sdk v0.4.4 h1:Z4CBzxfhiY6ikjRimOEeyEEnb3QT/BKw3OzNFH7Pe+U= -github.com/infisical/go-sdk v0.4.4/go.mod h1:6fWzAwTPIoKU49mQ2Oxu+aFnJu9n7k2JcNrZjzhHM2M= +github.com/infisical/go-sdk v0.5.97 h1:veOi6Hduda6emtwjdUI5SBg2qd2iDQc5xLKqZ15KSoM= +github.com/infisical/go-sdk v0.5.97/go.mod h1:ExjqFLRz7LSpZpGluqDLvFl6dFBLq5LKyLW7GBaMAIs= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= @@ -479,8 +481,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= -golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=