diff --git a/.env.example b/.env.example index 4029cb141..8a714f77d 100644 --- a/.env.example +++ b/.env.example @@ -19,10 +19,6 @@ POSTGRES_DB=infisical # Redis REDIS_URL=redis://redis:6379 -# Optional credentials for MongoDB container instance and Mongo-Express -MONGO_USERNAME=root -MONGO_PASSWORD=example - # Website URL # Required SITE_URL=http://localhost:8080 diff --git a/.github/resources/changelog-generator.py b/.github/resources/changelog-generator.py new file mode 100644 index 000000000..1e8c67871 --- /dev/null +++ b/.github/resources/changelog-generator.py @@ -0,0 +1,176 @@ +# inspired by https://www.photoroom.com/inside-photoroom/how-we-automated-our-changelog-thanks-to-chatgpt +import os +import requests +import re +from openai import OpenAI +import subprocess +from datetime import datetime + +import uuid + +# Constants +REPO_OWNER = "infisical" +REPO_NAME = "infisical" +TOKEN = os.environ["GITHUB_TOKEN"] +# SLACK_WEBHOOK_URL = os.environ["SLACK_WEBHOOK_URL"] +OPENAI_API_KEY = os.environ["OPENAI_API_KEY"] +SLACK_MSG_COLOR = "#36a64f" + +headers = { + "Authorization": f"Bearer {TOKEN}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", +} + + +def set_multiline_output(name, value): + with open(os.environ['GITHUB_OUTPUT'], 'a') as fh: + delimiter = uuid.uuid1() + print(f'{name}<<{delimiter}', file=fh) + print(value, file=fh) + print(delimiter, file=fh) + +def find_previous_release_tag(release_tag:str): + previous_tag = subprocess.check_output(["git", "describe", "--tags", "--abbrev=0", f"{release_tag}^"]).decode("utf-8").strip() + while not(previous_tag.startswith("infisical/")): + previous_tag = subprocess.check_output(["git", "describe", "--tags", "--abbrev=0", f"{previous_tag}^"]).decode("utf-8").strip() + return previous_tag + +def get_tag_creation_date(tag_name): + url = f"https://api.github.com/repos/{REPO_OWNER}/{REPO_NAME}/git/refs/tags/{tag_name}" + response = requests.get(url, headers=headers) + response.raise_for_status() + commit_sha = response.json()['object']['sha'] + + commit_url = f"https://api.github.com/repos/{REPO_OWNER}/{REPO_NAME}/commits/{commit_sha}" + commit_response = requests.get(commit_url, headers=headers) + commit_response.raise_for_status() + creation_date = commit_response.json()['commit']['author']['date'] + + return datetime.strptime(creation_date, '%Y-%m-%dT%H:%M:%SZ') + + +def fetch_prs_between_tags(previous_tag_date:datetime, release_tag_date:datetime): + # Use GitHub API to fetch PRs merged between the commits + url = f"https://api.github.com/repos/{REPO_OWNER}/{REPO_NAME}/pulls?state=closed&merged=true" + response = requests.get(url, headers=headers) + + if response.status_code != 200: + raise Exception("Error fetching PRs from GitHub API!") + + prs = [] + for pr in response.json(): + # the idea is as tags happen recently we get last 100 closed PRs and then filter by tag creation date + if pr["merged_at"] and datetime.strptime(pr["merged_at"],'%Y-%m-%dT%H:%M:%SZ') < release_tag_date and datetime.strptime(pr["merged_at"],'%Y-%m-%dT%H:%M:%SZ') > previous_tag_date: + prs.append(pr) + + return prs + + +def extract_commit_details_from_prs(prs): + commit_details = [] + for pr in prs: + commit_message = pr["title"] + commit_url = pr["html_url"] + pr_number = pr["number"] + branch_name = pr["head"]["ref"] + issue_numbers = re.findall(r"(www-\d+|web-\d+)", branch_name) + + # If no issue numbers are found, add the PR details without issue numbers and URLs + if not issue_numbers: + commit_details.append( + { + "message": commit_message, + "pr_number": pr_number, + "pr_url": commit_url, + "issue_number": None, + "issue_url": None, + } + ) + continue + + for issue in issue_numbers: + commit_details.append( + { + "message": commit_message, + "pr_number": pr_number, + "pr_url": commit_url, + "issue_number": issue, + } + ) + + return commit_details + +# Function to generate changelog using OpenAI +def generate_changelog_with_openai(commit_details): + commit_messages = [] + for details in commit_details: + base_message = f"{details['pr_url']} - {details['message']}" + # Add the issue URL if available + # if details["issue_url"]: + # base_message += f" (Linear Issue: {details['issue_url']})" + commit_messages.append(base_message) + + commit_list = "\n".join(commit_messages) + prompt = """ +Generate a changelog for Infisical, opensource secretops +The changelog should: +1. Be Informative: Using the provided list of GitHub commits, break them down into categories such as Features, Fixes & Improvements, and Technical Updates. Summarize each commit concisely, ensuring the key points are highlighted. +2. Have a Professional yet Friendly tone: The tone should be balanced, not too corporate or too informal. +3. Celebratory Introduction and Conclusion: Start the changelog with a celebratory note acknowledging the team's hard work and progress. End with a shoutout to the team and wishes for a pleasant weekend. +4. Formatting: you cannot use Markdown formatting, and you can only use emojis for the introductory paragraph or the conclusion paragraph, nowhere else. +5. Links: the syntax to create links is the following: ``. +6. Linear Links: note that the Linear link is optional, include it only if provided. +7. Do not wrap your answer in a codeblock. Just output the text, nothing else +Here's a good example to follow, please try to match the formatting as closely as possible, only changing the content of the changelog and have some liberty with the introduction. Notice the importance of the formatting of a changelog item: +``` +- : We optimize our ci to strip comments and minify production builds. ()) +``` +And here's an example of the full changelog: +``` +*Features* +• : We optimize our ci to strip comments and minify production builds. () +*Fixes & Improvements* +• : We optimize our ci to strip comments and minify production builds. () +*Technical Updates* +• : We optimize our ci to strip comments and minify production builds. () + +Stay tuned for more exciting updates coming soon! +``` +And here are the commits: +{} + """.format( + commit_list + ) + + client = OpenAI(api_key=OPENAI_API_KEY) + messages = [{"role": "user", "content": prompt}] + response = client.chat.completions.create(model="gpt-3.5-turbo", messages=messages) + + if "error" in response.choices[0].message: + raise Exception("Error generating changelog with OpenAI!") + + return response.choices[0].message.content.strip() + + +if __name__ == "__main__": + try: + # Get the latest and previous release tags + latest_tag = subprocess.check_output(["git", "describe", "--tags", "--abbrev=0"]).decode("utf-8").strip() + previous_tag = find_previous_release_tag(latest_tag) + + latest_tag_date = get_tag_creation_date(latest_tag) + previous_tag_date = get_tag_creation_date(previous_tag) + + prs = fetch_prs_between_tags(previous_tag_date,latest_tag_date) + pr_details = extract_commit_details_from_prs(prs) + + # Generate changelog + changelog = f"## Infisical - {latest_tag}\n\n{generate_changelog_with_openai(pr_details)}" + + # Print or post changelog to Slack + set_multiline_output("changelog", changelog) + + except Exception as e: + print(str(e)) + diff --git a/.github/values.yaml b/.github/values.yaml index 90bf2ce0a..d5f0202b5 100644 --- a/.github/values.yaml +++ b/.github/values.yaml @@ -13,11 +13,10 @@ fullnameOverride: "" ## infisical: - ## @param backend.enabled Enable backend - ## + autoDatabaseSchemaMigration: false + enabled: false - ## @param backend.name Backend name - ## + name: infisical replicaCount: 3 image: @@ -50,3 +49,9 @@ ingress: - secretName: letsencrypt-prod hosts: - gamma.infisical.com + +postgresql: + enabled: false + +redis: + enabled: false diff --git a/.github/workflows/generate-release-changelog.yml b/.github/workflows/generate-release-changelog.yml new file mode 100644 index 000000000..48e35497d --- /dev/null +++ b/.github/workflows/generate-release-changelog.yml @@ -0,0 +1,36 @@ +name: Generate Changelog +permissions: + contents: write + +on: [workflow_dispatch] + +jobs: + generate_changelog: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12.0" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install requests openai + - name: Generate Changelog and Post to Slack + id: gen-changelog + run: python .github/resources/changelog-generator.py + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + - name: Set git identity + run: | + git config user.name 'github-actions[bot]' + git config user.email 'github-actions[bot]@infisical.noreply.github.com' + - name: Save the changelog to file + run: | + echo "${{ steps.gen-changelog.outputs.changelog }}" >> CHANGELOG.md + git add CHANGELOG.md + git commit -m "chore: changelog update" --no-verify + git push origin main diff --git a/Makefile b/Makefile index 2b7f43c85..2bddeec0e 100644 --- a/Makefile +++ b/Makefile @@ -11,4 +11,4 @@ up-prod: docker-compose -f docker-compose.prod.yml up --build down: - docker-compose down + docker compose -f docker-compose.dev.yml down diff --git a/backend/e2e-test/mocks/keystore.ts b/backend/e2e-test/mocks/keystore.ts new file mode 100644 index 000000000..c85244129 --- /dev/null +++ b/backend/e2e-test/mocks/keystore.ts @@ -0,0 +1,30 @@ +import { TKeyStoreFactory } from "@app/keystore/keystore"; + +export const mockKeyStore = (): TKeyStoreFactory => { + const store: Record = {}; + + return { + setItem: async (key, value) => { + store[key] = value; + return "OK"; + }, + setItemWithExpiry: async (key, value) => { + store[key] = value; + return "OK"; + }, + deleteItem: async (key) => { + delete store[key]; + return 1; + }, + getItem: async (key) => { + const value = store[key]; + if (typeof value === "string") { + return value; + } + return null; + }, + incrementBy: async () => { + return 1; + } + }; +}; diff --git a/backend/e2e-test/vitest-environment-knex.ts b/backend/e2e-test/vitest-environment-knex.ts index afb1bbabe..c1c750225 100644 --- a/backend/e2e-test/vitest-environment-knex.ts +++ b/backend/e2e-test/vitest-environment-knex.ts @@ -14,6 +14,7 @@ import { AuthTokenType } from "@app/services/auth/auth-type"; import { mockQueue } from "./mocks/queue"; import { mockSmtpServer } from "./mocks/smtp"; +import { mockKeyStore } from "./mocks/keystore"; dotenv.config({ path: path.join(__dirname, "../../.env.test"), debug: true }); export default { @@ -41,7 +42,8 @@ export default { await db.seed.run(); const smtp = mockSmtpServer(); const queue = mockQueue(); - const server = await main({ db, smtp, logger, queue }); + const keyStore = mockKeyStore(); + const server = await main({ db, smtp, logger, queue, keyStore }); // @ts-expect-error type globalThis.testServer = server; // @ts-expect-error type diff --git a/backend/package-lock.json b/backend/package-lock.json index 34a875a01..8a8e9d13d 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -11,7 +11,7 @@ "dependencies": { "@aws-sdk/client-secrets-manager": "^3.504.0", "@casl/ability": "^6.5.0", - "@fastify/cookie": "^9.2.0", + "@fastify/cookie": "^9.3.1", "@fastify/cors": "^8.5.0", "@fastify/etag": "^5.1.0", "@fastify/formbody": "^7.4.0", @@ -29,11 +29,11 @@ "@ucast/mongo2js": "^1.3.4", "ajv": "^8.12.0", "argon2": "^0.31.2", - "aws-sdk": "^2.1549.0", + "aws-sdk": "^2.1553.0", "axios": "^1.6.7", "axios-retry": "^4.0.0", "bcrypt": "^5.1.1", - "bullmq": "^5.1.6", + "bullmq": "^5.3.3", "dotenv": "^16.4.1", "fastify": "^4.26.0", "fastify-plugin": "^4.5.1", @@ -47,7 +47,6 @@ "lodash.isequal": "^4.5.0", "mysql2": "^3.9.1", "nanoid": "^5.0.4", - "node-cache": "^5.1.2", "nodemailer": "^6.9.9", "ora": "^7.0.1", "passport-github": "^1.1.0", @@ -57,7 +56,7 @@ "pg": "^8.11.3", "picomatch": "^3.0.1", "pino": "^8.16.2", - "posthog-node": "^3.6.0", + "posthog-node": "^3.6.2", "probot": "^13.0.0", "smee-client": "^2.0.0", "tweetnacl": "^1.0.3", @@ -1688,9 +1687,9 @@ } }, "node_modules/@fastify/cookie": { - "version": "9.2.0", - "resolved": "https://registry.npmjs.org/@fastify/cookie/-/cookie-9.2.0.tgz", - "integrity": "sha512-fkg1yjjQRHPFAxSHeLC8CqYuNzvR6Lwlj/KjrzQcGjNBK+K82nW+UfCjfN71g1GkoVoc1GTOgIWkFJpcMfMkHQ==", + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/@fastify/cookie/-/cookie-9.3.1.tgz", + "integrity": "sha512-h1NAEhB266+ZbZ0e9qUE6NnNR07i7DnNXWG9VbbZ8uC6O/hxHpl+Zoe5sw1yfdZ2U6XhToUGDnzQtWJdCaPwfg==", "dependencies": { "cookie-signature": "^1.1.0", "fastify-plugin": "^4.0.0" @@ -2194,7 +2193,6 @@ "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", - "dev": true, "dependencies": { "@nodelib/fs.stat": "2.0.5", "run-parallel": "^1.1.9" @@ -2207,7 +2205,6 @@ "version": "2.0.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", - "dev": true, "engines": { "node": ">= 8" } @@ -2216,7 +2213,6 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", - "dev": true, "dependencies": { "@nodelib/fs.scandir": "2.1.5", "fastq": "^1.6.0" @@ -5214,9 +5210,9 @@ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "node_modules/aws-sdk": { - "version": "2.1549.0", - "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1549.0.tgz", - "integrity": "sha512-SoVfrrV3A2mxH+NV2tA0eMtG301glhewvhL3Ob4107qLWjvwjy/CoWLclMLmfXniTGxbI8tsgN0r5mLZUKey3Q==", + "version": "2.1553.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1553.0.tgz", + "integrity": "sha512-CfZaw8dR9e642aBOeFhkFL7KoQApeLR15uH2IQqfL/12snWYayAAesYh0tEaU+XbhrH0CUsf2Zro5IraEXEZMg==", "dependencies": { "buffer": "4.9.2", "events": "1.1.1", @@ -5483,7 +5479,6 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", - "dev": true, "dependencies": { "fill-range": "^7.0.1" }, @@ -5533,14 +5528,15 @@ } }, "node_modules/bullmq": { - "version": "5.1.6", - "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.1.6.tgz", - "integrity": "sha512-VkLfig+xm4U3hc4QChzuuAy0NGQ9dfPB8o54hmcZHCX9ofp0Zn6bEY+W3Ytkk76eYwPAgXfywDBlAb2Unjl1Rg==", + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.3.3.tgz", + "integrity": "sha512-Gc/68HxiCHLMPBiGIqtINxcf8HER/5wvBYMY/6x3tFejlvldUBFaAErMTLDv4TnPsTyzNPrfBKmFCEM58uVnJg==", "dependencies": { "cron-parser": "^4.6.0", - "glob": "^8.0.3", + "fast-glob": "^3.3.2", "ioredis": "^5.3.2", "lodash": "^4.17.21", + "minimatch": "^9.0.3", "msgpackr": "^1.10.1", "node-abort-controller": "^3.1.1", "semver": "^7.5.4", @@ -5548,6 +5544,28 @@ "uuid": "^9.0.0" } }, + "node_modules/bullmq/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/bullmq/node_modules/minimatch": { + "version": "9.0.3", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.3.tgz", + "integrity": "sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg==", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/bundle-require": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/bundle-require/-/bundle-require-4.0.2.tgz", @@ -5728,14 +5746,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/clone": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz", - "integrity": "sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==", - "engines": { - "node": ">=0.8" - } - }, "node_modules/cluster-key-slot": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", @@ -6960,7 +6970,6 @@ "version": "3.3.2", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.2.tgz", "integrity": "sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==", - "dev": true, "dependencies": { "@nodelib/fs.stat": "^2.0.2", "@nodelib/fs.walk": "^1.2.3", @@ -7112,7 +7121,6 @@ "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", - "dev": true, "dependencies": { "to-regex-range": "^5.0.1" }, @@ -7564,7 +7572,6 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "dev": true, "dependencies": { "is-glob": "^4.0.1" }, @@ -8165,7 +8172,6 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", - "dev": true, "engines": { "node": ">=0.10.0" } @@ -8196,7 +8202,6 @@ "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", - "dev": true, "dependencies": { "is-extglob": "^2.1.1" }, @@ -8231,7 +8236,6 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "dev": true, "engines": { "node": ">=0.12.0" } @@ -9039,7 +9043,6 @@ "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", - "dev": true, "engines": { "node": ">= 8" } @@ -9056,7 +9059,6 @@ "version": "4.0.5", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", - "dev": true, "dependencies": { "braces": "^3.0.2", "picomatch": "^2.3.1" @@ -9069,7 +9071,6 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", - "dev": true, "engines": { "node": ">=8.6" }, @@ -9353,17 +9354,6 @@ "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-5.1.0.tgz", "integrity": "sha512-eh0GgfEkpnoWDq+VY8OyvYhFEzBk6jIYbRKdIlyTiAXIVJ8PyBaKb0rp7oDtoddbdoHWhq8wwr+XZ81F1rpNdA==" }, - "node_modules/node-cache": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/node-cache/-/node-cache-5.1.2.tgz", - "integrity": "sha512-t1QzWwnk4sjLWaQAS8CHgOJ+RAfmHpxFWmc36IWTiWHQfs0w5JDMBS1b1ZxQteo0vVVuWJvIUKHDkkeK7vIGCg==", - "dependencies": { - "clone": "2.x" - }, - "engines": { - "node": ">= 8.0.0" - } - }, "node_modules/node-fetch": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", @@ -10425,9 +10415,9 @@ "dev": true }, "node_modules/posthog-node": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/posthog-node/-/posthog-node-3.6.0.tgz", - "integrity": "sha512-N/4//SIQR4fhwbHnDdJ2rQCYdu9wo0EVPK4lVgZswp5R/E42RKlpuO6ZfPsBl+Bcg06OYiOd/WR/jLV90FCoSw==", + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/posthog-node/-/posthog-node-3.6.2.tgz", + "integrity": "sha512-tVIaShR3SxBx17AlAUS86jQTweKuJIFRedBB504fCz7YPnXJTYSrVcUHn5IINE2wu4jUQimQK6ihQr90Djrdrg==", "dependencies": { "axios": "^1.6.2", "rusha": "^0.8.14" @@ -10682,7 +10672,6 @@ "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", - "dev": true, "funding": [ { "type": "github", @@ -11029,7 +11018,6 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", - "dev": true, "funding": [ { "type": "github", @@ -11830,7 +11818,6 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "dev": true, "dependencies": { "is-number": "^7.0.0" }, diff --git a/backend/package.json b/backend/package.json index 11facff86..954f1181d 100644 --- a/backend/package.json +++ b/backend/package.json @@ -72,7 +72,7 @@ "dependencies": { "@aws-sdk/client-secrets-manager": "^3.504.0", "@casl/ability": "^6.5.0", - "@fastify/cookie": "^9.2.0", + "@fastify/cookie": "^9.3.1", "@fastify/cors": "^8.5.0", "@fastify/etag": "^5.1.0", "@fastify/formbody": "^7.4.0", @@ -90,11 +90,11 @@ "@ucast/mongo2js": "^1.3.4", "ajv": "^8.12.0", "argon2": "^0.31.2", - "aws-sdk": "^2.1549.0", + "aws-sdk": "^2.1553.0", "axios": "^1.6.7", "axios-retry": "^4.0.0", "bcrypt": "^5.1.1", - "bullmq": "^5.1.6", + "bullmq": "^5.3.3", "dotenv": "^16.4.1", "fastify": "^4.26.0", "fastify-plugin": "^4.5.1", @@ -108,7 +108,6 @@ "lodash.isequal": "^4.5.0", "mysql2": "^3.9.1", "nanoid": "^5.0.4", - "node-cache": "^5.1.2", "nodemailer": "^6.9.9", "ora": "^7.0.1", "passport-github": "^1.1.0", @@ -118,7 +117,7 @@ "pg": "^8.11.3", "picomatch": "^3.0.1", "pino": "^8.16.2", - "posthog-node": "^3.6.0", + "posthog-node": "^3.6.2", "probot": "^13.0.0", "smee-client": "^2.0.0", "tweetnacl": "^1.0.3", diff --git a/backend/scripts/generate-schema-types.ts b/backend/scripts/generate-schema-types.ts index 68330613c..8c913991f 100644 --- a/backend/scripts/generate-schema-types.ts +++ b/backend/scripts/generate-schema-types.ts @@ -44,7 +44,7 @@ const getZodDefaultValue = (type: unknown, value: string | number | boolean | Ob if (!value || value === "null") return; switch (type) { case "uuid": - return; + return `.default("00000000-0000-0000-0000-000000000000")`; case "character varying": { if (value === "gen_random_uuid()") return; if (typeof value === "string" && value.includes("::")) { @@ -100,7 +100,8 @@ const main = async () => { const columnName = columnNames[colNum]; const colInfo = columns[columnName]; let ztype = getZodPrimitiveType(colInfo.type); - if (colInfo.defaultValue) { + // don't put optional on id + if (colInfo.defaultValue && columnName !== "id") { const { defaultValue } = colInfo; const zSchema = getZodDefaultValue(colInfo.type, defaultValue); if (zSchema) { @@ -120,6 +121,7 @@ const main = async () => { .split("_") .reduce((prev, curr) => prev + `${curr.at(0)?.toUpperCase()}${curr.slice(1).toLowerCase()}`, ""); + // the insert and update are changed to zod input type to use default cases writeFileSync( path.join(__dirname, "../src/db/schemas", `${dashcase}.ts`), `// Code generated by automation script, DO NOT EDIT. @@ -134,8 +136,8 @@ import { TImmutableDBKeys } from "./models"; export const ${pascalCase}Schema = z.object({${schema}}); export type T${pascalCase} = z.infer; -export type T${pascalCase}Insert = Omit; -export type T${pascalCase}Update = Partial>; +export type T${pascalCase}Insert = Omit, TImmutableDBKeys>; +export type T${pascalCase}Update = Partial, TImmutableDBKeys>>; ` ); } diff --git a/backend/src/cache/redis.ts b/backend/src/cache/redis.ts deleted file mode 100644 index 4e856fac1..000000000 --- a/backend/src/cache/redis.ts +++ /dev/null @@ -1,6 +0,0 @@ -import Redis from "ioredis"; - -export const initRedisConnection = (redisUrl: string) => { - const redis = new Redis(redisUrl); - return redis; -}; diff --git a/backend/src/db/knexfile.ts b/backend/src/db/knexfile.ts index b81285d22..532ab496d 100644 --- a/backend/src/db/knexfile.ts +++ b/backend/src/db/knexfile.ts @@ -17,7 +17,15 @@ dotenv.config({ export default { development: { client: "postgres", - connection: process.env.DB_CONNECTION_URI, + connection: { + connectionString: process.env.DB_CONNECTION_URI, + ssl: process.env.DB_ROOT_CERT + ? { + rejectUnauthorized: true, + ca: Buffer.from(process.env.DB_ROOT_CERT, "base64").toString("ascii") + } + : false + }, pool: { min: 2, max: 10 @@ -31,7 +39,15 @@ export default { }, production: { client: "postgres", - connection: process.env.DB_CONNECTION_URI, + connection: { + connectionString: process.env.DB_CONNECTION_URI, + ssl: process.env.DB_ROOT_CERT + ? { + rejectUnauthorized: true, + ca: Buffer.from(process.env.DB_ROOT_CERT, "base64").toString("ascii") + } + : false + }, pool: { min: 2, max: 10 diff --git a/backend/src/db/migrations/20240226094411_instance-id.ts b/backend/src/db/migrations/20240226094411_instance-id.ts new file mode 100644 index 000000000..094defdc8 --- /dev/null +++ b/backend/src/db/migrations/20240226094411_instance-id.ts @@ -0,0 +1,25 @@ +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-nocheck +import { Knex } from "knex"; + +import { TableName } from "../schemas"; + +const ADMIN_CONFIG_UUID = "00000000-0000-0000-0000-000000000000"; + +export async function up(knex: Knex): Promise { + await knex.schema.alterTable(TableName.SuperAdmin, (t) => { + t.uuid("instanceId").notNullable().defaultTo(knex.fn.uuid()); + }); + + const superUserConfigExists = await knex(TableName.SuperAdmin).where("id", ADMIN_CONFIG_UUID).first(); + if (!superUserConfigExists) { + // eslint-disable-next-line + await knex(TableName.SuperAdmin).update({ id: ADMIN_CONFIG_UUID }).whereNotNull("id").limit(1); + } +} + +export async function down(knex: Knex): Promise { + await knex.schema.alterTable(TableName.SuperAdmin, (t) => { + t.dropColumn("instanceId"); + }); +} diff --git a/backend/src/db/schemas/api-keys.ts b/backend/src/db/schemas/api-keys.ts index ff29a54e1..cf836fd88 100644 --- a/backend/src/db/schemas/api-keys.ts +++ b/backend/src/db/schemas/api-keys.ts @@ -19,5 +19,5 @@ export const ApiKeysSchema = z.object({ }); export type TApiKeys = z.infer; -export type TApiKeysInsert = Omit; -export type TApiKeysUpdate = Partial>; +export type TApiKeysInsert = Omit, TImmutableDBKeys>; +export type TApiKeysUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/audit-logs.ts b/backend/src/db/schemas/audit-logs.ts index f7143bb57..b8906698b 100644 --- a/backend/src/db/schemas/audit-logs.ts +++ b/backend/src/db/schemas/audit-logs.ts @@ -24,5 +24,5 @@ export const AuditLogsSchema = z.object({ }); export type TAuditLogs = z.infer; -export type TAuditLogsInsert = Omit; -export type TAuditLogsUpdate = Partial>; +export type TAuditLogsInsert = Omit, TImmutableDBKeys>; +export type TAuditLogsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/auth-token-sessions.ts b/backend/src/db/schemas/auth-token-sessions.ts index 46ed7c201..3a9376c83 100644 --- a/backend/src/db/schemas/auth-token-sessions.ts +++ b/backend/src/db/schemas/auth-token-sessions.ts @@ -20,5 +20,5 @@ export const AuthTokenSessionsSchema = z.object({ }); export type TAuthTokenSessions = z.infer; -export type TAuthTokenSessionsInsert = Omit; -export type TAuthTokenSessionsUpdate = Partial>; +export type TAuthTokenSessionsInsert = Omit, TImmutableDBKeys>; +export type TAuthTokenSessionsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/auth-tokens.ts b/backend/src/db/schemas/auth-tokens.ts index 9ae8eed44..dd8563b85 100644 --- a/backend/src/db/schemas/auth-tokens.ts +++ b/backend/src/db/schemas/auth-tokens.ts @@ -21,5 +21,5 @@ export const AuthTokensSchema = z.object({ }); export type TAuthTokens = z.infer; -export type TAuthTokensInsert = Omit; -export type TAuthTokensUpdate = Partial>; +export type TAuthTokensInsert = Omit, TImmutableDBKeys>; +export type TAuthTokensUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/backup-private-key.ts b/backend/src/db/schemas/backup-private-key.ts index 5930bbd4a..5a2148aa1 100644 --- a/backend/src/db/schemas/backup-private-key.ts +++ b/backend/src/db/schemas/backup-private-key.ts @@ -22,5 +22,5 @@ export const BackupPrivateKeySchema = z.object({ }); export type TBackupPrivateKey = z.infer; -export type TBackupPrivateKeyInsert = Omit; -export type TBackupPrivateKeyUpdate = Partial>; +export type TBackupPrivateKeyInsert = Omit, TImmutableDBKeys>; +export type TBackupPrivateKeyUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/git-app-install-sessions.ts b/backend/src/db/schemas/git-app-install-sessions.ts index 6c6db40ea..986ae9d8e 100644 --- a/backend/src/db/schemas/git-app-install-sessions.ts +++ b/backend/src/db/schemas/git-app-install-sessions.ts @@ -17,5 +17,5 @@ export const GitAppInstallSessionsSchema = z.object({ }); export type TGitAppInstallSessions = z.infer; -export type TGitAppInstallSessionsInsert = Omit; -export type TGitAppInstallSessionsUpdate = Partial>; +export type TGitAppInstallSessionsInsert = Omit, TImmutableDBKeys>; +export type TGitAppInstallSessionsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/git-app-org.ts b/backend/src/db/schemas/git-app-org.ts index 57e0d474a..627df6b4c 100644 --- a/backend/src/db/schemas/git-app-org.ts +++ b/backend/src/db/schemas/git-app-org.ts @@ -17,5 +17,5 @@ export const GitAppOrgSchema = z.object({ }); export type TGitAppOrg = z.infer; -export type TGitAppOrgInsert = Omit; -export type TGitAppOrgUpdate = Partial>; +export type TGitAppOrgInsert = Omit, TImmutableDBKeys>; +export type TGitAppOrgUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/identities.ts b/backend/src/db/schemas/identities.ts index 005adf025..adf3a6ef2 100644 --- a/backend/src/db/schemas/identities.ts +++ b/backend/src/db/schemas/identities.ts @@ -16,5 +16,5 @@ export const IdentitiesSchema = z.object({ }); export type TIdentities = z.infer; -export type TIdentitiesInsert = Omit; -export type TIdentitiesUpdate = Partial>; +export type TIdentitiesInsert = Omit, TImmutableDBKeys>; +export type TIdentitiesUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/identity-access-tokens.ts b/backend/src/db/schemas/identity-access-tokens.ts index cbd71e5c5..18dbb8193 100644 --- a/backend/src/db/schemas/identity-access-tokens.ts +++ b/backend/src/db/schemas/identity-access-tokens.ts @@ -23,5 +23,5 @@ export const IdentityAccessTokensSchema = z.object({ }); export type TIdentityAccessTokens = z.infer; -export type TIdentityAccessTokensInsert = Omit; -export type TIdentityAccessTokensUpdate = Partial>; +export type TIdentityAccessTokensInsert = Omit, TImmutableDBKeys>; +export type TIdentityAccessTokensUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/identity-org-memberships.ts b/backend/src/db/schemas/identity-org-memberships.ts index 647ec7124..2f29c52e4 100644 --- a/backend/src/db/schemas/identity-org-memberships.ts +++ b/backend/src/db/schemas/identity-org-memberships.ts @@ -18,5 +18,7 @@ export const IdentityOrgMembershipsSchema = z.object({ }); export type TIdentityOrgMemberships = z.infer; -export type TIdentityOrgMembershipsInsert = Omit; -export type TIdentityOrgMembershipsUpdate = Partial>; +export type TIdentityOrgMembershipsInsert = Omit, TImmutableDBKeys>; +export type TIdentityOrgMembershipsUpdate = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/identity-project-memberships.ts b/backend/src/db/schemas/identity-project-memberships.ts index 866324c8b..276c9581e 100644 --- a/backend/src/db/schemas/identity-project-memberships.ts +++ b/backend/src/db/schemas/identity-project-memberships.ts @@ -18,5 +18,10 @@ export const IdentityProjectMembershipsSchema = z.object({ }); export type TIdentityProjectMemberships = z.infer; -export type TIdentityProjectMembershipsInsert = Omit; -export type TIdentityProjectMembershipsUpdate = Partial>; +export type TIdentityProjectMembershipsInsert = Omit< + z.input, + TImmutableDBKeys +>; +export type TIdentityProjectMembershipsUpdate = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/identity-ua-client-secrets.ts b/backend/src/db/schemas/identity-ua-client-secrets.ts index 60f8d862f..bd549ca5b 100644 --- a/backend/src/db/schemas/identity-ua-client-secrets.ts +++ b/backend/src/db/schemas/identity-ua-client-secrets.ts @@ -23,5 +23,7 @@ export const IdentityUaClientSecretsSchema = z.object({ }); export type TIdentityUaClientSecrets = z.infer; -export type TIdentityUaClientSecretsInsert = Omit; -export type TIdentityUaClientSecretsUpdate = Partial>; +export type TIdentityUaClientSecretsInsert = Omit, TImmutableDBKeys>; +export type TIdentityUaClientSecretsUpdate = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/identity-universal-auths.ts b/backend/src/db/schemas/identity-universal-auths.ts index 5a8f0c7ec..eeec2f666 100644 --- a/backend/src/db/schemas/identity-universal-auths.ts +++ b/backend/src/db/schemas/identity-universal-auths.ts @@ -21,5 +21,7 @@ export const IdentityUniversalAuthsSchema = z.object({ }); export type TIdentityUniversalAuths = z.infer; -export type TIdentityUniversalAuthsInsert = Omit; -export type TIdentityUniversalAuthsUpdate = Partial>; +export type TIdentityUniversalAuthsInsert = Omit, TImmutableDBKeys>; +export type TIdentityUniversalAuthsUpdate = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/incident-contacts.ts b/backend/src/db/schemas/incident-contacts.ts index 431bf05ab..23c8503b0 100644 --- a/backend/src/db/schemas/incident-contacts.ts +++ b/backend/src/db/schemas/incident-contacts.ts @@ -16,5 +16,5 @@ export const IncidentContactsSchema = z.object({ }); export type TIncidentContacts = z.infer; -export type TIncidentContactsInsert = Omit; -export type TIncidentContactsUpdate = Partial>; +export type TIncidentContactsInsert = Omit, TImmutableDBKeys>; +export type TIncidentContactsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/integration-auths.ts b/backend/src/db/schemas/integration-auths.ts index db602c0af..185beae36 100644 --- a/backend/src/db/schemas/integration-auths.ts +++ b/backend/src/db/schemas/integration-auths.ts @@ -33,5 +33,5 @@ export const IntegrationAuthsSchema = z.object({ }); export type TIntegrationAuths = z.infer; -export type TIntegrationAuthsInsert = Omit; -export type TIntegrationAuthsUpdate = Partial>; +export type TIntegrationAuthsInsert = Omit, TImmutableDBKeys>; +export type TIntegrationAuthsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/integrations.ts b/backend/src/db/schemas/integrations.ts index 62f73d190..cf8c88154 100644 --- a/backend/src/db/schemas/integrations.ts +++ b/backend/src/db/schemas/integrations.ts @@ -31,5 +31,5 @@ export const IntegrationsSchema = z.object({ }); export type TIntegrations = z.infer; -export type TIntegrationsInsert = Omit; -export type TIntegrationsUpdate = Partial>; +export type TIntegrationsInsert = Omit, TImmutableDBKeys>; +export type TIntegrationsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/org-bots.ts b/backend/src/db/schemas/org-bots.ts index b328f1aaf..77be907ec 100644 --- a/backend/src/db/schemas/org-bots.ts +++ b/backend/src/db/schemas/org-bots.ts @@ -27,5 +27,5 @@ export const OrgBotsSchema = z.object({ }); export type TOrgBots = z.infer; -export type TOrgBotsInsert = Omit; -export type TOrgBotsUpdate = Partial>; +export type TOrgBotsInsert = Omit, TImmutableDBKeys>; +export type TOrgBotsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/org-memberships.ts b/backend/src/db/schemas/org-memberships.ts index b2fffa117..585addb7c 100644 --- a/backend/src/db/schemas/org-memberships.ts +++ b/backend/src/db/schemas/org-memberships.ts @@ -20,5 +20,5 @@ export const OrgMembershipsSchema = z.object({ }); export type TOrgMemberships = z.infer; -export type TOrgMembershipsInsert = Omit; -export type TOrgMembershipsUpdate = Partial>; +export type TOrgMembershipsInsert = Omit, TImmutableDBKeys>; +export type TOrgMembershipsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/org-roles.ts b/backend/src/db/schemas/org-roles.ts index 72b582f96..ca01c6574 100644 --- a/backend/src/db/schemas/org-roles.ts +++ b/backend/src/db/schemas/org-roles.ts @@ -19,5 +19,5 @@ export const OrgRolesSchema = z.object({ }); export type TOrgRoles = z.infer; -export type TOrgRolesInsert = Omit; -export type TOrgRolesUpdate = Partial>; +export type TOrgRolesInsert = Omit, TImmutableDBKeys>; +export type TOrgRolesUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/organizations.ts b/backend/src/db/schemas/organizations.ts index a91b93ea5..f2933af86 100644 --- a/backend/src/db/schemas/organizations.ts +++ b/backend/src/db/schemas/organizations.ts @@ -19,5 +19,5 @@ export const OrganizationsSchema = z.object({ }); export type TOrganizations = z.infer; -export type TOrganizationsInsert = Omit; -export type TOrganizationsUpdate = Partial>; +export type TOrganizationsInsert = Omit, TImmutableDBKeys>; +export type TOrganizationsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/project-bots.ts b/backend/src/db/schemas/project-bots.ts index c68576943..1fa59eb78 100644 --- a/backend/src/db/schemas/project-bots.ts +++ b/backend/src/db/schemas/project-bots.ts @@ -26,5 +26,5 @@ export const ProjectBotsSchema = z.object({ }); export type TProjectBots = z.infer; -export type TProjectBotsInsert = Omit; -export type TProjectBotsUpdate = Partial>; +export type TProjectBotsInsert = Omit, TImmutableDBKeys>; +export type TProjectBotsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/project-environments.ts b/backend/src/db/schemas/project-environments.ts index 8b95dbba0..76556b7e9 100644 --- a/backend/src/db/schemas/project-environments.ts +++ b/backend/src/db/schemas/project-environments.ts @@ -18,5 +18,5 @@ export const ProjectEnvironmentsSchema = z.object({ }); export type TProjectEnvironments = z.infer; -export type TProjectEnvironmentsInsert = Omit; -export type TProjectEnvironmentsUpdate = Partial>; +export type TProjectEnvironmentsInsert = Omit, TImmutableDBKeys>; +export type TProjectEnvironmentsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/project-keys.ts b/backend/src/db/schemas/project-keys.ts index 720cd79bf..924918b12 100644 --- a/backend/src/db/schemas/project-keys.ts +++ b/backend/src/db/schemas/project-keys.ts @@ -19,5 +19,5 @@ export const ProjectKeysSchema = z.object({ }); export type TProjectKeys = z.infer; -export type TProjectKeysInsert = Omit; -export type TProjectKeysUpdate = Partial>; +export type TProjectKeysInsert = Omit, TImmutableDBKeys>; +export type TProjectKeysUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/project-memberships.ts b/backend/src/db/schemas/project-memberships.ts index b9f191a84..8576a318e 100644 --- a/backend/src/db/schemas/project-memberships.ts +++ b/backend/src/db/schemas/project-memberships.ts @@ -18,5 +18,5 @@ export const ProjectMembershipsSchema = z.object({ }); export type TProjectMemberships = z.infer; -export type TProjectMembershipsInsert = Omit; -export type TProjectMembershipsUpdate = Partial>; +export type TProjectMembershipsInsert = Omit, TImmutableDBKeys>; +export type TProjectMembershipsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/project-roles.ts b/backend/src/db/schemas/project-roles.ts index 1946ab5e1..e10f6fd4c 100644 --- a/backend/src/db/schemas/project-roles.ts +++ b/backend/src/db/schemas/project-roles.ts @@ -19,5 +19,5 @@ export const ProjectRolesSchema = z.object({ }); export type TProjectRoles = z.infer; -export type TProjectRolesInsert = Omit; -export type TProjectRolesUpdate = Partial>; +export type TProjectRolesInsert = Omit, TImmutableDBKeys>; +export type TProjectRolesUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/projects.ts b/backend/src/db/schemas/projects.ts index 3834d6d58..3965e24c0 100644 --- a/backend/src/db/schemas/projects.ts +++ b/backend/src/db/schemas/projects.ts @@ -20,5 +20,5 @@ export const ProjectsSchema = z.object({ }); export type TProjects = z.infer; -export type TProjectsInsert = Omit; -export type TProjectsUpdate = Partial>; +export type TProjectsInsert = Omit, TImmutableDBKeys>; +export type TProjectsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/saml-configs.ts b/backend/src/db/schemas/saml-configs.ts index 6891d8add..67171469a 100644 --- a/backend/src/db/schemas/saml-configs.ts +++ b/backend/src/db/schemas/saml-configs.ts @@ -27,5 +27,5 @@ export const SamlConfigsSchema = z.object({ }); export type TSamlConfigs = z.infer; -export type TSamlConfigsInsert = Omit; -export type TSamlConfigsUpdate = Partial>; +export type TSamlConfigsInsert = Omit, TImmutableDBKeys>; +export type TSamlConfigsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/scim-tokens.ts b/backend/src/db/schemas/scim-tokens.ts index 593e4f7d9..ab6e10d27 100644 --- a/backend/src/db/schemas/scim-tokens.ts +++ b/backend/src/db/schemas/scim-tokens.ts @@ -17,5 +17,5 @@ export const ScimTokensSchema = z.object({ }); export type TScimTokens = z.infer; -export type TScimTokensInsert = Omit; -export type TScimTokensUpdate = Partial>; +export type TScimTokensInsert = Omit, TImmutableDBKeys>; +export type TScimTokensUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/secret-approval-policies-approvers.ts b/backend/src/db/schemas/secret-approval-policies-approvers.ts index 503299d30..12a3119e6 100644 --- a/backend/src/db/schemas/secret-approval-policies-approvers.ts +++ b/backend/src/db/schemas/secret-approval-policies-approvers.ts @@ -16,5 +16,10 @@ export const SecretApprovalPoliciesApproversSchema = z.object({ }); export type TSecretApprovalPoliciesApprovers = z.infer; -export type TSecretApprovalPoliciesApproversInsert = Omit; -export type TSecretApprovalPoliciesApproversUpdate = Partial>; +export type TSecretApprovalPoliciesApproversInsert = Omit< + z.input, + TImmutableDBKeys +>; +export type TSecretApprovalPoliciesApproversUpdate = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/secret-approval-policies.ts b/backend/src/db/schemas/secret-approval-policies.ts index 6c331f1b1..d907ef1e0 100644 --- a/backend/src/db/schemas/secret-approval-policies.ts +++ b/backend/src/db/schemas/secret-approval-policies.ts @@ -18,5 +18,7 @@ export const SecretApprovalPoliciesSchema = z.object({ }); export type TSecretApprovalPolicies = z.infer; -export type TSecretApprovalPoliciesInsert = Omit; -export type TSecretApprovalPoliciesUpdate = Partial>; +export type TSecretApprovalPoliciesInsert = Omit, TImmutableDBKeys>; +export type TSecretApprovalPoliciesUpdate = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/secret-approval-request-secret-tags.ts b/backend/src/db/schemas/secret-approval-request-secret-tags.ts index f5e7ba632..2851321e2 100644 --- a/backend/src/db/schemas/secret-approval-request-secret-tags.ts +++ b/backend/src/db/schemas/secret-approval-request-secret-tags.ts @@ -16,5 +16,10 @@ export const SecretApprovalRequestSecretTagsSchema = z.object({ }); export type TSecretApprovalRequestSecretTags = z.infer; -export type TSecretApprovalRequestSecretTagsInsert = Omit; -export type TSecretApprovalRequestSecretTagsUpdate = Partial>; +export type TSecretApprovalRequestSecretTagsInsert = Omit< + z.input, + TImmutableDBKeys +>; +export type TSecretApprovalRequestSecretTagsUpdate = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/secret-approval-requests-reviewers.ts b/backend/src/db/schemas/secret-approval-requests-reviewers.ts index a3657f1f9..f3ff88047 100644 --- a/backend/src/db/schemas/secret-approval-requests-reviewers.ts +++ b/backend/src/db/schemas/secret-approval-requests-reviewers.ts @@ -17,5 +17,10 @@ export const SecretApprovalRequestsReviewersSchema = z.object({ }); export type TSecretApprovalRequestsReviewers = z.infer; -export type TSecretApprovalRequestsReviewersInsert = Omit; -export type TSecretApprovalRequestsReviewersUpdate = Partial>; +export type TSecretApprovalRequestsReviewersInsert = Omit< + z.input, + TImmutableDBKeys +>; +export type TSecretApprovalRequestsReviewersUpdate = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/secret-approval-requests-secrets.ts b/backend/src/db/schemas/secret-approval-requests-secrets.ts index 810a4f2cf..b795b47b4 100644 --- a/backend/src/db/schemas/secret-approval-requests-secrets.ts +++ b/backend/src/db/schemas/secret-approval-requests-secrets.ts @@ -35,5 +35,10 @@ export const SecretApprovalRequestsSecretsSchema = z.object({ }); export type TSecretApprovalRequestsSecrets = z.infer; -export type TSecretApprovalRequestsSecretsInsert = Omit; -export type TSecretApprovalRequestsSecretsUpdate = Partial>; +export type TSecretApprovalRequestsSecretsInsert = Omit< + z.input, + TImmutableDBKeys +>; +export type TSecretApprovalRequestsSecretsUpdate = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/secret-approval-requests.ts b/backend/src/db/schemas/secret-approval-requests.ts index 590c283f5..6ee97fbb6 100644 --- a/backend/src/db/schemas/secret-approval-requests.ts +++ b/backend/src/db/schemas/secret-approval-requests.ts @@ -22,5 +22,7 @@ export const SecretApprovalRequestsSchema = z.object({ }); export type TSecretApprovalRequests = z.infer; -export type TSecretApprovalRequestsInsert = Omit; -export type TSecretApprovalRequestsUpdate = Partial>; +export type TSecretApprovalRequestsInsert = Omit, TImmutableDBKeys>; +export type TSecretApprovalRequestsUpdate = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/secret-blind-indexes.ts b/backend/src/db/schemas/secret-blind-indexes.ts index fa919babd..474e0aa54 100644 --- a/backend/src/db/schemas/secret-blind-indexes.ts +++ b/backend/src/db/schemas/secret-blind-indexes.ts @@ -20,5 +20,5 @@ export const SecretBlindIndexesSchema = z.object({ }); export type TSecretBlindIndexes = z.infer; -export type TSecretBlindIndexesInsert = Omit; -export type TSecretBlindIndexesUpdate = Partial>; +export type TSecretBlindIndexesInsert = Omit, TImmutableDBKeys>; +export type TSecretBlindIndexesUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/secret-folder-versions.ts b/backend/src/db/schemas/secret-folder-versions.ts index 8c550d065..8bef6e83f 100644 --- a/backend/src/db/schemas/secret-folder-versions.ts +++ b/backend/src/db/schemas/secret-folder-versions.ts @@ -18,5 +18,5 @@ export const SecretFolderVersionsSchema = z.object({ }); export type TSecretFolderVersions = z.infer; -export type TSecretFolderVersionsInsert = Omit; -export type TSecretFolderVersionsUpdate = Partial>; +export type TSecretFolderVersionsInsert = Omit, TImmutableDBKeys>; +export type TSecretFolderVersionsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/secret-folders.ts b/backend/src/db/schemas/secret-folders.ts index 648238b0a..0f9684d0e 100644 --- a/backend/src/db/schemas/secret-folders.ts +++ b/backend/src/db/schemas/secret-folders.ts @@ -18,5 +18,5 @@ export const SecretFoldersSchema = z.object({ }); export type TSecretFolders = z.infer; -export type TSecretFoldersInsert = Omit; -export type TSecretFoldersUpdate = Partial>; +export type TSecretFoldersInsert = Omit, TImmutableDBKeys>; +export type TSecretFoldersUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/secret-imports.ts b/backend/src/db/schemas/secret-imports.ts index 9c1ee905f..9d42d8da5 100644 --- a/backend/src/db/schemas/secret-imports.ts +++ b/backend/src/db/schemas/secret-imports.ts @@ -19,5 +19,5 @@ export const SecretImportsSchema = z.object({ }); export type TSecretImports = z.infer; -export type TSecretImportsInsert = Omit; -export type TSecretImportsUpdate = Partial>; +export type TSecretImportsInsert = Omit, TImmutableDBKeys>; +export type TSecretImportsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/secret-rotation-outputs.ts b/backend/src/db/schemas/secret-rotation-outputs.ts index 3b594365a..3ac5c2c9e 100644 --- a/backend/src/db/schemas/secret-rotation-outputs.ts +++ b/backend/src/db/schemas/secret-rotation-outputs.ts @@ -15,5 +15,5 @@ export const SecretRotationOutputsSchema = z.object({ }); export type TSecretRotationOutputs = z.infer; -export type TSecretRotationOutputsInsert = Omit; -export type TSecretRotationOutputsUpdate = Partial>; +export type TSecretRotationOutputsInsert = Omit, TImmutableDBKeys>; +export type TSecretRotationOutputsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/secret-rotations.ts b/backend/src/db/schemas/secret-rotations.ts index 4c65712fa..b491edc46 100644 --- a/backend/src/db/schemas/secret-rotations.ts +++ b/backend/src/db/schemas/secret-rotations.ts @@ -26,5 +26,5 @@ export const SecretRotationsSchema = z.object({ }); export type TSecretRotations = z.infer; -export type TSecretRotationsInsert = Omit; -export type TSecretRotationsUpdate = Partial>; +export type TSecretRotationsInsert = Omit, TImmutableDBKeys>; +export type TSecretRotationsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/secret-scanning-git-risks.ts b/backend/src/db/schemas/secret-scanning-git-risks.ts index 85cfcd376..08ba690e4 100644 --- a/backend/src/db/schemas/secret-scanning-git-risks.ts +++ b/backend/src/db/schemas/secret-scanning-git-risks.ts @@ -42,5 +42,7 @@ export const SecretScanningGitRisksSchema = z.object({ }); export type TSecretScanningGitRisks = z.infer; -export type TSecretScanningGitRisksInsert = Omit; -export type TSecretScanningGitRisksUpdate = Partial>; +export type TSecretScanningGitRisksInsert = Omit, TImmutableDBKeys>; +export type TSecretScanningGitRisksUpdate = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/secret-snapshot-folders.ts b/backend/src/db/schemas/secret-snapshot-folders.ts index acf11ab0a..3e2853cbe 100644 --- a/backend/src/db/schemas/secret-snapshot-folders.ts +++ b/backend/src/db/schemas/secret-snapshot-folders.ts @@ -17,5 +17,5 @@ export const SecretSnapshotFoldersSchema = z.object({ }); export type TSecretSnapshotFolders = z.infer; -export type TSecretSnapshotFoldersInsert = Omit; -export type TSecretSnapshotFoldersUpdate = Partial>; +export type TSecretSnapshotFoldersInsert = Omit, TImmutableDBKeys>; +export type TSecretSnapshotFoldersUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/secret-snapshot-secrets.ts b/backend/src/db/schemas/secret-snapshot-secrets.ts index 6a83d1155..121c5d56e 100644 --- a/backend/src/db/schemas/secret-snapshot-secrets.ts +++ b/backend/src/db/schemas/secret-snapshot-secrets.ts @@ -17,5 +17,5 @@ export const SecretSnapshotSecretsSchema = z.object({ }); export type TSecretSnapshotSecrets = z.infer; -export type TSecretSnapshotSecretsInsert = Omit; -export type TSecretSnapshotSecretsUpdate = Partial>; +export type TSecretSnapshotSecretsInsert = Omit, TImmutableDBKeys>; +export type TSecretSnapshotSecretsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/secret-snapshots.ts b/backend/src/db/schemas/secret-snapshots.ts index ed255cb77..7f070075f 100644 --- a/backend/src/db/schemas/secret-snapshots.ts +++ b/backend/src/db/schemas/secret-snapshots.ts @@ -17,5 +17,5 @@ export const SecretSnapshotsSchema = z.object({ }); export type TSecretSnapshots = z.infer; -export type TSecretSnapshotsInsert = Omit; -export type TSecretSnapshotsUpdate = Partial>; +export type TSecretSnapshotsInsert = Omit, TImmutableDBKeys>; +export type TSecretSnapshotsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/secret-tag-junction.ts b/backend/src/db/schemas/secret-tag-junction.ts index 1d25574c5..d14384fab 100644 --- a/backend/src/db/schemas/secret-tag-junction.ts +++ b/backend/src/db/schemas/secret-tag-junction.ts @@ -14,5 +14,5 @@ export const SecretTagJunctionSchema = z.object({ }); export type TSecretTagJunction = z.infer; -export type TSecretTagJunctionInsert = Omit; -export type TSecretTagJunctionUpdate = Partial>; +export type TSecretTagJunctionInsert = Omit, TImmutableDBKeys>; +export type TSecretTagJunctionUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/secret-tags.ts b/backend/src/db/schemas/secret-tags.ts index 622c29bd3..f94e1e262 100644 --- a/backend/src/db/schemas/secret-tags.ts +++ b/backend/src/db/schemas/secret-tags.ts @@ -19,5 +19,5 @@ export const SecretTagsSchema = z.object({ }); export type TSecretTags = z.infer; -export type TSecretTagsInsert = Omit; -export type TSecretTagsUpdate = Partial>; +export type TSecretTagsInsert = Omit, TImmutableDBKeys>; +export type TSecretTagsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/secret-version-tag-junction.ts b/backend/src/db/schemas/secret-version-tag-junction.ts index 2c9a24fee..b36e28c72 100644 --- a/backend/src/db/schemas/secret-version-tag-junction.ts +++ b/backend/src/db/schemas/secret-version-tag-junction.ts @@ -14,5 +14,7 @@ export const SecretVersionTagJunctionSchema = z.object({ }); export type TSecretVersionTagJunction = z.infer; -export type TSecretVersionTagJunctionInsert = Omit; -export type TSecretVersionTagJunctionUpdate = Partial>; +export type TSecretVersionTagJunctionInsert = Omit, TImmutableDBKeys>; +export type TSecretVersionTagJunctionUpdate = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/secret-versions.ts b/backend/src/db/schemas/secret-versions.ts index d1675e3d2..d60db9b75 100644 --- a/backend/src/db/schemas/secret-versions.ts +++ b/backend/src/db/schemas/secret-versions.ts @@ -36,5 +36,5 @@ export const SecretVersionsSchema = z.object({ }); export type TSecretVersions = z.infer; -export type TSecretVersionsInsert = Omit; -export type TSecretVersionsUpdate = Partial>; +export type TSecretVersionsInsert = Omit, TImmutableDBKeys>; +export type TSecretVersionsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/secrets.ts b/backend/src/db/schemas/secrets.ts index 3fe5ad8d8..f261c40bb 100644 --- a/backend/src/db/schemas/secrets.ts +++ b/backend/src/db/schemas/secrets.ts @@ -34,5 +34,5 @@ export const SecretsSchema = z.object({ }); export type TSecrets = z.infer; -export type TSecretsInsert = Omit; -export type TSecretsUpdate = Partial>; +export type TSecretsInsert = Omit, TImmutableDBKeys>; +export type TSecretsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/service-tokens.ts b/backend/src/db/schemas/service-tokens.ts index 24720f3e4..720c8fd6f 100644 --- a/backend/src/db/schemas/service-tokens.ts +++ b/backend/src/db/schemas/service-tokens.ts @@ -25,5 +25,5 @@ export const ServiceTokensSchema = z.object({ }); export type TServiceTokens = z.infer; -export type TServiceTokensInsert = Omit; -export type TServiceTokensUpdate = Partial>; +export type TServiceTokensInsert = Omit, TImmutableDBKeys>; +export type TServiceTokensUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/super-admin.ts b/backend/src/db/schemas/super-admin.ts index b4631195b..958fed0ab 100644 --- a/backend/src/db/schemas/super-admin.ts +++ b/backend/src/db/schemas/super-admin.ts @@ -13,9 +13,10 @@ export const SuperAdminSchema = z.object({ allowSignUp: z.boolean().default(true).nullable().optional(), createdAt: z.date(), updatedAt: z.date(), - allowedSignUpDomain: z.string().nullable().optional() + allowedSignUpDomain: z.string().nullable().optional(), + instanceId: z.string().uuid().default("00000000-0000-0000-0000-000000000000") }); export type TSuperAdmin = z.infer; -export type TSuperAdminInsert = Omit; -export type TSuperAdminUpdate = Partial>; +export type TSuperAdminInsert = Omit, TImmutableDBKeys>; +export type TSuperAdminUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/trusted-ips.ts b/backend/src/db/schemas/trusted-ips.ts index 6d9018db3..f9973a843 100644 --- a/backend/src/db/schemas/trusted-ips.ts +++ b/backend/src/db/schemas/trusted-ips.ts @@ -20,5 +20,5 @@ export const TrustedIpsSchema = z.object({ }); export type TTrustedIps = z.infer; -export type TTrustedIpsInsert = Omit; -export type TTrustedIpsUpdate = Partial>; +export type TTrustedIpsInsert = Omit, TImmutableDBKeys>; +export type TTrustedIpsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/user-actions.ts b/backend/src/db/schemas/user-actions.ts index eaa03ba98..89d269847 100644 --- a/backend/src/db/schemas/user-actions.ts +++ b/backend/src/db/schemas/user-actions.ts @@ -16,5 +16,5 @@ export const UserActionsSchema = z.object({ }); export type TUserActions = z.infer; -export type TUserActionsInsert = Omit; -export type TUserActionsUpdate = Partial>; +export type TUserActionsInsert = Omit, TImmutableDBKeys>; +export type TUserActionsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/user-encryption-keys.ts b/backend/src/db/schemas/user-encryption-keys.ts index 8f35b09fe..693b73b4c 100644 --- a/backend/src/db/schemas/user-encryption-keys.ts +++ b/backend/src/db/schemas/user-encryption-keys.ts @@ -25,5 +25,5 @@ export const UserEncryptionKeysSchema = z.object({ }); export type TUserEncryptionKeys = z.infer; -export type TUserEncryptionKeysInsert = Omit; -export type TUserEncryptionKeysUpdate = Partial>; +export type TUserEncryptionKeysInsert = Omit, TImmutableDBKeys>; +export type TUserEncryptionKeysUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/users.ts b/backend/src/db/schemas/users.ts index ea031cda4..c53e96b82 100644 --- a/backend/src/db/schemas/users.ts +++ b/backend/src/db/schemas/users.ts @@ -26,5 +26,5 @@ export const UsersSchema = z.object({ }); export type TUsers = z.infer; -export type TUsersInsert = Omit; -export type TUsersUpdate = Partial>; +export type TUsersInsert = Omit, TImmutableDBKeys>; +export type TUsersUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/webhooks.ts b/backend/src/db/schemas/webhooks.ts index 7abfb3772..44aa8c5da 100644 --- a/backend/src/db/schemas/webhooks.ts +++ b/backend/src/db/schemas/webhooks.ts @@ -25,5 +25,5 @@ export const WebhooksSchema = z.object({ }); export type TWebhooks = z.infer; -export type TWebhooksInsert = Omit; -export type TWebhooksUpdate = Partial>; +export type TWebhooksInsert = Omit, TImmutableDBKeys>; +export type TWebhooksUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/ee/routes/v1/saml-router.ts b/backend/src/ee/routes/v1/saml-router.ts index b2696a40e..09dc3247c 100644 --- a/backend/src/ee/routes/v1/saml-router.ts +++ b/backend/src/ee/routes/v1/saml-router.ts @@ -27,6 +27,7 @@ type TSAMLConfig = { cert: string; audience: string; wantAuthnResponseSigned?: boolean; + wantAssertionsSigned?: boolean; disableRequestedAuthnContext?: boolean; }; @@ -82,6 +83,10 @@ export const registerSamlRouter = async (server: FastifyZodProvider) => { samlConfig.audience = `spn:${ssoConfig.issuer}`; } } + if (ssoConfig.authProvider === SamlProviders.GOOGLE_SAML) { + samlConfig.wantAssertionsSigned = false; + } + (req as unknown as FastifyRequest).ssoConfig = ssoConfig; done(null, samlConfig); } catch (error) { diff --git a/backend/src/ee/services/audit-log/audit-log-queue.ts b/backend/src/ee/services/audit-log/audit-log-queue.ts index 6f2c93221..afffd463d 100644 --- a/backend/src/ee/services/audit-log/audit-log-queue.ts +++ b/backend/src/ee/services/audit-log/audit-log-queue.ts @@ -24,7 +24,7 @@ export const auditLogQueueServiceFactory = ({ const pushToLog = async (data: TCreateAuditLogDTO) => { await queueService.queue(QueueName.AuditLog, QueueJobs.AuditLog, data, { removeOnFail: { - count: 5 + count: 3 }, removeOnComplete: true }); @@ -46,6 +46,7 @@ export const auditLogQueueServiceFactory = ({ const ttl = plan.auditLogsRetentionDays * MS_IN_DAY; // skip inserting if audit log retention is 0 meaning its not supported if (ttl === 0) return; + await auditLogDAL.create({ actor: actor.type, actorMetadata: actor.metadata, diff --git a/backend/src/ee/services/license/license-service.ts b/backend/src/ee/services/license/license-service.ts index 5f935c6a2..49609e8c9 100644 --- a/backend/src/ee/services/license/license-service.ts +++ b/backend/src/ee/services/license/license-service.ts @@ -5,8 +5,8 @@ // TODO(akhilmhdh): With tony find out the api structure and fill it here import { ForbiddenError } from "@casl/ability"; -import NodeCache from "node-cache"; +import { TKeyStoreFactory } from "@app/keystore/keystore"; import { getConfig } from "@app/lib/config/env"; import { BadRequestError } from "@app/lib/errors"; import { logger } from "@app/lib/logger"; @@ -39,6 +39,7 @@ type TLicenseServiceFactoryDep = { orgDAL: Pick; permissionService: Pick; licenseDAL: TLicenseDALFactory; + keyStore: Pick; }; export type TLicenseServiceFactory = ReturnType; @@ -46,12 +47,18 @@ export type TLicenseServiceFactory = ReturnType; const LICENSE_SERVER_CLOUD_LOGIN = "/api/auth/v1/license-server-login"; const LICENSE_SERVER_ON_PREM_LOGIN = "/api/auth/v1/license-login"; -const FEATURE_CACHE_KEY = (orgId: string, projectId?: string) => `${orgId}-${projectId || ""}`; -export const licenseServiceFactory = ({ orgDAL, permissionService, licenseDAL }: TLicenseServiceFactoryDep) => { +const LICENSE_SERVER_CLOUD_PLAN_TTL = 30; // 30 second +const FEATURE_CACHE_KEY = (orgId: string) => `infisical-cloud-plan-${orgId}`; + +export const licenseServiceFactory = ({ + orgDAL, + permissionService, + licenseDAL, + keyStore +}: TLicenseServiceFactoryDep) => { let isValidLicense = false; let instanceType = InstanceType.OnPrem; let onPremFeatures: TFeatureSet = getDefaultOnPremFeatures(); - const featureStore = new NodeCache({ stdTTL: 60 }); const appCfg = getConfig(); const licenseServerCloudApi = setupLicenceRequestWithStore( @@ -75,6 +82,7 @@ export const licenseServiceFactory = ({ orgDAL, permissionService, licenseDAL }: isValidLicense = true; return; } + if (appCfg.LICENSE_KEY) { const token = await licenseServerOnPremApi.refreshLicence(); if (token) { @@ -100,22 +108,21 @@ export const licenseServiceFactory = ({ orgDAL, permissionService, licenseDAL }: logger.info(`getPlan: attempting to fetch plan for [orgId=${orgId}] [projectId=${projectId}]`); try { if (instanceType === InstanceType.Cloud) { - const cachedPlan = featureStore.get(FEATURE_CACHE_KEY(orgId, projectId)); - if (cachedPlan) return cachedPlan; + const cachedPlan = await keyStore.getItem(FEATURE_CACHE_KEY(orgId)); + if (cachedPlan) return JSON.parse(cachedPlan) as TFeatureSet; const org = await orgDAL.findOrgById(orgId); if (!org) throw new BadRequestError({ message: "Org not found" }); const { data: { currentPlan } } = await licenseServerCloudApi.request.get<{ currentPlan: TFeatureSet }>( - `/api/license-server/v1/customers/${org.customerId}/cloud-plan`, - { - params: { - workspaceId: projectId - } - } + `/api/license-server/v1/customers/${org.customerId}/cloud-plan` + ); + await keyStore.setItemWithExpiry( + FEATURE_CACHE_KEY(org.id), + LICENSE_SERVER_CLOUD_PLAN_TTL, + JSON.stringify(currentPlan) ); - featureStore.set(FEATURE_CACHE_KEY(org.id, projectId), currentPlan); return currentPlan; } } catch (error) { @@ -123,15 +130,20 @@ export const licenseServiceFactory = ({ orgDAL, permissionService, licenseDAL }: `getPlan: encountered an error when fetching pan [orgId=${orgId}] [projectId=${projectId}] [error]`, error ); + await keyStore.setItemWithExpiry( + FEATURE_CACHE_KEY(orgId), + LICENSE_SERVER_CLOUD_PLAN_TTL, + JSON.stringify(onPremFeatures) + ); return onPremFeatures; } return onPremFeatures; }; - const refreshPlan = async (orgId: string, projectId?: string) => { + const refreshPlan = async (orgId: string) => { if (instanceType === InstanceType.Cloud) { - featureStore.del(FEATURE_CACHE_KEY(orgId, projectId)); - await getPlan(orgId, projectId); + await keyStore.deleteItem(FEATURE_CACHE_KEY(orgId)); + await getPlan(orgId); } }; @@ -166,7 +178,7 @@ export const licenseServiceFactory = ({ orgDAL, permissionService, licenseDAL }: quantity: count }); } - featureStore.del(orgId); + await keyStore.deleteItem(FEATURE_CACHE_KEY(orgId)); } else if (instanceType === InstanceType.EnterpriseOnPrem) { const usedSeats = await licenseDAL.countOfOrgMembers(null); await licenseServerOnPremApi.request.patch(`/api/license/v1/license`, { usedSeats }); @@ -215,7 +227,7 @@ export const licenseServiceFactory = ({ orgDAL, permissionService, licenseDAL }: `/api/license-server/v1/customers/${organization.customerId}/session/trial`, { success_url } ); - featureStore.del(FEATURE_CACHE_KEY(orgId)); + await keyStore.deleteItem(FEATURE_CACHE_KEY(orgId)); return { url }; }; @@ -505,6 +517,9 @@ export const licenseServiceFactory = ({ orgDAL, permissionService, licenseDAL }: get isValidLicense() { return isValidLicense; }, + getInstanceType() { + return instanceType; + }, getPlan, updateSubscriptionOrgMemberCount, refreshPlan, diff --git a/backend/src/ee/services/saml-config/saml-config-types.ts b/backend/src/ee/services/saml-config/saml-config-types.ts index ef2f4c45b..ec7c066fc 100644 --- a/backend/src/ee/services/saml-config/saml-config-types.ts +++ b/backend/src/ee/services/saml-config/saml-config-types.ts @@ -4,7 +4,8 @@ import { ActorType } from "@app/services/auth/auth-type"; export enum SamlProviders { OKTA_SAML = "okta-saml", AZURE_SAML = "azure-saml", - JUMPCLOUD_SAML = "jumpcloud-saml" + JUMPCLOUD_SAML = "jumpcloud-saml", + GOOGLE_SAML = "google-saml" } export type TCreateSamlCfgDTO = { diff --git a/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts b/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts index 9e69f0a8f..330f6beca 100644 --- a/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts +++ b/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts @@ -240,7 +240,7 @@ export const secretRotationQueueFactory = ({ ); }); - telemetryService.sendPostHogEvents({ + await telemetryService.sendPostHogEvents({ event: PostHogEventTypes.SecretRotated, distinctId: "", properties: { diff --git a/backend/src/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue.ts b/backend/src/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue.ts index 04de3e7ef..b776d3d90 100644 --- a/backend/src/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue.ts +++ b/backend/src/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue.ts @@ -158,7 +158,7 @@ export const secretScanningQueueFactory = ({ }); } - telemetryService.sendPostHogEvents({ + await telemetryService.sendPostHogEvents({ event: PostHogEventTypes.SecretScannerPush, distinctId: repository.fullName, properties: { @@ -228,7 +228,7 @@ export const secretScanningQueueFactory = ({ }); } - telemetryService.sendPostHogEvents({ + await telemetryService.sendPostHogEvents({ event: PostHogEventTypes.SecretScannerFull, distinctId: repository.fullName, properties: { diff --git a/backend/src/keystore/keystore.ts b/backend/src/keystore/keystore.ts new file mode 100644 index 000000000..5e2c3aab3 --- /dev/null +++ b/backend/src/keystore/keystore.ts @@ -0,0 +1,20 @@ +import { Redis } from "ioredis"; + +export type TKeyStoreFactory = ReturnType; + +export const keyStoreFactory = (redisUrl: string) => { + const redis = new Redis(redisUrl); + + const setItem = async (key: string, value: string | number | Buffer) => redis.set(key, value); + + const getItem = async (key: string) => redis.get(key); + + const setItemWithExpiry = async (key: string, exp: number | string, value: string | number | Buffer) => + redis.setex(key, exp, value); + + const deleteItem = async (key: string) => redis.del(key); + + const incrementBy = async (key: string, value: number) => redis.incrby(key, value); + + return { setItem, getItem, setItemWithExpiry, deleteItem, incrementBy }; +}; diff --git a/backend/src/lib/config/env.ts b/backend/src/lib/config/env.ts index 4542c7fc3..655e402df 100644 --- a/backend/src/lib/config/env.ts +++ b/backend/src/lib/config/env.ts @@ -94,14 +94,17 @@ const envSchema = z SECRET_SCANNING_WEBHOOK_SECRET: zpStr(z.string().optional()), SECRET_SCANNING_GIT_APP_ID: zpStr(z.string().optional()), SECRET_SCANNING_PRIVATE_KEY: zpStr(z.string().optional()), - // LICENCE + // LICENSE LICENSE_SERVER_URL: zpStr(z.string().optional().default("https://portal.infisical.com")), LICENSE_SERVER_KEY: zpStr(z.string().optional()), LICENSE_KEY: zpStr(z.string().optional()), + + // GENERIC STANDALONE_MODE: z .enum(["true", "false"]) .transform((val) => val === "true") - .optional() + .optional(), + INFISICAL_CLOUD: zodStrBool.default("false") }) .transform((data) => ({ ...data, diff --git a/backend/src/main.ts b/backend/src/main.ts index fab576d3b..86681ef33 100644 --- a/backend/src/main.ts +++ b/backend/src/main.ts @@ -1,6 +1,7 @@ import dotenv from "dotenv"; import { initDbConnection } from "./db"; +import { keyStoreFactory } from "./keystore/keystore"; import { formatSmtpConfig, initEnvConfig } from "./lib/config/env"; import { initLogger } from "./lib/logger"; import { queueServiceFactory } from "./queue"; @@ -19,8 +20,9 @@ const run = async () => { const smtp = smtpServiceFactory(formatSmtpConfig()); const queue = queueServiceFactory(appCfg.REDIS_URL); + const keyStore = keyStoreFactory(appCfg.REDIS_URL); - const server = await main({ db, smtp, logger, queue }); + const server = await main({ db, smtp, logger, queue, keyStore }); const bootstrap = await bootstrapCheck({ db }); // eslint-disable-next-line process.on("SIGINT", async () => { diff --git a/backend/src/queue/queue-service.ts b/backend/src/queue/queue-service.ts index e11e0b590..45c135b77 100644 --- a/backend/src/queue/queue-service.ts +++ b/backend/src/queue/queue-service.ts @@ -13,6 +13,7 @@ export enum QueueName { SecretReminder = "secret-reminder", AuditLog = "audit-log", AuditLogPrune = "audit-log-prune", + TelemetryInstanceStats = "telemtry-self-hosted-stats", IntegrationSync = "sync-integrations", SecretWebhook = "secret-webhook", SecretFullRepoScan = "secret-full-repo-scan", @@ -26,6 +27,7 @@ export enum QueueJobs { AuditLog = "audit-log-job", AuditLogPrune = "audit-log-prune-job", SecWebhook = "secret-webhook-trigger", + TelemetryInstanceStats = "telemetry-self-hosted-stats", IntegrationSync = "secret-integration-pull", SecretScan = "secret-scan", UpgradeProjectToGhost = "upgrade-project-to-ghost-job" @@ -67,7 +69,6 @@ export type TQueueJobTypes = { payload: TScanFullRepoEventPayload; }; [QueueName.SecretPushEventScan]: { name: QueueJobs.SecretScan; payload: TScanPushEventPayload }; - [QueueName.UpgradeProjectToGhost]: { name: QueueJobs.UpgradeProjectToGhost; payload: { @@ -81,6 +82,10 @@ export type TQueueJobTypes = { }; }; }; + [QueueName.TelemetryInstanceStats]: { + name: QueueJobs.TelemetryInstanceStats; + payload: undefined; + }; }; export type TQueueServiceFactory = ReturnType; diff --git a/backend/src/server/app.ts b/backend/src/server/app.ts index 969d2fe43..556a88d7c 100644 --- a/backend/src/server/app.ts +++ b/backend/src/server/app.ts @@ -14,6 +14,7 @@ import fasitfy from "fastify"; import { Knex } from "knex"; import { Logger } from "pino"; +import { TKeyStoreFactory } from "@app/keystore/keystore"; import { getConfig } from "@app/lib/config/env"; import { TQueueServiceFactory } from "@app/queue"; import { TSmtpService } from "@app/services/smtp/smtp-service"; @@ -31,10 +32,11 @@ type TMain = { smtp: TSmtpService; logger?: Logger; queue: TQueueServiceFactory; + keyStore: TKeyStoreFactory; }; // Run the server! -export const main = async ({ db, smtp, logger, queue }: TMain) => { +export const main = async ({ db, smtp, logger, queue, keyStore }: TMain) => { const appCfg = getConfig(); const server = fasitfy({ logger: appCfg.NODE_ENV === "test" ? false : logger, @@ -70,7 +72,7 @@ export const main = async ({ db, smtp, logger, queue }: TMain) => { } await server.register(helmet, { contentSecurityPolicy: false }); - await server.register(registerRoutes, { smtp, queue, db }); + await server.register(registerRoutes, { smtp, queue, db, keyStore }); if (appCfg.isProductionMode) { await server.register(registerExternalNextjs, { diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index b4cbb9b9d..9c0e90a6e 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -36,6 +36,7 @@ import { snapshotFolderDALFactory } from "@app/ee/services/secret-snapshot/snaps import { snapshotSecretDALFactory } from "@app/ee/services/secret-snapshot/snapshot-secret-dal"; import { trustedIpDALFactory } from "@app/ee/services/trusted-ip/trusted-ip-dal"; import { trustedIpServiceFactory } from "@app/ee/services/trusted-ip/trusted-ip-service"; +import { TKeyStoreFactory } from "@app/keystore/keystore"; import { getConfig } from "@app/lib/config/env"; import { TQueueServiceFactory } from "@app/queue"; import { apiKeyDALFactory } from "@app/services/api-key/api-key-dal"; @@ -98,6 +99,8 @@ import { serviceTokenServiceFactory } from "@app/services/service-token/service- import { TSmtpService } from "@app/services/smtp/smtp-service"; import { superAdminDALFactory } from "@app/services/super-admin/super-admin-dal"; import { getServerCfg, superAdminServiceFactory } from "@app/services/super-admin/super-admin-service"; +import { telemetryDALFactory } from "@app/services/telemetry/telemetry-dal"; +import { telemetryQueueServiceFactory } from "@app/services/telemetry/telemetry-queue"; import { telemetryServiceFactory } from "@app/services/telemetry/telemetry-service"; import { userDALFactory } from "@app/services/user/user-dal"; import { userServiceFactory } from "@app/services/user/user-service"; @@ -114,7 +117,12 @@ import { registerV3Routes } from "./v3"; export const registerRoutes = async ( server: FastifyZodProvider, - { db, smtp: smtpService, queue: queueService }: { db: Knex; smtp: TSmtpService; queue: TQueueServiceFactory } + { + db, + smtp: smtpService, + queue: queueService, + keyStore + }: { db: Knex; smtp: TSmtpService; queue: TQueueServiceFactory; keyStore: TKeyStoreFactory } ) => { await server.register(registerSecretScannerGhApp, { prefix: "/ss-webhook" }); @@ -160,6 +168,7 @@ export const registerRoutes = async ( const auditLogDAL = auditLogDALFactory(db); const trustedIpDAL = trustedIpDALFactory(db); + const telemetryDAL = telemetryDALFactory(db); // ee db layer ops const permissionDAL = permissionDALFactory(db); @@ -188,7 +197,7 @@ export const registerRoutes = async ( projectRoleDAL, serviceTokenDAL }); - const licenseService = licenseServiceFactory({ permissionService, orgDAL, licenseDAL }); + const licenseService = licenseServiceFactory({ permissionService, orgDAL, licenseDAL, keyStore }); const trustedIpService = trustedIpServiceFactory({ licenseService, projectDAL, @@ -229,6 +238,7 @@ export const registerRoutes = async ( smtpService }); +<<<<<<< HEAD const ldapService = ldapConfigServiceFactory({ ldapConfigDAL, orgDAL, @@ -239,6 +249,18 @@ export const registerRoutes = async ( }); const telemetryService = telemetryServiceFactory(); +======= + const telemetryService = telemetryServiceFactory({ + keyStore, + licenseService + }); + const telemetryQueue = telemetryQueueServiceFactory({ + keyStore, + telemetryDAL, + queueService + }); + +>>>>>>> origin const tokenService = tokenServiceFactory({ tokenDAL: authTokenDAL, userDAL }); const userService = userServiceFactory({ userDAL }); const loginService = authLoginServiceFactory({ userDAL, smtpService, tokenService }); @@ -275,7 +297,8 @@ export const registerRoutes = async ( userDAL, authService: loginService, serverCfgDAL: superAdminDAL, - orgService + orgService, + keyStore }); const apiKeyService = apiKeyServiceFactory({ apiKeyDAL, userDAL }); @@ -503,9 +526,13 @@ export const registerRoutes = async ( }); await superAdminService.initServerCfg(); - await auditLogQueue.startAuditLogPruneJob(); + // // setup the communication with license key server await licenseService.init(); + + await auditLogQueue.startAuditLogPruneJob(); + await telemetryQueue.startTelemetryCheck(); + // inject all services server.decorate("services", { login: loginService, diff --git a/backend/src/server/routes/v1/admin-router.ts b/backend/src/server/routes/v1/admin-router.ts index 1e04980f7..d8431a62f 100644 --- a/backend/src/server/routes/v1/admin-router.ts +++ b/backend/src/server/routes/v1/admin-router.ts @@ -16,7 +16,7 @@ export const registerAdminRouter = async (server: FastifyZodProvider) => { schema: { response: { 200: z.object({ - config: SuperAdminSchema + config: SuperAdminSchema.omit({ createdAt: true, updatedAt: true }) }) } }, @@ -90,7 +90,7 @@ export const registerAdminRouter = async (server: FastifyZodProvider) => { userAgent: req.headers["user-agent"] || "" }); - server.services.telemetry.sendPostHogEvents({ + await server.services.telemetry.sendPostHogEvents({ event: PostHogEventTypes.AdminInit, distinctId: user.user.email ?? user.user.username ?? "", properties: { diff --git a/backend/src/server/routes/v1/identity-router.ts b/backend/src/server/routes/v1/identity-router.ts index a03cde62f..0ec27a98b 100644 --- a/backend/src/server/routes/v1/identity-router.ts +++ b/backend/src/server/routes/v1/identity-router.ts @@ -51,7 +51,7 @@ export const registerIdentityRouter = async (server: FastifyZodProvider) => { } }); - server.services.telemetry.sendPostHogEvents({ + await server.services.telemetry.sendPostHogEvents({ event: PostHogEventTypes.MachineIdentityCreated, distinctId: getTelemetryDistinctId(req), properties: { diff --git a/backend/src/server/routes/v1/identity-ua.ts b/backend/src/server/routes/v1/identity-ua.ts index 11b8e0e8d..4499a88e7 100644 --- a/backend/src/server/routes/v1/identity-ua.ts +++ b/backend/src/server/routes/v1/identity-ua.ts @@ -39,11 +39,12 @@ export const registerIdentityUaRouter = async (server: FastifyZodProvider) => { } }, handler: async (req) => { - const { identityUa, accessToken, identityAccessToken, validClientSecretInfo } = + const { identityUa, accessToken, identityAccessToken, validClientSecretInfo, identityMembershipOrg } = await server.services.identityUa.login(req.body.clientId, req.body.clientSecret, req.realIp); await server.services.auditLog.createAuditLog({ ...req.auditLogInfo, + orgId: identityMembershipOrg?.orgId, event: { type: EventType.LOGIN_IDENTITY_UNIVERSAL_AUTH, metadata: { diff --git a/backend/src/server/routes/v1/integration-router.ts b/backend/src/server/routes/v1/integration-router.ts index 1ccf75e94..ec712a543 100644 --- a/backend/src/server/routes/v1/integration-router.ts +++ b/backend/src/server/routes/v1/integration-router.ts @@ -82,7 +82,7 @@ export const registerIntegrationRouter = async (server: FastifyZodProvider) => { } }); - server.services.telemetry.sendPostHogEvents({ + await server.services.telemetry.sendPostHogEvents({ event: PostHogEventTypes.IntegrationCreated, distinctId: getTelemetryDistinctId(req), properties: { diff --git a/backend/src/server/routes/v1/invite-org-router.ts b/backend/src/server/routes/v1/invite-org-router.ts index dd2fd8157..5956b53df 100644 --- a/backend/src/server/routes/v1/invite-org-router.ts +++ b/backend/src/server/routes/v1/invite-org-router.ts @@ -32,7 +32,7 @@ export const registerInviteOrgRouter = async (server: FastifyZodProvider) => { actorOrgId: req.permission.orgId }); - server.services.telemetry.sendPostHogEvents({ + await server.services.telemetry.sendPostHogEvents({ event: PostHogEventTypes.UserOrgInvitation, distinctId: getTelemetryDistinctId(req), properties: { diff --git a/backend/src/server/routes/v1/organization-router.ts b/backend/src/server/routes/v1/organization-router.ts index c39fed49c..d31682d88 100644 --- a/backend/src/server/routes/v1/organization-router.ts +++ b/backend/src/server/routes/v1/organization-router.ts @@ -88,11 +88,12 @@ export const registerOrgRouter = async (server: FastifyZodProvider) => { schema: { params: z.object({ organizationId: z.string().trim() }), body: z.object({ - name: z.string().trim().optional(), + name: z.string().trim().max(64, { message: "Name must be 64 or fewer characters" }).optional(), slug: z .string() .trim() - .regex(/^[a-zA-Z0-9-]+$/, "Name must only contain alphanumeric characters or hyphens") + .max(64, { message: "Slug must be 64 or fewer characters" }) + .regex(/^[a-zA-Z0-9-]+$/, "Slug must only contain alphanumeric characters or hyphens") .optional(), authEnforced: z.boolean().optional(), scimEnabled: z.boolean().optional() diff --git a/backend/src/server/routes/v1/project-router.ts b/backend/src/server/routes/v1/project-router.ts index 4835b665e..ef928a4dc 100644 --- a/backend/src/server/routes/v1/project-router.ts +++ b/backend/src/server/routes/v1/project-router.ts @@ -223,7 +223,7 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { workspaceId: z.string().trim() }), body: z.object({ - name: z.string().trim().optional(), + name: z.string().trim().max(64, { message: "Name must be 64 or fewer characters" }).optional(), autoCapitalization: z.boolean().optional() }), response: { diff --git a/backend/src/server/routes/v2/project-router.ts b/backend/src/server/routes/v2/project-router.ts index 62f4c7cf5..fe1254b2b 100644 --- a/backend/src/server/routes/v2/project-router.ts +++ b/backend/src/server/routes/v2/project-router.ts @@ -154,7 +154,7 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { slug: req.body.slug }); - server.services.telemetry.sendPostHogEvents({ + await server.services.telemetry.sendPostHogEvents({ event: PostHogEventTypes.ProjectCreated, distinctId: getTelemetryDistinctId(req), properties: { diff --git a/backend/src/server/routes/v3/secret-router.ts b/backend/src/server/routes/v3/secret-router.ts index cfe4a87d7..6b3dd6041 100644 --- a/backend/src/server/routes/v3/secret-router.ts +++ b/backend/src/server/routes/v3/secret-router.ts @@ -95,7 +95,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { } }); - server.services.telemetry.sendPostHogEvents({ + await server.services.telemetry.sendPostHogEvents({ event: PostHogEventTypes.SecretPulled, distinctId: getTelemetryDistinctId(req), properties: { @@ -185,7 +185,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { } }); - server.services.telemetry.sendPostHogEvents({ + await server.services.telemetry.sendPostHogEvents({ event: PostHogEventTypes.SecretPulled, distinctId: getTelemetryDistinctId(req), properties: { @@ -261,7 +261,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { } }); - server.services.telemetry.sendPostHogEvents({ + await server.services.telemetry.sendPostHogEvents({ event: PostHogEventTypes.SecretCreated, distinctId: getTelemetryDistinctId(req), properties: { @@ -336,7 +336,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { } }); - server.services.telemetry.sendPostHogEvents({ + await server.services.telemetry.sendPostHogEvents({ event: PostHogEventTypes.SecretUpdated, distinctId: getTelemetryDistinctId(req), properties: { @@ -406,7 +406,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { } }); - server.services.telemetry.sendPostHogEvents({ + await server.services.telemetry.sendPostHogEvents({ event: PostHogEventTypes.SecretDeleted, distinctId: getTelemetryDistinctId(req), properties: { @@ -512,7 +512,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { (req.headers["user-agent"] !== "k8-operator" || shouldRecordK8Event); const approximateNumberTotalSecrets = secrets.length * 20; if (shouldCapture) { - server.services.telemetry.sendPostHogEvents({ + await server.services.telemetry.sendPostHogEvents({ event: PostHogEventTypes.SecretPulled, distinctId: getTelemetryDistinctId(req), properties: { @@ -589,7 +589,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { } }); - server.services.telemetry.sendPostHogEvents({ + await server.services.telemetry.sendPostHogEvents({ event: PostHogEventTypes.SecretPulled, distinctId: getTelemetryDistinctId(req), properties: { @@ -752,7 +752,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { } }); - server.services.telemetry.sendPostHogEvents({ + await server.services.telemetry.sendPostHogEvents({ event: PostHogEventTypes.SecretCreated, distinctId: getTelemetryDistinctId(req), properties: { @@ -934,7 +934,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { } }); - server.services.telemetry.sendPostHogEvents({ + await server.services.telemetry.sendPostHogEvents({ event: PostHogEventTypes.SecretUpdated, distinctId: getTelemetryDistinctId(req), properties: { @@ -1052,7 +1052,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { } }); - server.services.telemetry.sendPostHogEvents({ + await server.services.telemetry.sendPostHogEvents({ event: PostHogEventTypes.SecretDeleted, distinctId: getTelemetryDistinctId(req), properties: { @@ -1172,7 +1172,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { } }); - server.services.telemetry.sendPostHogEvents({ + await server.services.telemetry.sendPostHogEvents({ event: PostHogEventTypes.SecretCreated, distinctId: getTelemetryDistinctId(req), properties: { @@ -1292,7 +1292,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { } }); - server.services.telemetry.sendPostHogEvents({ + await server.services.telemetry.sendPostHogEvents({ event: PostHogEventTypes.SecretUpdated, distinctId: getTelemetryDistinctId(req), properties: { @@ -1400,7 +1400,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { } }); - server.services.telemetry.sendPostHogEvents({ + await server.services.telemetry.sendPostHogEvents({ event: PostHogEventTypes.SecretDeleted, distinctId: getTelemetryDistinctId(req), properties: { diff --git a/backend/src/services/identity-ua/identity-ua-service.ts b/backend/src/services/identity-ua/identity-ua-service.ts index 3b73da6d4..d375a8fa5 100644 --- a/backend/src/services/identity-ua/identity-ua-service.ts +++ b/backend/src/services/identity-ua/identity-ua-service.ts @@ -54,6 +54,8 @@ export const identityUaServiceFactory = ({ const identityUa = await identityUaDAL.findOne({ clientId }); if (!identityUa) throw new UnauthorizedError(); + const identityMembershipOrg = await identityOrgMembershipDAL.findOne({ identityId: identityUa.identityId }); + checkIPAgainstBlocklist({ ipAddress: ip, trustedIps: identityUa.clientSecretTrustedIps as TIp[] @@ -131,7 +133,7 @@ export const identityUaServiceFactory = ({ } ); - return { accessToken, identityUa, validClientSecretInfo, identityAccessToken }; + return { accessToken, identityUa, validClientSecretInfo, identityAccessToken, identityMembershipOrg }; }; const attachUa = async ({ diff --git a/backend/src/services/project-membership/project-membership-service.ts b/backend/src/services/project-membership/project-membership-service.ts index b049d0279..4b569976d 100644 --- a/backend/src/services/project-membership/project-membership-service.ts +++ b/backend/src/services/project-membership/project-membership-service.ts @@ -243,6 +243,8 @@ export const projectMembershipServiceFactory = ({ if (orgMembers.length !== emails.length) throw new BadRequestError({ message: "Some users are not part of org" }); + if (!orgMembers.length) return []; + const existingMembers = await projectMembershipDAL.find({ projectId, $in: { userId: orgMembers.map(({ user }) => user.id).filter(Boolean) } diff --git a/backend/src/services/secret-tag/secret-tag-service.ts b/backend/src/services/secret-tag/secret-tag-service.ts index 62ebd8a23..1007ec4c3 100644 --- a/backend/src/services/secret-tag/secret-tag-service.ts +++ b/backend/src/services/secret-tag/secret-tag-service.ts @@ -19,7 +19,7 @@ export const secretTagServiceFactory = ({ secretTagDAL, permissionService }: TSe const { permission } = await permissionService.getProjectPermission(actor, actorId, projectId, actorOrgId); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Create, ProjectPermissionSub.Tags); - const existingTag = await secretTagDAL.findOne({ slug }); + const existingTag = await secretTagDAL.findOne({ slug, projectId }); if (existingTag) throw new BadRequestError({ message: "Tag already exist" }); const newTag = await secretTagDAL.create({ diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index 96d851a63..4c14edc0c 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -7,7 +7,7 @@ import { TSecretSnapshotServiceFactory } from "@app/ee/services/secret-snapshot/ import { getConfig } from "@app/lib/config/env"; import { buildSecretBlindIndexFromName, encryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto"; import { BadRequestError } from "@app/lib/errors"; -import { groupBy, pick } from "@app/lib/fn"; +import { groupBy, pick, unique } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; import { ActorType } from "../auth/auth-type"; @@ -202,12 +202,13 @@ export const secretServiceFactory = ({ return deletedSecrets; }; - // this is a utility function for secret modification - // this will check given secret name blind index exist or not - // if its a created secret set isNew to true - // thus if these blindindex exist it will throw an error - // vice versa when u need to check for updated secret - // this will also return the blind index grouped by secretName + /** + * Checks and handles secrets using a blind index method. + * The function generates mappings between secret names and their blind indexes, validates user IDs for personal secrets, and retrieves secrets from the database based on their blind indexes. + * For new secrets (isNew = true), it ensures they don't already exist in the database. + * For existing secrets, it verifies their presence in the database. + * If discrepancies are found, errors are thrown. The function returns mappings and the fetched secrets. + */ const fnSecretBlindIndexCheck = async ({ inputSecrets, folderId, @@ -242,10 +243,18 @@ export const secretServiceFactory = ({ if (isNew) { if (secrets.length) throw new BadRequestError({ message: "Secret already exist" }); - } else if (secrets.length !== inputSecrets.length) - throw new BadRequestError({ - message: `Secret not found: blind index ${JSON.stringify(keyName2BlindIndex)}` - }); + } else { + const secretKeysInDB = unique(secrets, (el) => el.secretBlindIndex as string).map( + (el) => blindIndex2KeyName[el.secretBlindIndex as string] + ); + const hasUnknownSecretsProvided = secretKeysInDB.length !== inputSecrets.length; + if (hasUnknownSecretsProvided) { + const keysMissingInDB = Object.keys(keyName2BlindIndex).filter((key) => !secretKeysInDB.includes(key)); + throw new BadRequestError({ + message: `Secret not found: blind index ${keysMissingInDB.join(",")}` + }); + } + } return { blindIndex2KeyName, keyName2BlindIndex, secrets }; }; diff --git a/backend/src/services/super-admin/super-admin-service.ts b/backend/src/services/super-admin/super-admin-service.ts index 41fd14d85..d76ad3ec3 100644 --- a/backend/src/services/super-admin/super-admin-service.ts +++ b/backend/src/services/super-admin/super-admin-service.ts @@ -1,4 +1,5 @@ import { TSuperAdmin, TSuperAdminUpdate } from "@app/db/schemas"; +import { TKeyStoreFactory } from "@app/keystore/keystore"; import { getConfig } from "@app/lib/config/env"; import { BadRequestError } from "@app/lib/errors"; @@ -14,6 +15,7 @@ type TSuperAdminServiceFactoryDep = { userDAL: TUserDALFactory; authService: Pick; orgService: Pick; + keyStore: Pick; }; export type TSuperAdminServiceFactory = ReturnType; @@ -21,26 +23,53 @@ export type TSuperAdminServiceFactory = ReturnType Promise; +const ADMIN_CONFIG_KEY = "infisical-admin-cfg"; +const ADMIN_CONFIG_KEY_EXP = 60; // 60s +const ADMIN_CONFIG_DB_UUID = "00000000-0000-0000-0000-000000000000"; + export const superAdminServiceFactory = ({ serverCfgDAL, userDAL, authService, - orgService + orgService, + keyStore }: TSuperAdminServiceFactoryDep) => { const initServerCfg = async () => { // TODO(akhilmhdh): bad pattern time less change this later to me itself - getServerCfg = () => serverCfgDAL.findOne({}); + getServerCfg = async () => { + const config = await keyStore.getItem(ADMIN_CONFIG_KEY); + // missing in keystore means fetch from db + if (!config) { + const serverCfg = await serverCfgDAL.findById(ADMIN_CONFIG_DB_UUID); + if (serverCfg) { + await keyStore.setItemWithExpiry(ADMIN_CONFIG_KEY, ADMIN_CONFIG_KEY_EXP, JSON.stringify(serverCfg)); // insert it back to keystore + } + return serverCfg; + } - const serverCfg = await serverCfgDAL.findOne({}); + const keyStoreServerCfg = JSON.parse(config) as TSuperAdmin; + return { + ...keyStoreServerCfg, + // this is to allow admin router to work + createdAt: new Date(keyStoreServerCfg.createdAt), + updatedAt: new Date(keyStoreServerCfg.updatedAt) + }; + }; + + // reset on initialized + await keyStore.deleteItem(ADMIN_CONFIG_KEY); + const serverCfg = await serverCfgDAL.findById(ADMIN_CONFIG_DB_UUID); if (serverCfg) return; - const newCfg = await serverCfgDAL.create({ initialized: false, allowSignUp: true }); + + // @ts-expect-error id is kept as fixed for idempotence and to avoid race condition + const newCfg = await serverCfgDAL.create({ initialized: false, allowSignUp: true, id: ADMIN_CONFIG_DB_UUID }); return newCfg; }; const updateServerCfg = async (data: TSuperAdminUpdate) => { - const serverCfg = await getServerCfg(); - const cfg = await serverCfgDAL.updateById(serverCfg.id, data); - return cfg; + const updatedServerCfg = await serverCfgDAL.updateById(ADMIN_CONFIG_DB_UUID, data); + await keyStore.setItemWithExpiry(ADMIN_CONFIG_KEY, ADMIN_CONFIG_KEY_EXP, JSON.stringify(updatedServerCfg)); + return updatedServerCfg; }; const adminSignUp = async ({ diff --git a/backend/src/services/telemetry/telemetry-dal.ts b/backend/src/services/telemetry/telemetry-dal.ts new file mode 100644 index 000000000..9fac4f1ef --- /dev/null +++ b/backend/src/services/telemetry/telemetry-dal.ts @@ -0,0 +1,39 @@ +import { TDbClient } from "@app/db"; +import { TableName } from "@app/db/schemas"; +import { DatabaseError } from "@app/lib/errors"; + +export type TTelemetryDALFactory = ReturnType; + +export const telemetryDALFactory = (db: TDbClient) => { + const getTelemetryInstanceStats = async () => { + try { + const userCount = (await db(TableName.Users).where({ isGhost: false }).count().first())?.count as string; + const users = parseInt(userCount || "0", 10); + + const identityCount = (await db(TableName.Identity).count().first())?.count as string; + const identities = parseInt(identityCount || "0", 10); + + const projectCount = (await db(TableName.Project).count().first())?.count as string; + const projects = parseInt(projectCount || "0", 10); + + const secretCount = (await db(TableName.Secret).count().first())?.count as string; + const secrets = parseInt(secretCount || "0", 10); + + const organizationNames = await db(TableName.Organization).select("name"); + const organizations = organizationNames.length; + + return { + users, + identities, + projects, + secrets, + organizations, + organizationNames: organizationNames.map(({ name }) => name) + }; + } catch (error) { + throw new DatabaseError({ error, name: "TelemtryInstanceStats" }); + } + }; + + return { getTelemetryInstanceStats }; +}; diff --git a/backend/src/services/telemetry/telemetry-queue.ts b/backend/src/services/telemetry/telemetry-queue.ts new file mode 100644 index 000000000..02e906fe6 --- /dev/null +++ b/backend/src/services/telemetry/telemetry-queue.ts @@ -0,0 +1,78 @@ +import { PostHog } from "posthog-node"; + +import { TKeyStoreFactory } from "@app/keystore/keystore"; +import { getConfig } from "@app/lib/config/env"; +import { logger } from "@app/lib/logger"; +import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; + +import { getServerCfg } from "../super-admin/super-admin-service"; +import { TTelemetryDALFactory } from "./telemetry-dal"; +import { TELEMETRY_SECRET_OPERATIONS_KEY, TELEMETRY_SECRET_PROCESSED_KEY } from "./telemetry-service"; +import { PostHogEventTypes } from "./telemetry-types"; + +type TTelemetryQueueServiceFactoryDep = { + queueService: TQueueServiceFactory; + keyStore: Pick; + telemetryDAL: TTelemetryDALFactory; +}; + +export type TTelemetryQueueServiceFactory = ReturnType; + +export const telemetryQueueServiceFactory = ({ + queueService, + keyStore, + telemetryDAL +}: TTelemetryQueueServiceFactoryDep) => { + const appCfg = getConfig(); + const postHog = + appCfg.isProductionMode && appCfg.TELEMETRY_ENABLED + ? new PostHog(appCfg.POSTHOG_PROJECT_API_KEY, { host: appCfg.POSTHOG_HOST, flushAt: 1, flushInterval: 0 }) + : undefined; + + queueService.start(QueueName.TelemetryInstanceStats, async () => { + const { instanceId } = await getServerCfg(); + const telemtryStats = await telemetryDAL.getTelemetryInstanceStats(); + // parse the redis values into integer + const numberOfSecretOperationsMade = parseInt((await keyStore.getItem(TELEMETRY_SECRET_OPERATIONS_KEY)) || "0", 10); + const numberOfSecretProcessed = parseInt((await keyStore.getItem(TELEMETRY_SECRET_PROCESSED_KEY)) || "0", 10); + const stats = { ...telemtryStats, numberOfSecretProcessed, numberOfSecretOperationsMade }; + + // send to postHog + postHog?.capture({ + event: PostHogEventTypes.TelemetryInstanceStats, + distinctId: instanceId, + properties: stats + }); + // reset the stats + await keyStore.deleteItem(TELEMETRY_SECRET_PROCESSED_KEY); + await keyStore.deleteItem(TELEMETRY_SECRET_OPERATIONS_KEY); + }); + + // every day at midnight a telemetry job executes on self hosted + // this sends some telemetry information like instance id secrets operated etc + const startTelemetryCheck = async () => { + // this is a fast way to check its cloud or not + if (appCfg.INFISICAL_CLOUD) return; + // clear previous job + await queueService.stopRepeatableJob( + QueueName.TelemetryInstanceStats, + QueueJobs.TelemetryInstanceStats, + { pattern: "0 0 * * *", utc: true }, + QueueName.TelemetryInstanceStats // just a job id + ); + if (postHog) { + await queueService.queue(QueueName.TelemetryInstanceStats, QueueJobs.TelemetryInstanceStats, undefined, { + jobId: QueueName.TelemetryInstanceStats, + repeat: { pattern: "0 0 * * *", utc: true } + }); + } + }; + + queueService.listen(QueueName.TelemetryInstanceStats, "failed", (err) => { + logger.error(err?.failedReason, `${QueueName.TelemetryInstanceStats}: failed`); + }); + + return { + startTelemetryCheck + }; +}; diff --git a/backend/src/services/telemetry/telemetry-service.ts b/backend/src/services/telemetry/telemetry-service.ts index a1a4b78f7..9912e0101 100644 --- a/backend/src/services/telemetry/telemetry-service.ts +++ b/backend/src/services/telemetry/telemetry-service.ts @@ -1,15 +1,24 @@ import { PostHog } from "posthog-node"; +import { TLicenseServiceFactory } from "@app/ee/services/license/license-service"; +import { InstanceType } from "@app/ee/services/license/license-types"; +import { TKeyStoreFactory } from "@app/keystore/keystore"; import { getConfig } from "@app/lib/config/env"; import { request } from "@app/lib/config/request"; import { logger } from "@app/lib/logger"; -import { TPostHogEvent } from "./telemetry-types"; +import { PostHogEventTypes, TPostHogEvent, TSecretModifiedEvent } from "./telemetry-types"; + +export const TELEMETRY_SECRET_PROCESSED_KEY = "telemetry-secret-processed"; +export const TELEMETRY_SECRET_OPERATIONS_KEY = "telemetry-secret-operations"; export type TTelemetryServiceFactory = ReturnType; +export type TTelemetryServiceFactoryDep = { + keyStore: Pick; + licenseService: Pick; +}; -// type TTelemetryServiceFactoryDep = {}; -export const telemetryServiceFactory = () => { +export const telemetryServiceFactory = ({ keyStore, licenseService }: TTelemetryServiceFactoryDep) => { const appCfg = getConfig(); if (appCfg.isProductionMode && !appCfg.TELEMETRY_ENABLED) { @@ -21,10 +30,9 @@ To opt into telemetry, you can set "TELEMETRY_ENABLED=true" within the environme `); } - const postHog = - appCfg.isProductionMode && appCfg.TELEMETRY_ENABLED - ? new PostHog(appCfg.POSTHOG_PROJECT_API_KEY, { host: appCfg.POSTHOG_HOST }) - : undefined; + const postHog = appCfg.TELEMETRY_ENABLED + ? new PostHog(appCfg.POSTHOG_PROJECT_API_KEY, { host: appCfg.POSTHOG_HOST }) + : undefined; // used for email marketting email sending purpose const sendLoopsEvent = async (email: string, firstName?: string, lastName?: string) => { @@ -51,13 +59,33 @@ To opt into telemetry, you can set "TELEMETRY_ENABLED=true" within the environme } }; - const sendPostHogEvents = (event: TPostHogEvent) => { + const sendPostHogEvents = async (event: TPostHogEvent) => { if (postHog) { - postHog.capture({ - event: event.event, - distinctId: event.distinctId, - properties: event.properties - }); + const instanceType = licenseService.getInstanceType(); + // capture posthog only when its cloud or signup event happens in self hosted + if (instanceType === InstanceType.Cloud || event.event === PostHogEventTypes.UserSignedUp) { + postHog.capture({ + event: event.event, + distinctId: event.distinctId, + properties: event.properties + }); + return; + } + + if ( + [ + PostHogEventTypes.SecretPulled, + PostHogEventTypes.SecretCreated, + PostHogEventTypes.SecretDeleted, + PostHogEventTypes.SecretUpdated + ].includes(event.event) + ) { + await keyStore.incrementBy( + TELEMETRY_SECRET_PROCESSED_KEY, + (event as TSecretModifiedEvent).properties.numberOfSecrets + ); + await keyStore.incrementBy(TELEMETRY_SECRET_OPERATIONS_KEY, 1); + } } }; diff --git a/backend/src/services/telemetry/telemetry-types.ts b/backend/src/services/telemetry/telemetry-types.ts index 0f92da4da..947403783 100644 --- a/backend/src/services/telemetry/telemetry-types.ts +++ b/backend/src/services/telemetry/telemetry-types.ts @@ -12,7 +12,8 @@ export enum PostHogEventTypes { ProjectCreated = "Project Created", IntegrationCreated = "Integration Created", MachineIdentityCreated = "Machine Identity Created", - UserOrgInvitation = "User Org Invitation" + UserOrgInvitation = "User Org Invitation", + TelemetryInstanceStats = "Self Hosted Instance Stats" } export type TSecretModifiedEvent = { @@ -101,6 +102,20 @@ export type TUserOrgInvitedEvent = { }; }; +export type TTelemetryInstanceStatsEvent = { + event: PostHogEventTypes.TelemetryInstanceStats; + properties: { + users: number; + identities: number; + projects: number; + secrets: number; + organizations: number; + organizationNames: number; + numberOfSecretOperationsMade: number; + numberOfSecretProcessed: number; + }; +}; + export type TPostHogEvent = { distinctId: string } & ( | TSecretModifiedEvent | TAdminInitEvent @@ -110,4 +125,5 @@ export type TPostHogEvent = { distinctId: string } & ( | TMachineIdentityCreatedEvent | TIntegrationCreatedEvent | TProjectCreateEvent + | TTelemetryInstanceStatsEvent ); diff --git a/cli/agent-config.yaml b/cli/agent-config.yaml index ae130d3a8..e767fdca5 100644 --- a/cli/agent-config.yaml +++ b/cli/agent-config.yaml @@ -1,5 +1,5 @@ infisical: - address: "http://localhost:8080" + address: "https://app.infisical.com/" auth: type: "universal-auth" config: @@ -13,3 +13,12 @@ sinks: templates: - source-path: my-dot-ev-secret-template destination-path: my-dot-env.env + config: + polling-interval: 60s + execute: + command: docker-compose -f docker-compose.prod.yml down && docker-compose -f docker-compose.prod.yml up -d + - source-path: my-dot-ev-secret-template1 + destination-path: my-dot-env-1.env + config: + exec: + command: mkdir hello-world1 diff --git a/cli/packages/api/api.go b/cli/packages/api/api.go index 9ae356cd0..1b69b8f23 100644 --- a/cli/packages/api/api.go +++ b/cli/packages/api/api.go @@ -145,6 +145,25 @@ func CallLogin2V2(httpClient *resty.Client, request GetLoginTwoV2Request) (GetLo return loginTwoV2Response, nil } +func CallGetAllOrganizations(httpClient *resty.Client) (GetOrganizationsResponse, error) { + var orgResponse GetOrganizationsResponse + response, err := httpClient. + R(). + SetResult(&orgResponse). + SetHeader("User-Agent", USER_AGENT). + Get(fmt.Sprintf("%v/v1/organization", config.INFISICAL_URL)) + + if err != nil { + return GetOrganizationsResponse{}, err + } + + if response.IsError() { + return GetOrganizationsResponse{}, fmt.Errorf("CallGetAllOrganizations: Unsuccessful response: [response=%v]", response) + } + + return orgResponse, nil +} + func CallGetAllWorkSpacesUserBelongsTo(httpClient *resty.Client) (GetWorkSpacesResponse, error) { var workSpacesResponse GetWorkSpacesResponse response, err := httpClient. @@ -490,5 +509,7 @@ func CallGetRawSecretsV3(httpClient *resty.Client, request GetRawSecretsV3Reques return GetRawSecretsV3Response{}, fmt.Errorf("CallGetRawSecretsV3: Unsuccessful response [%v %v] [status-code=%v] [response=%v]", response.Request.Method, response.Request.URL, response.StatusCode(), response.String()) } + getRawSecretsV3Response.ETag = response.Header().Get(("etag")) + return getRawSecretsV3Response, nil } diff --git a/cli/packages/api/model.go b/cli/packages/api/model.go index 3c6466382..586a23511 100644 --- a/cli/packages/api/model.go +++ b/cli/packages/api/model.go @@ -120,14 +120,21 @@ type PullSecretsByInfisicalTokenResponse struct { type GetWorkSpacesResponse struct { Workspaces []struct { - ID string `json:"_id"` - Name string `json:"name"` - Plan string `json:"plan,omitempty"` - V int `json:"__v"` - Organization string `json:"organization,omitempty"` + ID string `json:"_id"` + Name string `json:"name"` + Plan string `json:"plan,omitempty"` + V int `json:"__v"` + OrganizationId string `json:"orgId"` } `json:"workspaces"` } +type GetOrganizationsResponse struct { + Organizations []struct { + ID string `json:"id"` + Name string `json:"name"` + } `json:"organizations"` +} + type Secret struct { SecretKeyCiphertext string `json:"secretKeyCiphertext,omitempty"` SecretKeyIV string `json:"secretKeyIV,omitempty"` @@ -505,4 +512,5 @@ type GetRawSecretsV3Response struct { SecretComment string `json:"secretComment"` } `json:"secrets"` Imports []any `json:"imports"` + ETag string } diff --git a/cli/packages/cmd/agent.go b/cli/packages/cmd/agent.go index 8857f5806..750727df1 100644 --- a/cli/packages/cmd/agent.go +++ b/cli/packages/cmd/agent.go @@ -5,12 +5,15 @@ package cmd import ( "bytes" + "context" "encoding/base64" "fmt" "io/ioutil" "os" + "os/exec" "os/signal" "path" + "runtime" "strings" "sync" "syscall" @@ -71,12 +74,56 @@ type Template struct { SourcePath string `yaml:"source-path"` Base64TemplateContent string `yaml:"base64-template-content"` DestinationPath string `yaml:"destination-path"` + + Config struct { // Configurations for the template + PollingInterval string `yaml:"polling-interval"` // How often to poll for changes in the secret + Execute struct { + Command string `yaml:"command"` // Command to execute once the template has been rendered + Timeout int64 `yaml:"timeout"` // Timeout for the command + } `yaml:"execute"` // Command to execute once the template has been rendered + } `yaml:"config"` } func ReadFile(filePath string) ([]byte, error) { return ioutil.ReadFile(filePath) } +func ExecuteCommandWithTimeout(command string, timeout int64) error { + + shell := [2]string{"sh", "-c"} + if runtime.GOOS == "windows" { + shell = [2]string{"cmd", "/C"} + } else { + currentShell := os.Getenv("SHELL") + if currentShell != "" { + shell[0] = currentShell + } + } + + ctx := context.Background() + if timeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second) + defer cancel() + } + + cmd := exec.CommandContext(ctx, shell[0], shell[1], command) + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + if err := cmd.Run(); err != nil { + if exitError, ok := err.(*exec.ExitError); ok { // type assertion + if exitError.ProcessState.ExitCode() == -1 { + return fmt.Errorf("command timed out") + } + } + return err + } else { + return nil + } +} + func FileExists(filepath string) bool { info, err := os.Stat(filepath) if os.IsNotExist(err) { @@ -170,20 +217,24 @@ func ParseAgentConfig(configFile []byte) (*Config, error) { return config, nil } -func secretTemplateFunction(accessToken string) func(string, string, string) ([]models.SingleEnvironmentVariable, error) { +func secretTemplateFunction(accessToken string, existingEtag string, currentEtag *string) func(string, string, string) ([]models.SingleEnvironmentVariable, error) { return func(projectID, envSlug, secretPath string) ([]models.SingleEnvironmentVariable, error) { - secrets, err := util.GetPlainTextSecretsViaMachineIdentity(accessToken, projectID, envSlug, secretPath, false) + res, err := util.GetPlainTextSecretsViaMachineIdentity(accessToken, projectID, envSlug, secretPath, false) if err != nil { return nil, err } - return secrets, nil + if existingEtag != res.Etag { + *currentEtag = res.Etag + } + + return res.Secrets, nil } } -func ProcessTemplate(templatePath string, data interface{}, accessToken string) (*bytes.Buffer, error) { +func ProcessTemplate(templatePath string, data interface{}, accessToken string, existingEtag string, currentEtag *string) (*bytes.Buffer, error) { // custom template function to fetch secrets from Infisical - secretFunction := secretTemplateFunction(accessToken) + secretFunction := secretTemplateFunction(accessToken, existingEtag, currentEtag) funcs := template.FuncMap{ "secret": secretFunction, } @@ -203,7 +254,7 @@ func ProcessTemplate(templatePath string, data interface{}, accessToken string) return &buf, nil } -func ProcessBase64Template(encodedTemplate string, data interface{}, accessToken string) (*bytes.Buffer, error) { +func ProcessBase64Template(encodedTemplate string, data interface{}, accessToken string, existingEtag string, currentEtag *string) (*bytes.Buffer, error) { // custom template function to fetch secrets from Infisical decoded, err := base64.StdEncoding.DecodeString(encodedTemplate) if err != nil { @@ -212,7 +263,7 @@ func ProcessBase64Template(encodedTemplate string, data interface{}, accessToken templateString := string(decoded) - secretFunction := secretTemplateFunction(accessToken) + secretFunction := secretTemplateFunction(accessToken, existingEtag, currentEtag) // TODO: Fix this funcs := template.FuncMap{ "secret": secretFunction, } @@ -250,7 +301,16 @@ type TokenManager struct { } func NewTokenManager(fileDeposits []Sink, templates []Template, clientIdPath string, clientSecretPath string, newAccessTokenNotificationChan chan bool, removeClientSecretOnRead bool, exitAfterAuth bool) *TokenManager { - return &TokenManager{filePaths: fileDeposits, templates: templates, clientIdPath: clientIdPath, clientSecretPath: clientSecretPath, newAccessTokenNotificationChan: newAccessTokenNotificationChan, removeClientSecretOnRead: removeClientSecretOnRead, exitAfterAuth: exitAfterAuth} + return &TokenManager{ + filePaths: fileDeposits, + templates: templates, + clientIdPath: clientIdPath, + clientSecretPath: clientSecretPath, + newAccessTokenNotificationChan: newAccessTokenNotificationChan, + removeClientSecretOnRead: removeClientSecretOnRead, + exitAfterAuth: exitAfterAuth, + } + } func (tm *TokenManager) SetToken(token string, accessTokenTTL time.Duration, accessTokenMaxTTL time.Duration) { @@ -428,38 +488,80 @@ func (tm *TokenManager) WriteTokenToFiles() { } } -func (tm *TokenManager) FetchSecrets() { - log.Info().Msgf("template engine started...") +func (tm *TokenManager) WriteTemplateToFile(bytes *bytes.Buffer, template *Template) { + if err := WriteBytesToFile(bytes, template.DestinationPath); err != nil { + log.Error().Msgf("template engine: unable to write secrets to path because %s. Will try again on next cycle", err) + return + } + log.Info().Msgf("template engine: secret template at path %s has been rendered and saved to path %s", template.SourcePath, template.DestinationPath) +} + +func (tm *TokenManager) MonitorSecretChanges(secretTemplate Template, sigChan chan os.Signal) { + + pollingInterval := time.Duration(5 * time.Minute) + + if secretTemplate.Config.PollingInterval != "" { + interval, err := util.ConvertPollingIntervalToTime(secretTemplate.Config.PollingInterval) + + if err != nil { + log.Error().Msgf("unable to convert polling interval to time because %v", err) + sigChan <- syscall.SIGINT + return + + } else { + pollingInterval = interval + } + } + + var existingEtag string + var currentEtag string + var firstRun = true + + execTimeout := secretTemplate.Config.Execute.Timeout + execCommand := secretTemplate.Config.Execute.Command + for { token := tm.GetToken() + if token != "" { - for _, secretTemplate := range tm.templates { - var processedTemplate *bytes.Buffer - var err error - if secretTemplate.SourcePath != "" { - processedTemplate, err = ProcessTemplate(secretTemplate.SourcePath, nil, token) - } else { - processedTemplate, err = ProcessBase64Template(secretTemplate.Base64TemplateContent, nil, token) - } - if err != nil { - log.Error().Msgf("template engine: unable to render secrets because %s. Will try again on next cycle", err) + var processedTemplate *bytes.Buffer + var err error - continue - } - - if err := WriteBytesToFile(processedTemplate, secretTemplate.DestinationPath); err != nil { - log.Error().Msgf("template engine: unable to write secrets to path because %s. Will try again on next cycle", err) - - continue - } - - log.Info().Msgf("template engine: secret template at path %s has been rendered and saved to path %s", secretTemplate.SourcePath, secretTemplate.DestinationPath) + if secretTemplate.SourcePath != "" { + processedTemplate, err = ProcessTemplate(secretTemplate.SourcePath, nil, token, existingEtag, ¤tEtag) + } else { + processedTemplate, err = ProcessBase64Template(secretTemplate.Base64TemplateContent, nil, token, existingEtag, ¤tEtag) } - // fetch new secrets every 5 minutes (TODO: add PubSub in the future ) - time.Sleep(5 * time.Minute) + if err != nil { + log.Error().Msgf("unable to process template because %v", err) + } else { + if (existingEtag != currentEtag) || firstRun { + + tm.WriteTemplateToFile(processedTemplate, &secretTemplate) + existingEtag = currentEtag + + if !firstRun && execCommand != "" { + log.Info().Msgf("executing command: %s", execCommand) + err := ExecuteCommandWithTimeout(execCommand, execTimeout) + + if err != nil { + log.Error().Msgf("unable to execute command because %v", err) + } + + } + if firstRun { + firstRun = false + } + } + } + time.Sleep(pollingInterval) + } else { + // It fails to get the access token. So we will re-try in 3 seconds. We do this because if we don't, the user will have to wait for the next polling interval to get the first secret render. + time.Sleep(3 * time.Second) } + } } @@ -544,7 +646,11 @@ var agentCmd = &cobra.Command{ tm := NewTokenManager(filePaths, agentConfig.Templates, configUniversalAuthType.ClientIDPath, configUniversalAuthType.ClientSecretPath, tokenRefreshNotifier, configUniversalAuthType.RemoveClientSecretOnRead, agentConfig.Infisical.ExitAfterAuth) go tm.ManageTokenLifecycle() - go tm.FetchSecrets() + + for i, template := range agentConfig.Templates { + log.Info().Msgf("template engine started for template %v...", i+1) + go tm.MonitorSecretChanges(template, sigChan) + } for { select { diff --git a/cli/packages/cmd/export.go b/cli/packages/cmd/export.go index d0db485b3..26cc45f65 100644 --- a/cli/packages/cmd/export.go +++ b/cli/packages/cmd/export.go @@ -87,16 +87,12 @@ var exportCmd = &cobra.Command{ var output string if shouldExpandSecrets { - substitutions := util.ExpandSecrets(secrets, infisicalToken, "") - output, err = formatEnvs(substitutions, format) - if err != nil { - util.HandleError(err) - } - } else { - output, err = formatEnvs(secrets, format) - if err != nil { - util.HandleError(err) - } + secrets = util.ExpandSecrets(secrets, infisicalToken, "") + } + secrets = util.FilterSecretsByTag(secrets, tagSlugs) + output, err = formatEnvs(secrets, format) + if err != nil { + util.HandleError(err) } fmt.Print(output) diff --git a/cli/packages/cmd/init.go b/cli/packages/cmd/init.go index 070074fa9..e47eb447e 100644 --- a/cli/packages/cmd/init.go +++ b/cli/packages/cmd/init.go @@ -5,7 +5,6 @@ package cmd import ( "encoding/json" - "fmt" "github.com/Infisical/infisical-merge/packages/api" "github.com/Infisical/infisical-merge/packages/models" @@ -52,25 +51,19 @@ var initCmd = &cobra.Command{ httpClient := resty.New() httpClient.SetAuthToken(userCreds.UserCredentials.JTWToken) - workspaceResponse, err := api.CallGetAllWorkSpacesUserBelongsTo(httpClient) + + organizationResponse, err := api.CallGetAllOrganizations(httpClient) if err != nil { - util.HandleError(err, "Unable to pull projects that belong to you") + util.HandleError(err, "Unable to pull organizations that belong to you") } - workspaces := workspaceResponse.Workspaces - if len(workspaces) == 0 { - message := fmt.Sprintf("You don't have any projects created in Infisical. You must first create a project at %s", util.INFISICAL_TOKEN_NAME) - util.PrintErrorMessageAndExit(message) - } + organizations := organizationResponse.Organizations - var workspaceNames []string - for _, workspace := range workspaces { - workspaceNames = append(workspaceNames, workspace.Name) - } + organizationNames := util.GetOrganizationsNameList(organizationResponse) prompt := promptui.Select{ - Label: "Which of your Infisical projects would you like to connect this project to?", - Items: workspaceNames, + Label: "Which Infisical organization would you like to select a project from?", + Items: organizationNames, Size: 7, } @@ -79,7 +72,27 @@ var initCmd = &cobra.Command{ util.HandleError(err) } - err = writeWorkspaceFile(workspaces[index]) + selectedOrganization := organizations[index] + + workspaceResponse, err := api.CallGetAllWorkSpacesUserBelongsTo(httpClient) + if err != nil { + util.HandleError(err, "Unable to pull projects that belong to you") + } + + filteredWorkspaces, workspaceNames := util.GetWorkspacesInOrganization(workspaceResponse, selectedOrganization.ID) + + prompt = promptui.Select{ + Label: "Which of your Infisical projects would you like to connect this project to?", + Items: workspaceNames, + Size: 7, + } + + index, _, err = prompt.Run() + if err != nil { + util.HandleError(err) + } + + err = writeWorkspaceFile(filteredWorkspaces[index]) if err != nil { util.HandleError(err) } diff --git a/cli/packages/models/cli.go b/cli/packages/models/cli.go index 165982a77..c3999f68b 100644 --- a/cli/packages/models/cli.go +++ b/cli/packages/models/cli.go @@ -34,17 +34,22 @@ type SingleEnvironmentVariable struct { Comment string `json:"comment"` } +type PlaintextSecretResult struct { + Secrets []SingleEnvironmentVariable + Etag string +} + type SingleFolder struct { ID string `json:"_id"` Name string `json:"name"` } type Workspace struct { - ID string `json:"_id"` - Name string `json:"name"` - Plan string `json:"plan,omitempty"` - V int `json:"__v"` - Organization string `json:"organization,omitempty"` + ID string `json:"_id"` + Name string `json:"name"` + Plan string `json:"plan,omitempty"` + V int `json:"__v"` + OrganizationId string `json:"orgId"` } type WorkspaceConfigFile struct { diff --git a/cli/packages/util/agent.go b/cli/packages/util/agent.go new file mode 100644 index 000000000..188ae5de2 --- /dev/null +++ b/cli/packages/util/agent.go @@ -0,0 +1,41 @@ +package util + +import ( + "fmt" + "strconv" + "time" +) + +// ConvertPollingIntervalToTime converts a string representation of a polling interval to a time.Duration +func ConvertPollingIntervalToTime(pollingInterval string) (time.Duration, error) { + length := len(pollingInterval) + if length < 2 { + return 0, fmt.Errorf("invalid format") + } + + unit := pollingInterval[length-1:] + numberPart := pollingInterval[:length-1] + + number, err := strconv.Atoi(numberPart) + if err != nil { + return 0, err + } + + switch unit { + case "s": + if number < 60 { + return 0, fmt.Errorf("polling interval should be at least 60 seconds") + } + return time.Duration(number) * time.Second, nil + case "m": + return time.Duration(number) * time.Minute, nil + case "h": + return time.Duration(number) * time.Hour, nil + case "d": + return time.Duration(number) * 24 * time.Hour, nil + case "w": + return time.Duration(number) * 7 * 24 * time.Hour, nil + default: + return 0, fmt.Errorf("invalid time unit") + } +} diff --git a/cli/packages/util/init.go b/cli/packages/util/init.go new file mode 100644 index 000000000..33350f3b7 --- /dev/null +++ b/cli/packages/util/init.go @@ -0,0 +1,45 @@ +package util + +import ( + "fmt" + + "github.com/Infisical/infisical-merge/packages/api" + "github.com/Infisical/infisical-merge/packages/models" +) + +func GetOrganizationsNameList(organizationResponse api.GetOrganizationsResponse) []string { + organizations := organizationResponse.Organizations + + if len(organizations) == 0 { + message := fmt.Sprintf("You don't have any organization created in Infisical. You must first create a organization at %s", INFISICAL_DEFAULT_URL) + PrintErrorMessageAndExit(message) + } + + var organizationNames []string + for _, workspace := range organizations { + organizationNames = append(organizationNames, workspace.Name) + } + + return organizationNames +} + +func GetWorkspacesInOrganization(workspaceResponse api.GetWorkSpacesResponse, orgId string) ([]models.Workspace, []string) { + workspaces := workspaceResponse.Workspaces + + var filteredWorkspaces []models.Workspace + var workspaceNames []string + + for _, workspace := range workspaces { + if workspace.OrganizationId == orgId { + filteredWorkspaces = append(filteredWorkspaces, workspace) + workspaceNames = append(workspaceNames, workspace.Name) + } + } + + if len(filteredWorkspaces) == 0 { + message := fmt.Sprintf("You don't have any projects created in Infisical organization. You must first create a project at %s", INFISICAL_DEFAULT_URL) + PrintErrorMessageAndExit(message) + } + + return filteredWorkspaces, workspaceNames +} diff --git a/cli/packages/util/secrets.go b/cli/packages/util/secrets.go index a260aaaec..cc75681e8 100644 --- a/cli/packages/util/secrets.go +++ b/cli/packages/util/secrets.go @@ -152,7 +152,7 @@ func GetPlainTextSecretsViaJTW(JTWToken string, receiversPrivateKey string, work return plainTextSecrets, nil } -func GetPlainTextSecretsViaMachineIdentity(accessToken string, workspaceId string, environmentName string, secretsPath string, includeImports bool) ([]models.SingleEnvironmentVariable, error) { +func GetPlainTextSecretsViaMachineIdentity(accessToken string, workspaceId string, environmentName string, secretsPath string, includeImports bool) (models.PlaintextSecretResult, error) { httpClient := resty.New() httpClient.SetAuthToken(accessToken). SetHeader("Accept", "application/json") @@ -170,12 +170,12 @@ func GetPlainTextSecretsViaMachineIdentity(accessToken string, workspaceId strin rawSecrets, err := api.CallGetRawSecretsV3(httpClient, api.GetRawSecretsV3Request{WorkspaceId: workspaceId, SecretPath: secretsPath, Environment: environmentName}) if err != nil { - return nil, err + return models.PlaintextSecretResult{}, err } plainTextSecrets := []models.SingleEnvironmentVariable{} if err != nil { - return nil, fmt.Errorf("unable to decrypt your secrets [err=%v]", err) + return models.PlaintextSecretResult{}, fmt.Errorf("unable to decrypt your secrets [err=%v]", err) } for _, secret := range rawSecrets.Secrets { @@ -189,7 +189,10 @@ func GetPlainTextSecretsViaMachineIdentity(accessToken string, workspaceId strin // } // } - return plainTextSecrets, nil + return models.PlaintextSecretResult{ + Secrets: plainTextSecrets, + Etag: rawSecrets.ETag, + }, nil } func InjectImportedSecret(plainTextWorkspaceKey []byte, secrets []models.SingleEnvironmentVariable, importedSecrets []api.ImportedSecretV3) ([]models.SingleEnvironmentVariable, error) { @@ -220,6 +223,30 @@ func InjectImportedSecret(plainTextWorkspaceKey []byte, secrets []models.SingleE return secrets, nil } +func FilterSecretsByTag(plainTextSecrets []models.SingleEnvironmentVariable, tagSlugs string) []models.SingleEnvironmentVariable { + if tagSlugs == "" { + return plainTextSecrets + } + + tagSlugsMap := make(map[string]bool) + tagSlugsList := strings.Split(tagSlugs, ",") + for _, slug := range tagSlugsList { + tagSlugsMap[slug] = true + } + + filteredSecrets := []models.SingleEnvironmentVariable{} + for _, secret := range plainTextSecrets { + for _, tag := range secret.Tags { + if tagSlugsMap[tag.Slug] { + filteredSecrets = append(filteredSecrets, secret) + break + } + } + } + + return filteredSecrets +} + func GetAllEnvironmentVariables(params models.GetAllSecretsParameters, projectConfigFilePath string) ([]models.SingleEnvironmentVariable, error) { var infisicalToken string if params.InfisicalToken == "" { diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 8c1848ad3..656448705 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -86,6 +86,7 @@ services: environment: - NODE_ENV=development - DB_CONNECTION_URI=postgres://infisical:infisical@db/infisical?sslmode=disable + - TELEMETRY_ENABLED=false volumes: - ./backend/src:/app/src diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index f26f17284..86a8e4cca 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -52,7 +52,7 @@ services: restart: always env_file: .env volumes: - - pg_data:/data/db + - pg_data:/var/lib/postgresql/data networks: - infisical healthcheck: diff --git a/docs/changelog/overview.mdx b/docs/changelog/overview.mdx index ca62fe94b..d73c0bb14 100644 --- a/docs/changelog/overview.mdx +++ b/docs/changelog/overview.mdx @@ -4,7 +4,22 @@ title: "Changelog" The changelog below reflects new product developments and updates on a monthly basis. -## January 2024 +## Feb 2024 +- Added org-scoped authentication enforcement for SAML +- Added support for [SCIM](https://infisical.com/docs/documentation/platform/scim/overview) along with instructions for setting it up with [Okta](https://infisical.com/docs/documentation/platform/scim/okta), [Azure](https://infisical.com/docs/documentation/platform/scim/azure), and [JumpCloud](https://infisical.com/docs/documentation/platform/scim/jumpcloud). +- Pushed out project update for non-E2EE w/ new endpoints like for project creation and member invitation. +- Added API Integration testing for new backend. +- Added capability to create projects in Terraform. +- Added slug-based capabilities to both organizations and projects to gradually make the API more developer-friendly moving forward. +- Fixed + improved various analytics/telemetry-related items. +- Fixed various issues associated with the Python SDK: build during installation on Mac OS, Rust dependency. +- Updated self-hosting documentation to reflect [new backend](https://infisical.com/docs/self-hosting/overview). +- Released [Postgres-based Infisical helm chart](https://cloudsmith.io/~infisical/repos/helm-charts/packages/detail/helm/infisical-standalone/). +- Added checks to ensure that breaking API changes don't get released. +- Automated API reference documentation to be inline with latest releases of Infisical. + +## Jan 2024 +- Completed Postgres migration initiative with restructed Fastify-based backend. - Reduced size of Infisical Node.js SDK by ≈90%. - Added secret fallback support to all SDK's. - Added Machine Identity support to [Terraform Provider](https://github.com/Infisical/terraform-provider-infisical). @@ -12,21 +27,21 @@ The changelog below reflects new product developments and updates on a monthly b - Added symmetric encryption support to all SDK's. - Fixed secret reminders bug, where reminders were not being updated correctly. -## December 2023 +## Dec 2023 - Released [(machine) identities](https://infisical.com/docs/documentation/platform/identities/overview) and [universal auth](https://infisical.com/docs/documentation/platform/identities/universal-auth) features. - Created new cross-language SDKs for [Python](https://infisical.com/docs/sdks/languages/python), [Node](https://infisical.com/docs/sdks/languages/node), and [Java](https://infisical.com/docs/sdks/languages/java). - Released first version of the [Infisical Agent](https://infisical.com/docs/infisical-agent/overview) - Added ability to [manage folders via CLI](https://infisical.com/docs/cli/commands/secrets). -## November 2023 +## Nov 2023 - Replaced internal [Winston](https://github.com/winstonjs/winston) with [Pino](https://github.com/pinojs/pino) logging library with external logging to AWS CloudWatch - Added admin panel to self-hosting experience. - Released [secret rotation](https://infisical.com/docs/documentation/platform/secret-rotation/overview) feature with preliminary support for rotating [SendGrid](https://infisical.com/docs/documentation/platform/secret-rotation/sendgrid), [PostgreSQL/CockroachDB](https://infisical.com/docs/documentation/platform/secret-rotation/postgres), and [MySQL/MariaDB](https://infisical.com/docs/documentation/platform/secret-rotation/mysql) credentials. - Released secret reminders feature. -## October 2023 +## Oct 2023 - Added support for [GitLab SSO](https://infisical.com/docs/documentation/platform/sso/gitlab). - Became SOC 2 (Type II) certified. @@ -35,7 +50,7 @@ The changelog below reflects new product developments and updates on a monthly b - Added native [Hasura Cloud integration](https://infisical.com/docs/integrations/cloud/hasura-cloud). - Updated resource deletion logic for user, organization, and project deletion. -## September 2023 +## Sep 2023 - Released [secret approvals](https://infisical.com/docs/documentation/platform/pr-workflows) feature. - Released an update to access controls; every user role now clearly defines and enforces a certain set of conditions across Infisical. @@ -43,7 +58,7 @@ The changelog below reflects new product developments and updates on a monthly b - Added a native integration with [Qovery](https://infisical.com/docs/integrations/cloud/qovery). - Added service token generation capability for the CLI. -## August 2023 +## Aug 2023 - Release Audit Logs V2. - Add support for [GitHub SSO](https://infisical.com/docs/documentation/platform/sso/github). @@ -171,7 +186,7 @@ The changelog below reflects new product developments and updates on a monthly b - Added sorting capability to sort keys by name alphabetically in dashboard. - Added downloading secrets back as `.env` file capability. -## August 2022 +## Aug 2022 - Released first version of the Infisical platform with push/pull capability and end-to-end encryption. - Improved security handling of authentication tokens by storing refresh tokens in HttpOnly cookies. diff --git a/docs/contributing/platform/developing.mdx b/docs/contributing/platform/developing.mdx index 7a43c8250..a6675b6f6 100644 --- a/docs/contributing/platform/developing.mdx +++ b/docs/contributing/platform/developing.mdx @@ -16,49 +16,7 @@ git checkout -b MY_BRANCH_NAME ## Set up environment variables -Start by creating a .env file at the root of the Infisical directory then copy the contents of the file below into the .env file. - - - ```env - # Keys - # Required key for platform encryption/decryption ops - ENCRYPTION_KEY=6c1fe4e407b8911c104518103505b218 - - # JWT - # Required secrets to sign JWT tokens - JWT_SIGNUP_SECRET=3679e04ca949f914c03332aaaeba805a - JWT_REFRESH_SECRET=5f2f3c8f0159068dc2bbb3a652a716ff - JWT_AUTH_SECRET=4be6ba5602e0fa0ac6ac05c3cd4d247f - JWT_SERVICE_SECRET=f32f716d70a42c5703f4656015e76200 - - # MongoDB - # Backend will connect to the MongoDB instance at connection string MONGO_URL which can either be a ref - # to the MongoDB container instance or Mongo Cloud - # Required - MONGO_URL=mongodb://root:example@mongo:27017/?authSource=admin - - # Optional credentials for MongoDB container instance and Mongo-Express - MONGO_USERNAME=root - MONGO_PASSWORD=example - - # Website URL - # Required - SITE_URL=http://localhost:8080 - - # Mail/SMTP - SMTP_HOST='smtp-server' - SMTP_PORT='1025' - SMTP_NAME='local' - SMTP_USERNAME='team@infisical.com' - SMTP_PASSWORD= - ``` - - - - The pre-populated environment variable values above are meant to be used in development only. They should never be used in production. - - -View all available [environment variables](https://infisical.com/docs/self-hosting/configuration/envars) and guidance for each. +Start by creating a .env file at the root of the Infisical directory then copy the contents of the file linked [here](https://github.com/Infisical/infisical/blob/main/.env.example). View all available [environment variables](https://infisical.com/docs/self-hosting/configuration/envars) and guidance for each. ## Starting Infisical for development @@ -72,10 +30,7 @@ docker-compose -f docker-compose.dev.yml up --build --force-recreate ``` #### Access local server -Once all the services have spun up, browse to http://localhost:8080. To sign in, you may use the default credentials listed below. - -Email: `test@localhost.local` -Password: `testInfisical1` +Once all the services have spun up, browse to http://localhost:8080. #### Shutdown local server diff --git a/docs/documentation/getting-started/introduction.mdx b/docs/documentation/getting-started/introduction.mdx index 84d34e871..4c96b5a78 100644 --- a/docs/documentation/getting-started/introduction.mdx +++ b/docs/documentation/getting-started/introduction.mdx @@ -2,7 +2,7 @@ title: "Introduction" --- -Infisical is an [open-source](https://opensource.com/resources/what-open-source), [end-to-end encrypted](https://en.wikipedia.org/wiki/End-to-end_encryption) secret management platform for storing, managing, and syncing +Infisical is an [open-source](https://opensource.com/resources/what-open-source), [end-to-end encrypted](https://en.wikipedia.org/wiki/End-to-end_encryption) secrets management platform for storing, managing, and syncing application configuration and secrets like API keys, database credentials, and environment variables across applications and infrastructure. Start syncing environment variables with [Infisical Cloud](https://app.infisical.com) or learn how to [host Infisical](/self-hosting/overview) yourself. diff --git a/docs/documentation/platform/ldap/general.mdx b/docs/documentation/platform/ldap/general.mdx index 81c291c22..137538cff 100644 --- a/docs/documentation/platform/ldap/general.mdx +++ b/docs/documentation/platform/ldap/general.mdx @@ -5,7 +5,6 @@ description: "Log in to Infisical with LDAP" LDAP is a paid feature. - If you're using Infisical Cloud, then it is available under the **Enterprise Tier**. If you're self-hosting Infisical, then you should contact team@infisical.com to purchase an enterprise license to use it. @@ -15,11 +14,11 @@ You can configure your organization in Infisical to have members authenticate wi In Infisical, head to your Organization Settings > Authentication > LDAP Configuration and select **Set up LDAP**. - + Next, input your LDAP server settings. - + ![LDAP configuration](/images/platform/ldap/ldap-config.png) - + Here's some guidance for each field: - URL: The LDAP server to connect to such as `ldap://ldap.your-org.com`, `ldaps://ldap.myorg.com:636` (for connection over SSL/TLS), etc. @@ -30,7 +29,6 @@ You can configure your organization in Infisical to have members authenticate wi Enabling LDAP allows members in your organization to log into Infisical via LDAP. - ![LDAP toggle](/images/platform/ldap/ldap-toggle.png) \ No newline at end of file diff --git a/docs/documentation/platform/ldap/jumpcloud.mdx b/docs/documentation/platform/ldap/jumpcloud.mdx index b80cb49a1..5e7e42ca8 100644 --- a/docs/documentation/platform/ldap/jumpcloud.mdx +++ b/docs/documentation/platform/ldap/jumpcloud.mdx @@ -5,7 +5,6 @@ description: "Configure JumpCloud LDAP for Logging into Infisical" LDAP is a paid feature. - If you're using Infisical Cloud, then it is available under the **Enterprise Tier**. If you're self-hosting Infisical, then you should contact team@infisical.com to purchase an enterprise license to use it. @@ -14,24 +13,24 @@ description: "Configure JumpCloud LDAP for Logging into Infisical" In JumpCloud, head to USER MANAGEMENT > Users and create a new user via the **Manual user entry** option. This user will be used as a privileged service account to facilitate Infisical's ability to bind/search the LDAP directory. - + When creating the user, input their **First Name**, **Last Name**, **Username** (required), **Company Email** (required), and **Description**. Also, create a password for the user. - + Next, under User Security Settings and Permissions > Permission Settings, check the box next to **Enable as LDAP Bind DN**. - + ![LDAP JumpCloud](/images/platform/ldap/jumpcloud/ldap-jumpcloud-enable-bind-dn.png) In Infisical, head to your Organization Settings > Authentication > LDAP Configuration and select **Set up LDAP**. - + Next, input your JumpCloud LDAP server settings. - + ![LDAP configuration](/images/platform/ldap/ldap-config.png) - + Here's some guidance for each field: - + - URL: The LDAP server to connect to (`ldaps://ldap.jumpcloud.com:636`). - Bind DN: The distinguished name of object to bind when performing the user search (`uid=,ou=Users,o=,dc=jumpcloud,dc=com`). - Bind Pass: The password to use along with `Bind DN` when performing the user search. @@ -47,7 +46,6 @@ description: "Configure JumpCloud LDAP for Logging into Infisical" Enabling LDAP allows members in your organization to log into Infisical via LDAP. - ![LDAP toggle](/images/platform/ldap/ldap-toggle.png) diff --git a/docs/documentation/platform/ldap/overview.mdx b/docs/documentation/platform/ldap/overview.mdx index 1c5723b7e..d19095b7c 100644 --- a/docs/documentation/platform/ldap/overview.mdx +++ b/docs/documentation/platform/ldap/overview.mdx @@ -4,9 +4,9 @@ description: "Log in to Infisical with LDAP" --- LDAP is a paid feature. - + If you're using Infisical Cloud, then it is available under the **Enterprise Tier**. If you're self-hosting Infisical, - then you should contact team@infisical.com to purchase an enterprise license to use it. + then you should contact sales@infisical.com to purchase an enterprise license to use it. You can configure your organization in Infisical to have members authenticate with the platform via [LDAP](https://en.wikipedia.org/wiki/Lightweight_Directory_Access_Protocol) @@ -20,4 +20,4 @@ LDAP providers: - AWS Directory Service - Foxpass -Check out the general instructions for configuring LDAP [here](/documentation/platform/ldap/general). \ No newline at end of file +Check out the general instructions for configuring LDAP [here](/documentation/platform/ldap/general). diff --git a/docs/documentation/platform/project-upgrade.mdx b/docs/documentation/platform/project-upgrade.mdx new file mode 100644 index 000000000..8286342d0 --- /dev/null +++ b/docs/documentation/platform/project-upgrade.mdx @@ -0,0 +1,20 @@ +--- +title: "Enhancing Security and Usability: Project Upgrades" +--- + +At Infisical, we're constantly striving to elevate the security and usability standards of our platform to better serve our users. +With this commitment in mind, we're excited to introduce our latest addition, non-E2EE projects, aimed at addressing two significant issues while enhancing how clients interact with Infisical programmatically. + +Previously, users encountered a challenge where projects risked becoming inaccessible if the project creator deleted their account. +Additionally, our API lacked the capability to interact with projects without dealing with complex cryptographic operations. +These obstacles made API driven automation and collaboration a painful experience for a majority of our users. + +To overcome these limitations, our upgrade focuses on disabling end-to-end encryption (E2EE) for projects. +While this may raise eyebrows, it's important to understand that this decision is a strategic move to make Infisical easier to use and interact with. + +But what does this mean for our users? Essentially nothing, there are no changes required on your end. +Rest assured, all sensitive data remains encrypted at rest according to the latest industry standards. +Our commitment to security remains unwavering, and this upgrade is a testament to our dedication to delivering on our promises in both security and usability when it comes to secrets management. + +To increase consistency with existing and future integrations, all projects created on Infisical from now on will have end-to-end encryption (E2EE) disabled by default. +This will not only reduce confusion for end users, but will also make the Infisical API seamless to use. diff --git a/docs/documentation/platform/secret-rotation/postgres.mdx b/docs/documentation/platform/secret-rotation/postgres.mdx index f70e8a70b..b11ae1d76 100644 --- a/docs/documentation/platform/secret-rotation/postgres.mdx +++ b/docs/documentation/platform/secret-rotation/postgres.mdx @@ -1,21 +1,17 @@ --- title: "PostgreSQL/CockroachDB" -description: "Rotated database user password of a postgreSQL or cockroach db" +description: "Rotated database user password of a PostgreSQL or Cockroach DB" --- Infisical will update periodically the provided database user's password. - - At present Infisical do require access to your database. We will soon be released Infisical agent based rotation which would help you rotate without direct database access from Infisical cloud. - - ## Working -1. User's has to create the two user's for Infisical to rotate and provide them required database access -2. Infisical will connect with your database with admin access -3. If last rotated one was username1, then username2 is chosen to be rotated -5. Update it's password with random value -6. After testing it gets saved to the provided secret mapping +1. User's has to create the two user's for Infisical to rotate and provide them required database access. +2. Infisical will connect with your database with admin access. +3. If last rotated one was username1, then username2 is chosen to be rotated. +5. Update it's password with random value. +6. After testing it gets saved to the provided secret mapping. ## Rotation Configuration @@ -34,4 +30,4 @@ Infisical will update periodically the provided database user's password. - Finally select the secrets in your provided board to replace with new secret after each rotation - Your done and good to go. -Congrats. You have 10x your PostgreSQL/CockroachDB access security. +Congratulations. You have improved your PostgreSQL/CockroachDB access security. diff --git a/docs/documentation/platform/sso/google-saml.mdx b/docs/documentation/platform/sso/google-saml.mdx new file mode 100644 index 000000000..743c4e3ff --- /dev/null +++ b/docs/documentation/platform/sso/google-saml.mdx @@ -0,0 +1,95 @@ +--- +title: "Google SAML" +description: "Configure Google SAML for Infisical SSO" +--- + + + Google SAML SSO feature is a paid feature. + + If you're using Infisical Cloud, then it is available under the **Pro Tier**. If you're self-hosting Infisical, + then you should contact team@infisical.com to purchase an enterprise license to use it. + + + + + In Infisical, head to your Organization Settings > Authentication > SAML SSO Configuration and select **Set up SAML SSO**. + + Next, note the **ACS URL** and **SP Entity ID** to use when configuring the Google SAML application. + + ![Google SAML initial configuration](../../../images/sso/google-saml/init-config.png) + + + 2.1. In your [Google Admin console](https://support.google.com/a/answer/182076), head to Menu > Apps > Web and mobile apps and + create a **custom SAML app**. + + ![Google SAML app creation](../../../images/sso/google-saml/create-custom-saml-app.png) + + 2.2. In the **App details** tab, give the application a unique name like Infisical. + + ![Google SAML app naming](../../../images/sso/google-saml/name-custom-saml-app.png) + + 2.3. In the **Google Identity Provider details** tab, copy the **SSO URL**, **Entity ID** and **Certificate**. + + ![Google SAML custom app details](../../../images/sso/google-saml/custom-saml-app-config.png) + + 2.4. Back in Infisical, set **SSO URL**, **IdP Entity ID**, and **Certificate** to the corresponding items from step 2.3. + + ![Google SAML Infisical config](../../../images/sso/google-saml/infisical-config.png) + + 2.5. Back in the Google Admin console, in the **Service provider details** tab, set the **ACS URL** and **Entity ID** to the corresponding items from step 1. + + Also, check the **Signed response** checkbox. + + ![Google SAML app config 2](../../../images/sso/google-saml/custom-saml-app-config-2.png) + + 2.6. In the **Attribute mapping** tab, configure the following map: + + - **First name** -> **firstName** + - **Last name** -> **lastName** + - **Primary email** -> **email** + + ![Google SAML attribute mapping](../../../images/sso/google-saml/attribute-mapping.png) + + Click **Finish**. + + + Back in your [Google Admin console](https://support.google.com/a/answer/182076), head to Menu > Apps > Web and mobile apps > your SAML app + and press on **User access**. + + ![Google SAML user access](../../../images/sso/google-saml/user-access.png) + + To assign everyone in your organization to the application, click **On for everyone** or **Off for everyone** and then click **Save**. + + You can also assign an organizational unit or set of users to an application; you can learn more about that [here](https://support.google.com/a/answer/6087519?hl=en#add_custom_saml&turn_on&verify_sso&&zippy=%2Cstep-add-the-custom-saml-app%2Cstep-turn-on-your-saml-app%2Cstep-verify-that-sso-is-working-with-your-custom-app). + + ![Google SAML user access assignment](../../../images/sso/google-saml/user-access-assign.png) + + + Enabling SAML SSO allows members in your organization to log into Infisical via Google Workspace. + + ![Google SAML enable](../../../images/sso/google-saml/enable-saml.png) + + + Enforcing SAML SSO ensures that members in your organization can only access Infisical + by logging into the organization via Google. + + To enforce SAML SSO, you're required to test out the SAML connection by successfully authenticating at least one Google user with Infisical; + Once you've completed this requirement, you can toggle the **Enforce SAML SSO** button to enforce SAML SSO. + + + We recommend ensuring that your account is provisioned the application in Google + prior to enforcing SAML SSO to prevent any unintended issues. + + + + + + If you're configuring SAML SSO on a self-hosted instance of Infisical, make sure to + set the `AUTH_SECRET` and `SITE_URL` environment variable for it to work: + + - `AUTH_SECRET`: A secret key used for signing and verifying JWT. This can be a random 32-byte base64 string generated with `openssl rand -base64 32`. + - `SITE_URL`: The URL of your self-hosted instance of Infisical - should be an absolute URL including the protocol (e.g. https://app.infisical.com) + + +References: +- Google's guide to [set up your own custom SAML app](https://support.google.com/a/answer/6087519?hl=en#add_custom_saml&turn_on&verify_sso&&zippy=%2Cstep-add-the-custom-saml-app%2Cstep-turn-on-your-saml-app%2Cstep-verify-that-sso-is-working-with-your-custom-app). \ No newline at end of file diff --git a/docs/documentation/platform/sso/overview.mdx b/docs/documentation/platform/sso/overview.mdx index cd2f8ff31..e1fd25957 100644 --- a/docs/documentation/platform/sso/overview.mdx +++ b/docs/documentation/platform/sso/overview.mdx @@ -22,3 +22,4 @@ your IdP cannot and will not have access to the decryption key needed to decrypt - [Okta SAML](/documentation/platform/sso/okta) - [Azure SAML](/documentation/platform/sso/azure) - [JumpCloud SAML](/documentation/platform/sso/jumpcloud) +- [Google SAML](/documentation/platform/sso/google-saml) diff --git a/docs/images/self-hosting/applicable-to-all/selfhost-signup.png b/docs/images/self-hosting/applicable-to-all/selfhost-signup.png index ec73105a0..745c32a44 100644 Binary files a/docs/images/self-hosting/applicable-to-all/selfhost-signup.png and b/docs/images/self-hosting/applicable-to-all/selfhost-signup.png differ diff --git a/docs/images/self-hosting/guides/mongo-postgres/mongo-migration.png b/docs/images/self-hosting/guides/mongo-postgres/mongo-migration.png new file mode 100644 index 000000000..b74cd4963 Binary files /dev/null and b/docs/images/self-hosting/guides/mongo-postgres/mongo-migration.png differ diff --git a/docs/images/sso/google-saml/attribute-mapping.png b/docs/images/sso/google-saml/attribute-mapping.png new file mode 100644 index 000000000..b5702cd2b Binary files /dev/null and b/docs/images/sso/google-saml/attribute-mapping.png differ diff --git a/docs/images/sso/google-saml/create-custom-saml-app.png b/docs/images/sso/google-saml/create-custom-saml-app.png new file mode 100644 index 000000000..6139932f1 Binary files /dev/null and b/docs/images/sso/google-saml/create-custom-saml-app.png differ diff --git a/docs/images/sso/google-saml/custom-saml-app-config-2.png b/docs/images/sso/google-saml/custom-saml-app-config-2.png new file mode 100644 index 000000000..9839dd0c4 Binary files /dev/null and b/docs/images/sso/google-saml/custom-saml-app-config-2.png differ diff --git a/docs/images/sso/google-saml/custom-saml-app-config.png b/docs/images/sso/google-saml/custom-saml-app-config.png new file mode 100644 index 000000000..8f4ad5928 Binary files /dev/null and b/docs/images/sso/google-saml/custom-saml-app-config.png differ diff --git a/docs/images/sso/google-saml/enable-saml.png b/docs/images/sso/google-saml/enable-saml.png new file mode 100644 index 000000000..7a90eed55 Binary files /dev/null and b/docs/images/sso/google-saml/enable-saml.png differ diff --git a/docs/images/sso/google-saml/infisical-config.png b/docs/images/sso/google-saml/infisical-config.png new file mode 100644 index 000000000..250b4ed37 Binary files /dev/null and b/docs/images/sso/google-saml/infisical-config.png differ diff --git a/docs/images/sso/google-saml/init-config.png b/docs/images/sso/google-saml/init-config.png new file mode 100644 index 000000000..c4b967e54 Binary files /dev/null and b/docs/images/sso/google-saml/init-config.png differ diff --git a/docs/images/sso/google-saml/name-custom-saml-app.png b/docs/images/sso/google-saml/name-custom-saml-app.png new file mode 100644 index 000000000..580896d05 Binary files /dev/null and b/docs/images/sso/google-saml/name-custom-saml-app.png differ diff --git a/docs/images/sso/google-saml/user-access-assign.png b/docs/images/sso/google-saml/user-access-assign.png new file mode 100644 index 000000000..afa115c65 Binary files /dev/null and b/docs/images/sso/google-saml/user-access-assign.png differ diff --git a/docs/images/sso/google-saml/user-access.png b/docs/images/sso/google-saml/user-access.png new file mode 100644 index 000000000..bd69c2277 Binary files /dev/null and b/docs/images/sso/google-saml/user-access.png differ diff --git a/docs/infisical-agent/overview.mdx b/docs/infisical-agent/overview.mdx index c98090289..f62194ad8 100644 --- a/docs/infisical-agent/overview.mdx +++ b/docs/infisical-agent/overview.mdx @@ -1,5 +1,5 @@ --- -title: "Infisical Agent" +title: "Overview" --- Infisical Agent is a client daemon that simplifies the adoption of Infisical by providing a more scalable and user-friendly approach for applications to interact with Infisical. @@ -51,6 +51,9 @@ While specifying an authentication method is mandatory to start the agent, confi | `sinks[].config.path` | The file path where the access token should be stored for each sink in the list. | | `templates[].source-path` | The path to the template file that should be used to render secrets. | | `templates[].destination-path` | The path where the rendered secrets from the source template will be saved to. | +| `templates[].config.polling-interval` | How frequently to check for secret changes. Default: `60s` (optional) | +| `templates[].config.execute.command` | The command to execute when secret change is detected (optional) | +| `templates[].config.execute.timeout` | How long in seconds to wait for command to execute before timing out (optional) | ## Quick start Infisical Agent @@ -76,6 +79,11 @@ sinks: templates: - source-path: my-dot-ev-secret-template destination-path: /some/path/.env + config: + polling-interval: 60s + execute: + timeout: 30 + command: ./reload-app.sh ``` Above is an example agent configuration file that defines the token authentication method, one sink location (where to deposit access tokens after renewal) and a secret template. diff --git a/docs/integrations/platforms/docker-intro.mdx b/docs/integrations/platforms/docker-intro.mdx index 5f23584c8..2823fc8ca 100644 --- a/docs/integrations/platforms/docker-intro.mdx +++ b/docs/integrations/platforms/docker-intro.mdx @@ -1,25 +1,25 @@ --- title: "Docker" -description: "Learn how to feed secrets from Infisical into your docker application" +description: "Learn how to feed secrets from Infisical into your Docker application" --- -There are many methods to inject Infisical secrets to docker-based applications. -Regardless of which method you choose, these methods will inject secrets from Infisical as environment variables into your Docker container. +There are many methods to inject Infisical secrets into Docker-based applications. +Regardless of the method you choose, they all inject secrets from Infisical as environment variables into your Docker container. Install and run your app start command with Infisical CLI - Feed secrets via `--env-file` flag in docker run command + Feed secrets with the `--env-file` flag when using the + `docker run` command - Inject secrets to multiple services using Docker Compose + Inject secrets into multiple services using Docker Compose The main difference between the "Docker Entrypoint" and "Docker run" approach is where the Infisical CLI is installed. -In most production settings, it's typically inconvenient to have the Infisical CLI installed and executed externally. -As a result, we suggest using the "Docker Entrypoint" method for production purposes. +In most production settings, it's typically less convenient to have the Infisical CLI installed and executed externally, so we suggest using the "Docker Entrypoint" method for production purposes. However, if this limitation doesn't apply to you, select the method that best fits your needs. \ No newline at end of file diff --git a/docs/internals/components.mdx b/docs/internals/components.mdx index 02c6d3692..29522b0bb 100644 --- a/docs/internals/components.mdx +++ b/docs/internals/components.mdx @@ -9,9 +9,7 @@ The Infisical API (sometimes referred to as the **backend**) contains the core p ## Storage backend -Infisical relies on a storage backend to store data including users and secrets. - -Currently, the only supported storage backend is [MongoDB](https://www.mongodb.com) but we plan to add support for other options including PostgreSQL in Q1 2024. +Infisical relies on a storage backend to store data including users and secrets. Infisical's storage backend is Postgres. ## Redis @@ -27,4 +25,4 @@ Clients are any application or infrastructure that connecting to the Infisical A - Public API: Making API requests directly to the Infisical API. - Client SDK: A platform-specific library with method abstractions for working with secrets. Currently, there are three official SDKs: [Node SDK](https://infisical.com/docs/sdks/languages/node), [Python SDK](https://infisical.com/docs/sdks/languages/python), and [Java SDK](https://infisical.com/docs/sdks/languages/java). - CLI: A terminal-based interface for interacting with the Infisical API. -- Kubernetes Operator: This operator retrieves secrets from Infisical and securely store \ No newline at end of file +- Kubernetes Operator: This operator retrieves secrets from Infisical and securely store diff --git a/docs/mint.json b/docs/mint.json index dc56e3971..d316ff81a 100644 --- a/docs/mint.json +++ b/docs/mint.json @@ -146,10 +146,18 @@ "documentation/platform/sso/gitlab", "documentation/platform/sso/okta", "documentation/platform/sso/azure", - "documentation/platform/sso/jumpcloud" + "documentation/platform/sso/jumpcloud", + "documentation/platform/sso/google-saml" + ] + }, + { + "group": "LDAP", + "pages": [ + "documentation/platform/ldap/overview", + "documentation/platform/ldap/jumpcloud", + "documentation/platform/ldap/general" ] }, - "documentation/platform/ldap", { "group": "LDAP", "pages": [ @@ -174,7 +182,6 @@ "pages": [ "self-hosting/overview", "self-hosting/configuration/requirements", - "self-hosting/configuration/schema-migrations", { "group": "Installation methods", "pages": [ @@ -184,6 +191,13 @@ ] }, "self-hosting/configuration/envars", + { + "group": "Guides", + "pages": [ + "self-hosting/configuration/schema-migrations", + "self-hosting/guides/mongo-to-postgres" + ] + }, "self-hosting/faq" ] }, diff --git a/docs/self-hosting/configuration/requirements.mdx b/docs/self-hosting/configuration/requirements.mdx index 45b1b92ff..262c7fb7c 100644 --- a/docs/self-hosting/configuration/requirements.mdx +++ b/docs/self-hosting/configuration/requirements.mdx @@ -58,7 +58,7 @@ Redis requirements: - Use Redis versions 6.x or 7.x. We advise upgrading to at least Redis 6.2. - Redis Cluster mode is currently not supported; use Redis Standalone, with or without High Availability (HA). -- Redis storage needs are minimal: a setup with 1 vCPU, 1 GB RAM, and 1GB SSD will be sufficient for most deployments. +- Redis storage needs are minimal: a setup with 1 vCPU, 1 GB RAM, and 1GB SSD will be sufficient for small deployments. ## Supported Web Browsers @@ -68,4 +68,4 @@ Infisical supports a range of web browsers. However, features such as browser-ba - [Google Chrome](https://www.google.com/chrome/) - [Chromium](https://www.chromium.org/getting-involved/dev-channel/) - [Apple Safari](https://www.apple.com/safari/) -- [Microsoft Edge](https://www.microsoft.com/en-us/edge?form=MA13FJ) \ No newline at end of file +- [Microsoft Edge](https://www.microsoft.com/en-us/edge?form=MA13FJ) diff --git a/docs/self-hosting/deployment-options/kubernetes-helm.mdx b/docs/self-hosting/deployment-options/kubernetes-helm.mdx index 300cb3db9..ae27207cd 100644 --- a/docs/self-hosting/deployment-options/kubernetes-helm.mdx +++ b/docs/self-hosting/deployment-options/kubernetes-helm.mdx @@ -46,27 +46,34 @@ description: "Use Helm chart to install Infisical on your Kubernetes cluster" - For test or proof-of-concept purposes, you may omit `DB_CONNECTION_URI` and `REDIS_URL` from `infisical-secrets`. This is because the Helm chart will automatically provision and connect to the in-cluster instances of Postgres and Redis by default. - + For test or proof-of-concept purposes, you may omit `DB_CONNECTION_URI` and `REDIS_URL` from `infisical-secrets`. This is because the Helm chart will automatically provision and connect to the in-cluster instances of Postgres and Redis by default. + ```yaml simple-values-example.yaml + apiVersion: v1 + kind: Secret + metadata: + name: infisical-secrets + type: Opaque + stringData: + AUTH_SECRET: <> + ENCRYPTION_KEY: <> + ``` For production environments, we recommend using Cloud-based Platform as a Service (PaaS) solutions for PostgreSQL and Redis to ensure high availability. In on-premise setups, it's recommended to configure Redis and Postgres for high availability, either by using Bitnami charts or a custom configuration. - + ```yaml simple-values-example.yaml + apiVersion: v1 + kind: Secret + metadata: + name: infisical-secrets + type: Opaque + stringData: + AUTH_SECRET: <> + ENCRYPTION_KEY: <> + REDIS_URL: <> + DB_CONNECTION_URI: <> + ``` - - ```yaml simple-values-example.yaml - apiVersion: v1 - kind: Secret - metadata: - name: infisical-secrets - type: Opaque - stringData: - AUTH_SECRET: <> - ENCRYPTION_KEY: <> - REDIS_URL: <> - DB_CONNECTION_URI: <> - ``` diff --git a/docs/self-hosting/faq.mdx b/docs/self-hosting/faq.mdx index 5bbc426e3..598d408ae 100644 --- a/docs/self-hosting/faq.mdx +++ b/docs/self-hosting/faq.mdx @@ -15,3 +15,7 @@ However, in the event you choose to use Infisical without SSL, you can do so by [Learn more about secure cookies](https://really-simple-ssl.com/definition/what-are-secure-cookies/) + + Follow the step by step guide [here](self-hosting/guides/mongo-to-postgres) to learn how. + + diff --git a/docs/self-hosting/guides/mongo-to-postgres.mdx b/docs/self-hosting/guides/mongo-to-postgres.mdx new file mode 100644 index 000000000..f8a6cca0f --- /dev/null +++ b/docs/self-hosting/guides/mongo-to-postgres.mdx @@ -0,0 +1,201 @@ +--- +title: "Migrate Mongo to Postgres" +description: "How to migrate from MongoDB to PostgreSQL for Infisical" +--- + +This guide will provide step by step instructions on migrating your Infisical instance running on MongoDB to the newly released PostgreSQL version of Infisical. +The newly released Postgres version of Infisical is the only version of Infisical that will receive feature updates and patches going forward. + + + If you have a small set of secrets, we recommend you to download the secrets and upload them to your new instance of Infisical instead of running the migration script. + + +## Prerequisites + +Before starting the migration, ensure you have the following command line tools installed: + +- [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) +- [pg_dump](https://www.postgresql.org/docs/current/app-pgrestore.html) +- [pg_restore](https://www.postgresql.org/docs/current/app-pgdump.html) +- [mongodump](https://www.mongodb.com/docs/database-tools/mongodump/) +- [mongorestore](https://www.mongodb.com/docs/database-tools/mongorestore/) +- [Docker](https://docs.docker.com/engine/install/) + +## Prepare for migration + + + + While the migration script will not mutate any MongoDB production data, we recommend you to take a backup of your MongoDB instance if possible. + + + To prevent new data entries during the migration, set your Infisical instance to migration mode by setting the environment variable `MIGRATION_MODE=true` and redeploying your instance. + This mode will block all write operations, only allowing GET requests. It also disables user logins and sets up a migration page to prevent UI interactions. + ![migration mode](/images/self-hosting/guides/mongo-postgres/mongo-migration.png) + + + Start local instances of MongoDB and Postgres. This will be used in later steps to process and transform the data locally. + + To start local instances of the two databases, create a file called `docker-compose.yaml` as shown below. + + ```yaml docker-compose.yaml + version: '3.1' + + services: + mongodb: + image: mongo + restart: always + environment: + MONGO_INITDB_ROOT_USERNAME: root + MONGO_INITDB_ROOT_PASSWORD: example + ports: + - "27017:27017" + volumes: + - mongodb_data:/data/db + + postgres: + image: postgres + restart: always + environment: + POSTGRES_PASSWORD: example + ports: + - "5432:5432" + volumes: + - postgres_data:/var/lib/postgresql/data + + volumes: + mongodb_data: + postgres_data: + ``` + + Next, run the command below in the same working directory where the `docker-compose.yaml` file resides to start both services. + + ``` + docker-compose up + ``` + + + + +## Dump MongoDB +To speed up the data transformation process, the first step involves transferring the production data from Infisical's MongoDB to a local machine. +This is achieved by creating a dump of the production database and then uploading this dumped data into a local Mongo instance. +By having a running local instance of the production database, we will significantly reduce the time it takes to run the migration script. + + + + + ``` + mongodump --uri= --archive="mongodump-db" --db= --excludeCollection=auditlogs + ``` + + + + ``` + mongorestore --uri=mongodb://root:example@localhost:27017/ --archive="mongodump-db" + ``` + + + +## Start the migration + +Once started, the migration script will transform MongoDB data into an equivalent PostgreSQL format. + + + + Clone the Infisical MongoDB repository. + ``` + git clone -b infisical/v0.46.11-postgres https://github.com/Infisical/infisical.git + ``` + + + ``` + cd backend + ``` + + ``` + npm install + ``` + + + ``` + cd pg-migrator + ``` + + ``` + npm install + ``` + + + ``` + npm run migration + ``` + + When executing the above command, you'll be asked to provide the MongoDB connection string for the database containing your production Infisical data. Since our production Mongo data is transferred to a local Mongo instance, you should input the connection string for this local instance. + + ``` + mongodb://root:example@localhost:27017/?authSource=admin + ``` + + + Remember to replace `` with the name of the MongoDB database. If you are not sure the name, you can use [Compass](https://www.mongodb.com/products/tools/compass) to view the available databases. + + + + Next, you will be asked to enter the Postgres connection string for the database where the transformed data should be stored. + Input the connection string of the local Postgres instance that was set up earlier in the guide. + + ``` + postgres://infisical:infisical@localhost/infisical?sslmode=disable + ``` + + + + Once the script has completed, you will notice a new folder has been created called `db` in the `pg-migrator` folder. + This folder contains meta data for schema mapping and can be helpful when debugging migration related issues. + We highly recommend you to make a copy of this folder in case you need assistance from the Infisical team during your migration process. + + + The `db` folder does not contain any sensitive data + + + + +## Finalizing Migration +At this stage, the data from the Mongo instance of Infisical should have been successfully converted into its Postgres equivalent. +The remaining step involves transferring the local Postgres database, which now contains all the migrated data, to your chosen production Postgres environment. +Rather than transferring the data row-by-row from your local machine to the production Postgres database, we will first create a dump file from the local Postgres and then upload this file to your production Postgres instance. + + + + ``` + pg_dump -h localhost -U infisical -Fc -b -v -f dumpfilelocation.sql -d infisical + ``` + + + ``` + pg_restore --clean -v -h -U -d -j 2 dumpfilelocation.sql + ``` + + + Remember to replace ``, ``, `` with the corresponding details of your production Postgres database. + + + + Use a tool like Beekeeper Studio to confirm that the data has been successfully transferred to your production Postgres DB. + + + +## Post-Migration Steps + +Once the data migration to PostgreSQL is complete, you're ready to deploy Infisical using the deployment method of your choice. +For guidance on deployment options, please visit the [self-hosting documentation](/self-hosting/overview). +Remember to transfer the necessary [environment variables](/self-hosting/configuration/envars) from the MongoDB version of Infisical to the new Postgres based Infisical; rest assured, they are fully compatible. + + +The first deployment of Postgres based Infisical must be deployed with Docker image tag `v0.46.11-postgres`. +After deploying this version, you can proceed to update to any subsequent versions. + + +## Additional discussion +- When you visit Infisical's [docker hub](https://hub.docker.com/r/infisical/infisical) page, you will notice that image tags end with `-postgres`. +This is to indicate that this version of Infisical runs on the new Postgres backend. Any image tag that does not end in `postgres` runs on MongoDB. \ No newline at end of file diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 0ab91f92f..72607a272 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,5 +1,5 @@ { - "name": "npm-proj-1708687711895-0.8280111363176879xoEiUg", + "name": "npm-proj-1709146141702-0.772936286416932EMIzNi", "lockfileVersion": 3, "requires": true, "packages": { @@ -68,7 +68,7 @@ "next": "^12.3.4", "nprogress": "^0.2.0", "picomatch": "^2.3.1", - "posthog-js": "^1.103.0", + "posthog-js": "^1.105.4", "query-string": "^7.1.3", "react": "^17.0.2", "react-beautiful-dnd": "^13.1.1", @@ -19065,9 +19065,9 @@ "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==" }, "node_modules/posthog-js": { - "version": "1.103.0", - "resolved": "https://registry.npmjs.org/posthog-js/-/posthog-js-1.103.0.tgz", - "integrity": "sha512-NldabkbCB9a/2JLszoB7vk5XZr93iBoGEgEEKn201oDD3QkRn1nC+c+e1HQ3S9oMs5oZIhKmPNBCje1J9prYRg==", + "version": "1.105.4", + "resolved": "https://registry.npmjs.org/posthog-js/-/posthog-js-1.105.4.tgz", + "integrity": "sha512-hazxQYi4nxSqktu0Hh1xCV+sJCpN8mp5E5Ei/cfEa2nsb13xQbzn81Lf3VIDA0xMU1mXxNRStntlY267eQVC/w==", "dependencies": { "fflate": "^0.4.8", "preact": "^10.19.3" diff --git a/frontend/package.json b/frontend/package.json index d28ca4bf6..5871c9599 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -76,7 +76,7 @@ "next": "^12.3.4", "nprogress": "^0.2.0", "picomatch": "^2.3.1", - "posthog-js": "^1.103.0", + "posthog-js": "^1.105.4", "query-string": "^7.1.3", "react": "^17.0.2", "react-beautiful-dnd": "^13.1.1", diff --git a/frontend/src/components/v2/Menu/Menu.tsx b/frontend/src/components/v2/Menu/Menu.tsx index e4771e551..dd3407557 100644 --- a/frontend/src/components/v2/Menu/Menu.tsx +++ b/frontend/src/components/v2/Menu/Menu.tsx @@ -42,27 +42,27 @@ export const MenuItem = ({ const iconRef = useRef(); return ( - iconRef.current?.play()} onMouseLeave={() => iconRef.current?.stop()}> + ); }; @@ -103,16 +103,16 @@ export const SubMenuItem = ({ iconRef.current?.play()} onMouseLeave={() => iconRef.current?.stop()}>
  • - + diff --git a/frontend/src/components/v2/UpgradeProjectAlert/UpgradeProjectAlert.tsx b/frontend/src/components/v2/UpgradeProjectAlert/UpgradeProjectAlert.tsx index 121f77067..4b5cf0fba 100644 --- a/frontend/src/components/v2/UpgradeProjectAlert/UpgradeProjectAlert.tsx +++ b/frontend/src/components/v2/UpgradeProjectAlert/UpgradeProjectAlert.tsx @@ -1,4 +1,5 @@ import { useCallback, useState } from "react"; +import Link from "next/link"; import { useRouter } from "next/router"; import { faWarning } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; @@ -97,16 +98,31 @@ export const UpgradeProjectAlert = ({ project }: UpgradeProjectAlertProps): JSX.
    Upgrade your project {membership.role === "admin" ? ( -

    - Upgrade your project version to continue receiving the latest improvements and patches. -

    + <> +

    + Upgrade your project version to continue receiving the latest improvements and + patches. +

    + +     + Learn more + + + ) : ( -

    - Please ask a project admin to upgrade the project. -
    - Upgrading the project version is required to continue receiving the latest improvements - and patches. -

    + <> +

    + Please ask a project admin to upgrade the project. +
    + Upgrading the project version is required to continue receiving the latest + improvements and patches. +

    + + + Learn more + + + )} {currentStatus &&

    Status: {currentStatus}

    }
    diff --git a/frontend/src/i18n.ts b/frontend/src/i18n.ts index 2ccfa2b8d..8eed25a42 100644 --- a/frontend/src/i18n.ts +++ b/frontend/src/i18n.ts @@ -1,6 +1,6 @@ import { initReactI18next } from "react-i18next"; import i18n from "i18next"; -import LanguageDetector from "i18next-browser-languagedetector"; +// import LanguageDetector from "i18next-browser-languagedetector"; import Backend from "i18next-http-backend"; // don't want to use this? // have a look at the Quick start guide @@ -13,17 +13,18 @@ i18n .use(Backend) // detect user language // learn more: https://github.com/i18next/i18next-browser-languageDetector - .use(LanguageDetector) + // .use(LanguageDetector) // pass the i18n instance to react-i18next. .use(initReactI18next) // init i18next // for all options read: https://www.i18next.com/overview/configuration-options .init({ + lng:"en", fallbackLng: "en", - supportedLngs: ["en", "ko", "fr", "pt-BR", "pt-PT", "es"], + // supportedLngs: ["en", "ko", "fr", "pt-BR", "pt-PT", "es"], debug: process.env.NODE_ENV === "development", detection: { - lookupLocalStorage: "lang" + // lookupLocalStorage: "lang" }, ns: ["translations"], interpolation: { diff --git a/frontend/src/pages/integrations/cloudflare-workers/create.tsx b/frontend/src/pages/integrations/cloudflare-workers/create.tsx index a977920d2..6dd87cbb7 100644 --- a/frontend/src/pages/integrations/cloudflare-workers/create.tsx +++ b/frontend/src/pages/integrations/cloudflare-workers/create.tsx @@ -1,10 +1,12 @@ import { useEffect, useState } from "react"; import { useRouter } from "next/router"; +import axios from "axios"; import queryString from "query-string"; +import { useNotificationContext } from "@app/components/context/Notifications/NotificationProvider"; import { useCreateIntegration, useGetWorkspaceById } from "@app/hooks/api"; -import { Button, Card, CardTitle, FormControl, Select, SelectItem } from "../../../components/v2"; +import { Button, Card, CardTitle, FormControl, Input, Select, SelectItem } from "../../../components/v2"; import { useGetIntegrationAuthApps, useGetIntegrationAuthById @@ -13,6 +15,7 @@ import { export default function CloudflareWorkersIntegrationPage() { const router = useRouter(); const { mutateAsync } = useCreateIntegration(); + const { createNotification } = useNotificationContext(); const { integrationAuthId } = queryString.parse(router.asPath.split("?")[1]); const { data: workspace } = useGetWorkspaceById(localStorage.getItem("projectData.id") ?? ""); @@ -22,6 +25,8 @@ export default function CloudflareWorkersIntegrationPage() { }); const [selectedSourceEnvironment, setSelectedSourceEnvironment] = useState(""); + const [secretPath, setSecretPath] = useState("/"); + const [targetApp, setTargetApp] = useState(""); const [targetAppId, setTargetAppId] = useState(""); @@ -56,7 +61,7 @@ export default function CloudflareWorkersIntegrationPage() { app: targetApp, appId: targetAppId, sourceEnvironment: selectedSourceEnvironment, - secretPath: "/" + secretPath }); setIsLoading(false); @@ -64,6 +69,18 @@ export default function CloudflareWorkersIntegrationPage() { router.push(`/integrations/${localStorage.getItem("projectData.id")}`); } catch (err) { console.error(err); + + let errorMessage: string = "Something went wrong!"; + if (axios.isAxiosError(err)) { + const { message } = err?.response?.data as { message: string }; + errorMessage = message; + } + + createNotification({ + text: errorMessage, + type: "error" + }); + setIsLoading(false); } }; @@ -96,6 +113,13 @@ export default function CloudflareWorkersIntegrationPage() { ))} + + setSecretPath(evt.target.value)} + placeholder="Provide a path, default is /" + /> +