diff --git a/.github/workflows/validate-upgrade-path.yml b/.github/workflows/validate-upgrade-path.yml index 88728be70..edc494dc3 100644 --- a/.github/workflows/validate-upgrade-path.yml +++ b/.github/workflows/validate-upgrade-path.yml @@ -10,7 +10,7 @@ on: jobs: validate-upgrade-path: - name: Validate upgrade-path.yaml format + name: Validate upgrade-path.yaml runs-on: ubuntu-latest timeout-minutes: 5 @@ -38,204 +38,135 @@ jobs: fi fi - - name: Setup Python for YAML validation + - name: Setup Node.js for YAML validation + if: steps.check-changes.outputs.changed == 'true' + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Create lightweight validation script if: steps.check-changes.outputs.changed == 'true' run: | - # Use system python3 and install PyYAML - python3 --version || echo "Python3 not found" - python3 -m pip install --user PyYAML || pip3 install PyYAML || echo "PyYAML installation failed" + # Create a temporary package.json with only the required dependencies + cat > package.json << 'EOF' + { + "name": "upgrade-path-validator", + "version": "1.0.0", + "dependencies": { + "js-yaml": "^4.1.0", + "zod": "^3.22.0" + } + } + EOF + + - name: Install minimal validation dependencies + if: steps.check-changes.outputs.changed == 'true' + run: | + npm install --no-package-lock --production - name: Validate upgrade-path.yaml format if: steps.check-changes.outputs.changed == 'true' run: | echo "Running upgrade-path.yaml validation..." + node << 'EOF' + const fs = require('fs'); + const yaml = require('js-yaml'); + const { z } = require('zod'); - # Debug: Check if file exists - ls -la backend/upgrade-path.yaml || echo "File not found" + // Validation schemas matching backend service + const versionSchema = z + .string() + .min(1) + .max(50) + .regex(/^[a-zA-Z0-9._/-]+$/, "Invalid version format"); - # Debug: Show last few lines - echo "Last 5 lines of file:" - tail -5 backend/upgrade-path.yaml || echo "Cannot read file" + const breakingChangeSchema = z.object({ + title: z.string().min(1).max(200), + description: z.string().min(1).max(1000), + action: z.string().min(1).max(500) + }); - echo "Starting Python validation..." + const versionConfigSchema = z.object({ + breaking_changes: z.array(breakingChangeSchema).optional(), + db_schema_changes: z.string().max(1000).optional(), + notes: z.string().max(2000).optional() + }); - python3 << 'EOF' - import yaml - import re - import sys + const upgradePathConfigSchema = z.object({ + versions: z.record(versionSchema, versionConfigSchema).optional().nullable() + }); - def validate_upgrade_path(): - try: - print("Reading upgrade-path.yaml...") - with open('backend/upgrade-path.yaml', 'r') as file: - raw_lines = file.readlines() + function validateUpgradePathConfig() { + try { + const yamlPath = './backend/upgrade-path.yaml'; - # Remove comments and empty lines more thoroughly - content_lines = [] - for line in raw_lines: - # Remove inline comments but preserve quoted strings - stripped = line.strip() - if not stripped or stripped.startswith("#"): - # Skip empty lines and full comment lines - continue + if (!fs.existsSync(yamlPath)) { + console.log('Warning: No upgrade-path.yaml file found'); + return true; + } - # Handle inline comments - scan character by character to respect quotes - if "#" in line: - result = "" - in_single_quote = False - in_double_quote = False - i = 0 - while i < len(line): - char = line[i] - if char == "'" and not in_double_quote: - in_single_quote = not in_single_quote - elif char == '"' and not in_single_quote: - in_double_quote = not in_double_quote - elif char == "#" and not in_single_quote and not in_double_quote: - # Found unquoted #, rest is comment - break - result += char - i += 1 - line = result + const yamlContent = fs.readFileSync(yamlPath, 'utf8'); - # Only add non-empty lines after comment removal - if line.strip(): - content_lines.append(line.rstrip() + "\n") + if (yamlContent.length > 1024 * 1024) { + throw new Error('Config file too large (>1MB)'); + } - content = "".join(content_lines) + // Parse YAML safely + const config = yaml.load(yamlContent, { schema: yaml.FAILSAFE_SCHEMA }); - if not content.strip(): - raise ValueError("File is empty") + if (!config) { + console.log('Warning: Empty configuration file'); + return true; + } - if len(content) > 1024 * 1024: - raise ValueError("File is too large (>1MB)") + // Validate schema + const result = upgradePathConfigSchema.safeParse(config); - print("Parsing YAML content...") - try: - config = yaml.safe_load(content) - print("YAML parsed successfully") - except yaml.YAMLError as e: - print(f"YAML parsing failed: {e}") - raise ValueError(f"Invalid YAML syntax: {e}") + if (!result.success) { + console.log('Validation failed with the following errors:'); + result.error.issues.forEach(issue => { + const path = issue.path.length > 0 ? `[${issue.path.join('.')}]` : ''; + console.log(` - ${path}: ${issue.message}`); + }); + return false; + } - if not isinstance(config, dict): - raise ValueError("Root level must be an object") + const versions = config.versions || {}; + const versionCount = Object.keys(versions).length; - print("YAML syntax is valid") - print("Validating schema structure...") + if (versionCount === 0) { + console.log('Warning: No versions found in the configuration'); + } else { + console.log(`Validated ${versionCount} version configuration(s)`); - if 'versions' not in config: - print("Warning: No versions found in the configuration") - return True + // Check for common version patterns + const commonPatterns = [ + /^v?\d+\.\d+\.\d+$/, // v1.2.3 or 1.2.3 + /^v?\d+\.\d+\.\d+\.\d+$/, // v1.2.3.4 or 1.2.3.4 + /^infisical\/v?\d+\.\d+\.\d+$/, // infisical/v1.2.3 + /^infisical\/v?\d+\.\d+\.\d+-\w+$/ // infisical/v1.2.3-postgres + ]; - versions = config['versions'] - if versions is None: - # Empty versions section is valid - print("Empty versions section found - this is valid") - return True - if not isinstance(versions, dict): - raise ValueError("'versions' must be an object") + for (const versionKey of Object.keys(versions)) { + const isCommonPattern = commonPatterns.some(pattern => pattern.test(versionKey)); + if (!isCommonPattern) { + console.log(`Warning: Version key '${versionKey}' doesn't match common patterns. This may be intentional.`); + } + } + } - print(f"Found {len(versions)} version(s) to validate") + console.log('upgrade-path.yaml format is valid'); + return true; - # Version key pattern validation - version_pattern = re.compile(r'^[a-zA-Z0-9._/-]+$') - common_patterns = [ - re.compile(r'^v?\d+\.\d+\.\d+$'), # v1.2.3 or 1.2.3 - re.compile(r'^v?\d+\.\d+\.\d+\.\d+$'), # v1.2.3.4 or 1.2.3.4 - re.compile(r'^infisical/v?\d+\.\d+\.\d+$'), # infisical/v1.2.3 - re.compile(r'^infisical/v?\d+\.\d+\.\d+-\w+$') # infisical/v1.2.3-postgres - ] + } catch (error) { + console.log(`Validation failed: ${error.message}`); + return false; + } + } - errors = [] - - for version_key, version_config in versions.items(): - print(f"Validating version key: {version_key}") - - # Validate version key format - if not version_pattern.match(version_key): - errors.append(f"Invalid version key '{version_key}': contains invalid characters") - continue - - if len(version_key) > 50: - errors.append(f"Version key '{version_key}' is too long (max 50 characters)") - continue - - if not isinstance(version_config, dict): - errors.append(f"Version '{version_key}' configuration must be an object") - continue - - # Validate breaking_changes - if 'breaking_changes' in version_config: - breaking_changes = version_config['breaking_changes'] - if not isinstance(breaking_changes, list): - errors.append(f"Version '{version_key}': breaking_changes must be a list") - elif len(breaking_changes) == 0: - errors.append(f"Version '{version_key}': breaking_changes is empty (remove field or add items)") - else: - for i, change in enumerate(breaking_changes): - if not isinstance(change, dict): - errors.append(f"Version '{version_key}': breaking_changes[{i}] must be an object") - continue - - # Validate required fields - for field in ['title', 'description', 'action']: - if field not in change: - errors.append(f"Version '{version_key}': breaking_changes[{i}] missing '{field}'") - elif not isinstance(change[field], str): - errors.append(f"Version '{version_key}': breaking_changes[{i}].{field} must be string") - elif not change[field].strip(): - errors.append(f"Version '{version_key}': breaking_changes[{i}].{field} cannot be empty") - elif field == 'title' and len(change[field]) > 200: - errors.append(f"Version '{version_key}': breaking_changes[{i}].title too long (max 200)") - elif field == 'description' and len(change[field]) > 1000: - errors.append(f"Version '{version_key}': breaking_changes[{i}].description too long (max 1000)") - elif field == 'action' and len(change[field]) > 500: - errors.append(f"Version '{version_key}': breaking_changes[{i}].action too long (max 500)") - - # Validate db_schema_changes - if 'db_schema_changes' in version_config: - db_changes = version_config['db_schema_changes'] - if not isinstance(db_changes, str): - errors.append(f"Version '{version_key}': db_schema_changes must be string") - elif db_changes == "": - errors.append(f"Version '{version_key}': db_schema_changes is empty (remove field or add content)") - elif len(db_changes) > 1000: - errors.append(f"Version '{version_key}': db_schema_changes too long (max 1000)") - - # Validate notes - if 'notes' in version_config: - notes = version_config['notes'] - if not isinstance(notes, str): - errors.append(f"Version '{version_key}': notes must be string") - elif notes == "": - errors.append(f"Version '{version_key}': notes is empty (remove field or add content)") - elif len(notes) > 2000: - errors.append(f"Version '{version_key}': notes too long (max 2000)") - - # Check if version follows common patterns - is_common_pattern = any(pattern.match(version_key) for pattern in common_patterns) - if not is_common_pattern: - print(f"Warning: Version key '{version_key}' doesn't match common patterns. This may be intentional.") - - print(f"Version '{version_key}' is valid") - - if errors: - print("Validation failed with the following errors:") - for error in errors: - print(f" - {error}") - return False - - print("All validations passed!") - print("upgrade-path.yaml format is valid") - return True - - except Exception as e: - print(f"Validation failed: {e}") - return False - - if not validate_upgrade_path(): - sys.exit(1) + if (!validateUpgradePathConfig()) { + process.exit(1); + } EOF - name: Validation completed diff --git a/backend/src/lib/config/env.ts b/backend/src/lib/config/env.ts index d8e39e718..15f878323 100644 --- a/backend/src/lib/config/env.ts +++ b/backend/src/lib/config/env.ts @@ -129,6 +129,8 @@ const envSchema = z POSTHOG_HOST: zpStr(z.string().optional().default("https://app.posthog.com")), POSTHOG_PROJECT_API_KEY: zpStr(z.string().optional().default("phc_nSin8j5q2zdhpFDI1ETmFNUIuTG4DwKVyIigrY10XiE")), LOOPS_API_KEY: zpStr(z.string().optional()), + // GitHub API token for upgrade path tool + GITHUB_API_TOKEN: zpStr(z.string().optional()), // jwt options AUTH_SECRET: zpStr(z.string()).default(process.env.JWT_AUTH_SECRET), // for those still using old JWT_AUTH_SECRET JWT_AUTH_LIFETIME: zpStr(z.string().default("10d")), diff --git a/backend/src/services/upgrade-path/github-client.ts b/backend/src/services/upgrade-path/github-client.ts index 3fcd5593f..44aacca6d 100644 --- a/backend/src/services/upgrade-path/github-client.ts +++ b/backend/src/services/upgrade-path/github-client.ts @@ -1,6 +1,8 @@ /* eslint-disable no-await-in-loop */ import RE2 from "re2"; +import { getConfig } from "@app/lib/config/env"; + import { FormattedRelease, GitHubApiError, GitHubRelease } from "./types"; interface GitHubClientConfig { @@ -20,7 +22,7 @@ interface RateLimitInfo { } const getDefaultConfig = (): GitHubClientConfig => ({ - token: process.env.GITHUB_TOKEN, + token: getConfig().GITHUB_API_TOKEN, timeout: 30000, maxRetries: 3, retryDelay: 1000, @@ -56,7 +58,15 @@ const isMainInfisicalRelease = (tagName: string): boolean => { ) { return false; } - return tagName.startsWith("v") || tagName.startsWith("infisical/v") || new RE2(/^\d+\.\d+\.\d+/).test(tagName); + + const patterns = [ + new RE2(/^v\d+\.\d+\.\d+/), + new RE2(/^\d+\.\d+\.\d+/), + new RE2(/^infisical\/v?\d+\.\d+\.\d+/), + new RE2(/^infisical\/v?\d+\.\d+\.\d+[-\w]*/) + ]; + + return patterns.some((pattern) => pattern.test(tagName)); }; const normalizeVersion = (tagName: string): string => { @@ -72,6 +82,39 @@ const normalizeVersion = (tagName: string): string => { return tagName.replace(new RE2(/-[a-zA-Z]+$/), ""); }; +const compareVersions = (v1: string, v2: string): number => { + const normalize = (v: string) => { + const versionMatch = v.match(new RE2(/(\d+\.\d+\.\d+(?:\.\d+)?)/)); + if (versionMatch) { + return versionMatch[1]; + } + if (v.startsWith("infisical/")) { + return v.replace(new RE2(/^infisical\/v?/), "").replace(new RE2(/-[a-zA-Z]+$/), ""); + } + return v.replace(new RE2(/^v/), "").replace(new RE2(/-[a-zA-Z]+$/), ""); + }; + + const clean1 = normalize(v1); + const clean2 = normalize(v2); + + const parts1 = clean1.split(".").map(Number); + const parts2 = clean2.split(".").map(Number); + + const maxLength = Math.max(parts1.length, parts2.length); + while (parts1.length < maxLength) parts1.push(0); + while (parts2.length < maxLength) parts2.push(0); + + for (let i = 0; i < maxLength; i += 1) { + if (parts1[i] > parts2[i]) return 1; + if (parts1[i] < parts2[i]) return -1; + } + return 0; +}; + +const isVersionAtLeastMinimum = (tagName: string, minimumVersion = "0.147.0"): boolean => { + return compareVersions(tagName, minimumVersion) >= 0; +}; + const makeRequest = async ( url: string, config: GitHubClientConfig, @@ -142,10 +185,11 @@ export const fetchReleases = async (includePrerelease = false): Promise[] = []; for (let i = 0; i < maxConcurrentRequests && page <= config.maxPagesPerRequest; i += 1, page += 1) { @@ -160,7 +204,16 @@ export const fetchReleases = async (includePrerelease = false): Promise 0) { - allReleases.push(...data); + for (const release of data) { + if (!release.draft && isMainInfisicalRelease(release.tag_name)) { + if (isVersionAtLeastMinimum(release.tag_name)) { + allReleases.push(release); + } else { + reachedMinimumVersion = true; + break; + } + } + } hasData = true; } } @@ -172,8 +225,6 @@ export const fetchReleases = async (includePrerelease = false): Promise !release.draft) - .filter((release) => isMainInfisicalRelease(release.tag_name)) .map( (release): FormattedRelease => ({ tagName: release.tag_name, diff --git a/backend/src/services/upgrade-path/upgrade-path-service.ts b/backend/src/services/upgrade-path/upgrade-path-service.ts index b12eb52d3..ebba09de6 100644 --- a/backend/src/services/upgrade-path/upgrade-path-service.ts +++ b/backend/src/services/upgrade-path/upgrade-path-service.ts @@ -120,35 +120,6 @@ export const upgradePathServiceFactory = ({ keyStore }: TUpgradePathServiceFacto return version.replace(new RE2(/^v/), "").replace(new RE2(/-[a-zA-Z]+$/), ""); }; - const findBreakingChangesForVersion = ( - version: FormattedRelease, - config: Record> - ): BreakingChange[] => { - // Check multiple key variations for breaking changes configuration - const versionNumber = normalizeVersion(version.tagName); - const cleanVersionNumber = versionNumber.replace(new RE2(/^v/), ""); - - const possibleKeys = [ - version.tagName, - version.normalizedTagName, - versionNumber, - `v${cleanVersionNumber}`, - cleanVersionNumber, - version.tagName.replace(new RE2(/^infisical\//), ""), - version.tagName.replace(new RE2(/^infisical\/v?/), "").replace(new RE2(/-[a-zA-Z]+$/), ""), - `v${cleanVersionNumber}`, - cleanVersionNumber - ]; - - for (const key of possibleKeys) { - const versionConfig = config[key]; - if (versionConfig?.breaking_changes?.length) { - return versionConfig.breaking_changes as BreakingChange[]; - } - } - return []; - }; - const validateParams = (params: CalculateUpgradePathParams) => { const { fromVersion, toVersion } = params; @@ -182,59 +153,69 @@ export const upgradePathServiceFactory = ({ keyStore }: TUpgradePathServiceFacto const cleanFrom = normalizeVersion(fromVersion); const cleanTo = normalizeVersion(toVersion); + const compareVersions = (v1: string, v2: string): number => { + const normalize = (v: string) => normalizeVersion(v); + const clean1 = normalize(v1); + const clean2 = normalize(v2); + + const parts1 = clean1.split(".").map(Number); + const parts2 = clean2.split(".").map(Number); + + const maxLength = Math.max(parts1.length, parts2.length); + while (parts1.length < maxLength) parts1.push(0); + while (parts2.length < maxLength) parts2.push(0); + + for (let i = 0; i < maxLength; i += 1) { + if (parts1[i] > parts2[i]) return 1; + if (parts1[i] < parts2[i]) return -1; + } + return 0; + }; + + if (compareVersions(cleanFrom, cleanTo) >= 0) { + throw new Error("fromVersion must be older than toVersion"); + } + const fromIdx = releases.findIndex((r) => normalizeVersion(r.normalizedTagName) === cleanFrom); const toIdx = releases.findIndex((r) => normalizeVersion(r.normalizedTagName) === cleanTo); - if (fromIdx === -1) throw new Error(`Version ${fromVersion} not found`); - if (toIdx === -1) throw new Error(`Version ${toVersion} not found`); - if (fromIdx <= toIdx) throw new Error("Invalid version order"); + let upgradePath: FormattedRelease[] = []; + const filteredPath: FormattedRelease[] = []; - const upgradePath = releases.slice(toIdx, fromIdx + 1).reverse(); - const [first, last] = [upgradePath[0], upgradePath[upgradePath.length - 1]]; + if (fromIdx !== -1 && toIdx !== -1) { + if (fromIdx <= toIdx) throw new Error("Invalid version order"); + upgradePath = releases.slice(toIdx, fromIdx + 1).reverse(); + const [first, last] = [upgradePath[0], upgradePath[upgradePath.length - 1]]; - // Find all versions with breaking changes in the upgrade path - const withBreakingChanges = upgradePath.filter((version) => { - const breakingChanges = findBreakingChangesForVersion(version, config); - return breakingChanges.length > 0; - }); - - // Build the filtered path with breaking change versions - const filteredPath = [first]; - - // Get intermediate versions with breaking changes (excluding first and last) - const allIntermediateWithBreaking = withBreakingChanges - .filter((v) => v !== first && v !== last) - .sort((a, b) => new Date(a.publishedAt).getTime() - new Date(b.publishedAt).getTime()); - - // Limit intermediate steps to avoid overly complex upgrade paths - const maxIntermediateSteps = 8; - const intermediate = - allIntermediateWithBreaking.length > maxIntermediateSteps - ? allIntermediateWithBreaking.slice(-maxIntermediateSteps) - : allIntermediateWithBreaking; - - filteredPath.push(...intermediate); - if (last !== first) filteredPath.push(last); + filteredPath.push(first); + if (last !== first) filteredPath.push(last); + } const breakingChanges: Array<{ version: string; changes: BreakingChange[] }> = []; const features: Array<{ version: string; name: string; body: string; publishedAt: string }> = []; let hasDbMigration = false; - // Process versions in upgrade path + const isVersionInRange = (version: string, fromVer: string, toVer: string): boolean => { + const versionComp = compareVersions(version, fromVer); + const toVersionComp = compareVersions(version, toVer); + return versionComp > 0 && toVersionComp < 0; + }; + + Object.keys(config).forEach((configVersion) => { + const versionConfig = config[configVersion]; + if (versionConfig?.breaking_changes?.length) { + if (isVersionInRange(configVersion, cleanFrom, cleanTo)) { + breakingChanges.push({ + version: configVersion, + changes: versionConfig.breaking_changes + }); + } + } + }); for (let i = 0; i < upgradePath.length; i += 1) { const version = upgradePath[i]; const isFromVersion = normalizeVersion(version.normalizedTagName) === cleanFrom; - // Process breaking changes for all versions in the upgrade path - const versionBreakingChanges = findBreakingChangesForVersion(version, config); - if (versionBreakingChanges.length > 0) { - breakingChanges.push({ - version: version.tagName, - changes: versionBreakingChanges - }); - } - - // Process database migrations for intermediate versions only (excluding starting version) if (!isFromVersion) { const versionNumber = normalizeVersion(version.tagName); const possibleKeys = [ diff --git a/frontend/src/layouts/OrganizationLayout/components/NavBar/Navbar.tsx b/frontend/src/layouts/OrganizationLayout/components/NavBar/Navbar.tsx index 71be38b11..128760ab4 100644 --- a/frontend/src/layouts/OrganizationLayout/components/NavBar/Navbar.tsx +++ b/frontend/src/layouts/OrganizationLayout/components/NavBar/Navbar.tsx @@ -13,6 +13,7 @@ import { faInfoCircle, faServer, faSignOut, + faToolbox, faUser, faUsers } from "@fortawesome/free-solid-svg-icons"; @@ -104,6 +105,11 @@ export const INFISICAL_SUPPORT_OPTIONS = [ , "Instance Admins", () => "server-admins" + ], + [ + , + "Version Upgrade Tool", + () => "/upgrade-path" ] ] as const; @@ -345,6 +351,9 @@ export const Navbar = () => { if (url === "server-admins" && isInfisicalCloud()) { return null; } + if (url === "upgrade-path" && isInfisicalCloud()) { + return null; + } return ( {url === "server-admins" ? ( diff --git a/frontend/src/pages/public/UpgradePathPage/UpgradePathPage.tsx b/frontend/src/pages/public/UpgradePathPage/UpgradePathPage.tsx index ff1f60530..79dbed9ef 100644 --- a/frontend/src/pages/public/UpgradePathPage/UpgradePathPage.tsx +++ b/frontend/src/pages/public/UpgradePathPage/UpgradePathPage.tsx @@ -328,9 +328,48 @@ export const UpgradePathPage = () => {
- {upgradeResult.path.map((step, index) => { + {(() => { + const pathSteps = upgradeResult.path.map((step) => ({ + ...step, + hasGithubRelease: true + })); + + const breakingChangeSteps = upgradeResult.breakingChanges + .filter( + (bc) => + !upgradeResult.path.some((step) => { + const normalizeVersion = (v: string) => + v.replace(/^(infisical\/)?v?/, "").replace(/-[a-zA-Z]+$/, ""); + return ( + normalizeVersion(step.version) === normalizeVersion(bc.version) + ); + }) + ) + .map((bc) => ({ + version: bc.version, + name: bc.version, + publishedAt: new Date().toISOString(), + prerelease: false, + hasGithubRelease: false + })); + + const allSteps = [...pathSteps, ...breakingChangeSteps]; + + allSteps.sort((a, b) => { + const normalizeForSort = (v: string) => { + const cleaned = v + .replace(/^(infisical\/)?v?/, "") + .replace(/-[a-zA-Z]+$/, ""); + const parts = cleaned.split(".").map(Number); + return parts[0] * 1000000 + (parts[1] || 0) * 1000 + (parts[2] || 0); + }; + return normalizeForSort(a.version) - normalizeForSort(b.version); + }); + + return allSteps; + })().map((step, index, allSteps) => { const isFirst = index === 0; - const isLast = index === upgradeResult.path.length - 1; + const isLast = index === allSteps.length - 1; const versionChanges = upgradeResult.breakingChanges.find((bc) => { if (bc.version === step.version) return true; @@ -412,15 +451,21 @@ export const UpgradePathPage = () => { Target Version )} - - - View Changelog - + {step.hasGithubRelease ? ( + + + View Changelog + + ) : ( + + No GitHub Release + + )}
{/* Version Notes */}