Move yaml check to use nodejs, limit versions to 0.147.0 and add UI option to redirect to the new upgrade tool

This commit is contained in:
Carlos Monastyrski
2025-09-25 22:24:47 -03:00
parent af085db685
commit 3c80456c32
6 changed files with 277 additions and 258 deletions

View File

@@ -10,7 +10,7 @@ on:
jobs:
validate-upgrade-path:
name: Validate upgrade-path.yaml format
name: Validate upgrade-path.yaml
runs-on: ubuntu-latest
timeout-minutes: 5
@@ -38,204 +38,135 @@ jobs:
fi
fi
- name: Setup Python for YAML validation
- name: Setup Node.js for YAML validation
if: steps.check-changes.outputs.changed == 'true'
uses: actions/setup-node@v4
with:
node-version: '20'
- name: Create lightweight validation script
if: steps.check-changes.outputs.changed == 'true'
run: |
# Use system python3 and install PyYAML
python3 --version || echo "Python3 not found"
python3 -m pip install --user PyYAML || pip3 install PyYAML || echo "PyYAML installation failed"
# Create a temporary package.json with only the required dependencies
cat > package.json << 'EOF'
{
"name": "upgrade-path-validator",
"version": "1.0.0",
"dependencies": {
"js-yaml": "^4.1.0",
"zod": "^3.22.0"
}
}
EOF
- name: Install minimal validation dependencies
if: steps.check-changes.outputs.changed == 'true'
run: |
npm install --no-package-lock --production
- name: Validate upgrade-path.yaml format
if: steps.check-changes.outputs.changed == 'true'
run: |
echo "Running upgrade-path.yaml validation..."
node << 'EOF'
const fs = require('fs');
const yaml = require('js-yaml');
const { z } = require('zod');
# Debug: Check if file exists
ls -la backend/upgrade-path.yaml || echo "File not found"
// Validation schemas matching backend service
const versionSchema = z
.string()
.min(1)
.max(50)
.regex(/^[a-zA-Z0-9._/-]+$/, "Invalid version format");
# Debug: Show last few lines
echo "Last 5 lines of file:"
tail -5 backend/upgrade-path.yaml || echo "Cannot read file"
const breakingChangeSchema = z.object({
title: z.string().min(1).max(200),
description: z.string().min(1).max(1000),
action: z.string().min(1).max(500)
});
echo "Starting Python validation..."
const versionConfigSchema = z.object({
breaking_changes: z.array(breakingChangeSchema).optional(),
db_schema_changes: z.string().max(1000).optional(),
notes: z.string().max(2000).optional()
});
python3 << 'EOF'
import yaml
import re
import sys
const upgradePathConfigSchema = z.object({
versions: z.record(versionSchema, versionConfigSchema).optional().nullable()
});
def validate_upgrade_path():
try:
print("Reading upgrade-path.yaml...")
with open('backend/upgrade-path.yaml', 'r') as file:
raw_lines = file.readlines()
function validateUpgradePathConfig() {
try {
const yamlPath = './backend/upgrade-path.yaml';
# Remove comments and empty lines more thoroughly
content_lines = []
for line in raw_lines:
# Remove inline comments but preserve quoted strings
stripped = line.strip()
if not stripped or stripped.startswith("#"):
# Skip empty lines and full comment lines
continue
if (!fs.existsSync(yamlPath)) {
console.log('Warning: No upgrade-path.yaml file found');
return true;
}
# Handle inline comments - scan character by character to respect quotes
if "#" in line:
result = ""
in_single_quote = False
in_double_quote = False
i = 0
while i < len(line):
char = line[i]
if char == "'" and not in_double_quote:
in_single_quote = not in_single_quote
elif char == '"' and not in_single_quote:
in_double_quote = not in_double_quote
elif char == "#" and not in_single_quote and not in_double_quote:
# Found unquoted #, rest is comment
break
result += char
i += 1
line = result
const yamlContent = fs.readFileSync(yamlPath, 'utf8');
# Only add non-empty lines after comment removal
if line.strip():
content_lines.append(line.rstrip() + "\n")
if (yamlContent.length > 1024 * 1024) {
throw new Error('Config file too large (>1MB)');
}
content = "".join(content_lines)
// Parse YAML safely
const config = yaml.load(yamlContent, { schema: yaml.FAILSAFE_SCHEMA });
if not content.strip():
raise ValueError("File is empty")
if (!config) {
console.log('Warning: Empty configuration file');
return true;
}
if len(content) > 1024 * 1024:
raise ValueError("File is too large (>1MB)")
// Validate schema
const result = upgradePathConfigSchema.safeParse(config);
print("Parsing YAML content...")
try:
config = yaml.safe_load(content)
print("YAML parsed successfully")
except yaml.YAMLError as e:
print(f"YAML parsing failed: {e}")
raise ValueError(f"Invalid YAML syntax: {e}")
if (!result.success) {
console.log('Validation failed with the following errors:');
result.error.issues.forEach(issue => {
const path = issue.path.length > 0 ? `[${issue.path.join('.')}]` : '';
console.log(` - ${path}: ${issue.message}`);
});
return false;
}
if not isinstance(config, dict):
raise ValueError("Root level must be an object")
const versions = config.versions || {};
const versionCount = Object.keys(versions).length;
print("YAML syntax is valid")
print("Validating schema structure...")
if (versionCount === 0) {
console.log('Warning: No versions found in the configuration');
} else {
console.log(`Validated ${versionCount} version configuration(s)`);
if 'versions' not in config:
print("Warning: No versions found in the configuration")
return True
// Check for common version patterns
const commonPatterns = [
/^v?\d+\.\d+\.\d+$/, // v1.2.3 or 1.2.3
/^v?\d+\.\d+\.\d+\.\d+$/, // v1.2.3.4 or 1.2.3.4
/^infisical\/v?\d+\.\d+\.\d+$/, // infisical/v1.2.3
/^infisical\/v?\d+\.\d+\.\d+-\w+$/ // infisical/v1.2.3-postgres
];
versions = config['versions']
if versions is None:
# Empty versions section is valid
print("Empty versions section found - this is valid")
return True
if not isinstance(versions, dict):
raise ValueError("'versions' must be an object")
for (const versionKey of Object.keys(versions)) {
const isCommonPattern = commonPatterns.some(pattern => pattern.test(versionKey));
if (!isCommonPattern) {
console.log(`Warning: Version key '${versionKey}' doesn't match common patterns. This may be intentional.`);
}
}
}
print(f"Found {len(versions)} version(s) to validate")
console.log('upgrade-path.yaml format is valid');
return true;
# Version key pattern validation
version_pattern = re.compile(r'^[a-zA-Z0-9._/-]+$')
common_patterns = [
re.compile(r'^v?\d+\.\d+\.\d+$'), # v1.2.3 or 1.2.3
re.compile(r'^v?\d+\.\d+\.\d+\.\d+$'), # v1.2.3.4 or 1.2.3.4
re.compile(r'^infisical/v?\d+\.\d+\.\d+$'), # infisical/v1.2.3
re.compile(r'^infisical/v?\d+\.\d+\.\d+-\w+$') # infisical/v1.2.3-postgres
]
} catch (error) {
console.log(`Validation failed: ${error.message}`);
return false;
}
}
errors = []
for version_key, version_config in versions.items():
print(f"Validating version key: {version_key}")
# Validate version key format
if not version_pattern.match(version_key):
errors.append(f"Invalid version key '{version_key}': contains invalid characters")
continue
if len(version_key) > 50:
errors.append(f"Version key '{version_key}' is too long (max 50 characters)")
continue
if not isinstance(version_config, dict):
errors.append(f"Version '{version_key}' configuration must be an object")
continue
# Validate breaking_changes
if 'breaking_changes' in version_config:
breaking_changes = version_config['breaking_changes']
if not isinstance(breaking_changes, list):
errors.append(f"Version '{version_key}': breaking_changes must be a list")
elif len(breaking_changes) == 0:
errors.append(f"Version '{version_key}': breaking_changes is empty (remove field or add items)")
else:
for i, change in enumerate(breaking_changes):
if not isinstance(change, dict):
errors.append(f"Version '{version_key}': breaking_changes[{i}] must be an object")
continue
# Validate required fields
for field in ['title', 'description', 'action']:
if field not in change:
errors.append(f"Version '{version_key}': breaking_changes[{i}] missing '{field}'")
elif not isinstance(change[field], str):
errors.append(f"Version '{version_key}': breaking_changes[{i}].{field} must be string")
elif not change[field].strip():
errors.append(f"Version '{version_key}': breaking_changes[{i}].{field} cannot be empty")
elif field == 'title' and len(change[field]) > 200:
errors.append(f"Version '{version_key}': breaking_changes[{i}].title too long (max 200)")
elif field == 'description' and len(change[field]) > 1000:
errors.append(f"Version '{version_key}': breaking_changes[{i}].description too long (max 1000)")
elif field == 'action' and len(change[field]) > 500:
errors.append(f"Version '{version_key}': breaking_changes[{i}].action too long (max 500)")
# Validate db_schema_changes
if 'db_schema_changes' in version_config:
db_changes = version_config['db_schema_changes']
if not isinstance(db_changes, str):
errors.append(f"Version '{version_key}': db_schema_changes must be string")
elif db_changes == "":
errors.append(f"Version '{version_key}': db_schema_changes is empty (remove field or add content)")
elif len(db_changes) > 1000:
errors.append(f"Version '{version_key}': db_schema_changes too long (max 1000)")
# Validate notes
if 'notes' in version_config:
notes = version_config['notes']
if not isinstance(notes, str):
errors.append(f"Version '{version_key}': notes must be string")
elif notes == "":
errors.append(f"Version '{version_key}': notes is empty (remove field or add content)")
elif len(notes) > 2000:
errors.append(f"Version '{version_key}': notes too long (max 2000)")
# Check if version follows common patterns
is_common_pattern = any(pattern.match(version_key) for pattern in common_patterns)
if not is_common_pattern:
print(f"Warning: Version key '{version_key}' doesn't match common patterns. This may be intentional.")
print(f"Version '{version_key}' is valid")
if errors:
print("Validation failed with the following errors:")
for error in errors:
print(f" - {error}")
return False
print("All validations passed!")
print("upgrade-path.yaml format is valid")
return True
except Exception as e:
print(f"Validation failed: {e}")
return False
if not validate_upgrade_path():
sys.exit(1)
if (!validateUpgradePathConfig()) {
process.exit(1);
}
EOF
- name: Validation completed

View File

@@ -129,6 +129,8 @@ const envSchema = z
POSTHOG_HOST: zpStr(z.string().optional().default("https://app.posthog.com")),
POSTHOG_PROJECT_API_KEY: zpStr(z.string().optional().default("phc_nSin8j5q2zdhpFDI1ETmFNUIuTG4DwKVyIigrY10XiE")),
LOOPS_API_KEY: zpStr(z.string().optional()),
// GitHub API token for upgrade path tool
GITHUB_API_TOKEN: zpStr(z.string().optional()),
// jwt options
AUTH_SECRET: zpStr(z.string()).default(process.env.JWT_AUTH_SECRET), // for those still using old JWT_AUTH_SECRET
JWT_AUTH_LIFETIME: zpStr(z.string().default("10d")),

View File

@@ -1,6 +1,8 @@
/* eslint-disable no-await-in-loop */
import RE2 from "re2";
import { getConfig } from "@app/lib/config/env";
import { FormattedRelease, GitHubApiError, GitHubRelease } from "./types";
interface GitHubClientConfig {
@@ -20,7 +22,7 @@ interface RateLimitInfo {
}
const getDefaultConfig = (): GitHubClientConfig => ({
token: process.env.GITHUB_TOKEN,
token: getConfig().GITHUB_API_TOKEN,
timeout: 30000,
maxRetries: 3,
retryDelay: 1000,
@@ -56,7 +58,15 @@ const isMainInfisicalRelease = (tagName: string): boolean => {
) {
return false;
}
return tagName.startsWith("v") || tagName.startsWith("infisical/v") || new RE2(/^\d+\.\d+\.\d+/).test(tagName);
const patterns = [
new RE2(/^v\d+\.\d+\.\d+/),
new RE2(/^\d+\.\d+\.\d+/),
new RE2(/^infisical\/v?\d+\.\d+\.\d+/),
new RE2(/^infisical\/v?\d+\.\d+\.\d+[-\w]*/)
];
return patterns.some((pattern) => pattern.test(tagName));
};
const normalizeVersion = (tagName: string): string => {
@@ -72,6 +82,39 @@ const normalizeVersion = (tagName: string): string => {
return tagName.replace(new RE2(/-[a-zA-Z]+$/), "");
};
const compareVersions = (v1: string, v2: string): number => {
const normalize = (v: string) => {
const versionMatch = v.match(new RE2(/(\d+\.\d+\.\d+(?:\.\d+)?)/));
if (versionMatch) {
return versionMatch[1];
}
if (v.startsWith("infisical/")) {
return v.replace(new RE2(/^infisical\/v?/), "").replace(new RE2(/-[a-zA-Z]+$/), "");
}
return v.replace(new RE2(/^v/), "").replace(new RE2(/-[a-zA-Z]+$/), "");
};
const clean1 = normalize(v1);
const clean2 = normalize(v2);
const parts1 = clean1.split(".").map(Number);
const parts2 = clean2.split(".").map(Number);
const maxLength = Math.max(parts1.length, parts2.length);
while (parts1.length < maxLength) parts1.push(0);
while (parts2.length < maxLength) parts2.push(0);
for (let i = 0; i < maxLength; i += 1) {
if (parts1[i] > parts2[i]) return 1;
if (parts1[i] < parts2[i]) return -1;
}
return 0;
};
const isVersionAtLeastMinimum = (tagName: string, minimumVersion = "0.147.0"): boolean => {
return compareVersions(tagName, minimumVersion) >= 0;
};
const makeRequest = async <T>(
url: string,
config: GitHubClientConfig,
@@ -142,10 +185,11 @@ export const fetchReleases = async (includePrerelease = false): Promise<Formatte
const allReleases: GitHubRelease[] = [];
let page = 1;
let hasMorePages = true;
let reachedMinimumVersion = false;
const maxConcurrentRequests = Math.min(3, config.maxPagesPerRequest);
while (hasMorePages && page <= config.maxPagesPerRequest) {
while (hasMorePages && page <= config.maxPagesPerRequest && !reachedMinimumVersion) {
const requests: Promise<{ data: GitHubRelease[]; rateLimit: RateLimitInfo }>[] = [];
for (let i = 0; i < maxConcurrentRequests && page <= config.maxPagesPerRequest; i += 1, page += 1) {
@@ -160,7 +204,16 @@ export const fetchReleases = async (includePrerelease = false): Promise<Formatte
if (result.status === "fulfilled") {
const { data } = result.value;
if (data.length > 0) {
allReleases.push(...data);
for (const release of data) {
if (!release.draft && isMainInfisicalRelease(release.tag_name)) {
if (isVersionAtLeastMinimum(release.tag_name)) {
allReleases.push(release);
} else {
reachedMinimumVersion = true;
break;
}
}
}
hasData = true;
}
}
@@ -172,8 +225,6 @@ export const fetchReleases = async (includePrerelease = false): Promise<Formatte
}
const formattedReleases = allReleases
.filter((release) => !release.draft)
.filter((release) => isMainInfisicalRelease(release.tag_name))
.map(
(release): FormattedRelease => ({
tagName: release.tag_name,

View File

@@ -120,35 +120,6 @@ export const upgradePathServiceFactory = ({ keyStore }: TUpgradePathServiceFacto
return version.replace(new RE2(/^v/), "").replace(new RE2(/-[a-zA-Z]+$/), "");
};
const findBreakingChangesForVersion = (
version: FormattedRelease,
config: Record<string, z.infer<typeof versionConfigSchema>>
): BreakingChange[] => {
// Check multiple key variations for breaking changes configuration
const versionNumber = normalizeVersion(version.tagName);
const cleanVersionNumber = versionNumber.replace(new RE2(/^v/), "");
const possibleKeys = [
version.tagName,
version.normalizedTagName,
versionNumber,
`v${cleanVersionNumber}`,
cleanVersionNumber,
version.tagName.replace(new RE2(/^infisical\//), ""),
version.tagName.replace(new RE2(/^infisical\/v?/), "").replace(new RE2(/-[a-zA-Z]+$/), ""),
`v${cleanVersionNumber}`,
cleanVersionNumber
];
for (const key of possibleKeys) {
const versionConfig = config[key];
if (versionConfig?.breaking_changes?.length) {
return versionConfig.breaking_changes as BreakingChange[];
}
}
return [];
};
const validateParams = (params: CalculateUpgradePathParams) => {
const { fromVersion, toVersion } = params;
@@ -182,59 +153,69 @@ export const upgradePathServiceFactory = ({ keyStore }: TUpgradePathServiceFacto
const cleanFrom = normalizeVersion(fromVersion);
const cleanTo = normalizeVersion(toVersion);
const compareVersions = (v1: string, v2: string): number => {
const normalize = (v: string) => normalizeVersion(v);
const clean1 = normalize(v1);
const clean2 = normalize(v2);
const parts1 = clean1.split(".").map(Number);
const parts2 = clean2.split(".").map(Number);
const maxLength = Math.max(parts1.length, parts2.length);
while (parts1.length < maxLength) parts1.push(0);
while (parts2.length < maxLength) parts2.push(0);
for (let i = 0; i < maxLength; i += 1) {
if (parts1[i] > parts2[i]) return 1;
if (parts1[i] < parts2[i]) return -1;
}
return 0;
};
if (compareVersions(cleanFrom, cleanTo) >= 0) {
throw new Error("fromVersion must be older than toVersion");
}
const fromIdx = releases.findIndex((r) => normalizeVersion(r.normalizedTagName) === cleanFrom);
const toIdx = releases.findIndex((r) => normalizeVersion(r.normalizedTagName) === cleanTo);
if (fromIdx === -1) throw new Error(`Version ${fromVersion} not found`);
if (toIdx === -1) throw new Error(`Version ${toVersion} not found`);
if (fromIdx <= toIdx) throw new Error("Invalid version order");
let upgradePath: FormattedRelease[] = [];
const filteredPath: FormattedRelease[] = [];
const upgradePath = releases.slice(toIdx, fromIdx + 1).reverse();
const [first, last] = [upgradePath[0], upgradePath[upgradePath.length - 1]];
if (fromIdx !== -1 && toIdx !== -1) {
if (fromIdx <= toIdx) throw new Error("Invalid version order");
upgradePath = releases.slice(toIdx, fromIdx + 1).reverse();
const [first, last] = [upgradePath[0], upgradePath[upgradePath.length - 1]];
// Find all versions with breaking changes in the upgrade path
const withBreakingChanges = upgradePath.filter((version) => {
const breakingChanges = findBreakingChangesForVersion(version, config);
return breakingChanges.length > 0;
});
// Build the filtered path with breaking change versions
const filteredPath = [first];
// Get intermediate versions with breaking changes (excluding first and last)
const allIntermediateWithBreaking = withBreakingChanges
.filter((v) => v !== first && v !== last)
.sort((a, b) => new Date(a.publishedAt).getTime() - new Date(b.publishedAt).getTime());
// Limit intermediate steps to avoid overly complex upgrade paths
const maxIntermediateSteps = 8;
const intermediate =
allIntermediateWithBreaking.length > maxIntermediateSteps
? allIntermediateWithBreaking.slice(-maxIntermediateSteps)
: allIntermediateWithBreaking;
filteredPath.push(...intermediate);
if (last !== first) filteredPath.push(last);
filteredPath.push(first);
if (last !== first) filteredPath.push(last);
}
const breakingChanges: Array<{ version: string; changes: BreakingChange[] }> = [];
const features: Array<{ version: string; name: string; body: string; publishedAt: string }> = [];
let hasDbMigration = false;
// Process versions in upgrade path
const isVersionInRange = (version: string, fromVer: string, toVer: string): boolean => {
const versionComp = compareVersions(version, fromVer);
const toVersionComp = compareVersions(version, toVer);
return versionComp > 0 && toVersionComp < 0;
};
Object.keys(config).forEach((configVersion) => {
const versionConfig = config[configVersion];
if (versionConfig?.breaking_changes?.length) {
if (isVersionInRange(configVersion, cleanFrom, cleanTo)) {
breakingChanges.push({
version: configVersion,
changes: versionConfig.breaking_changes
});
}
}
});
for (let i = 0; i < upgradePath.length; i += 1) {
const version = upgradePath[i];
const isFromVersion = normalizeVersion(version.normalizedTagName) === cleanFrom;
// Process breaking changes for all versions in the upgrade path
const versionBreakingChanges = findBreakingChangesForVersion(version, config);
if (versionBreakingChanges.length > 0) {
breakingChanges.push({
version: version.tagName,
changes: versionBreakingChanges
});
}
// Process database migrations for intermediate versions only (excluding starting version)
if (!isFromVersion) {
const versionNumber = normalizeVersion(version.tagName);
const possibleKeys = [

View File

@@ -13,6 +13,7 @@ import {
faInfoCircle,
faServer,
faSignOut,
faToolbox,
faUser,
faUsers
} from "@fortawesome/free-solid-svg-icons";
@@ -104,6 +105,11 @@ export const INFISICAL_SUPPORT_OPTIONS = [
<FontAwesomeIcon key={5} className="pr-4 text-sm" icon={faUsers} />,
"Instance Admins",
() => "server-admins"
],
[
<FontAwesomeIcon key={6} className="pr-4 text-sm" icon={faToolbox} />,
"Version Upgrade Tool",
() => "/upgrade-path"
]
] as const;
@@ -345,6 +351,9 @@ export const Navbar = () => {
if (url === "server-admins" && isInfisicalCloud()) {
return null;
}
if (url === "upgrade-path" && isInfisicalCloud()) {
return null;
}
return (
<DropdownMenuItem key={url as string}>
{url === "server-admins" ? (

View File

@@ -328,9 +328,48 @@ export const UpgradePathPage = () => {
</div>
<div className="space-y-6">
{upgradeResult.path.map((step, index) => {
{(() => {
const pathSteps = upgradeResult.path.map((step) => ({
...step,
hasGithubRelease: true
}));
const breakingChangeSteps = upgradeResult.breakingChanges
.filter(
(bc) =>
!upgradeResult.path.some((step) => {
const normalizeVersion = (v: string) =>
v.replace(/^(infisical\/)?v?/, "").replace(/-[a-zA-Z]+$/, "");
return (
normalizeVersion(step.version) === normalizeVersion(bc.version)
);
})
)
.map((bc) => ({
version: bc.version,
name: bc.version,
publishedAt: new Date().toISOString(),
prerelease: false,
hasGithubRelease: false
}));
const allSteps = [...pathSteps, ...breakingChangeSteps];
allSteps.sort((a, b) => {
const normalizeForSort = (v: string) => {
const cleaned = v
.replace(/^(infisical\/)?v?/, "")
.replace(/-[a-zA-Z]+$/, "");
const parts = cleaned.split(".").map(Number);
return parts[0] * 1000000 + (parts[1] || 0) * 1000 + (parts[2] || 0);
};
return normalizeForSort(a.version) - normalizeForSort(b.version);
});
return allSteps;
})().map((step, index, allSteps) => {
const isFirst = index === 0;
const isLast = index === upgradeResult.path.length - 1;
const isLast = index === allSteps.length - 1;
const versionChanges = upgradeResult.breakingChanges.find((bc) => {
if (bc.version === step.version) return true;
@@ -412,15 +451,21 @@ export const UpgradePathPage = () => {
Target Version
</span>
)}
<a
href={`https://github.com/Infisical/infisical/releases/tag/${step.version}`}
target="_blank"
rel="noopener noreferrer"
className="inline-flex items-center space-x-1 text-xs text-primary transition-colors hover:text-primary/80"
>
<FontAwesomeIcon icon={faExternalLink} className="h-3 w-3" />
<span>View Changelog</span>
</a>
{step.hasGithubRelease ? (
<a
href={`https://github.com/Infisical/infisical/releases/tag/${step.version}`}
target="_blank"
rel="noopener noreferrer"
className="inline-flex items-center space-x-1 text-xs text-primary transition-colors hover:text-primary/80"
>
<FontAwesomeIcon icon={faExternalLink} className="h-3 w-3" />
<span>View Changelog</span>
</a>
) : (
<span className="inline-flex items-center space-x-1 text-xs text-bunker-400">
<span>No GitHub Release</span>
</span>
)}
</div>
{/* Version Notes */}