awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-10-08 06:27:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: resolved trimming keys but keeping last line break for ssh keys and added skip encoding on integration sync

Browse Source
This commit is contained in:
Akhil Mohan
2023-09-28 21:28:43 +05:30
parent 85378e25aa
commit 4929022523
6 changed files with 159 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/helpers/bot.ts
+8 -1
View File
@@ -103,7 +103,10 @@ export const getSecretsBotHelper = async ({
environment: string; environment: string;
secretPath: string; secretPath: string;
}) => { }) => {
const content: Record<string, { value: string; comment?: string }> = {}; const content: Record<
string,
{ value: string; comment?: string; skipMultilineEncoding?: boolean }
> = {};
const key = await getKey({ workspaceId: workspaceId }); const key = await getKey({ workspaceId: workspaceId });
let folderId = "root"; let folderId = "root";
@@ -165,6 +168,8 @@ export const getSecretsBotHelper = async ({
}); });
content[secretKey].comment = commentValue; content[secretKey].comment = commentValue;
} }
content[secretKey].skipMultilineEncoding = secret.skipMultilineEncoding;
}); });
}); });
@@ -194,6 +199,8 @@ export const getSecretsBotHelper = async ({
}); });
content[secretKey].comment = commentValue; content[secretKey].comment = commentValue;
} }
content[secretKey].skipMultilineEncoding = secret.skipMultilineEncoding;
}); });
await expandSecrets(workspaceId.toString(), key, content); await expandSecrets(workspaceId.toString(), key, content);
backend/src/helpers/secrets.ts
+8 -3
View File
@@ -1203,7 +1203,7 @@ const formatMultiValueEnv = (val?: string) => {
export const expandSecrets = async ( export const expandSecrets = async (
workspaceId: string, workspaceId: string,
rootEncKey: string, rootEncKey: string,
secrets: Record<string, { value: string; comment?: string }> secrets: Record<string, { value: string; comment?: string; skipMultilineEncoding?: boolean }>
) => { ) => {
const expandedSec: Record<string, string> = {}; const expandedSec: Record<string, string> = {};
const interpolatedSec: Record<string, string> = {}; const interpolatedSec: Record<string, string> = {};
@@ -1221,7 +1221,10 @@ export const expandSecrets = async (
for (const key of Object.keys(secrets)) { for (const key of Object.keys(secrets)) {
if (expandedSec?.[key]) { if (expandedSec?.[key]) {
secrets[key].value = formatMultiValueEnv(expandedSec[key]); // should not do multi line encoding if user has set it to skip
secrets[key].value = secrets[key].skipMultilineEncoding
? expandedSec[key]
: formatMultiValueEnv(expandedSec[key]);
continue; continue;
} }
@@ -1236,7 +1239,9 @@ export const expandSecrets = async (
key key
); );
secrets[key].value = formatMultiValueEnv(expandedVal); secrets[key].value = secrets[key].skipMultilineEncoding
? expandedVal
: formatMultiValueEnv(expandedVal);
} }
return secrets; return secrets;
backend/src/integrations/sync.ts
+112 -96
View File
@@ -65,10 +65,10 @@ import sodium from "libsodium-wrappers";
import { standardRequest } from "../config/request"; import { standardRequest } from "../config/request";
const getSecretKeyValuePair = ( const getSecretKeyValuePair = (
secrets: Record<string, { value: string; comment?: string } | null> secrets: Record<string, { value: string | null; comment?: string } | null>
) => ) =>
Object.keys(secrets).reduce<Record<string, string>>((prev, key) => { Object.keys(secrets).reduce<Record<string, string | null | undefined>>((prev, key) => {
if (secrets[key]) prev[key] = secrets[key]?.value || ""; prev[key] = secrets?.[key] === null ? null : secrets?.[key]?.value;
return prev; return prev;
}, {}); }, {});
@@ -325,40 +325,42 @@ const syncSecretsGCPSecretManager = async ({
name: string; name: string;
createTime: string; createTime: string;
} }
interface GCPSMListSecretsRes { interface GCPSMListSecretsRes {
secrets?: GCPSecret[]; secrets?: GCPSecret[];
totalSize?: number; totalSize?: number;
nextPageToken?: string; nextPageToken?: string;
} }
let gcpSecrets: GCPSecret[] = []; let gcpSecrets: GCPSecret[] = [];
const pageSize = 100; const pageSize = 100;
let pageToken: string | undefined; let pageToken: string | undefined;
let hasMorePages = true; let hasMorePages = true;
const filterParam = integration.metadata.secretGCPLabel const filterParam = integration.metadata.secretGCPLabel
? `?filter=labels.${integration.metadata.secretGCPLabel.labelName}=${integration.metadata.secretGCPLabel.labelValue}` ? `?filter=labels.${integration.metadata.secretGCPLabel.labelName}=${integration.metadata.secretGCPLabel.labelValue}`
: ""; : "";
while (hasMorePages) { while (hasMorePages) {
const params = new URLSearchParams({ const params = new URLSearchParams({
pageSize: String(pageSize), pageSize: String(pageSize),
...(pageToken ? { pageToken } : {}) ...(pageToken ? { pageToken } : {})
}); });
const res: GCPSMListSecretsRes = (await standardRequest.get( const res: GCPSMListSecretsRes = (
`${INTEGRATION_GCP_SECRET_MANAGER_URL}/v1/projects/${integration.appId}/secrets${filterParam}`, await standardRequest.get(
{ `${INTEGRATION_GCP_SECRET_MANAGER_URL}/v1/projects/${integration.appId}/secrets${filterParam}`,
params, {
headers: { params,
"Authorization": `Bearer ${accessToken}`, headers: {
"Accept-Encoding": "application/json" Authorization: `Bearer ${accessToken}`,
"Accept-Encoding": "application/json"
}
} }
} )
)).data; ).data;
if (res.secrets) { if (res.secrets) {
const filteredSecrets = res.secrets?.filter((gcpSecret) => { const filteredSecrets = res.secrets?.filter((gcpSecret) => {
const arr = gcpSecret.name.split("/"); const arr = gcpSecret.name.split("/");
@@ -366,54 +368,58 @@ const syncSecretsGCPSecretManager = async ({
let isValid = true; let isValid = true;
if (integration.metadata.secretPrefix && !key.startsWith(integration.metadata.secretPrefix)) { if (
integration.metadata.secretPrefix &&
!key.startsWith(integration.metadata.secretPrefix)
) {
isValid = false; isValid = false;
} }
if (integration.metadata.secretSuffix && !key.endsWith(integration.metadata.secretSuffix)) { if (integration.metadata.secretSuffix && !key.endsWith(integration.metadata.secretSuffix)) {
isValid = false; isValid = false;
} }
return isValid; return isValid;
}); });
gcpSecrets = gcpSecrets.concat(filteredSecrets); gcpSecrets = gcpSecrets.concat(filteredSecrets);
} }
if (!res.nextPageToken) { if (!res.nextPageToken) {
hasMorePages = false; hasMorePages = false;
} }
pageToken = res.nextPageToken; pageToken = res.nextPageToken;
} }
const res: { [key: string]: string; } = {}; const res: { [key: string]: string } = {};
interface GCPLatestSecretVersionAccess { interface GCPLatestSecretVersionAccess {
name: string; name: string;
payload: { payload: {
data: string; data: string;
} };
} }
for await (const gcpSecret of gcpSecrets) { for await (const gcpSecret of gcpSecrets) {
const arr = gcpSecret.name.split("/"); const arr = gcpSecret.name.split("/");
const key = arr[arr.length - 1]; const key = arr[arr.length - 1];
const secretLatest: GCPLatestSecretVersionAccess = (await standardRequest.get( const secretLatest: GCPLatestSecretVersionAccess = (
`${INTEGRATION_GCP_SECRET_MANAGER_URL}/v1/projects/${integration.appId}/secrets/${key}/versions/latest:access`, await standardRequest.get(
{ `${INTEGRATION_GCP_SECRET_MANAGER_URL}/v1/projects/${integration.appId}/secrets/${key}/versions/latest:access`,
headers: { {
Authorization: `Bearer ${accessToken}`, headers: {
"Accept-Encoding": "application/json" Authorization: `Bearer ${accessToken}`,
"Accept-Encoding": "application/json"
}
} }
} )
)).data; ).data;
res[key] = Buffer.from(secretLatest.payload.data, "base64").toString("utf-8"); res[key] = Buffer.from(secretLatest.payload.data, "base64").toString("utf-8");
} }
for await (const key of Object.keys(secrets)) { for await (const key of Object.keys(secrets)) {
if (!(key in res)) { if (!(key in res)) {
// case: create secret // case: create secret
@@ -423,11 +429,14 @@ const syncSecretsGCPSecretManager = async ({
replication: { replication: {
automatic: {} automatic: {}
}, },
...(integration.metadata.secretGCPLabel ? { ...(integration.metadata.secretGCPLabel
labels: { ? {
[integration.metadata.secretGCPLabel.labelName]: integration.metadata.secretGCPLabel.labelValue labels: {
} [integration.metadata.secretGCPLabel.labelName]:
} : {}) integration.metadata.secretGCPLabel.labelValue
}
}
: {})
}, },
{ {
params: { params: {
@@ -439,7 +448,7 @@ const syncSecretsGCPSecretManager = async ({
} }
} }
); );
await standardRequest.post( await standardRequest.post(
`${INTEGRATION_GCP_SECRET_MANAGER_URL}/v1/projects/${integration.appId}/secrets/${key}:addVersion`, `${INTEGRATION_GCP_SECRET_MANAGER_URL}/v1/projects/${integration.appId}/secrets/${key}:addVersion`,
{ {
@@ -456,7 +465,7 @@ const syncSecretsGCPSecretManager = async ({
); );
} }
} }
for await (const key of Object.keys(res)) { for await (const key of Object.keys(res)) {
if (!(key in secrets)) { if (!(key in secrets)) {
// case: delete secret // case: delete secret
@@ -489,7 +498,7 @@ const syncSecretsGCPSecretManager = async ({
} }
} }
} }
} };
/** /**
* Sync/push [secrets] to Azure Key Vault with vault URI [integration.app] * Sync/push [secrets] to Azure Key Vault with vault URI [integration.app]
@@ -729,15 +738,12 @@ const syncSecretsAWSParameterStore = async ({
} = {}; } = {};
if (parameterList) { if (parameterList) {
awsParameterStoreSecretsObj = parameterList.reduce( awsParameterStoreSecretsObj = parameterList.reduce((obj: any, secret: any) => {
(obj: any, secret: any) => { return {
return ({ ...obj,
...obj, [secret.Name.substring(integration.path.length)]: secret
[secret.Name.substring(integration.path.length)]: secret };
}); }, {});
},
{}
);
} }
// Identify secrets to create // Identify secrets to create
@@ -1869,8 +1875,10 @@ const syncSecretsGitLab = async ({
value: string; value: string;
environment_scope: string; environment_scope: string;
} }
const gitLabApiUrl = integrationAuth.url ? `${integrationAuth.url}/api` : INTEGRATION_GITLAB_API_URL; const gitLabApiUrl = integrationAuth.url
? `${integrationAuth.url}/api`
: INTEGRATION_GITLAB_API_URL;
const getAllEnvVariables = async (integrationAppId: string, accessToken: string) => { const getAllEnvVariables = async (integrationAppId: string, accessToken: string) => {
const headers = { const headers = {
@@ -1880,7 +1888,9 @@ const syncSecretsGitLab = async ({
}; };
let allEnvVariables: GitLabSecret[] = []; let allEnvVariables: GitLabSecret[] = [];
let url: string | null = `${gitLabApiUrl}/v4/projects/${integrationAppId}/variables?per_page=100`; let url:
| string
| null = `${gitLabApiUrl}/v4/projects/${integrationAppId}/variables?per_page=100`;
while (url) { while (url) {
const response: any = await standardRequest.get(url, { headers }); const response: any = await standardRequest.get(url, { headers });
@@ -1901,23 +1911,27 @@ const syncSecretsGitLab = async ({
const allEnvVariables = await getAllEnvVariables(integration?.appId, accessToken); const allEnvVariables = await getAllEnvVariables(integration?.appId, accessToken);
const getSecretsRes: GitLabSecret[] = allEnvVariables const getSecretsRes: GitLabSecret[] = allEnvVariables
.filter( .filter((secret: GitLabSecret) => secret.environment_scope === integration.targetEnvironment)
(secret: GitLabSecret) => secret.environment_scope === integration.targetEnvironment
)
.filter((gitLabSecret) => { .filter((gitLabSecret) => {
let isValid = true; let isValid = true;
if (integration.metadata.secretPrefix && !gitLabSecret.key.startsWith(integration.metadata.secretPrefix)) { if (
integration.metadata.secretPrefix &&
!gitLabSecret.key.startsWith(integration.metadata.secretPrefix)
) {
isValid = false; isValid = false;
} }
if (integration.metadata.secretSuffix && !gitLabSecret.key.endsWith(integration.metadata.secretSuffix)) { if (
integration.metadata.secretSuffix &&
!gitLabSecret.key.endsWith(integration.metadata.secretSuffix)
) {
isValid = false; isValid = false;
} }
return isValid; return isValid;
}); });
for await (const key of Object.keys(secrets)) { for await (const key of Object.keys(secrets)) {
const existingSecret = getSecretsRes.find((s: any) => s.key == key); const existingSecret = getSecretsRes.find((s: any) => s.key == key);
if (!existingSecret) { if (!existingSecret) {
@@ -2371,41 +2385,43 @@ const syncSecretsTeamCity = async ({
if (integration.targetEnvironment && integration.targetEnvironmentId) { if (integration.targetEnvironment && integration.targetEnvironmentId) {
// case: sync to specific build-config in TeamCity project // case: sync to specific build-config in TeamCity project
const res = (await standardRequest.get<GetTeamCityBuildConfigParametersRes>( const res = (
`${integrationAuth.url}/app/rest/buildTypes/${integration.targetEnvironmentId}/parameters`, await standardRequest.get<GetTeamCityBuildConfigParametersRes>(
{ `${integrationAuth.url}/app/rest/buildTypes/${integration.targetEnvironmentId}/parameters`,
headers: { {
Authorization: `Bearer ${accessToken}`, headers: {
Accept: "application/json", Authorization: `Bearer ${accessToken}`,
}, Accept: "application/json"
} }
)) }
.data )
.property ).data.property
.filter((parameter) => !parameter.inherited) .filter((parameter) => !parameter.inherited)
.reduce((obj: any, secret: TeamCitySecret) => { .reduce((obj: any, secret: TeamCitySecret) => {
const secretName = secret.name.replace(/^env\./, ""); const secretName = secret.name.replace(/^env\./, "");
return { return {
...obj, ...obj,
[secretName]: secret.value [secretName]: secret.value
}; };
}, {}); }, {});
for await (const key of Object.keys(secrets)) { for await (const key of Object.keys(secrets)) {
if (!(key in res) || (key in res && secrets[key].value !== res[key])) { if (!(key in res) || (key in res && secrets[key].value !== res[key])) {
// case: secret does not exist in TeamCity or secret value has changed // case: secret does not exist in TeamCity or secret value has changed
// -> create/update secret // -> create/update secret
await standardRequest.post(`${integrationAuth.url}/app/rest/buildTypes/${integration.targetEnvironmentId}/parameters`, await standardRequest.post(
{ `${integrationAuth.url}/app/rest/buildTypes/${integration.targetEnvironmentId}/parameters`,
name:`env.${key}`, {
value: secrets[key].value name: `env.${key}`,
}, value: secrets[key].value
{
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: "application/json",
}, },
}); {
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: "application/json"
}
}
);
} }
} }
@@ -3034,4 +3050,4 @@ const syncSecretsNorthflank = async ({
); );
}; };
export { syncSecrets }; export { syncSecrets };
backend/src/validation/secrets.ts
+6 -2
View File
@@ -257,7 +257,9 @@ export const CreateSecretRawV3 = z.object({
workspaceId: z.string().trim(), workspaceId: z.string().trim(),
environment: z.string().trim(), environment: z.string().trim(),
secretPath: z.string().trim().default("/"), secretPath: z.string().trim().default("/"),
secretValue: z.string().trim(), secretValue: z
.string()
.transform((val) => (val.at(-1) === "\n" ? `${val.trim()}\n` : val.trim())),
secretComment: z.string().trim(), secretComment: z.string().trim(),
skipMultilineEncoding: z.boolean().optional(), skipMultilineEncoding: z.boolean().optional(),
type: z.enum([SECRET_SHARED, SECRET_PERSONAL]) type: z.enum([SECRET_SHARED, SECRET_PERSONAL])
@@ -274,7 +276,9 @@ export const UpdateSecretByNameRawV3 = z.object({
body: z.object({ body: z.object({
workspaceId: z.string().trim(), workspaceId: z.string().trim(),
environment: z.string().trim(), environment: z.string().trim(),
secretValue: z.string().trim(), secretValue: z
.string()
.transform((val) => (val.at(-1) === "\n" ? `${val.trim()}\n` : val.trim())),
secretPath: z.string().trim().default("/"), secretPath: z.string().trim().default("/"),
skipMultilineEncoding: z.boolean().optional(), skipMultilineEncoding: z.boolean().optional(),
type: z.enum([SECRET_SHARED, SECRET_PERSONAL]).default(SECRET_SHARED) type: z.enum([SECRET_SHARED, SECRET_PERSONAL]).default(SECRET_SHARED)
frontend/src/views/SecretMainPage/components/SecretListView/SecretListView.tsx
+14 -1
View File
@@ -11,6 +11,8 @@ import { secretKeys } from "@app/hooks/api/secrets/queries";
import { DecryptedSecret } from "@app/hooks/api/secrets/types"; import { DecryptedSecret } from "@app/hooks/api/secrets/types";
import { UserWsKeyPair, WsTag } from "@app/hooks/api/types"; import { UserWsKeyPair, WsTag } from "@app/hooks/api/types";
import { secretSnapshotKeys } from "~/hooks/api/secretSnapshots/queries";
import { Filter, GroupBy, SortDir } from "../../SecretMainPage.types"; import { Filter, GroupBy, SortDir } from "../../SecretMainPage.types";
import { SecretDetailSidebar } from "./SecretDetaiSidebar"; import { SecretDetailSidebar } from "./SecretDetaiSidebar";
import { SecretItem } from "./SecretItem"; import { SecretItem } from "./SecretItem";
@@ -218,8 +220,13 @@ export const SecretListView = ({
queryClient.invalidateQueries( queryClient.invalidateQueries(
secretKeys.getProjectSecret({ workspaceId, environment, secretPath }) secretKeys.getProjectSecret({ workspaceId, environment, secretPath })
); );
queryClient.invalidateQueries(
secretSnapshotKeys.list({ workspaceId, environment, directory: secretPath })
);
queryClient.invalidateQueries(
secretSnapshotKeys.count({ workspaceId, environment, directory: secretPath })
);
handlePopUpClose("secretDetail"); handlePopUpClose("secretDetail");
createNotification({ createNotification({
type: "success", type: "success",
text: "Successfully saved secrets" text: "Successfully saved secrets"
@@ -242,6 +249,12 @@ export const SecretListView = ({
queryClient.invalidateQueries( queryClient.invalidateQueries(
secretKeys.getProjectSecret({ workspaceId, environment, secretPath }) secretKeys.getProjectSecret({ workspaceId, environment, secretPath })
); );
queryClient.invalidateQueries(
secretSnapshotKeys.list({ workspaceId, environment, directory: secretPath })
);
queryClient.invalidateQueries(
secretSnapshotKeys.count({ workspaceId, environment, directory: secretPath })
);
handlePopUpClose("deleteSecret"); handlePopUpClose("deleteSecret");
handlePopUpClose("secretDetail"); handlePopUpClose("secretDetail");
createNotification({ createNotification({
frontend/src/views/SecretMainPage/components/SecretListView/SecretListView.utils.ts
+11 -5
View File
@@ -1,3 +1,4 @@
/* eslint-disable no-nested-ternary */
import { z } from "zod"; import { z } from "zod";
export enum SecretActionType { export enum SecretActionType {
@@ -7,11 +8,16 @@ export enum SecretActionType {
} }
export const formSchema = z.object({ export const formSchema = z.object({
key: z.string(), key: z.string().trim(),
value: z.string(), value: z.string().transform((val) => (val.at(-1) === "\n" ? `${val.trim()}\n` : val.trim())),
idOverride: z.string().optional(), idOverride: z.string().trim().optional(),
valueOverride: z.string().optional(), valueOverride: z
overrideAction: z.string().optional(), .string()
.optional()
.transform((val) =>
typeof val === "string" ? (val.at(-1) === "\n" ? `${val.trim()}\n` : val.trim()) : val
),
overrideAction: z.string().trim().optional(),
comment: z.string().trim().optional(), comment: z.string().trim().optional(),
skipMultilineEncoding: z.boolean().optional(), skipMultilineEncoding: z.boolean().optional(),
tags: z tags: z