feat(secret-sync): Add gitlab secret sync

This commit is contained in:
carlosmonastyrski
2025-06-20 21:13:14 -03:00
parent e5352e7aa8
commit e201d77a8f
99 changed files with 2912 additions and 56 deletions
@@ -0,0 +1,10 @@
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
import { TSecretSyncListItem } from "@app/services/secret-sync/secret-sync-types";
export const GITLAB_SYNC_LIST_OPTION: TSecretSyncListItem = {
name: "GitLab",
destination: SecretSync.GitLab,
connection: AppConnection.GitLab,
canImportSecrets: false
};
@@ -0,0 +1,4 @@
export enum GitLabSyncScope {
Individual = "individual",
Group = "group"
}
@@ -0,0 +1,369 @@
/* eslint-disable no-await-in-loop */
import { request } from "@app/lib/config/request";
import { TAppConnectionDALFactory } from "@app/services/app-connection/app-connection-dal";
import { GitLabConnectionMethod, refreshGitLabToken, TGitLabConnection } from "@app/services/app-connection/gitlab";
import { IntegrationUrls } from "@app/services/integration-auth/integration-list";
import { TKmsServiceFactory } from "@app/services/kms/kms-service";
import { TGitLabSyncWithCredentials, TGitLabVariable } from "@app/services/secret-sync/gitlab/gitlab-sync-types";
import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors";
import { matchesSchema } from "@app/services/secret-sync/secret-sync-fns";
import { TSecretMap } from "@app/services/secret-sync/secret-sync-types";
import { SECRET_SYNC_NAME_MAP } from "../secret-sync-maps";
interface TGitLabVariablePayload {
key?: string;
value: string;
variable_type?: "env_var" | "file";
environment_scope?: string;
protected?: boolean;
masked?: boolean;
masked_and_hidden?: boolean;
description?: string;
}
interface TGitLabVariableCreate extends TGitLabVariablePayload {
key: string;
}
interface TGitLabVariableUpdate extends Omit<TGitLabVariablePayload, "key"> {}
type TGitLabSyncFactoryDeps = {
appConnectionDAL: Pick<TAppConnectionDALFactory, "updateById">;
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
};
const getValidAccessToken = async (
connection: TGitLabConnection,
appConnectionDAL: Pick<TAppConnectionDALFactory, "updateById">,
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">
): Promise<string> => {
if (
connection.method === GitLabConnectionMethod.OAuth &&
connection.credentials.refreshToken &&
connection.credentials.expiresAt < new Date()
) {
const accessToken = await refreshGitLabToken(
connection.credentials.refreshToken,
connection.id,
connection.orgId,
appConnectionDAL,
kmsService
);
return accessToken;
}
return connection.credentials.accessToken;
};
const getGitLabApiUrl = (connection: TGitLabConnection): string => {
const baseUrl = connection.credentials.instanceUrl || IntegrationUrls.GITLAB_API_URL;
return baseUrl.includes("/api") ? baseUrl : `${baseUrl}/api`;
};
const buildVariablesEndpoint = (apiUrl: string, projectId: string): string => {
return `${apiUrl}/v4/projects/${encodeURIComponent(projectId)}/variables`;
};
const getGitLabVariables = async ({
accessToken,
connection,
projectId,
targetEnvironment
}: {
accessToken: string;
connection: TGitLabConnection;
projectId: string;
targetEnvironment?: string;
}): Promise<TGitLabVariable[]> => {
try {
const apiUrl = getGitLabApiUrl(connection);
const baseEndpoint = buildVariablesEndpoint(apiUrl, projectId);
const headers = {
Authorization: `Bearer ${accessToken}`,
"Accept-Encoding": "application/json",
"Content-Type": "application/json"
};
let allVariables: TGitLabVariable[] = [];
let url: string | null = `${baseEndpoint}?per_page=100`;
if (targetEnvironment) {
url += `&filter[environment_scope]=${encodeURIComponent(targetEnvironment)}`;
}
while (url) {
const response = await request.get<TGitLabVariable[]>(url, { headers });
allVariables = [...allVariables, ...(response.data || [])];
const linkHeader = response.headers.link as string;
const nextLink = linkHeader?.split(",").find((part: string) => part.includes('rel="next"'));
if (nextLink) {
url = nextLink.trim().split(";")[0].slice(1, -1);
} else {
url = null;
}
}
if (targetEnvironment) {
return allVariables.filter((variable) => variable.environment_scope === targetEnvironment);
}
return allVariables;
} catch (error) {
throw new SecretSyncError({
error,
secretKey: "list_variables"
});
}
};
const createGitLabVariable = async ({
accessToken,
connection,
projectId,
variable
}: {
accessToken: string;
connection: TGitLabConnection;
projectId: string;
variable: TGitLabVariableCreate;
}): Promise<void> => {
try {
const apiUrl = getGitLabApiUrl(connection);
const endpoint = buildVariablesEndpoint(apiUrl, projectId);
const payload = {
key: variable.key,
value: variable.value,
variable_type: variable.variable_type || "env_var",
environment_scope: variable.environment_scope || "*",
protected: variable.protected || false,
masked: variable.masked || false,
masked_and_hidden: variable.masked_and_hidden || false,
raw: false,
...(variable.description && { description: variable.description })
};
await request.post(endpoint, payload, {
headers: {
Authorization: `Bearer ${accessToken}`,
"Accept-Encoding": "application/json",
"Content-Type": "application/json"
}
});
} catch (error) {
throw new SecretSyncError({
error,
secretKey: variable.key
});
}
};
const updateGitLabVariable = async ({
accessToken,
connection,
projectId,
key,
variable,
targetEnvironment
}: {
accessToken: string;
connection: TGitLabConnection;
projectId: string;
key: string;
variable: TGitLabVariableUpdate;
targetEnvironment?: string;
}): Promise<void> => {
try {
const apiUrl = getGitLabApiUrl(connection);
const baseEndpoint = buildVariablesEndpoint(apiUrl, projectId);
let url = `${baseEndpoint}/${encodeURIComponent(key)}`;
if (targetEnvironment) {
url += `?filter[environment_scope]=${encodeURIComponent(targetEnvironment)}`;
}
const payload = {
value: variable.value,
...(variable.variable_type && { variable_type: variable.variable_type }),
...(variable.environment_scope && { environment_scope: variable.environment_scope }),
...(variable.protected !== undefined && { protected: variable.protected }),
...(variable.masked !== undefined && { masked: variable.masked }),
...(variable.masked_and_hidden !== undefined && { masked_and_hidden: variable.masked_and_hidden }),
...(variable.description !== undefined && { description: variable.description || "" })
};
await request.put(url, payload, {
headers: {
Authorization: `Bearer ${accessToken}`,
"Accept-Encoding": "application/json",
"Content-Type": "application/json"
}
});
} catch (error) {
throw new SecretSyncError({
error,
secretKey: key
});
}
};
const deleteGitLabVariable = async ({
accessToken,
connection,
projectId,
key,
targetEnvironment
}: {
accessToken: string;
connection: TGitLabConnection;
projectId: string;
key: string;
targetEnvironment?: string;
}): Promise<void> => {
try {
const apiUrl = getGitLabApiUrl(connection);
const baseEndpoint = buildVariablesEndpoint(apiUrl, projectId);
let url = `${baseEndpoint}/${encodeURIComponent(key)}`;
if (targetEnvironment) {
url += `?filter[environment_scope]=${encodeURIComponent(targetEnvironment)}`;
}
await request.delete(url, {
headers: {
Authorization: `Bearer ${accessToken}`,
"Accept-Encoding": "application/json",
"Content-Type": "application/json"
}
});
} catch (error: unknown) {
throw new SecretSyncError({
error
});
}
};
export const GitLabSyncFns = {
syncSecrets: async (
secretSync: TGitLabSyncWithCredentials,
secretMap: TSecretMap,
{ appConnectionDAL, kmsService }: TGitLabSyncFactoryDeps
): Promise<void> => {
const { connection, environment, destinationConfig } = secretSync;
const { projectId, targetEnvironment } = destinationConfig;
const accessToken = await getValidAccessToken(connection, appConnectionDAL, kmsService);
try {
const currentVariables = await getGitLabVariables({
accessToken,
connection,
projectId,
targetEnvironment
});
const currentVariableMap = new Map(currentVariables.map((v) => [v.key, v]));
for (const [key, { value }] of Object.entries(secretMap)) {
const existingVariable = currentVariableMap.get(key);
if (existingVariable) {
if (existingVariable.value !== value) {
await updateGitLabVariable({
accessToken,
connection,
projectId,
key,
variable: {
value,
variable_type: existingVariable.variable_type,
environment_scope: targetEnvironment || existingVariable.environment_scope,
protected: destinationConfig.shouldProtectSecrets ?? existingVariable.protected,
...(!existingVariable.masked && destinationConfig.shouldMaskSecrets && { masked: value?.length > 8 }),
...(!existingVariable.hidden &&
destinationConfig.shouldHideSecrets && { masked_and_hidden: value?.length > 8 }),
description: existingVariable.description ?? undefined
},
targetEnvironment
});
}
} else {
await createGitLabVariable({
accessToken,
connection,
projectId,
variable: {
key,
value,
variable_type: "env_var",
environment_scope: targetEnvironment || "*",
protected: destinationConfig.shouldProtectSecrets || false,
masked: value?.length > 8 ? destinationConfig.shouldMaskSecrets || false : false,
masked_and_hidden: value?.length > 8 ? destinationConfig.shouldHideSecrets || false : false
}
});
}
}
if (!secretSync.syncOptions.disableSecretDeletion) {
for (const variable of currentVariables) {
const shouldDelete =
matchesSchema(variable.key, environment?.slug || "", secretSync.syncOptions.keySchema) &&
!(variable.key in secretMap);
if (shouldDelete) {
await deleteGitLabVariable({
accessToken,
connection,
projectId,
key: variable.key,
targetEnvironment
});
}
}
}
} catch (error) {
throw new SecretSyncError({
error,
secretKey: "batch_sync"
});
}
},
removeSecrets: async (
secretSync: TGitLabSyncWithCredentials,
secretMap: TSecretMap,
{ appConnectionDAL, kmsService }: TGitLabSyncFactoryDeps
): Promise<void> => {
const { connection, destinationConfig } = secretSync;
const { projectId, targetEnvironment } = destinationConfig;
const accessToken = await getValidAccessToken(connection, appConnectionDAL, kmsService);
try {
for (const key of Object.keys(secretMap)) {
await deleteGitLabVariable({
accessToken,
connection,
projectId,
key,
targetEnvironment
});
}
} catch (error) {
throw new SecretSyncError({
error,
secretKey: "batch_remove"
});
}
},
getSecrets: async (secretSync: TGitLabSyncWithCredentials): Promise<TSecretMap> => {
throw new Error(`${SECRET_SYNC_NAME_MAP[secretSync.destination]} does not support importing secrets.`);
}
};
@@ -0,0 +1,101 @@
import { z } from "zod";
import { SecretSyncs } from "@app/lib/api-docs";
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
import {
BaseSecretSyncSchema,
GenericCreateSecretSyncFieldsSchema,
GenericUpdateSecretSyncFieldsSchema
} from "@app/services/secret-sync/secret-sync-schemas";
import { TSyncOptionsConfig } from "@app/services/secret-sync/secret-sync-types";
import { GitLabSyncScope } from "./gitlab-sync-enums";
const GitLabSyncDestinationConfigSchema = z.discriminatedUnion("scope", [
z.object({
scope: z.literal(GitLabSyncScope.Individual).describe(SecretSyncs.DESTINATION_CONFIG.GITLAB.scope),
projectId: z.string().min(1, "Project ID is required").describe(SecretSyncs.DESTINATION_CONFIG.GITLAB.projectId),
projectName: z
.string()
.min(1, "Project name is required")
.describe(SecretSyncs.DESTINATION_CONFIG.GITLAB.projectName),
targetEnvironment: z
.string()
.optional()
.default("*")
.describe(SecretSyncs.DESTINATION_CONFIG.GITLAB.targetEnvironment),
shouldProtectSecrets: z
.boolean()
.optional()
.default(false)
.describe(SecretSyncs.DESTINATION_CONFIG.GITLAB.shouldProtectSecrets),
shouldMaskSecrets: z
.boolean()
.optional()
.default(false)
.describe(SecretSyncs.DESTINATION_CONFIG.GITLAB.shouldMaskSecrets),
shouldHideSecrets: z
.boolean()
.optional()
.default(false)
.describe(SecretSyncs.DESTINATION_CONFIG.GITLAB.shouldHideSecrets)
}),
z.object({
scope: z.literal(GitLabSyncScope.Group).describe(SecretSyncs.DESTINATION_CONFIG.GITLAB.scope),
groupId: z.string().min(1, "Group ID is required").describe(SecretSyncs.DESTINATION_CONFIG.GITLAB.groupId),
projectId: z.string().min(1, "Project ID is required").describe(SecretSyncs.DESTINATION_CONFIG.GITLAB.projectId),
projectName: z
.string()
.min(1, "Project name is required")
.describe(SecretSyncs.DESTINATION_CONFIG.GITLAB.projectName),
targetEnvironment: z
.string()
.optional()
.default("*")
.describe(SecretSyncs.DESTINATION_CONFIG.GITLAB.targetEnvironment),
shouldProtectSecrets: z
.boolean()
.optional()
.default(false)
.describe(SecretSyncs.DESTINATION_CONFIG.GITLAB.shouldProtectSecrets),
shouldMaskSecrets: z
.boolean()
.optional()
.default(false)
.describe(SecretSyncs.DESTINATION_CONFIG.GITLAB.shouldMaskSecrets),
shouldHideSecrets: z
.boolean()
.optional()
.default(false)
.describe(SecretSyncs.DESTINATION_CONFIG.GITLAB.shouldHideSecrets)
})
]);
const GitLabSyncOptionsConfig: TSyncOptionsConfig = { canImportSecrets: false };
export const GitLabSyncSchema = BaseSecretSyncSchema(SecretSync.GitLab, GitLabSyncOptionsConfig).extend({
destination: z.literal(SecretSync.GitLab),
destinationConfig: GitLabSyncDestinationConfigSchema
});
export const CreateGitLabSyncSchema = GenericCreateSecretSyncFieldsSchema(
SecretSync.GitLab,
GitLabSyncOptionsConfig
).extend({
destinationConfig: GitLabSyncDestinationConfigSchema
});
export const UpdateGitLabSyncSchema = GenericUpdateSecretSyncFieldsSchema(
SecretSync.GitLab,
GitLabSyncOptionsConfig
).extend({
destinationConfig: GitLabSyncDestinationConfigSchema.optional()
});
export const GitLabSyncListItemSchema = z.object({
name: z.literal("GitLab"),
connection: z.literal(AppConnection.GitLab),
destination: z.literal(SecretSync.GitLab),
canImportSecrets: z.literal(false)
});
@@ -0,0 +1,64 @@
import { z } from "zod";
import { TGitLabConnection } from "@app/services/app-connection/gitlab";
import { CreateGitLabSyncSchema, GitLabSyncListItemSchema, GitLabSyncSchema } from "./gitlab-sync-schemas";
export type TGitLabSync = z.infer<typeof GitLabSyncSchema>;
export type TGitLabSyncInput = z.infer<typeof CreateGitLabSyncSchema>;
export type TGitLabSyncListItem = z.infer<typeof GitLabSyncListItemSchema>;
export type TGitLabSyncWithCredentials = TGitLabSync & {
connection: TGitLabConnection;
};
// GitLab CI/CD Variable structure based on API documentation
export type TGitLabVariable = {
key: string;
value: string;
variable_type: "env_var" | "file";
protected: boolean;
masked: boolean;
hidden: boolean;
raw: boolean;
environment_scope: string;
description: string | null;
};
// Type for creating a new variable
export type TGitLabVariableCreate = {
key: string;
value: string;
variable_type?: "env_var" | "file";
protected?: boolean;
masked?: boolean;
raw?: boolean;
environment_scope?: string;
description?: string;
};
// Type for updating an existing variable
export type TGitLabVariableUpdate = {
value: string;
variable_type?: "env_var" | "file";
protected?: boolean;
masked?: boolean;
raw?: boolean;
environment_scope?: string;
description?: string | null;
};
export type TGitLabListVariables = {
accessToken: string;
projectId: string;
environmentScope?: string;
};
export type TGitLabCreateVariable = TGitLabListVariables & {
variable: TGitLabVariableCreate;
};
export type TGitLabUpdateVariable = TGitLabListVariables & {
key: string;
variable: TGitLabVariableUpdate;
};
@@ -0,0 +1,4 @@
export * from "./gitlab-sync-constants";
export * from "./gitlab-sync-fns";
export * from "./gitlab-sync-schemas";
export * from "./gitlab-sync-types";
@@ -18,7 +18,8 @@ export enum SecretSync {
OnePass = "1password",
Heroku = "heroku",
Render = "render",
Flyio = "flyio"
Flyio = "flyio",
GitLab = "gitlab"
}
export enum SecretSyncInitialSyncBehavior {
@@ -32,6 +32,7 @@ import { CAMUNDA_SYNC_LIST_OPTION, camundaSyncFactory } from "./camunda";
import { FLYIO_SYNC_LIST_OPTION, FlyioSyncFns } from "./flyio";
import { GCP_SYNC_LIST_OPTION } from "./gcp";
import { GcpSyncFns } from "./gcp/gcp-sync-fns";
import { GITLAB_SYNC_LIST_OPTION, GitLabSyncFns } from "./gitlab";
import { HC_VAULT_SYNC_LIST_OPTION, HCVaultSyncFns } from "./hc-vault";
import { HEROKU_SYNC_LIST_OPTION, HerokuSyncFns } from "./heroku";
import { HUMANITEC_SYNC_LIST_OPTION } from "./humanitec";
@@ -63,7 +64,8 @@ const SECRET_SYNC_LIST_OPTIONS: Record<SecretSync, TSecretSyncListItem> = {
[SecretSync.OnePass]: ONEPASS_SYNC_LIST_OPTION,
[SecretSync.Heroku]: HEROKU_SYNC_LIST_OPTION,
[SecretSync.Render]: RENDER_SYNC_LIST_OPTION,
[SecretSync.Flyio]: FLYIO_SYNC_LIST_OPTION
[SecretSync.Flyio]: FLYIO_SYNC_LIST_OPTION,
[SecretSync.GitLab]: GITLAB_SYNC_LIST_OPTION
};
export const listSecretSyncOptions = () => {
@@ -227,6 +229,8 @@ export const SecretSyncFns = {
return RenderSyncFns.syncSecrets(secretSync, schemaSecretMap);
case SecretSync.Flyio:
return FlyioSyncFns.syncSecrets(secretSync, schemaSecretMap);
case SecretSync.GitLab:
return GitLabSyncFns.syncSecrets(secretSync, schemaSecretMap, { appConnectionDAL, kmsService });
default:
throw new Error(
`Unhandled sync destination for sync secrets fns: ${(secretSync as TSecretSyncWithCredentials).destination}`
@@ -313,6 +317,9 @@ export const SecretSyncFns = {
case SecretSync.Flyio:
secretMap = await FlyioSyncFns.getSecrets(secretSync);
break;
case SecretSync.GitLab:
secretMap = await GitLabSyncFns.getSecrets(secretSync);
break;
default:
throw new Error(
`Unhandled sync destination for get secrets fns: ${(secretSync as TSecretSyncWithCredentials).destination}`
@@ -386,6 +393,8 @@ export const SecretSyncFns = {
return RenderSyncFns.removeSecrets(secretSync, schemaSecretMap);
case SecretSync.Flyio:
return FlyioSyncFns.removeSecrets(secretSync, schemaSecretMap);
case SecretSync.GitLab:
return GitLabSyncFns.removeSecrets(secretSync, schemaSecretMap, { appConnectionDAL, kmsService });
default:
throw new Error(
`Unhandled sync destination for remove secrets fns: ${(secretSync as TSecretSyncWithCredentials).destination}`
@@ -21,7 +21,8 @@ export const SECRET_SYNC_NAME_MAP: Record<SecretSync, string> = {
[SecretSync.OnePass]: "1Password",
[SecretSync.Heroku]: "Heroku",
[SecretSync.Render]: "Render",
[SecretSync.Flyio]: "Fly.io"
[SecretSync.Flyio]: "Fly.io",
[SecretSync.GitLab]: "GitLab"
};
export const SECRET_SYNC_CONNECTION_MAP: Record<SecretSync, AppConnection> = {
@@ -44,7 +45,8 @@ export const SECRET_SYNC_CONNECTION_MAP: Record<SecretSync, AppConnection> = {
[SecretSync.OnePass]: AppConnection.OnePass,
[SecretSync.Heroku]: AppConnection.Heroku,
[SecretSync.Render]: AppConnection.Render,
[SecretSync.Flyio]: AppConnection.Flyio
[SecretSync.Flyio]: AppConnection.Flyio,
[SecretSync.GitLab]: AppConnection.GitLab
};
export const SECRET_SYNC_PLAN_MAP: Record<SecretSync, SecretSyncPlanType> = {
@@ -67,5 +69,6 @@ export const SECRET_SYNC_PLAN_MAP: Record<SecretSync, SecretSyncPlanType> = {
[SecretSync.OnePass]: SecretSyncPlanType.Regular,
[SecretSync.Heroku]: SecretSyncPlanType.Regular,
[SecretSync.Render]: SecretSyncPlanType.Regular,
[SecretSync.Flyio]: SecretSyncPlanType.Regular
[SecretSync.Flyio]: SecretSyncPlanType.Regular,
[SecretSync.GitLab]: SecretSyncPlanType.Regular
};
@@ -74,6 +74,7 @@ import {
} from "./azure-key-vault";
import { TFlyioSync, TFlyioSyncInput, TFlyioSyncListItem, TFlyioSyncWithCredentials } from "./flyio/flyio-sync-types";
import { TGcpSync, TGcpSyncInput, TGcpSyncListItem, TGcpSyncWithCredentials } from "./gcp";
import { TGitLabSync, TGitLabSyncInput, TGitLabSyncListItem, TGitLabSyncWithCredentials } from "./gitlab";
import {
THCVaultSync,
THCVaultSyncInput,
@@ -127,7 +128,8 @@ export type TSecretSync =
| TOnePassSync
| THerokuSync
| TRenderSync
| TFlyioSync;
| TFlyioSync
| TGitLabSync;
export type TSecretSyncWithCredentials =
| TAwsParameterStoreSyncWithCredentials
@@ -149,7 +151,8 @@ export type TSecretSyncWithCredentials =
| TOnePassSyncWithCredentials
| THerokuSyncWithCredentials
| TRenderSyncWithCredentials
| TFlyioSyncWithCredentials;
| TFlyioSyncWithCredentials
| TGitLabSyncWithCredentials;
export type TSecretSyncInput =
| TAwsParameterStoreSyncInput
@@ -171,7 +174,8 @@ export type TSecretSyncInput =
| TOnePassSyncInput
| THerokuSyncInput
| TRenderSyncInput
| TFlyioSyncInput;
| TFlyioSyncInput
| TGitLabSyncInput;
export type TSecretSyncListItem =
| TAwsParameterStoreSyncListItem
@@ -193,7 +197,8 @@ export type TSecretSyncListItem =
| TOnePassSyncListItem
| THerokuSyncListItem
| TRenderSyncListItem
| TFlyioSyncListItem;
| TFlyioSyncListItem
| TGitLabSyncListItem;
export type TSyncOptionsConfig = {
canImportSecrets: boolean;