Merge pull request #2041 from Infisical/daniel/infisical-binary

feat: Infisical Binary

.github/workflows/build-binaries.yml

@@ -0,0 +1,99 @@
+name: Build Binaries and Deploy
+
+on:
+    workflow_dispatch:
+        inputs:
+            version:
+                description: "Version number"
+                required: true
+                type: string
+
+defaults:
+    run:
+        working-directory: ./backend
+
+jobs:
+    build-and-deploy:
+        runs-on: ubuntu-20.04
+        strategy:
+            matrix:
+                arch: [x64, arm64]
+                os: [linux, win]
+                include:
+                    - os: linux
+                      target: node20-linux
+                    - os: win
+                      target: node20-win
+
+        steps:
+            - name: Checkout code
+              uses: actions/checkout@v3
+
+            - name: Set up Node.js
+              uses: actions/setup-node@v3
+              with:
+                  node-version: 20
+
+            - name: Install pkg
+              run: npm install -g @yao-pkg/pkg
+
+            - name: Install dependencies (backend)
+              run: npm install
+
+            - name: Install dependencies (frontend)
+              run: npm install --prefix ../frontend
+
+            - name: Prerequisites for pkg
+              run: npm run binary:build
+
+            - name: Package into node binary
+              run: |
+                  if [ "${{ matrix.os }}" != "linux" ]; then
+                    pkg --no-bytecode --public-packages "*" --public --compress Brotli --target ${{ matrix.target }}-${{ matrix.arch }} --output ./binary/infisical-standalone-${{ matrix.os }}-${{ matrix.arch }} .
+                  else
+                    pkg --no-bytecode --public-packages "*" --public --compress Brotli --target ${{ matrix.target }}-${{ matrix.arch }} --output ./binary/infisical-standalone .
+                  fi
+
+            # Set up .deb package structure (Debian/Ubuntu only)
+            - name: Set up .deb package structure
+              if: matrix.os == 'linux'
+              run: |
+                  mkdir -p infisical-standalone/DEBIAN
+                  mkdir -p infisical-standalone/usr/local/bin
+                  cp ./binary/infisical-standalone infisical-standalone/usr/local/bin/
+                  chmod +x infisical-standalone/usr/local/bin/infisical-standalone
+
+            - name: Create control file
+              if: matrix.os == 'linux'
+              run: |
+                  cat <<EOF > infisical-standalone/DEBIAN/control
+                  Package: infisical-standalone
+                  Version: ${{ github.event.inputs.version }}
+                  Section: base
+                  Priority: optional
+                  Architecture: ${{ matrix.arch == 'x64' && 'amd64' || matrix.arch }}
+                  Maintainer: Infisical <daniel@infisical.com>
+                  Description: Infisical standalone executable (app.infisical.com)
+                  EOF
+
+            # Build .deb file (Debian/Ubunutu only)
+            - name: Build .deb package
+              if: matrix.os == 'linux'
+              run: |
+                  dpkg-deb --build infisical-standalone
+                  mv infisical-standalone.deb ./binary/infisical-standalone-${{matrix.arch}}.deb
+
+            - uses: actions/setup-python@v4
+            - run: pip install --upgrade cloudsmith-cli
+
+            # Publish .deb file to Cloudsmith (Debian/Ubuntu only)
+            - name: Publish to Cloudsmith (Debian/Ubuntu)
+              if: matrix.os == 'linux'
+              working-directory: ./backend
+              run: cloudsmith push deb --republish --no-wait-for-sync --api-key=${{ secrets.CLOUDSMITH_API_KEY }} infisical/infisical-standalone/any-distro/any-version ./binary/infisical-standalone-${{ matrix.arch }}.deb
+
+            # Publish .exe file to Cloudsmith (Windows only)
+            - name: Publish to Cloudsmith (Windows)
+              if: matrix.os == 'win'
+              working-directory: ./backend
+              run: cloudsmith push raw infisical/infisical-standalone ./binary/infisical-standalone-${{ matrix.os }}-${{ matrix.arch }}.exe --republish --no-wait-for-sync --version ${{ github.event.inputs.version }} --api-key ${{ secrets.CLOUDSMITH_API_KEY }}

.gitignore

@@ -69,3 +69,4 @@ frontend-build
 *.tgz
 cli/infisical-merge
 cli/test/infisical-merge
+/backend/binary

backend/babel.config.json

@@ -0,0 +1,4 @@
+{
+  "presets": ["@babel/preset-env", "@babel/preset-react"],
+  "plugins": ["@babel/plugin-syntax-import-attributes", "babel-plugin-transform-import-meta"]
+}

backend/package.json

@@ -3,11 +3,39 @@
   "version": "1.0.0",
   "description": "",
   "main": "./dist/main.mjs",
+  "bin": "dist/main.js",
+  "pkg": {
+    "scripts": [
+      "dist/**/*.js",
+      "../frontend/node_modules/next/**/*.js",
+      "../frontend/.next/*/**/*.js",
+      "../frontend/node_modules/next/dist/server/**/*.js",
+      "../frontend/node_modules/@fortawesome/fontawesome-svg-core/**/*.js"
+    ],
+    "assets": [
+      "dist/**",
+      "!dist/**/*.js",
+      "node_modules/**",
+      "../frontend/node_modules/**",
+      "../frontend/.next/**",
+      "!../frontend/node_modules/next/dist/server/**/*.js",
+      "../frontend/node_modules/@fortawesome/fontawesome-svg-core/**/*",
+      "../frontend/public/**"
+    ],
+    "outputPath": "binary"
+  },
   "scripts": {
+    "binary:build": "npm run binary:clean && npm run build:frontend && npm run build && npm run binary:babel-frontend && npm run binary:babel-backend && npm run binary:rename-imports",
+    "binary:package": "pkg --no-bytecode --public-packages \"*\" --public --target host .",
+    "binary:babel-backend": " babel ./dist -d ./dist",
+    "binary:babel-frontend": "babel --copy-files ../frontend/.next/server -d ../frontend/.next/server",
+    "binary:clean": "rm -rf ./dist && rm -rf ./binary",
+    "binary:rename-imports": "ts-node ./scripts/rename-mjs.ts",
     "test": "echo \"Error: no test specified\" && exit 1",
     "dev": "tsx watch --clear-screen=false ./src/main.ts  | pino-pretty --colorize --colorizeObjects --singleLine",
     "dev:docker": "nodemon",
     "build": "tsup",
+    "build:frontend": "npm run build --prefix ../frontend",
     "start": "node dist/main.mjs",
     "type:check": "tsc --noEmit",
     "lint:fix": "eslint --fix --ext js,ts ./src",
@@ -31,6 +59,11 @@
   "author": "",
   "license": "ISC",
   "devDependencies": {
+    "@babel/cli": "^7.18.10",
+    "@babel/core": "^7.18.10",
+    "@babel/plugin-syntax-import-attributes": "^7.24.7",
+    "@babel/preset-env": "^7.18.10",
+    "@babel/preset-react": "^7.24.7",
     "@types/bcrypt": "^5.0.2",
     "@types/jmespath": "^0.15.2",
     "@types/jsonwebtoken": "^9.0.5",
@@ -48,6 +81,8 @@
     "@types/uuid": "^9.0.7",
     "@typescript-eslint/eslint-plugin": "^6.20.0",
     "@typescript-eslint/parser": "^6.20.0",
+    "@yao-pkg/pkg": "^5.12.0",
+    "babel-plugin-transform-import-meta": "^2.2.1",
     "eslint": "^8.56.0",
     "eslint-config-airbnb-base": "^15.0.0",
     "eslint-config-airbnb-typescript": "^17.1.0",
@@ -60,7 +95,7 @@
     "pino-pretty": "^10.2.3",
     "prompt-sync": "^4.2.0",
     "rimraf": "^5.0.5",
-    "ts-node": "^10.9.1",
+    "ts-node": "^10.9.2",
     "tsc-alias": "^1.8.8",
     "tsconfig-paths": "^4.2.0",
     "tsup": "^8.0.1",
@@ -90,8 +125,8 @@
     "@peculiar/asn1-schema": "^2.3.8",
     "@peculiar/x509": "^1.10.0",
     "@serdnam/pino-cloudwatch-transport": "^1.0.4",
-    "@sindresorhus/slugify": "^2.2.1",
     "@team-plain/typescript-sdk": "^4.6.1",
+    "@sindresorhus/slugify": "1.1.0",
     "@ucast/mongo2js": "^1.3.4",
     "ajv": "^8.12.0",
     "argon2": "^0.31.2",
@@ -119,7 +154,7 @@
     "lodash.isequal": "^4.5.0",
     "ms": "^2.1.3",
     "mysql2": "^3.9.8",
-    "nanoid": "^5.0.4",
+    "nanoid": "^3.3.4",
     "nodemailer": "^6.9.9",
     "openid-client": "^5.6.5",
     "ora": "^7.0.1",

backend/scripts/rename-mjs.ts

@@ -0,0 +1,27 @@
+/* eslint-disable @typescript-eslint/no-shadow */
+import fs from "node:fs";
+import path from "node:path";
+
+function replaceMjsOccurrences(directory: string) {
+  fs.readdir(directory, (err, files) => {
+    if (err) throw err;
+    files.forEach((file) => {
+      const filePath = path.join(directory, file);
+      if (fs.statSync(filePath).isDirectory()) {
+        replaceMjsOccurrences(filePath);
+      } else {
+        fs.readFile(filePath, "utf8", (err, data) => {
+          if (err) throw err;
+          const result = data.replace(/\.mjs/g, ".js");
+          fs.writeFile(filePath, result, "utf8", (err) => {
+            if (err) throw err;
+            // eslint-disable-next-line no-console
+            console.log(`Updated: ${filePath}`);
+          });
+        });
+      }
+    });
+  });
+}
+
+replaceMjsOccurrences("dist");

backend/src/lib/config/env.ts

@@ -5,6 +5,9 @@ import { zpStr } from "../zod";
 
 export const GITLAB_URL = "https://gitlab.com";
 
+// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any -- If `process.pkg` is set, and it's true, then it means that the app is currently running in a packaged environment (a binary)
+export const IS_PACKAGED = (process as any)?.pkg !== undefined;
+
 const zodStrBool = z
   .enum(["true", "false"])
   .optional()
@@ -20,7 +23,7 @@ const databaseReadReplicaSchema = z
 
 const envSchema = z
   .object({
-    PORT: z.coerce.number().default(4000),
+    PORT: z.coerce.number().default(IS_PACKAGED ? 8080 : 4000),
     DISABLE_SECRET_SCANNING: z
       .enum(["true", "false"])
       .default("false")
@@ -131,7 +134,7 @@ const envSchema = z
     // GENERIC
     STANDALONE_MODE: z
       .enum(["true", "false"])
-      .transform((val) => val === "true")
+      .transform((val) => val === "true" || IS_PACKAGED)
       .optional(),
     INFISICAL_CLOUD: zodStrBool.default("false"),
     MAINTENANCE_MODE: zodStrBool.default("false"),
@@ -148,7 +151,7 @@ const envSchema = z
     isSmtpConfigured: Boolean(data.SMTP_HOST),
     isRedisConfigured: Boolean(data.REDIS_URL),
     isDevelopmentMode: data.NODE_ENV === "development",
-    isProductionMode: data.NODE_ENV === "production",
+    isProductionMode: data.NODE_ENV === "production" || IS_PACKAGED,
     isSecretScanningConfigured:
       Boolean(data.SECRET_SCANNING_GIT_APP_ID) &&
       Boolean(data.SECRET_SCANNING_PRIVATE_KEY) &&

backend/src/lib/fn/argv.ts

@@ -0,0 +1 @@
+export const isMigrationMode = () => !!process.argv.slice(2).find((arg) => arg === "migration:latest"); // example -> ./binary migration:latest

backend/src/lib/fn/index.ts

@@ -1,6 +1,7 @@
 // Some of the functions are taken from https://github.com/rayepps/radash
 // Full credits goes to https://github.com/rayapps to those functions
 // Code taken to keep in in house and to adjust somethings for our needs
+export * from "./argv";
 export * from "./array";
 export * from "./dates";
 export * from "./object";

backend/src/main.ts

@@ -1,8 +1,10 @@
 import dotenv from "dotenv";
+import path from "path";
 
 import { initDbConnection } from "./db";
 import { keyStoreFactory } from "./keystore/keystore";
-import { formatSmtpConfig, initEnvConfig } from "./lib/config/env";
+import { formatSmtpConfig, initEnvConfig, IS_PACKAGED } from "./lib/config/env";
+import { isMigrationMode } from "./lib/fn";
 import { initLogger } from "./lib/logger";
 import { queueServiceFactory } from "./queue";
 import { main } from "./server/app";
@@ -10,6 +12,7 @@ import { bootstrapCheck } from "./server/boot-strap-check";
 import { smtpServiceFactory } from "./services/smtp/smtp-service";
 
 dotenv.config();
+
 const run = async () => {
   const logger = await initLogger();
   const appCfg = initEnvConfig(logger);
@@ -22,12 +25,30 @@ const run = async () => {
     }))
   });
 
+  // Case: App is running in packaged mode (binary), and migration mode is enabled.
+  // Run the migrations and exit the process after completion.
+  if (IS_PACKAGED && isMigrationMode()) {
+    try {
+      logger.info("Running Postgres migrations..");
+      await db.migrate.latest({
+        directory: path.join(__dirname, "./db/migrations")
+      });
+      logger.info("Postgres migrations completed");
+    } catch (err) {
+      logger.error(err, "Failed to run migrations");
+      process.exit(1);
+    }
+
+    process.exit(0);
+  }
+
   const smtp = smtpServiceFactory(formatSmtpConfig());
   const queue = queueServiceFactory(appCfg.REDIS_URL);
   const keyStore = keyStoreFactory(appCfg.REDIS_URL);
 
   const server = await main({ db, smtp, logger, queue, keyStore });
   const bootstrap = await bootstrapCheck({ db });
+
   // eslint-disable-next-line
   process.on("SIGINT", async () => {
     await server.close();

backend/src/server/app.ts

@@ -15,7 +15,7 @@ import { Knex } from "knex";
 import { Logger } from "pino";
 
 import { TKeyStoreFactory } from "@app/keystore/keystore";
-import { getConfig } from "@app/lib/config/env";
+import { getConfig, IS_PACKAGED } from "@app/lib/config/env";
 import { TQueueServiceFactory } from "@app/queue";
 import { TSmtpService } from "@app/services/smtp/smtp-service";
 
@@ -80,8 +80,8 @@ export const main = async ({ db, smtp, logger, queue, keyStore }: TMain) => {
 
     if (appCfg.isProductionMode) {
       await server.register(registerExternalNextjs, {
-        standaloneMode: appCfg.STANDALONE_MODE,
+        standaloneMode: appCfg.STANDALONE_MODE || IS_PACKAGED,
-        dir: path.join(__dirname, "../../"),
+        dir: path.join(__dirname, IS_PACKAGED ? "../../../" : "../../"),
         port: appCfg.PORT
       });
     }

backend/src/server/plugins/external-nextjs.ts

@@ -1,9 +1,10 @@
 // this plugins allows to run infisical in standalone mode
 // standalone mode = infisical backend and nextjs frontend in one server
 // this way users don't need to deploy two things
-
 import path from "node:path";
 
+import { IS_PACKAGED } from "@app/lib/config/env";
+
 // to enabled this u need to set standalone mode to true
 export const registerExternalNextjs = async (
   server: FastifyZodProvider,
@@ -18,20 +19,33 @@ export const registerExternalNextjs = async (
   }
 ) => {
   if (standaloneMode) {
-    const nextJsBuildPath = path.join(dir, "frontend-build");
+    const frontendName = IS_PACKAGED ? "frontend" : "frontend-build";
+    const nextJsBuildPath = path.join(dir, frontendName);
 
     const { default: conf } = (await import(
-      path.join(dir, "frontend-build/.next/required-server-files.json"),
+      path.join(dir, `${frontendName}/.next/required-server-files.json`),
       // @ts-expect-error type
       {
         assert: { type: "json" }
       }
     )) as { default: { config: string } };
 
-    /* eslint-disable */
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    const { default: NextServer } = (
+    let NextServer: any;
-      await import(path.join(dir, "frontend-build/node_modules/next/dist/server/next-server.js"))
+
-    ).default;
+    if (!IS_PACKAGED) {
+      /* eslint-disable */
+      const { default: nextServer } = (
+        await import(path.join(dir, `${frontendName}/node_modules/next/dist/server/next-server.js`))
+      ).default;
+
+      NextServer = nextServer;
+    } else {
+      /* eslint-disable */
+      const nextServer = await import(path.join(dir, `${frontendName}/node_modules/next/dist/server/next-server.js`));
+
+      NextServer = nextServer.default;
+    }
 
     const nextApp = new NextServer({
       dev: false,