Feature/PluginSystem

CONTRIBUTING.md

@@ -12,7 +12,7 @@ Star and share the [Github Repo](https://github.com/FlowiseAI/Flowise).
 
 ## 🙋 Q&A
 
-Search up for any questions in [Q&A section](https://github.com/FlowiseAI/Flowise/discussions/categories/q-a), if you can't find one, don't hesitate to create one. It might helps others that have similar question.
+Search up for any questions in [Q&A section](https://github.com/FlowiseAI/Flowise/discussions/categories/q-a), if you can't find one, don't hesitate to create one. It might helps others that have similar question.
 
 ## 🙌 Share Chatflow
 
@@ -52,16 +52,13 @@ Flowise has 3 different modules in a single mono repository.
 #### Step by step
 
 1. Fork the official [Flowise Github Repository](https://github.com/FlowiseAI/Flowise).
-
 2. Clone your forked repository.
-
 3. Create a new branch, see [guide](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-and-deleting-branches-within-your-repository). Naming conventions:
 
     - For feature branch: `feature/<Your New Feature>`
     - For bug fix branch: `bugfix/<Your New Bugfix>`.
 
 4. Switch to the newly created branch.
-
 5. Go into repository folder
 
     ```bash
@@ -120,26 +117,27 @@ Flowise has 3 different modules in a single mono repository.
 
 Flowise support different environment variables to configure your instance. You can specify the following variables in the `.env` file inside `packages/server` folder. Read [more](https://docs.flowiseai.com/environment-variables)
 
-| Variable                    | Description                                                                  | Type                                             | Default                             |
-| --------------------------- | ---------------------------------------------------------------------------- | ------------------------------------------------ | ----------------------------------- | --- |
-| PORT                        | The HTTP port Flowise runs on                                                | Number                                           | 3000                                |
-| FLOWISE_USERNAME            | Username to login                                                            | String                                           |                                     |
-| FLOWISE_PASSWORD            | Password to login                                                            | String                                           |                                     |
-| DEBUG                       | Print logs from components                                                   | Boolean                                          |                                     |
-| LOG_PATH                    | Location where log files are stored                                          | String                                           | `your-path/Flowise/logs`            |
-| LOG_LEVEL                   | Different levels of logs                                                     | Enum String: `error`, `info`, `verbose`, `debug` | `info`                              |
-| APIKEY_PATH                 | Location where api keys are saved                                            | String                                           | `your-path/Flowise/packages/server` |
-| TOOL_FUNCTION_BUILTIN_DEP   | NodeJS built-in modules to be used for Tool Function                         | String                                           |                                     |
-| TOOL_FUNCTION_EXTERNAL_DEP  | External modules to be used for Tool Function                                | String                                           |                                     |     |
-| DATABASE_TYPE               | Type of database to store the flowise data                                   | Enum String: `sqlite`, `mysql`, `postgres`       | `sqlite`                            |
-| DATABASE_PATH               | Location where database is saved (When DATABASE_TYPE is sqlite)              | String                                           | `your-home-dir/.flowise`            |
-| DATABASE_HOST               | Host URL or IP address (When DATABASE_TYPE is not sqlite)                    | String                                           |                                     |
-| DATABASE_PORT               | Database port (When DATABASE_TYPE is not sqlite)                             | String                                           |                                     |
-| DATABASE_USER               | Database username (When DATABASE_TYPE is not sqlite)                         | String                                           |                                     |
-| DATABASE_PASSWORD           | Database password (When DATABASE_TYPE is not sqlite)                         | String                                           |                                     |
-| DATABASE_NAME               | Database name (When DATABASE_TYPE is not sqlite)                             | String                                           |                                     |
-| SECRETKEY_PATH              | Location where encryption key (used to encrypt/decrypt credentials) is saved | String                                           | `your-path/Flowise/packages/server` |
-| FLOWISE_SECRETKEY_OVERWRITE | Encryption key to be used instead of the key stored in SECRETKEY_PATH        | String                                           |
+| Variable                    | Description                                                                  | Type                                            | Default                             |     |
+| --------------------------- | ---------------------------------------------------------------------------- | ----------------------------------------------- | ----------------------------------- | --- |
+| PORT                        | The HTTP port Flowise runs on                                                | Number                                          | 3000                                |     |
+| FLOWISE_USERNAME            | Username to login                                                            | String                                          |                                     |     |
+| FLOWISE_PASSWORD            | Password to login                                                            | String                                          |                                     |     |
+| DEBUG                       | Print logs from components                                                   | Boolean                                         |                                     |     |
+| LOG_PATH                    | Location where log files are stored                                          | String                                          | `your-path/Flowise/logs`            |     |
+| LOG_LEVEL                   | Different levels of logs                                                     | Enum String:`error`, `info`, `verbose`, `debug` | `info`                              |     |
+| APIKEY_PATH                 | Location where api keys are saved                                            | String                                          | `your-path/Flowise/packages/server` |     |
+| TOOL_FUNCTION_BUILTIN_DEP   | NodeJS built-in modules to be used for Tool Function                         | String                                          |                                     |     |
+| TOOL_FUNCTION_EXTERNAL_DEP  | External modules to be used for Tool Function                                | String                                          |                                     |     |
+| DATABASE_TYPE               | Type of database to store the flowise data                                   | Enum String:`sqlite`, `mysql`, `postgres`       | `sqlite`                            |     |
+| DATABASE_PATH               | Location where database is saved (When DATABASE_TYPE is sqlite)              | String                                          | `your-home-dir/.flowise`            |     |
+| DATABASE_HOST               | Host URL or IP address (When DATABASE_TYPE is not sqlite)                    | String                                          |                                     |     |
+| DATABASE_PORT               | Database port (When DATABASE_TYPE is not sqlite)                             | String                                          |                                     |     |
+| DATABASE_USER               | Database username (When DATABASE_TYPE is not sqlite)                         | String                                          |                                     |     |
+| DATABASE_PASSWORD           | Database password (When DATABASE_TYPE is not sqlite)                         | String                                          |                                     |     |
+| DATABASE_NAME               | Database name (When DATABASE_TYPE is not sqlite)                             | String                                          |                                     |     |
+| SECRETKEY_PATH              | Location where encryption key (used to encrypt/decrypt credentials) is saved | String                                          | `your-path/Flowise/packages/server` |     |
+| FLOWISE_SECRETKEY_OVERWRITE | Encryption key to be used instead of the key stored in SECRETKEY_PATH        | String                                          |                                     |     |
+| PLUGIN_PATH                 | Location where plugins are loaded from.                                      | String                                          |                                     |     |
 
 You can also specify the env variables when using `npx`. For example:
 

docker/docker-compose.yml

@@ -15,6 +15,7 @@ services:
             - FLOWISE_SECRETKEY_OVERWRITE=${FLOWISE_SECRETKEY_OVERWRITE}
             - LOG_LEVEL=${LOG_LEVEL}
             - LOG_PATH=${LOG_PATH}
+            - PLUGIN_PATH=${PLUGIN_PATH}
         ports:
             - '${PORT}:${PORT}'
         volumes:

package.json

@@ -7,10 +7,12 @@
         "packages/*",
         "flowise",
         "ui",
-        "components"
+        "components",
+        "plugins/*"
     ],
     "scripts": {
         "build": "turbo run build",
+        "build-nocache": "turbo --no-cache run build",
         "build-force": "turbo run build --force",
         "dev": "turbo run dev --parallel",
         "start": "run-script-os",

packages/server/.env.example

@@ -3,6 +3,7 @@ PORT=3000
 # APIKEY_PATH=/your_api_key_path/.flowise
 # SECRETKEY_PATH=/your_api_key_path/.flowise
 # LOG_PATH=/your_log_path/.flowise/logs
+# PLUGIN_PATH=/your_plugin_path/.flowise/plugins
 
 # NUMBER_OF_PROXIES= 1
 

packages/server/nodemon.json

@@ -1,6 +1,6 @@
 {
     "ignore": ["**/*.spec.ts", ".git", "node_modules"],
-    "watch": ["commands", "index.ts", "src", "../components/nodes", "../components/src"],
+    "watch": ["commands", "index.ts", "src", "../components/nodes", "../components/src", "../../plugins"],
     "exec": "yarn oclif-dev",
     "ext": "ts"
 }

packages/server/src/NodesPool.ts

@@ -4,18 +4,47 @@ import { Dirent } from 'fs'
 import { getNodeModulesPackagePath } from './utils'
 import { promises } from 'fs'
 import { ICommonObject } from 'flowise-components'
+import { hooks } from './utils/hooks'
+
+/**
+ * Available hooks for NodesPool
+ */
+export enum NodesPoolHooks {
+    /**
+     * Action after nodes pool is initialized.
+     * @param {NodesPool} nodesPool
+     */
+    OnInitialize = 'flowise:on-initialize-nodes-pool',
+
+    /**
+     * Allows to add an additional (components) nodes path.
+     * Typically within a plugin
+     * @returns {string}
+     */
+    GetAdditionalNodesPath = 'flowise:get-additional-nodes-path',
+
+    /**
+     * Allows to add an additional credentials path.
+     * Typically within a plugin
+     * @returns {string}
+     */
+    GetAdditionalCredentialsPath = 'flowise:get-additional-credentials-path'
+}
 
 export class NodesPool {
     componentNodes: IComponentNodes = {}
     componentCredentials: IComponentCredentials = {}
     private credentialIconPath: ICommonObject = {}
 
+    constructor() {}
+
     /**
      * Initialize to get all nodes & credentials
      */
     async initialize() {
         await this.initializeNodes()
         await this.initializeCredentials()
+        hooks.emit(NodesPoolHooks.OnInitialize, this)
     }
 
     /**
@@ -24,7 +53,15 @@ export class NodesPool {
     private async initializeNodes() {
         const packagePath = getNodeModulesPackagePath('flowise-components')
         const nodesPath = path.join(packagePath, 'dist', 'nodes')
-        const nodeFiles = await this.getFiles(nodesPath)
+        let nodeFiles = await this.getFiles(nodesPath)
+
+        // Load additional nodes via hook (usually from within plugins)
+        const additionalNodesPathes = await hooks.call(NodesPoolHooks.GetAdditionalNodesPath)
+        for (const additionalNodesPath of additionalNodesPathes) {
+            const _nodeFiles = await this.getFiles(additionalNodesPath as string)
+            nodeFiles.push(..._nodeFiles)
+        }
+
         return Promise.all(
             nodeFiles.map(async (file) => {
                 if (file.endsWith('.js')) {
@@ -64,13 +101,23 @@ export class NodesPool {
         )
     }
 
+    public async addNode() {}
+
     /**
      * Initialize credentials
      */
     private async initializeCredentials() {
         const packagePath = getNodeModulesPackagePath('flowise-components')
         const nodesPath = path.join(packagePath, 'dist', 'credentials')
         const nodeFiles = await this.getFiles(nodesPath)
+
+        // Load additional nodes via hook (usually from within plugins)
+        const additionalCredentialsPathes = await hooks.call(NodesPoolHooks.GetAdditionalCredentialsPath)
+        for (const additionalCredentialsPath of additionalCredentialsPathes) {
+            const _nodeFiles = await this.getFiles(additionalCredentialsPath as string)
+            nodeFiles.push(..._nodeFiles)
+        }
+
         return Promise.all(
             nodeFiles.map(async (file) => {
                 if (file.endsWith('.credential.js')) {

packages/server/src/Plugin.ts

@@ -0,0 +1,148 @@
+import fs from 'fs'
+import path from 'path'
+import logger from './utils/logger'
+import { hooks } from './utils/hooks'
+import { NodesPoolHooks } from './NodesPool'
+
+export async function loadPlugins() {
+    await pluginManager.loadPlugins()
+    return pluginManager
+}
+
+class PluginManager {
+    plugins: IFlowisePlugin[] = []
+
+    isLoaded = false
+
+    isInitialized = false
+
+    /**
+     * Initialize the plugin manager.
+     * Then call await pluginManager.loadPlugins() to load plugins.
+     */
+    constructor() {
+        this.plugins = []
+    }
+
+    /**
+     * Initialize all plugins by calling their init() method.
+     */
+    async initialize() {
+        this.plugins.forEach((plugin) => {
+            plugin.initialize()
+        })
+        this.isInitialized = true
+    }
+
+    getPlugins() {
+        return this.plugins
+    }
+
+    /**
+     * Load plugins from the plugin directory without initializing them.
+     *
+     * @uses process.env.PLUGIN_PATH to specify the plugin directory
+     * @returns {Promise<void>}
+     */
+    async loadPlugins() {
+        const pluginDir = process.env.PLUGIN_PATH
+            ? path.join(__dirname, process.env.PLUGIN_PATH)
+            : path.join(__dirname, '..', '..', '..', 'plugins')
+
+        const pluginFolders = fs.readdirSync(pluginDir)
+        logger.debug(`Loading plugins from ${pluginDir}`)
+
+        for (const folder of pluginFolders) {
+            const pluginPath = path.join(pluginDir, folder)
+            const packageJsonPath = path.join(pluginPath, 'package.json')
+
+            // Check if package.json exists
+            if (!fs.existsSync(packageJsonPath)) {
+                logger.error(`package.json missing for ${pluginFolders} at ${pluginPath}`)
+                continue
+            }
+
+            // Load package.json
+            const packageJson = require(packageJsonPath)
+
+            // Check if package.json has main field
+            if (!packageJson.main) {
+                logger.error(`package.json missing main field for ${pluginFolders} at ${pluginPath}`)
+                continue
+            }
+
+            // Assume main field in package.json points to the compiled plugin file
+            const mainFilePath = path.join(pluginPath, packageJson.main)
+
+            // Dynamic import
+            const pluginModule = await import(mainFilePath)
+            const plugin: IFlowisePlugin = new pluginModule.default()
+
+            plugin.registerComponentNodes()
+            plugin.registerCredentials()
+
+            // Successfully loaded plugin
+            plugin.isLoaded = true
+            this.plugins.push(plugin)
+            logger.debug(`Loaded plugin ${plugin.name} from ${pluginPath}`)
+        }
+        this.isLoaded = true
+    }
+}
+
+export const pluginManager = new PluginManager()
+
+export class FlowisePlugin implements IFlowisePlugin {
+    name: string = 'FlowisePlugin'
+
+    isLoaded: boolean = false
+
+    isInitialized: boolean = false
+
+    dirname: string | null
+
+    nodesPath: string | null
+
+    credentailsPath: string | null
+
+    constructor() {}
+
+    protected logger = logger
+
+    /**
+     * Will be called after app and plugins are loaded and initialized, just before the server starts listening.
+     */
+    initialize() {
+        // Base init implementation
+    }
+
+    registerComponentNodes() {
+        // Register (component) nodes
+        if (this.nodesPath) {
+            this.logger.debug(`Plugin:${this.name}: Registering (component) nodes from ${this.nodesPath}`)
+            hooks.on(NodesPoolHooks.GetAdditionalNodesPath, () => this.nodesPath)
+        }
+    }
+
+    registerCredentials() {
+        // Register credentials nodes
+        if (this.credentailsPath) {
+            this.logger.debug(`Plugin:${this.name}: Registering credentials from ${this.credentailsPath}`)
+            hooks.on(NodesPoolHooks.GetAdditionalCredentialsPath, () => this.credentailsPath)
+        }
+    }
+}
+
+// ------------------------------
+// Interface definitions
+
+// Plugin System
+export interface IFlowisePlugin {
+    isLoaded: boolean
+    name: string
+    dirname: string | null
+    nodesPath: string | null
+    initialize: () => void
+    registerComponentNodes: () => void
+    registerCredentials: () => void
+}

packages/server/src/commands/start.ts

@@ -38,7 +38,8 @@ export default class Start extends Command {
         LANGCHAIN_TRACING_V2: Flags.string(),
         LANGCHAIN_ENDPOINT: Flags.string(),
         LANGCHAIN_API_KEY: Flags.string(),
-        LANGCHAIN_PROJECT: Flags.string()
+        LANGCHAIN_PROJECT: Flags.string(),
+        PLUGIN_PATH: Flags.string()
     }
 
     async stopProcess() {
@@ -107,6 +108,9 @@ export default class Start extends Command {
         if (flags.LANGCHAIN_API_KEY) process.env.LANGCHAIN_API_KEY = flags.LANGCHAIN_API_KEY
         if (flags.LANGCHAIN_PROJECT) process.env.LANGCHAIN_PROJECT = flags.LANGCHAIN_PROJECT
 
+        // Plugin config
+        if (flags.PLUGIN_PATH) process.env.PLUGIN_PATH = flags.PLUGIN_PATH
+
         await (async () => {
             try {
                 logger.info('Starting Flowise...')