UN-2930 [MISC] Add plugin infrastructure to unstract-core

.gitignore

@@ -620,29 +620,6 @@ $RECYCLE.BIN/
 
 ### Unstract ###
 
-# Authentication Plugins
-backend/plugins/authentication/*
-!backend/plugins/authentication/auth_sample
-
-# Processor Plugins
-backend/plugins/processor/*
-
-# Subscription Plugins
-backend/plugins/subscription/*
-
-
-# API Deployment Plugins
-backend/plugins/api/**
-
-# Notification Plugin
-backend/plugins/notification/**
-
-# Configuration Plugin
-backend/plugins/configuration/**
-
-# Verticals Usage Plugin
-backend/plugins/verticals_usage/**
-
 # BE pluggable-apps
 backend/pluggable_apps/*
 

backend/backend/settings/base.py

@@ -286,7 +286,7 @@ def filter(self, record):
     "drf_yasg",
     "docs",
     # Plugins
-    "plugins",
+    "plugins.apps.PluginsConfig",
     "feature_flag",
     "django_celery_beat",
     # For additional helper commands

backend/plugins/.gitignore

@@ -1 +1,12 @@
-modifier
+# Ignore all plugin implementation directories
+# Keep only infrastructure files (__init__.py, plugin_manager.py, etc.)
+*/
+
+# But allow infrastructure files to be tracked
+!__init__.py
+!apps.py
+
+# TODO: Make these as proper plugins
+!authentication/auth_sample
+!api/dto/__init__.py
+!workflow_manager

backend/plugins/README.md

@@ -1,5 +1,3 @@
 # Plugins
 
-## Authentication
-
-Enhance the authentication capabilities of the `account/authentication_controller.py` module by incorporating additional modules.
+Read [unstract-core's README](../../unstract/core/src/unstract/core/plugins/README.md) for more details on adding and using plugins.

backend/plugins/__init__.py

@@ -0,0 +1,60 @@
+"""Backend plugins initialization.
+
+This module initializes the Django plugin manager for all backend plugins.
+"""
+
+import logging
+from pathlib import Path
+
+from unstract.core.django import DjangoPluginManager
+
+logger = logging.getLogger(__name__)
+
+# Initialize plugin manager singleton
+_plugin_manager = None
+
+
+def get_plugin_manager() -> DjangoPluginManager:
+    """Get or initialize the plugin manager singleton.
+
+    Note: Plugins are loaded by PluginsConfig.ready() after Django initialization.
+    This function only creates the manager instance without loading plugins.
+
+    Returns:
+        DjangoPluginManager: The plugin manager instance
+    """
+    global _plugin_manager
+    if _plugin_manager is None:
+        plugins_dir = Path(__file__).parent
+        _plugin_manager = DjangoPluginManager(
+            plugins_dir=plugins_dir,
+            plugins_pkg="plugins",
+            logger=logger,
+        )
+    return _plugin_manager
+
+
+def get_plugin(plugin_name: str) -> dict:
+    """Get plugin metadata by name (simplified single-line access).
+
+    This is a convenience function that combines get_plugin_manager()
+    and plugin_manager.get_plugin() into one call.
+
+    Args:
+        plugin_name: Name of the plugin to retrieve
+
+    Returns:
+        Dictionary containing plugin metadata (version, module, service_class, etc.)
+        or empty dict if plugin not found
+
+    Example:
+        >>> from plugins import get_plugin
+        >>> plugin = get_plugin("subscription_usage")
+        >>> if plugin:
+        ...     service = plugin["service_class"]()
+    """
+    manager = get_plugin_manager()
+    return manager.get_plugin(plugin_name)
+
+
+__all__ = ["get_plugin_manager", "get_plugin"]

backend/plugins/apps.py

@@ -0,0 +1,34 @@
+"""Django AppConfig for backend plugins."""
+
+import logging
+
+from django.apps import AppConfig
+
+logger = logging.getLogger(__name__)
+
+
+class PluginsConfig(AppConfig):
+    """Django AppConfig for the plugins package.
+
+    This loads plugins after Django is fully initialized to avoid
+    AppRegistryNotReady errors.
+    """
+
+    name = "plugins"
+    verbose_name = "Backend Plugins"
+
+    def ready(self) -> None:
+        """Load plugins after Django apps are ready.
+
+        This method is called by Django after all apps are loaded,
+        ensuring that models and other Django components are available.
+        """
+        from plugins import get_plugin_manager
+
+        try:
+            plugin_manager = get_plugin_manager()
+            plugin_manager.load_plugins()
+            logger.info("Backend plugins loaded successfully")
+        except Exception as e:
+            logger.error(f"Failed to load plugins: {e}", exc_info=True)
+            # Don't raise - allow Django to continue even if plugins fail

unstract/core/src/unstract/core/cache/redis_client.py

@@ -252,6 +252,7 @@ def from_env(cls) -> "RedisClient":
             REDIS_HOST: Redis host (default: localhost)
             REDIS_PORT: Redis port (default: 6379)
             REDIS_USER: Redis username (optional)
+            REDIS_USERNAME: Redis username (optional, alternative to REDIS_USER)
             REDIS_PASSWORD: Redis password (optional)
             REDIS_DB: Redis database number (default: 0)
 
@@ -263,7 +264,7 @@ def from_env(cls) -> "RedisClient":
         return cls(
             host=os.getenv("REDIS_HOST", "localhost"),
             port=int(os.getenv("REDIS_PORT", "6379")),
-            username=os.getenv("REDIS_USER"),
+            username=os.getenv("REDIS_USER") or os.getenv("REDIS_USERNAME"),
             password=os.getenv("REDIS_PASSWORD"),
             db=int(os.getenv("REDIS_DB", "0")),
         )

unstract/core/src/unstract/core/django/__init__.py

@@ -0,0 +1 @@
+from .plugin import DjangoPluginManager, PluginManager, plugin_loader  # noqa: F401

unstract/core/src/unstract/core/django/plugin.py

@@ -0,0 +1,163 @@
+"""Django-specific plugin manager wrapper.
+
+This module provides Django integration for the generic PluginManager,
+handling Django-specific features like app registry integration and Django
+logging.
+"""
+
+import logging
+from pathlib import Path
+from typing import Any
+
+from unstract.core.plugins import PluginManager as GenericPluginManager
+
+
+class DjangoPluginManager:
+    """Django-specific plugin manager wrapper.
+
+    Wraps the generic PluginManager with Django-specific functionality like
+    Django app integration and logging.
+    """
+
+    _instance = None
+
+    def __new__(
+        cls,
+        plugins_dir: Path | str,
+        plugins_pkg: str,
+        logger: logging.Logger | None = None,
+    ) -> "DjangoPluginManager":
+        """Create or return the singleton DjangoPluginManager instance.
+
+        Args:
+            plugins_dir: Directory containing plugins
+            plugins_pkg: Python package path for plugins (e.g., 'myapp.plugins')
+            logger: Logger instance (defaults to module logger)
+
+        Returns:
+            DjangoPluginManager singleton instance
+        """
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+            cls._instance._initialized = False
+
+        # Initialize or update plugin manager if parameters change
+        if plugins_dir and plugins_pkg:
+            plugins_dir_path = (
+                Path(plugins_dir) if isinstance(plugins_dir, str) else plugins_dir
+            )
+
+            if (
+                not cls._instance._initialized
+                or cls._instance._plugins_dir != plugins_dir_path
+                or cls._instance._plugins_pkg != plugins_pkg
+            ):
+                cls._instance._plugins_dir = plugins_dir_path
+                cls._instance._plugins_pkg = plugins_pkg
+                cls._instance._logger = logger or logging.getLogger(__name__)
+                cls._instance._init_manager()
+
+        return cls._instance
+
+    def _init_manager(self) -> None:
+        """Initialize the generic plugin manager."""
+        self._manager = GenericPluginManager(
+            plugins_dir=self._plugins_dir,
+            plugins_pkg=self._plugins_pkg,
+            logger=self._logger,
+            use_singleton=True,
+            registration_callback=None,  # Django doesn't need special registration
+        )
+        self._initialized = True
+
+    def load_plugins(self) -> None:
+        """Load plugins using the generic manager."""
+        if not self._initialized:
+            raise RuntimeError(
+                "DjangoPluginManager not initialized. "
+                "Call with plugins_dir and plugins_pkg first."
+            )
+        self._manager.load_plugins()
+
+    def get_plugin(self, name: str) -> dict[str, Any]:
+        """Get plugin metadata by name.
+
+        Args:
+            name: Plugin name to retrieve
+
+        Returns:
+            Dictionary containing plugin metadata
+        """
+        if not self._initialized:
+            return {}
+        return self._manager.get_plugin(name)
+
+    def has_plugin(self, name: str) -> bool:
+        """Check if a plugin is loaded.
+
+        Args:
+            name: Plugin name to check
+
+        Returns:
+            bool: True if plugin exists
+        """
+        if not self._initialized:
+            return False
+        return self._manager.has_plugin(name)
+
+    def get_all_plugins(self) -> dict[str, dict[str, Any]]:
+        """Get all loaded plugins.
+
+        Returns:
+            Dictionary mapping plugin names to their metadata
+        """
+        if not self._initialized:
+            return {}
+        return self._manager.get_all_plugins()
+
+    @property
+    def plugins(self) -> dict[str, dict[str, Any]]:
+        """Get all loaded plugins."""
+        return self.get_all_plugins()
+
+
+# Maintain backward compatibility with common class name
+PluginManager = DjangoPluginManager
+
+
+def plugin_loader(
+    plugins_dir: Path | str,
+    plugins_pkg: str,
+    logger: logging.Logger | None = None,
+) -> DjangoPluginManager:
+    """Load plugins for a Django application.
+
+    Convenience function to create a DjangoPluginManager instance and load plugins.
+
+    Args:
+        plugins_dir: Directory containing plugins
+        plugins_pkg: Python package path for plugins (e.g., 'backend.plugins')
+        logger: Logger instance (optional)
+
+    Returns:
+        DjangoPluginManager: The plugin manager instance
+
+    Example:
+        # In your Django app initialization (e.g., apps.py or __init__.py):
+        from pathlib import Path
+        from unstract.core.django import plugin_loader
+
+        # Load plugins from backend/plugins directory
+        plugins_dir = Path(__file__).parent / 'plugins'
+        manager = plugin_loader(plugins_dir, 'backend.plugins')
+        manager.load_plugins()
+
+        # Later, check for plugins:
+        if manager.has_plugin('subscription_usage'):
+            plugin = manager.get_plugin('subscription_usage')
+            service = plugin['service_class']()
+            service.commit_usage(...)
+    """
+    manager = DjangoPluginManager(plugins_dir, plugins_pkg, logger)
+    manager.load_plugins()
+    return manager

unstract/core/src/unstract/core/flask/__init__.py

@@ -1,2 +1,3 @@
 from .exceptions import register_error_handlers  # noqa: F401
 from .middleware import register_request_id_middleware  # noqa: F401
+from .plugin import PluginManager, plugin_loader  # noqa: F401