Update Document segmentation for CLI

Author: Zongwei9888

cli/cli_app.py

@@ -37,6 +37,8 @@ def __init__(self):
         self.app = None  # Will be initialized by workflow adapter
         self.logger = None
         self.context = None
+        # Document segmentation configuration
+        self.segmentation_config = {"enabled": True, "size_threshold_chars": 50000}
 
     async def initialize_mcp_app(self):
         """初始化MCP应用 - 使用工作流适配器"""
@@ -47,9 +49,51 @@ async def cleanup_mcp_app(self):
         """清理MCP应用 - 使用工作流适配器"""
         await self.workflow_adapter.cleanup_mcp_app()
 
+    def update_segmentation_config(self):
+        """Update document segmentation configuration in mcp_agent.config.yaml"""
+        import yaml
+        import os
+
+        config_path = os.path.join(
+            os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+            "mcp_agent.config.yaml",
+        )
+
+        try:
+            # Read current config
+            with open(config_path, "r", encoding="utf-8") as f:
+                config = yaml.safe_load(f)
+
+            # Update document segmentation settings
+            if "document_segmentation" not in config:
+                config["document_segmentation"] = {}
+
+            config["document_segmentation"]["enabled"] = self.segmentation_config[
+                "enabled"
+            ]
+            config["document_segmentation"]["size_threshold_chars"] = (
+                self.segmentation_config["size_threshold_chars"]
+            )
+
+            # Write updated config
+            with open(config_path, "w", encoding="utf-8") as f:
+                yaml.dump(config, f, default_flow_style=False, allow_unicode=True)
+
+            self.cli.print_status(
+                "📄 Document segmentation configuration updated", "success"
+            )
+
+        except Exception as e:
+            self.cli.print_status(
+                f"⚠️ Failed to update segmentation config: {str(e)}", "warning"
+            )
+
     async def process_input(self, input_source: str, input_type: str):
         """处理输入源（URL或文件）- 使用升级版智能体编排引擎"""
         try:
+            # Update segmentation configuration before processing
+            self.update_segmentation_config()
+
             self.cli.print_separator()
             self.cli.print_status(
                 "🚀 Starting intelligent agent orchestration...", "processing"
@@ -237,8 +281,20 @@ async def run_interactive_session(self):
                     self.cli.show_history()
 
                 elif choice in ["c", "config", "configure"]:
+                    # Sync current segmentation config from CLI interface
+                    self.segmentation_config["enabled"] = self.cli.segmentation_enabled
+                    self.segmentation_config["size_threshold_chars"] = (
+                        self.cli.segmentation_threshold
+                    )
+
                     self.cli.show_configuration_menu()
 
+                    # Sync back from CLI interface after configuration changes
+                    self.segmentation_config["enabled"] = self.cli.segmentation_enabled
+                    self.segmentation_config["size_threshold_chars"] = (
+                        self.cli.segmentation_threshold
+                    )
+
                 else:
                     self.cli.print_status(
                         "Invalid choice. Please select U, F, T, C, H, or Q.", "warning"

cli/cli_interface.py

@@ -40,6 +40,8 @@ def __init__(self):
         self.is_running = True
         self.processing_history = []
         self.enable_indexing = True  # Default configuration
+        self.segmentation_enabled = True  # Default to smart segmentation
+        self.segmentation_threshold = 50000  # Default threshold
 
         # Check tkinter availability for file dialogs
         self.tkinter_available = True
@@ -125,6 +127,9 @@ def create_menu(self):
         # Display current configuration
         pipeline_mode = "🧠 COMPREHENSIVE" if self.enable_indexing else "⚡ OPTIMIZED"
         index_status = "✅ Enabled" if self.enable_indexing else "🔶 Disabled"
+        segmentation_mode = (
+            "📄 SMART" if self.segmentation_enabled else "📋 TRADITIONAL"
+        )
 
         menu = f"""
 {Colors.BOLD}{Colors.CYAN}╔═══════════════════════════════════════════════════════════════════════════════╗
@@ -135,6 +140,7 @@ def create_menu(self):
 ║                                                                               ║
 ║  {Colors.BOLD}🤖 Current Pipeline Mode: {pipeline_mode}{Colors.CYAN}                          ║
 ║  {Colors.BOLD}🗂️  Codebase Indexing: {index_status}{Colors.CYAN}                                    ║
+║  {Colors.BOLD}📄 Document Processing: {segmentation_mode}{Colors.CYAN}                               ║
 ║                                                                               ║
 ║  {Colors.YELLOW}📝 URL Processing:{Colors.CYAN}                                                         ║
 ║  {Colors.YELLOW}   ▶ Enter research paper URL (arXiv, IEEE, ACM, etc.)                    {Colors.CYAN}║
@@ -693,6 +699,11 @@ def show_history(self):
     def show_configuration_menu(self):
         """Show configuration options menu"""
         self.clear_screen()
+
+        # Get segmentation config status
+        segmentation_enabled = getattr(self, "segmentation_enabled", True)
+        segmentation_threshold = getattr(self, "segmentation_threshold", 50000)
+
         print(f"""
 {Colors.BOLD}{Colors.CYAN}╔═══════════════════════════════════════════════════════════════════════════════╗
 ║                           CONFIGURATION MENU                                  ║
@@ -716,9 +727,23 @@ def show_configuration_menu(self):
 ║         ✗ Repository Acquisition (Skipped)                                   ║
 ║         ✗ Codebase Intelligence Orchestration (Skipped)                     ║
 ║                                                                               ║
-║  {Colors.YELLOW}Current Setting:{Colors.CYAN} {'🧠 Comprehensive Mode' if self.enable_indexing else '⚡ Optimized Mode'}                              ║
+║  {Colors.OKCYAN}[2] Document Processing:{Colors.CYAN}                                                   ║
+║      {Colors.BOLD}📄 Smart Segmentation{Colors.CYAN} - Intelligent document analysis (Default)      ║
+║         ✓ Semantic boundary detection                                        ║
+║         ✓ Algorithm integrity preservation                                   ║
+║         ✓ Formula chain recognition                                          ║
+║         ✓ Adaptive character limits                                          ║
+║                                                                               ║
+║      {Colors.BOLD}📋 Traditional Processing{Colors.CYAN} - Full document reading                       ║
+║         ✓ Complete document analysis                                         ║
+║         ✗ Smart segmentation (Disabled)                                      ║
 ║                                                                               ║
-║  {Colors.OKGREEN}[T] Toggle Pipeline Mode    {Colors.CYAN}│  {Colors.FAIL}[B] Back to Main Menu{Colors.CYAN}            ║
+║  {Colors.YELLOW}Current Settings:{Colors.CYAN}                                                         ║
+║    Pipeline: {'🧠 Comprehensive Mode' if self.enable_indexing else '⚡ Optimized Mode'}                                          ║
+║    Document: {'📄 Smart Segmentation' if segmentation_enabled else '📋 Traditional Processing'}                                ║
+║    Threshold: {segmentation_threshold} characters                                    ║
+║                                                                               ║
+║  {Colors.OKGREEN}[T] Toggle Pipeline    {Colors.BLUE}[S] Toggle Segmentation    {Colors.FAIL}[B] Back{Colors.CYAN}     ║
 ╚═══════════════════════════════════════════════════════════════════════════════╝{Colors.ENDC}
 """)
 
@@ -737,8 +762,25 @@ def show_configuration_menu(self):
                 self.show_configuration_menu()
                 return
 
+            elif choice in ["s", "segmentation"]:
+                current_state = getattr(self, "segmentation_enabled", True)
+                self.segmentation_enabled = not current_state
+                seg_mode = (
+                    "📄 Smart Segmentation"
+                    if self.segmentation_enabled
+                    else "📋 Traditional Processing"
+                )
+                self.print_status(
+                    f"Document processing switched to: {seg_mode}", "success"
+                )
+                time.sleep(1)
+                self.show_configuration_menu()
+                return
+
             elif choice in ["b", "back"]:
                 return
 
             else:
-                self.print_status("Invalid choice. Please enter 'T' or 'B'.", "warning")
+                self.print_status(
+                    "Invalid choice. Please enter 'T', 'S', or 'B'.", "warning"
+                )

cli/main_cli.py

@@ -94,15 +94,21 @@ def parse_arguments():
         formatter_class=argparse.RawDescriptionHelpFormatter,
         epilog=f"""
 {Colors.BOLD}Examples:{Colors.ENDC}
-  {Colors.CYAN}python main_cli.py{Colors.ENDC}                    # Interactive mode
-  {Colors.CYAN}python main_cli.py --file paper.pdf{Colors.ENDC}     # Process file directly
-  {Colors.CYAN}python main_cli.py --url https://...{Colors.ENDC}    # Process URL directly
-  {Colors.CYAN}python main_cli.py --chat "Build a web app..."{Colors.ENDC} # Process chat requirements
-  {Colors.CYAN}python main_cli.py --optimized{Colors.ENDC}          # Use optimized mode
+  {Colors.CYAN}python main_cli.py{Colors.ENDC}                               # Interactive mode
+  {Colors.CYAN}python main_cli.py --file paper.pdf{Colors.ENDC}                # Process file directly
+  {Colors.CYAN}python main_cli.py --url https://...{Colors.ENDC}               # Process URL directly
+  {Colors.CYAN}python main_cli.py --chat "Build a web app..."{Colors.ENDC}     # Process chat requirements
+  {Colors.CYAN}python main_cli.py --optimized{Colors.ENDC}                     # Use optimized mode
+  {Colors.CYAN}python main_cli.py --disable-segmentation{Colors.ENDC}          # Disable document segmentation
+  {Colors.CYAN}python main_cli.py --segmentation-threshold 30000{Colors.ENDC}  # Custom segmentation threshold
 
 {Colors.BOLD}Pipeline Modes:{Colors.ENDC}
   {Colors.GREEN}Comprehensive{Colors.ENDC}: Full intelligence analysis with indexing
   {Colors.YELLOW}Optimized{Colors.ENDC}:     Fast processing without indexing
+
+{Colors.BOLD}Document Processing:{Colors.ENDC}
+  {Colors.BLUE}Smart Segmentation{Colors.ENDC}: Intelligent document segmentation for large papers
+  {Colors.MAGENTA}Supported Formats{Colors.ENDC}: PDF, DOCX, DOC, PPT, PPTX, XLS, XLSX, HTML, TXT, MD
         """,
     )
 
@@ -128,6 +134,19 @@ def parse_arguments():
         help="Use optimized mode (skip indexing for faster processing)",
     )
 
+    parser.add_argument(
+        "--disable-segmentation",
+        action="store_true",
+        help="Disable intelligent document segmentation (use traditional full-document processing)",
+    )
+
+    parser.add_argument(
+        "--segmentation-threshold",
+        type=int,
+        default=50000,
+        help="Document size threshold (characters) to trigger segmentation (default: 50000)",
+    )
+
     parser.add_argument(
         "--verbose", "-v", action="store_true", help="Enable verbose output"
     )
@@ -206,6 +225,24 @@ async def main():
                 f"\n{Colors.GREEN}🧠 Comprehensive mode enabled - full intelligence analysis{Colors.ENDC}"
             )
 
+        # Configure document segmentation settings
+        if hasattr(args, "disable_segmentation") and args.disable_segmentation:
+            print(
+                f"\n{Colors.MAGENTA}📄 Document segmentation disabled - using traditional processing{Colors.ENDC}"
+            )
+            app.segmentation_config = {
+                "enabled": False,
+                "size_threshold_chars": args.segmentation_threshold,
+            }
+        else:
+            print(
+                f"\n{Colors.BLUE}📄 Smart document segmentation enabled (threshold: {args.segmentation_threshold} chars){Colors.ENDC}"
+            )
+            app.segmentation_config = {
+                "enabled": True,
+                "size_threshold_chars": args.segmentation_threshold,
+            }
+
         # 检查是否为直接处理模式
         if args.file or args.url or args.chat:
             if args.file: