From 9e0fce87d35eca10293e7bb7f55d53586d9c28ae Mon Sep 17 00:00:00 2001 From: Mihai Criveti Date: Sun, 17 Aug 2025 10:37:53 +0100 Subject: [PATCH 01/13] feat: Bulk Import Tools modal wiring and backend implementation - Add modal UI in admin.html with bulk import button and dialog - Implement modal open/close/ESC functionality in admin.js - Add POST /admin/tools/import endpoint with rate limiting - Support both JSON textarea and file upload inputs - Validate JSON structure and enforce 200 tool limit - Return detailed success/failure information per tool - Include loading states and comprehensive error handling Refs #737 Signed-off-by: Mihai Criveti --- mcpgateway/admin.py | 137 ++++++++++++++++++++ mcpgateway/static/admin.js | 213 ++++++++++++++++++++++++++++++++ mcpgateway/templates/admin.html | 118 +++++++++++++++++- 3 files changed, 463 insertions(+), 5 deletions(-) diff --git a/mcpgateway/admin.py b/mcpgateway/admin.py index e36fdb7b..805c6821 100644 --- a/mcpgateway/admin.py +++ b/mcpgateway/admin.py @@ -1987,6 +1987,143 @@ async def admin_add_tool( return JSONResponse(content={"message": str(ex), "success": False}, status_code=500) +@admin_router.post("/tools/import/") +@admin_router.post("/tools/import") +@rate_limit(requests_per_minute=10) +async def admin_import_tools( + request: Request, + db: Session = Depends(get_db), + user: str = Depends(require_auth), +) -> JSONResponse: + """ + Bulk import multiple tools from JSON. + + Accepts either form data with: + - tools_file: Uploaded JSON file containing array of tool objects + - tools/tools_json: JSON string containing array of tool objects + + Returns detailed success/failure information for each tool. + + Args: + request: FastAPI request containing form data + db: Database session dependency + user: Authenticated user dependency + + Returns: + JSONResponse with success status, counts, and details of created/failed tools + + Raises: + HTTPException: For authentication or rate limiting failures + """ + # Check if bulk import is enabled + if not settings.mcpgateway_bulk_import_enabled: + LOGGER.warning("Bulk import attempted but feature is disabled") + raise HTTPException(status_code=403, detail="Bulk import feature is disabled. Enable MCPGATEWAY_BULK_IMPORT_ENABLED to use this endpoint.") + + LOGGER.debug("bulk tool import: user=%s", user) + try: + # ---------- robust payload parsing ---------- + ctype = (request.headers.get("content-type") or "").lower() + if "application/json" in ctype: + try: + payload = await request.json() + except Exception as ex: + LOGGER.exception("Invalid JSON body") + return JSONResponse({"success": False, "message": f"Invalid JSON: {ex}"}, status_code=422) + else: + try: + form = await request.form() + except Exception as ex: + LOGGER.exception("Invalid form body") + return JSONResponse({"success": False, "message": f"Invalid form data: {ex}"}, status_code=422) + + # Check for file upload first + if "tools_file" in form: + file = form["tools_file"] + if hasattr(file, "file"): + content = await file.read() + try: + payload = json.loads(content.decode("utf-8")) + except (json.JSONDecodeError, UnicodeDecodeError) as ex: + LOGGER.exception("Invalid JSON file") + return JSONResponse({"success": False, "message": f"Invalid JSON file: {ex}"}, status_code=422) + else: + return JSONResponse({"success": False, "message": "Invalid file upload"}, status_code=422) + else: + # Check for JSON in form fields + raw = form.get("tools") or form.get("tools_json") or form.get("json") or form.get("payload") + if not raw: + return JSONResponse({"success": False, "message": "Missing tools/tools_json/json/payload form field."}, status_code=422) + try: + payload = json.loads(raw) + except Exception as ex: + LOGGER.exception("Invalid JSON in form field") + return JSONResponse({"success": False, "message": f"Invalid JSON: {ex}"}, status_code=422) + + if not isinstance(payload, list): + return JSONResponse({"success": False, "message": "Payload must be a JSON array of tools."}, status_code=422) + + max_batch = 200 + if len(payload) > max_batch: + return JSONResponse({"success": False, "message": f"Too many tools ({len(payload)}). Max {max_batch}."}, status_code=413) + + created, errors = [], [] + for i, tool_data in enumerate(payload): + name = tool_data.get("name", f"tool_{i}") + try: + tool = ToolCreate(**tool_data) + await tool_service.register_tool(db, tool) + created.append({"index": i, "name": name}) + except ValidationError as ex: + try: + formatted = ErrorFormatter.format_validation_error(ex) + except Exception: + formatted = {"message": str(ex)} + errors.append({"index": i, "name": name, "error": formatted}) + except ToolError as ex: + errors.append({"index": i, "name": name, "error": {"message": str(ex)}}) + except Exception as ex: + LOGGER.exception("Unexpected error importing tool %r at index %d", name, i) + errors.append({"index": i, "name": name, "error": {"message": str(ex)}}) + + # Format response to match both frontend and test expectations + response_data = { + "success": len(errors) == 0, + # New format for frontend + "imported": len(created), + "failed": len(errors), + "total": len(payload), + # Original format for tests + "created_count": len(created), + "failed_count": len(errors), + "created": created, + "errors": errors, + # Detailed format for frontend + "details": { + "success": [item["name"] for item in created if item.get("name")], + "failed": [{"name": item["name"], "error": item["error"].get("message", str(item["error"]))} for item in errors], + }, + } + + if len(errors) == 0: + response_data["message"] = f"Successfully imported all {len(created)} tools" + else: + response_data["message"] = f"Imported {len(created)} of {len(payload)} tools. {len(errors)} failed." + + return JSONResponse( + response_data, + status_code=200, # Always return 200, success field indicates if all succeeded + ) + + except HTTPException: + # let FastAPI semantics (e.g., auth) pass through + raise + except Exception as ex: + # absolute catch-all: report instead of crashing + LOGGER.exception("Fatal error in admin_import_tools") + return JSONResponse({"success": False, "message": str(ex)}, status_code=500) + + @admin_router.post("/tools/{tool_id}/edit/", response_model=None) @admin_router.post("/tools/{tool_id}/edit", response_model=None) async def admin_edit_tool( diff --git a/mcpgateway/static/admin.js b/mcpgateway/static/admin.js index f6dfa818..529f9bd7 100644 --- a/mcpgateway/static/admin.js +++ b/mcpgateway/static/admin.js @@ -6905,3 +6905,216 @@ window.updateAuthHeadersJSON = updateAuthHeadersJSON; window.loadAuthHeaders = loadAuthHeaders; console.log("🛡️ ContextForge MCP Gateway admin.js initialized"); + + +// =================================================================== +// BULK IMPORT TOOLS — MODAL WIRING +// =================================================================== + +(function initBulkImportModal() { + // ensure it runs after the DOM is ready + window.addEventListener("DOMContentLoaded", function () { + const openBtn = safeGetElement("open-bulk-import", true); + const modalId = "bulk-import-modal"; + const modal = safeGetElement(modalId, true); + + if (!openBtn || !modal) { + console.warn("Bulk Import modal wiring skipped (missing button or modal)."); + return; + } + + // avoid double-binding if admin.js gets evaluated more than once + if (openBtn.dataset.wired === "1") return; + openBtn.dataset.wired = "1"; + + const closeBtn = safeGetElement("close-bulk-import", true); + const backdrop = safeGetElement("bulk-import-backdrop", true); + const resultEl = safeGetElement("import-result", true); + + const focusTarget = + modal.querySelector("#tools_json") || + modal.querySelector("#tools_file") || + modal.querySelector("[data-autofocus]"); + + // helpers + const open = (e) => { + if (e) e.preventDefault(); + // clear previous results each time we open + if (resultEl) resultEl.innerHTML = ""; + openModal(modalId); + // prevent background scroll + document.documentElement.classList.add("overflow-hidden"); + document.body.classList.add("overflow-hidden"); + if (focusTarget) setTimeout(() => focusTarget.focus(), 0); + return false; + }; + + const close = () => { + // also clear results on close to keep things tidy + closeModal(modalId, "import-result"); + document.documentElement.classList.remove("overflow-hidden"); + document.body.classList.remove("overflow-hidden"); + }; + + // wire events + openBtn.addEventListener("click", open); + + if (closeBtn) { + closeBtn.addEventListener("click", (e) => { + e.preventDefault(); + close(); + }); + } + + // click on backdrop only (not the dialog content) closes the modal + if (backdrop) { + backdrop.addEventListener("click", (e) => { + if (e.target === backdrop) close(); + }); + } + + // ESC to close + modal.addEventListener("keydown", (e) => { + if (e.key === "Escape") { + e.stopPropagation(); + close(); + } + }); + + // FORM SUBMISSION → handle bulk import + const form = safeGetElement("bulk-import-form", true); + if (form) { + form.addEventListener("submit", async (e) => { + e.preventDefault(); + e.stopPropagation(); + const resultEl = safeGetElement("import-result", true); + const indicator = safeGetElement("bulk-import-indicator", true); + + try { + const formData = new FormData(); + + // Get JSON from textarea or file + const jsonTextarea = form.querySelector('[name="tools_json"]'); + const fileInput = form.querySelector('[name="tools_file"]'); + + let hasData = false; + + // Check for file upload first (takes precedence) + if (fileInput && fileInput.files.length > 0) { + formData.append('tools_file', fileInput.files[0]); + hasData = true; + } else if (jsonTextarea && jsonTextarea.value.trim()) { + // Validate JSON before sending + try { + const toolsData = JSON.parse(jsonTextarea.value); + if (!Array.isArray(toolsData)) { + throw new Error("JSON must be an array of tools"); + } + formData.append('tools', jsonTextarea.value); + hasData = true; + } catch (err) { + if (resultEl) { + resultEl.innerHTML = ` +
+

Invalid JSON

+

${escapeHtml(err.message)}

+
+ `; + } + return; + } + } + + if (!hasData) { + if (resultEl) { + resultEl.innerHTML = ` +
+

Please provide JSON data or upload a file

+
+ `; + } + return; + } + + // Show loading state + if (indicator) { + indicator.style.display = 'flex'; + } + + // Submit to backend + const response = await fetchWithTimeout( + `${window.ROOT_PATH}/admin/tools/import`, + { + method: 'POST', + body: formData + } + ); + + const result = await response.json(); + + // Display results + if (resultEl) { + if (result.success) { + resultEl.innerHTML = ` +
+

Import Successful

+

${escapeHtml(result.message)}

+
+ `; + + // Close modal and refresh page after delay + setTimeout(() => { + closeModal('bulk-import-modal'); + window.location.reload(); + }, 2000); + } else if (result.imported > 0) { + // Partial success + let detailsHtml = ''; + if (result.details && result.details.failed) { + detailsHtml = ''; + } + + resultEl.innerHTML = ` +
+

Partial Import

+

${escapeHtml(result.message)}

+ ${detailsHtml} +
+ `; + } else { + // Complete failure + resultEl.innerHTML = ` +
+

Import Failed

+

${escapeHtml(result.message)}

+
+ `; + } + } + } catch (error) { + console.error('Bulk import error:', error); + if (resultEl) { + resultEl.innerHTML = ` +
+

Import Error

+

${escapeHtml(error.message || 'An unexpected error occurred')}

+
+ `; + } + } finally { + // Hide loading state + if (indicator) { + indicator.style.display = 'none'; + } + } + + return false; + }); + } + + }); +})(); diff --git a/mcpgateway/templates/admin.html b/mcpgateway/templates/admin.html index d4955113..5ab9c992 100644 --- a/mcpgateway/templates/admin.html +++ b/mcpgateway/templates/admin.html @@ -47,6 +47,11 @@ rel="stylesheet" /> + @@ -1078,11 +1083,63 @@

-
-

- Add New Tool -

+
+

Add New Tool

+ + + + +
+ +
@@ -1310,8 +1367,59 @@

-
+ + + - - + From 1ae2a55d318c6344f34b19dbecd727cadd1884a6 Mon Sep 17 00:00:00 2001 From: Mihai Criveti Date: Sun, 17 Aug 2025 11:59:29 +0100 Subject: [PATCH 12/13] feat: Add configurable bulk import settings Configuration additions: - MCPGATEWAY_BULK_IMPORT_MAX_TOOLS (default: 200) - MCPGATEWAY_BULK_IMPORT_RATE_LIMIT (default: 10) Implementation: - config.py: Add new settings with defaults - admin.py: Use configurable rate limit and batch size - .env.example: Document all bulk import environment variables - admin.html: Use dynamic max tools value in UI text - CLAUDE.md: Document configuration options for developers - docs: Update bulk import guide with configuration details This makes bulk import fully configurable for different deployment scenarios. Signed-off-by: Mihai Criveti --- .env.example | 6 ++++++ CLAUDE.md | 5 +++++ docs/docs/manage/bulk-import.md | 9 ++++++--- mcpgateway/admin.py | 5 +++-- mcpgateway/config.py | 2 ++ mcpgateway/templates/admin.html | 2 +- 6 files changed, 23 insertions(+), 6 deletions(-) diff --git a/.env.example b/.env.example index 06a5c0c7..ed2a311c 100644 --- a/.env.example +++ b/.env.example @@ -131,6 +131,12 @@ MCPGATEWAY_ADMIN_API_ENABLED=true # Enable bulk import endpoint for tools (true/false) MCPGATEWAY_BULK_IMPORT_ENABLED=true +# Maximum number of tools allowed per bulk import request +MCPGATEWAY_BULK_IMPORT_MAX_TOOLS=200 + +# Rate limiting for bulk import endpoint (requests per minute) +MCPGATEWAY_BULK_IMPORT_RATE_LIMIT=10 + ##################################### # Header Passthrough Configuration ##################################### diff --git a/CLAUDE.md b/CLAUDE.md index eabefb8e..99d92ef6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -137,6 +137,11 @@ AUTH_REQUIRED=true MCPGATEWAY_UI_ENABLED=true MCPGATEWAY_ADMIN_API_ENABLED=true +# Bulk Import (Admin UI feature) +MCPGATEWAY_BULK_IMPORT_ENABLED=true # Enable/disable bulk import endpoint +MCPGATEWAY_BULK_IMPORT_MAX_TOOLS=200 # Maximum tools per import batch +MCPGATEWAY_BULK_IMPORT_RATE_LIMIT=10 # Requests per minute limit + # Federation MCPGATEWAY_ENABLE_MDNS_DISCOVERY=true MCPGATEWAY_ENABLE_FEDERATION=true diff --git a/docs/docs/manage/bulk-import.md b/docs/docs/manage/bulk-import.md index 10e268fd..c1563257 100644 --- a/docs/docs/manage/bulk-import.md +++ b/docs/docs/manage/bulk-import.md @@ -2,9 +2,12 @@ The MCP Gateway provides a bulk import endpoint for efficiently loading multiple tools in a single request, perfect for migrations, environment setup, and team onboarding. -!!! info "Feature Flag Required" - This feature is controlled by the `MCPGATEWAY_BULK_IMPORT_ENABLED` environment variable. - Default: `true` (enabled). Set to `false` to disable this endpoint. +!!! info "Configuration Options" + This feature is controlled by several environment variables: + + - `MCPGATEWAY_BULK_IMPORT_ENABLED=true` - Enable/disable the endpoint (default: true) + - `MCPGATEWAY_BULK_IMPORT_MAX_TOOLS=200` - Maximum tools per batch (default: 200) + - `MCPGATEWAY_BULK_IMPORT_RATE_LIMIT=10` - Requests per minute limit (default: 10) --- diff --git a/mcpgateway/admin.py b/mcpgateway/admin.py index f6c6b811..a77c0860 100644 --- a/mcpgateway/admin.py +++ b/mcpgateway/admin.py @@ -1568,6 +1568,7 @@ async def admin_ui( "root_path": root_path, "max_name_length": max_name_length, "gateway_tool_name_separator": settings.gateway_tool_name_separator, + "bulk_import_max_tools": settings.mcpgateway_bulk_import_max_tools, }, ) @@ -4375,7 +4376,7 @@ async def admin_list_tags( @admin_router.post("/tools/import/") @admin_router.post("/tools/import") -@rate_limit(requests_per_minute=10) +@rate_limit(requests_per_minute=settings.mcpgateway_bulk_import_rate_limit) async def admin_import_tools( request: Request, db: Session = Depends(get_db), @@ -4444,7 +4445,7 @@ async def admin_import_tools( if not isinstance(payload, list): return JSONResponse({"success": False, "message": "Payload must be a JSON array of tools."}, status_code=422) - max_batch = 200 + max_batch = settings.mcpgateway_bulk_import_max_tools if len(payload) > max_batch: return JSONResponse({"success": False, "message": f"Too many tools ({len(payload)}). Max {max_batch}."}, status_code=413) diff --git a/mcpgateway/config.py b/mcpgateway/config.py index eb792062..0b3523b3 100644 --- a/mcpgateway/config.py +++ b/mcpgateway/config.py @@ -150,6 +150,8 @@ class Settings(BaseSettings): mcpgateway_ui_enabled: bool = False mcpgateway_admin_api_enabled: bool = False mcpgateway_bulk_import_enabled: bool = True + mcpgateway_bulk_import_max_tools: int = 200 + mcpgateway_bulk_import_rate_limit: int = 10 # Security skip_ssl_verify: bool = False diff --git a/mcpgateway/templates/admin.html b/mcpgateway/templates/admin.html index 70286cac..541ed2f9 100644 --- a/mcpgateway/templates/admin.html +++ b/mcpgateway/templates/admin.html @@ -1342,7 +1342,7 @@

Bulk Import Tools

enctype="multipart/form-data" class="space-y-4 p-5">

- Paste a JSON array or upload a .json file. Max 200 tools. + Paste a JSON array or upload a .json file. Max {{ bulk_import_max_tools }} tools.

From 8765e1bf8e375a96e5a7499a79b1b42d2ce2ba4f Mon Sep 17 00:00:00 2001 From: Mihai Criveti Date: Sun, 17 Aug 2025 12:09:13 +0100 Subject: [PATCH 13/13] Update docs Signed-off-by: Mihai Criveti --- docs/docs/manage/bulk-import.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/manage/bulk-import.md b/docs/docs/manage/bulk-import.md index c1563257..6b4af7e7 100644 --- a/docs/docs/manage/bulk-import.md +++ b/docs/docs/manage/bulk-import.md @@ -4,7 +4,7 @@ The MCP Gateway provides a bulk import endpoint for efficiently loading multiple !!! info "Configuration Options" This feature is controlled by several environment variables: - + - `MCPGATEWAY_BULK_IMPORT_ENABLED=true` - Enable/disable the endpoint (default: true) - `MCPGATEWAY_BULK_IMPORT_MAX_TOOLS=200` - Maximum tools per batch (default: 200) - `MCPGATEWAY_BULK_IMPORT_RATE_LIMIT=10` - Requests per minute limit (default: 10)