feat: support human in the loop for TS (#686)

* feat: support human in the loop for TS

* add example for custom workflow

* fix: need to request humanResponseEvent to save missing step to snapshot

* refactor: human response data should be any

* refactor runWorkflow function to support resume stream

* refactor: hitl

* fix: workflow

* add summary event

* send tool event

* use requestId from Vercel

* update chat route.ts

* fix copy utils/*

* refactor: workflow and stream

* Create eight-moons-perform.md

* update typo

* make schema simple

* fix typo

* use messages in startAgentEvent

* save to snapshots folder

* fix lint

* feat: workflowBaseEvent

* include response event in input event

* simplify type

* update readme

* update document

* fix typecheck

* bump: "@llamaindex/workflow": "~1.1.8"

* remove any

* use fixed tsx version to fix e2e

* fix wrong copy

* add cli hitl examples as a use case for both Python and TS

* update changeset to release create-llama also

* fix e2e

* fix e2e

* hitl frontend chat

* try disable hitl test

Author: thucpn

.changeset/eight-moons-perform.md

@@ -0,0 +1,6 @@
+---
+"@llamaindex/server": patch
+"create-llama": patch
+---
+
+feat: support human in the loop for TS

packages/create-llama/e2e/python/resolve_dependencies.spec.ts

@@ -20,6 +20,7 @@ const useCases: TemplateUseCase[] = [
   "financial_report",
   "code_generator",
   "document_generator",
+  "hitl",
 ];
 const dataSource: string = process.env.DATASOURCE
   ? process.env.DATASOURCE

packages/create-llama/e2e/shared/llamaindexserver_template.spec.ts

@@ -27,6 +27,7 @@ const templateUseCases = [
   "financial_report",
   "deep_research",
   "code_generator",
+  // "hitl",
 ];
 const ejectDir = "next";
 

packages/create-llama/e2e/typescript/resolve_dependencies.spec.ts

@@ -3,6 +3,7 @@ import { exec } from "child_process";
 import fs from "fs";
 import path from "path";
 import util from "util";
+import { NO_DATA_USE_CASES } from "../../helpers/constant";
 import {
   TemplateFramework,
   TemplateType,
@@ -25,6 +26,7 @@ const useCases: TemplateUseCase[] = [
   "financial_report",
   "code_generator",
   "document_generator",
+  "hitl",
 ];
 const dataSource: string = process.env.DATASOURCE
   ? process.env.DATASOURCE
@@ -83,7 +85,7 @@ test.describe("Test resolve TS dependencies", () => {
           });
         });
         // Skipping llamacloud for the use case doesn't use index.
-        if (useCase !== "code_generator" && useCase !== "document_generator") {
+        if (!useCase || !NO_DATA_USE_CASES.includes(useCase)) {
           test(`llamaParse - ${optionDescription}`, async () => {
             await runTest({
               templateType: templateType,

packages/create-llama/helpers/constant.ts

@@ -1,6 +1,15 @@
+import { TemplateUseCase } from "./types";
+
 export const COMMUNITY_OWNER = "run-llama";
 export const COMMUNITY_REPO = "create_llama_projects";
 export const LLAMA_PACK_OWNER = "run-llama";
 export const LLAMA_PACK_REPO = "llama_index";
 export const LLAMA_PACK_FOLDER = "llama-index-packs";
 export const LLAMA_PACK_FOLDER_PATH = `${LLAMA_PACK_OWNER}/${LLAMA_PACK_REPO}/main/${LLAMA_PACK_FOLDER}`;
+
+// these use cases don't have data folder, so no need to run generate and no need to getIndex
+export const NO_DATA_USE_CASES: TemplateUseCase[] = [
+  "code_generator",
+  "document_generator",
+  "hitl",
+];

packages/create-llama/helpers/index.ts

@@ -4,6 +4,7 @@ import path from "path";
 import picocolors, { cyan } from "picocolors";
 
 import fsExtra from "fs-extra";
+import { NO_DATA_USE_CASES } from "./constant";
 import { writeLoadersConfig } from "./datasources";
 import { createBackendEnvFile, createFrontendEnvFile } from "./env-variables";
 import { PackageManager } from "./get-pkg-manager";
@@ -98,8 +99,9 @@ async function generateContextData(
         }
       } else {
         console.log(`Running ${runGenerate} to generate the context data.`);
+
         const shouldRunGenerate =
-          useCase !== "code_generator" && useCase !== "document_generator"; // Artifact use case doesn't use index.
+          !useCase || !NO_DATA_USE_CASES.includes(useCase);
 
         if (shouldRunGenerate) {
           await callPackageManager(packageManager, true, ["run", "generate"]);

packages/create-llama/helpers/types.ts

@@ -59,7 +59,8 @@ export type TemplateUseCase =
   | "contract_review"
   | "agentic_rag"
   | "code_generator"
-  | "document_generator";
+  | "document_generator"
+  | "hitl";
 // Config for both file and folder
 export type FileSourceConfig =
   | {

packages/create-llama/helpers/typescript.ts

@@ -4,6 +4,7 @@ import path from "path";
 import { bold, cyan, red, yellow } from "picocolors";
 import { assetRelocator, copy } from "../helpers/copy";
 import { callPackageManager } from "../helpers/install";
+import { NO_DATA_USE_CASES } from "./constant";
 import { templatesDir } from "./dir";
 import { PackageManager } from "./get-pkg-manager";
 import { InstallTemplateArgs, ModelProvider, TemplateVectorDB } from "./types";
@@ -83,7 +84,7 @@ const installLlamaIndexServerTemplate = async ({
   }
 
   // Simplify use case code
-  if (useCase === "code_generator" || useCase === "document_generator") {
+  if (useCase && NO_DATA_USE_CASES.includes(useCase)) {
     // Artifact use case doesn't use index.
     // We don't need data.ts, generate.ts
     await fs.rm(path.join(root, "src", "app", "data.ts"));

packages/create-llama/questions/simple.ts

@@ -1,4 +1,5 @@
 import prompts from "prompts";
+import { NO_DATA_USE_CASES } from "../helpers/constant";
 import { EXAMPLE_10K_SEC_FILES, EXAMPLE_FILE } from "../helpers/datasources";
 import { askModelConfig } from "../helpers/providers";
 import { getTools } from "../helpers/tools";
@@ -11,7 +12,8 @@ type AppType =
   | "financial_report"
   | "deep_research"
   | "code_generator"
-  | "document_generator";
+  | "document_generator"
+  | "hitl";
 
 type SimpleAnswers = {
   appType: AppType;
@@ -57,6 +59,12 @@ export const askSimpleQuestions = async (
           value: "document_generator",
           description: "Build a OpenAI canvas-styled document generator.",
         },
+        {
+          title: "Human in the Loop",
+          value: "hitl",
+          description:
+            "Build a CLI command workflow that is reviewed by a human before execution",
+        },
       ],
     },
     questionHandlers,
@@ -81,7 +89,8 @@ export const askSimpleQuestions = async (
   );
   language = newLanguage;
 
-  if (appType !== "code_generator" && appType !== "document_generator") {
+  const shouldAskLlamaCloud = !NO_DATA_USE_CASES.includes(appType);
+  if (shouldAskLlamaCloud) {
     const { useLlamaCloud: newUseLlamaCloud } = await prompts(
       {
         type: "toggle",
@@ -170,6 +179,12 @@ const convertAnswers = async (
       tools: [],
       modelConfig: MODEL_GPT41,
     },
+    hitl: {
+      template: "llamaindexserver",
+      dataSources: [],
+      tools: [],
+      modelConfig: MODEL_GPT41,
+    },
   };
 
   const results = lookup[answers.appType];

packages/create-llama/templates/components/ui/use-cases/hitl/cli_human_input.tsx

@@ -0,0 +1,95 @@
+import { Button } from "@/components/ui/button";
+import { Card, CardContent, CardFooter } from "@/components/ui/card";
+import { JSONValue, useChatUI } from "@llamaindex/chat-ui";
+import React, { FC, useState } from "react";
+import { z } from "zod";
+
+// This schema is equivalent to the CLICommand model defined in events.py
+const CLIInputEventSchema = z.object({
+  command: z.string(),
+});
+type CLIInputEvent = z.infer<typeof CLIInputEventSchema>;
+
+const CLIHumanInput: FC<{
+  events: JSONValue[];
+}> = ({ events }) => {
+  const inputEvent = (events || [])
+    .map((ev) => {
+      const parseResult = CLIInputEventSchema.safeParse(ev);
+      return parseResult.success ? parseResult.data : null;
+    })
+    .filter((ev): ev is CLIInputEvent => ev !== null)
+    .at(-1);
+
+  const { append } = useChatUI();
+  const [confirmedValue, setConfirmedValue] = useState<boolean | null>(null);
+  const [editableCommand, setEditableCommand] = useState<string | undefined>(
+    inputEvent?.command,
+  );
+
+  // Update editableCommand if inputEvent changes (e.g. new event comes in)
+  React.useEffect(() => {
+    setEditableCommand(inputEvent?.command);
+  }, [inputEvent?.command]);
+
+  const handleConfirm = () => {
+    append({
+      content: "Yes",
+      role: "user",
+      annotations: [
+        {
+          type: "human_response",
+          data: {
+            execute: true,
+            command: editableCommand, // Use editable command
+          },
+        },
+      ],
+    });
+    setConfirmedValue(true);
+  };
+
+  const handleCancel = () => {
+    append({
+      content: "No",
+      role: "user",
+      annotations: [
+        {
+          type: "human_response",
+          data: {
+            execute: false,
+            command: inputEvent?.command,
+          },
+        },
+      ],
+    });
+    setConfirmedValue(false);
+  };
+
+  return (
+    <Card className="my-4">
+      <CardContent className="pt-6">
+        <p className="text-sm text-gray-700">
+          Do you want to execute the following command?
+        </p>
+        <input
+          disabled
+          type="text"
+          value={editableCommand || ""}
+          onChange={(e) => setEditableCommand(e.target.value)}
+          className="my-2 w-full overflow-x-auto rounded border border-gray-300 bg-gray-100 p-3 font-mono text-xs text-gray-800"
+        />
+      </CardContent>
+      {confirmedValue === null ? (
+        <CardFooter className="flex justify-end gap-2">
+          <>
+            <Button onClick={handleConfirm}>Yes</Button>
+            <Button onClick={handleCancel}>No</Button>
+          </>
+        </CardFooter>
+      ) : null}
+    </Card>
+  );
+};
+
+export default CLIHumanInput;