chore: Add make check-config command (#875)

tianjing-li · web-flow · commit 6fc3a08546d7 · 2024-12-23T10:05:34.000-05:00
* wip

* Add check config

* uncomment poetry command

* test

* uncomment

* update lock poetry
diff --git a/Makefile b/Makefile
@@ -93,17 +93,22 @@ install:
 .PHONY: setup
 setup:
 	poetry install --with setup,dev --verbose
-	poetry run python3 src/backend/cli/main.py
+	poetry run python3 src/backend/scripts/cli/main.py
 
 .PHONY: setup-use-community
 setup-use-community:
 	poetry install --with setup,community --verbose
-	poetry run python3 src/backend/cli/main.py --use-community
+	poetry run python3 src/backend/scripts/cli/main.py --use-community
 
 .PHONY: win-setup
 win-setup:
 	poetry install --with setup --verbose
-	poetry run python src/backend/cli/main.py
+	poetry run python src/backend/scripts/cli/main.py
+
+.PHONY: check-config
+check-config:
+	poetry install --with setup --verbose
+	poetry run python src/backend/scripts/config/check_config.py
 
 .PHONY: first-run
 first-run:
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -58,6 +58,7 @@ llama-index-llms-cohere = "^0.3.0"
 llama-index-embeddings-cohere = "^0.2.1"
 google-cloud-texttospeech = "^2.18.0"
 slack-sdk = "^3.33.1"
+onnxruntime = "1.19.2"
 
 [tool.poetry.group.dev]
 optional = true
diff --git a/src/backend/scripts/cli/__init__.py b/src/backend/scripts/cli/__init__.py
diff --git a/src/backend/scripts/cli/constants.py b/src/backend/scripts/cli/constants.py
diff --git a/src/backend/scripts/cli/main.py b/src/backend/scripts/cli/main.py
@@ -2,13 +2,13 @@
 
 from dotenv import dotenv_values
 
-from backend.cli.constants import (
+from backend.scripts.cli.constants import (
     COMMUNITY_TOOLS,
     CONFIG_FILE_PATH,
     SECRETS_FILE_PATH,
     TOOLS,
 )
-from backend.cli.prompts import (
+from backend.scripts.cli.prompts import (
     PROMPTS,
     community_tools_prompt,
     deployment_prompt,
@@ -19,13 +19,13 @@
     tool_prompt,
     update_variable_prompt,
 )
-from backend.cli.setters import (
+from backend.scripts.cli.setters import (
     delete_config_folders,
     write_config_files,
     write_env_file,
     write_template_config_files,
 )
-from backend.cli.utils import (
+from backend.scripts.cli.utils import (
     process_existing_yaml_config,
     show_examples,
     show_welcome_message,
diff --git a/src/backend/scripts/cli/prompts.py b/src/backend/scripts/cli/prompts.py
@@ -1,15 +1,15 @@
 import inquirer
 
-from backend.cli.constants import (
+from backend.scripts.cli.constants import (
     DATABASE_URL_DEFAULT,
     FRONTEND_HOSTNAME_DEFAULT,
     NEXT_PUBLIC_API_HOSTNAME_DEFAULT,
     REDIS_URL_DEFAULT,
     BuildTarget,
     bcolors,
 )
-from backend.cli.setters import write_env_file
-from backend.cli.utils import print_styled
+from backend.scripts.cli.setters import write_env_file
+from backend.scripts.cli.utils import print_styled
 
 
 def overwrite_config_prompt():
diff --git a/src/backend/scripts/cli/setters.py b/src/backend/scripts/cli/setters.py
@@ -3,7 +3,7 @@
 import yaml
 from dotenv import set_key
 
-from backend.cli.constants import (
+from backend.scripts.cli.constants import (
     CONFIG_FILE_PATH,
     CONFIG_TEMPLATE_PATH,
     DOT_ENV_FILE_PATH,
diff --git a/src/backend/scripts/cli/utils.py b/src/backend/scripts/cli/utils.py
@@ -3,13 +3,13 @@
 
 import yaml
 
-from backend.cli.constants import (
+from backend.scripts.cli.constants import (
     ENV_YAML_CONFIG_MAPPING,
     WELCOME_MESSAGE,
     DeploymentName,
     bcolors,
 )
-from backend.cli.setters import read_yaml
+from backend.scripts.cli.setters import read_yaml
 
 
 def print_styled(text: str, color: str = bcolors.ENDC):
diff --git a/src/backend/scripts/config/__init__.py b/src/backend/scripts/config/__init__.py
diff --git a/src/backend/scripts/config/check_config.py b/src/backend/scripts/config/check_config.py
@@ -0,0 +1,27 @@
+from backend.scripts.config.constants import (
+    CONFIG_FILE_PATH,
+    CONFIG_TEMPLATE_PATH,
+    SECRETS_FILE_PATH,
+    SECRETS_TEMPLATE_PATH,
+)
+from backend.scripts.config.utils import find_missing_keys, print_styled, read_yaml
+
+
+def check_config():
+    # Check configuration files
+    config_yaml = read_yaml(CONFIG_FILE_PATH)
+    config_template_yaml = read_yaml(CONFIG_TEMPLATE_PATH)
+    config_keys_missing = find_missing_keys(config_yaml, config_template_yaml)
+
+    # Check secrets files
+    secrets_yaml = read_yaml(SECRETS_FILE_PATH)
+    secrets_template_yaml = read_yaml(SECRETS_TEMPLATE_PATH)
+    secrets_keys_missing = find_missing_keys(secrets_yaml, secrets_template_yaml)
+
+    if config_keys_missing:
+        print_styled(f"Warning: Your configuration.yaml is missing the following keys: {config_keys_missing}")
+
+    if secrets_keys_missing:
+        print_styled(f"Warning: Your secrets.yaml is missing the following keys: {secrets_keys_missing}")
+
+check_config()
diff --git a/src/backend/scripts/config/constants.py b/src/backend/scripts/config/constants.py
@@ -0,0 +1,5 @@
+BASE_CONFIG_PATH = "src/backend/config"
+CONFIG_FILE_PATH = f"{BASE_CONFIG_PATH}/configuration.yaml"
+CONFIG_TEMPLATE_PATH = f"{BASE_CONFIG_PATH}/configuration.template.yaml"
+SECRETS_FILE_PATH = f"{BASE_CONFIG_PATH}/secrets.yaml"
+SECRETS_TEMPLATE_PATH = f"{BASE_CONFIG_PATH}/secrets.template.yaml"
diff --git a/src/backend/scripts/config/utils.py b/src/backend/scripts/config/utils.py
@@ -0,0 +1,54 @@
+import yaml
+
+
+class bcolors:
+    FAIL = "\033[91m"
+    ENDC = "\033[0m"
+
+def print_styled(text: str, color: str = bcolors.FAIL):
+    print(color + text + bcolors.ENDC)
+
+def read_yaml(file_path: str):
+    """
+    Loads a YAML file.
+
+    Args:
+        file_path (str): File path to a YAML file.
+
+    Returns:
+        dict: Dictionary representation of the YAML file.
+    """
+    with open(file_path, "r") as file:
+        return yaml.safe_load(file)
+
+def find_missing_keys(dict_a, dict_b, parent_key=''):
+    """
+    Recursively checks for keys in dict_b that are missing in dict_a.
+
+    Args:
+        dict_a (dict): The base dictionary to compare against.
+        dict_b (dict): The dictionary to check for missing keys.
+        parent_key (str): Used for tracking nested keys.
+
+    Returns:
+        list: A list of missing keys.
+    """
+    missing_keys = []
+
+    for key, value in dict_b.items():
+        # Construct the full key path for nested keys
+        full_key = f"{parent_key}.{key}" if parent_key else key
+
+        if isinstance(value, dict):
+            # If both dictionaries contain this key, go deeper
+            if isinstance(dict_a.get(key), dict):
+                missing_keys.extend(find_missing_keys(dict_a[key], value, full_key))
+            else:
+                # If dict_a has the key but it's not a dictionary, treat all nested keys as missing
+                missing_keys.extend(find_missing_keys({}, value, full_key))
+        else:
+            if key not in dict_a:
+                # If the key itself is missing in dict_a
+                missing_keys.append(full_key)
+
+    return missing_keys
diff --git a/src/backend/tests/unit/cli/test_main.py b/src/backend/tests/unit/cli/test_main.py
@@ -1,5 +1,5 @@
 # ruff: noqa
-from backend.cli.main import *  # NOQA
+from backend.scripts.cli.main import *  # NOQA
 
 """
 These tests are boilerplate and do not test the actual functionality of the code.
