diff --git a/examples-copier/.dockerignore b/examples-copier/.dockerignore new file mode 100644 index 0000000..4fb86bf --- /dev/null +++ b/examples-copier/.dockerignore @@ -0,0 +1,51 @@ +# Git +.git +.gitignore +.github + +# Documentation +*.md +!README.md +docs/ +readme_files/ + +# Test files +*_test.go +test-payloads/ +tests/ + +# Config examples (not needed in container) +configs/*.example +configs/*.production +configs/README.md +configs/.env* + +# Scripts (not needed in container) +scripts/ + +# Build artifacts +*.log +all.log + +# Environment files (will be set via Cloud Run env vars) +.env +.env.* +env.yaml +env-cloudrun.yaml +app.yaml + +# IDE +.vscode +.idea +*.swp +*.swo +*~ + +# OS +.DS_Store +Thumbs.db + +# Temporary files +tmp/ +temp/ + diff --git a/examples-copier/.gitignore b/examples-copier/.gitignore index d3a2616..99d319b 100644 --- a/examples-copier/.gitignore +++ b/examples-copier/.gitignore @@ -1,6 +1,7 @@ # Binaries examples-copier code-copier +copier *.exe *.exe~ *.dll @@ -19,13 +20,23 @@ vendor/ # Go workspace file go.work -# Environment files with secrets +# Environment files with secrets (working files - create from templates in configs/) +# Working files (create from templates): +# env.yaml - App Engine deployment config (from configs/env.yaml.*) +# env-cloudrun.yaml - Cloud Run deployment config (from configs/env.yaml.*) +# .env - Local development config (from configs/.env.local.example) env.yaml +env-cloudrun.yaml .env .env.local .env.production .env.*.local +# Explicitly keep template files in configs/ directory (these should be tracked) +!configs/env.yaml.example +!configs/env.yaml.production +!configs/.env.local.example + # Private keys *.pem *.key @@ -47,5 +58,3 @@ Thumbs.db # Temporary files tmp/ temp/ - -env.yaml diff --git a/examples-copier/Dockerfile b/examples-copier/Dockerfile new file mode 100644 index 0000000..c1e0ac5 --- /dev/null +++ b/examples-copier/Dockerfile @@ -0,0 +1,38 @@ +# Build stage +FROM golang:1.23.4-alpine AS builder + +# Install build dependencies +RUN apk add --no-cache git ca-certificates + +WORKDIR /app + +# Copy go mod files first for better caching +COPY go.mod go.sum ./ +RUN go mod download + +# Copy source code +COPY . . + +# Build the binary +# CGO_ENABLED=0 for static binary (no C dependencies) +# -ldflags="-w -s" strips debug info to reduce size +RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-w -s" -o examples-copier . + +# Runtime stage +FROM alpine:latest + +# Install ca-certificates for HTTPS requests +RUN apk --no-cache add ca-certificates + +WORKDIR /root/ + +# Copy binary from builder +COPY --from=builder /app/examples-copier . + +# Cloud Run sets PORT environment variable +# Our app reads it from config.Port (defaults to 8080) +EXPOSE 8080 + +# Run the binary +CMD ["./examples-copier"] + diff --git a/examples-copier/QUICK-REFERENCE.md b/examples-copier/QUICK-REFERENCE.md index d8280b8..8e8d289 100644 --- a/examples-copier/QUICK-REFERENCE.md +++ b/examples-copier/QUICK-REFERENCE.md @@ -31,7 +31,7 @@ ./config-validator test-pattern -type regex -pattern "^examples/(?P[^/]+)/.*$" -file "examples/go/main.go" # Test transformation -./config-validator test-transform -template "docs/${lang}/${file}" -file "examples/go/main.go" -pattern "^examples/(?P[^/]+)/(?P.+)$" +./config-validator test-transform -source "examples/go/main.go" -template "docs/${lang}/${file}" -vars "lang=go,file=main.go" # Initialize new config ./config-validator init -output copier-config.yaml @@ -63,6 +63,18 @@ source_pattern: pattern: "^examples/(?P[^/]+)/(?P.+)$" ``` +### Pattern with Exclusions +```yaml +source_pattern: + type: "prefix" + pattern: "examples/" + exclude_patterns: + - "\.gitignore$" + - "node_modules/" + - "\.env$" + - "/dist/" +``` + ## Path Transformations ### Built-in Variables @@ -102,9 +114,73 @@ commit_strategy: commit_message: "Update examples" pr_title: "Update code examples" pr_body: "Automated update" + use_pr_template: true # Fetch PR template from target repo auto_merge: true ``` +### Batch PRs by Repository +```yaml +batch_by_repo: true + +batch_pr_config: + pr_title: "Update from ${source_repo}" + pr_body: | + 🤖 Automated update + Files: ${file_count} + use_pr_template: true + commit_message: "Update from ${source_repo} PR #${pr_number}" +``` + +## Advanced Features + +### Exclude Patterns +Exclude unwanted files from being copied: + +```yaml +source_pattern: + type: "prefix" + pattern: "examples/" + exclude_patterns: + - "\.gitignore$" # Exclude .gitignore + - "node_modules/" # Exclude dependencies + - "\.env$" # Exclude .env files + - "\.env\\..*$" # Exclude .env.local, .env.production, etc. + - "/dist/" # Exclude build output + - "/build/" # Exclude build artifacts + - "\.test\.(js|ts)$" # Exclude test files +``` + +### PR Template Integration +Fetch and merge PR templates from target repos: + +```yaml +commit_strategy: + type: "pull_request" + pr_body: | + 🤖 Automated update + Files: ${file_count} + use_pr_template: true # Fetches .github/pull_request_template.md +``` + +**Result:** PR description shows: +1. Target repo's PR template (checklists, guidelines) +2. Separator (`---`) +3. Your configured content (automation info) + +### Batch Configuration +When `batch_by_repo: true`, use `batch_pr_config` for accurate file counts: + +```yaml +batch_by_repo: true + +batch_pr_config: + pr_title: "Update from ${source_repo}" + pr_body: | + Files: ${file_count} # Accurate count across all rules + Source: ${source_repo} PR #${pr_number} + use_pr_template: true +``` + ## Message Templates ### Available Variables diff --git a/examples-copier/README.md b/examples-copier/README.md index 674bafa..f9e7840 100644 --- a/examples-copier/README.md +++ b/examples-copier/README.md @@ -1,6 +1,8 @@ # GitHub Docs Code Example Copier -A GitHub app that automatically copies code examples and files from a source repository to one or more target repositories when pull requests are merged. Features advanced pattern matching, path transformations, audit logging, and comprehensive monitoring. +A GitHub app that automatically copies code examples and files from a source repository to one or more target +repositories when pull requests are merged. Features advanced pattern matching, path transformations, audit logging, +and comprehensive monitoring. ## Features @@ -15,6 +17,9 @@ A GitHub app that automatically copies code examples and files from a source rep ### Enhanced Features - **YAML Configuration** - Modern YAML config with JSON backward compatibility - **Message Templating** - Template-ized commit messages and PR titles +- **Batch PR Support** - Combine multiple rules into one PR per target repo +- **PR Template Integration** - Fetch and merge PR templates from target repos +- **File Exclusion** - Exclude patterns to filter out unwanted files (`.gitignore`, `node_modules`, etc.) - **Audit Logging** - MongoDB-based event tracking for all operations - **Health & Metrics** - `/health` and `/metrics` endpoints for monitoring - **Development Tools** - Dry-run mode, CLI validation, enhanced logging @@ -46,7 +51,7 @@ go build -o examples-copier . go build -o config-validator ./cmd/config-validator ``` -### Configuration +### Local Configuration 1. **Copy environment example file** @@ -96,12 +101,28 @@ Create `copier-config.yaml` in your source repository: ```yaml source_repo: "your-org/source-repo" source_branch: "main" +batch_by_repo: true # Optional: batch all changes into one PR per target repo + +# Optional: Customize batched PR metadata +batch_pr_config: + pr_title: "Update code examples from ${source_repo}" + pr_body: | + 🤖 Automated update of code examples + + **Source:** ${source_repo} PR #${pr_number} + **Files:** ${file_count} + use_pr_template: true # Fetch PR template from target repos + commit_message: "Update examples from ${source_repo} PR #${pr_number}" copy_rules: - name: "Copy Go examples" source_pattern: type: "regex" pattern: "^examples/(?P[^/]+)/(?P[^/]+)/(?P.+)$" + exclude_patterns: # Optional: exclude unwanted files + - "\.gitignore$" + - "node_modules/" + - "\.env$" targets: - repo: "your-org/target-repo" branch: "main" @@ -110,6 +131,8 @@ copy_rules: type: "pull_request" commit_message: "Update ${category} examples from ${lang}" pr_title: "Update ${category} examples" + pr_body: "Automated update of ${lang} examples" + use_pr_template: true # Merge with target repo's PR template auto_merge: false deprecation_check: enabled: true @@ -203,9 +226,71 @@ commit_strategy: commit_message: "Update examples" pr_title: "Update ${category} examples" pr_body: "Automated update from ${source_repo}" + use_pr_template: true # Fetch and merge PR template from target repo auto_merge: true ``` +### Advanced Features + +#### Batch PRs by Repository + +Combine all changes from a single source PR into one PR per target repository: + +```yaml +batch_by_repo: true + +batch_pr_config: + pr_title: "Update code examples from ${source_repo}" + pr_body: | + 🤖 Automated update + + Files: ${file_count} + Source: ${source_repo} PR #${pr_number} + use_pr_template: true + commit_message: "Update from ${source_repo} PR #${pr_number}" +``` + +**Benefits:** +- Single PR per target repo instead of multiple PRs +- Accurate `${file_count}` across all matched rules +- Easier review process for related changes + +#### PR Template Integration + +Automatically fetch and merge PR templates from target repositories: + +```yaml +commit_strategy: + type: "pull_request" + pr_body: | + 🤖 Automated update + Files: ${file_count} + use_pr_template: true # Fetches .github/pull_request_template.md +``` + +**Result:** PR description shows the target repo's template first (with checklists and guidelines), followed by your configured content. + +#### File Exclusion Patterns + +Exclude unwanted files from being copied: + +```yaml +source_pattern: + type: "prefix" + pattern: "examples/" + exclude_patterns: + - "\.gitignore$" # Exclude .gitignore files + - "node_modules/" # Exclude dependencies + - "\.env$" # Exclude environment files + - "/dist/" # Exclude build artifacts + - "\.test\.(js|ts)$" # Exclude test files +``` + +**Use cases:** +- Filter out configuration files +- Exclude build artifacts and dependencies +- Skip test files or documentation + ### Message Templates Use variables in commit messages and PR titles: @@ -432,7 +517,7 @@ container := NewServiceContainer(config) ## Deployment -See [DEPLOYMENT.md](./docs/DEPLOYMENT.md) for complete deployment guide and [DEPLOYMENT-CHECKLIST.md](./docs/DEPLOYMENT-CHECKLIST.md) for step-by-step checklist. +See [DEPLOYMENT.md](./docs/DEPLOYMENT.md) for complete deployment guide for step-by-step checklist. ### Google Cloud App Engine diff --git a/examples-copier/cmd/test-webhook/main.go b/examples-copier/cmd/test-webhook/main.go index e3bbeb2..a84c3b8 100644 --- a/examples-copier/cmd/test-webhook/main.go +++ b/examples-copier/cmd/test-webhook/main.go @@ -86,7 +86,7 @@ func main() { } func printHelp() { - fmt.Println(`Test Webhook Tool + fmt.Print(`Test Webhook Tool Usage: test-webhook [options] diff --git a/examples-copier/configs/README.md b/examples-copier/configs/README.md index 924adaa..068e6a1 100644 --- a/examples-copier/configs/README.md +++ b/examples-copier/configs/README.md @@ -1,15 +1,17 @@ -# Environment Files Comparison +# Environment Configuration Guide -Overview of the different environment configuration files and when to use each. +This directory contains **template files** for different deployment scenarios. Copy the appropriate template to create your working configuration file. -## Files Overview +## Template Files Overview -| File | Purpose | Use Case | +| Template File | Purpose | Use Case | |-----------------------|---------------------------------------|---------------------------------| | `env.yaml.example` | Complete reference with all variables | First-time setup, documentation | | `env.yaml.production` | Production-ready template | Quick deployment to production | | `.env.local.example` | Local development template | Local testing and development | +**Note:** Your actual working files (`env.yaml`, `.env`) should be created from these templates and are gitignored to protect secrets. + --- ## env.yaml.example @@ -99,6 +101,54 @@ env_variables: --- +## Deployment Targets + +This service supports **two Google Cloud deployment options**: + +### App Engine (Flexible Environment) + +**Config file:** `env.yaml` (with `env_variables:` wrapper) + +**Format:** +```yaml +env_variables: + GITHUB_APP_ID: "123456" + REPO_OWNER: "mongodb" +``` + +**Deploy:** +```bash +cp configs/env.yaml.production env.yaml +# Edit env.yaml with your values +gcloud app deploy app.yaml # Includes env.yaml automatically +``` + +**Best for:** Long-running services, always-on applications + +--- + +### Cloud Run (Serverless Containers) + +**Config file:** `env-cloudrun.yaml` (plain YAML, no wrapper) + +**Format:** +```yaml +GITHUB_APP_ID: "123456" +REPO_OWNER: "mongodb" +``` + +**Deploy:** +```bash +cp configs/env.yaml.production env-cloudrun.yaml +# Remove the 'env_variables:' wrapper +# Edit env-cloudrun.yaml with your values +gcloud run deploy examples-copier --source . --env-vars-file=env-cloudrun.yaml +``` + +**Best for:** Cost-effective, scales to zero, serverless + +--- + ## Usage Scenarios ### Scenario 1: First-Time Production Deployment @@ -196,6 +246,22 @@ env_variables: REPO_OWNER: "mongodb" ``` +### Between App Engine and Cloud Run formats + +Use the format conversion script: + +```bash +# Convert App Engine → Cloud Run +./scripts/convert-env-format.sh to-cloudrun env.yaml env-cloudrun.yaml + +# Convert Cloud Run → App Engine +./scripts/convert-env-format.sh to-appengine env-cloudrun.yaml env.yaml +``` + +**Key difference:** +- **App Engine**: Requires `env_variables:` wrapper with 2-space indentation +- **Cloud Run**: Plain YAML without wrapper + ### From env.yaml.production to env.yaml.example ```bash @@ -238,10 +304,16 @@ examples-copier/ │ ├── env.yaml.example # ← Complete reference (all variables) │ ├── env.yaml.production # ← Production template (essential only) │ └── .env.local.example # ← Local development template -├── env.yaml # ← Your actual config (gitignored) -└── .env # ← Your local config (gitignored) +├── env.yaml # ← App Engine config (create from template, gitignored) +├── env-cloudrun.yaml # ← Cloud Run config (create from template, gitignored) +└── .env # ← Local config (create from template, gitignored) ``` +**Working files (not in repo):** +- `env.yaml` - App Engine deployment (YAML with `env_variables:` wrapper) +- `env-cloudrun.yaml` - Cloud Run deployment (plain YAML, no wrapper) +- `.env` - Local development (KEY=value format) + --- ## Quick Reference diff --git a/examples-copier/configs/config.example.json b/examples-copier/configs/copier-config-examples/config.example.json similarity index 100% rename from examples-copier/configs/config.example.json rename to examples-copier/configs/copier-config-examples/config.example.json diff --git a/examples-copier/configs/copier-config-examples/copier-config-batch-example.yaml b/examples-copier/configs/copier-config-examples/copier-config-batch-example.yaml new file mode 100644 index 0000000..b89e304 --- /dev/null +++ b/examples-copier/configs/copier-config-examples/copier-config-batch-example.yaml @@ -0,0 +1,78 @@ +# Example configuration demonstrating batch_by_repo with batch_pr_config +# This shows how to batch multiple copy rules into a single PR per target repo +# with custom PR metadata and accurate file counts + +source_repo: "mongodb/code-examples" +source_branch: "main" + +# Enable batching - all rules targeting the same repo will be combined into one PR +batch_by_repo: true + +# Configure PR metadata for batched PRs +# These templates will be rendered with accurate file counts after all files are collected +batch_pr_config: + pr_title: "Update code examples from ${source_repo}" + pr_body: | + 🤖 Automated update of code examples + + **Source Information:** + - Repository: ${source_repo} + - Branch: ${source_branch} + - PR: #${pr_number} + - Commit: ${commit_sha} + + **Changes:** + - Total files: ${file_count} + - Target branch: ${target_branch} + + This PR includes files from multiple copy rules batched together for easier review. + use_pr_template: true # Optional: Fetch PR template from target repos + commit_message: "Update examples from ${source_repo} PR #${pr_number} (${file_count} files)" + +copy_rules: + # Rule 1: Copy client files + - name: "client-files" + source_pattern: + type: "prefix" + pattern: "examples/client/" + targets: + - repo: "mongodb/docs" + branch: "main" + path_transform: "source/code-examples/client/${relative_path}" + commit_strategy: + type: "pull_request" + # Individual rule commit messages are still used for commits + # But PR title/body come from batch_pr_config above + commit_message: "Update client examples" + + # Rule 2: Copy server files + - name: "server-files" + source_pattern: + type: "prefix" + pattern: "examples/server/" + targets: + - repo: "mongodb/docs" + branch: "main" + path_transform: "source/code-examples/server/${relative_path}" + commit_strategy: + type: "pull_request" + commit_message: "Update server examples" + + # Rule 3: Copy README files + - name: "readme-files" + source_pattern: + type: "glob" + pattern: "examples/**/README.md" + targets: + - repo: "mongodb/docs" + branch: "main" + path_transform: "source/code-examples/${matched_pattern}" + commit_strategy: + type: "pull_request" + commit_message: "Update README files" + +# Result: All three rules will be batched into ONE PR in mongodb/docs +# The PR will use the title/body from batch_pr_config with accurate file count +# For example, if rule 1 matches 10 files, rule 2 matches 5 files, and rule 3 matches 2 files, +# the PR will show "Total files: 17" in the body + diff --git a/examples-copier/configs/copier-config-examples/copier-config-exclude-example.yaml b/examples-copier/configs/copier-config-examples/copier-config-exclude-example.yaml new file mode 100644 index 0000000..0a23776 --- /dev/null +++ b/examples-copier/configs/copier-config-examples/copier-config-exclude-example.yaml @@ -0,0 +1,189 @@ +# Example Configuration with exclude_patterns +# This demonstrates how to use exclude_patterns to filter out specific files + +source_repo: "mongodb/code-examples" +source_branch: "main" +batch_by_repo: false # Create separate PRs for each rule + +copy_rules: + # Example 1: Exclude config files from examples + - name: "go-examples-no-config" + source_pattern: + type: "prefix" + pattern: "examples/go/" + exclude_patterns: + - "\.gitignore$" # Exclude .gitignore + - "\.env$" # Exclude .env files + - "\.env\\..*$" # Exclude .env.local, .env.production, etc. + - "config\\.local\\." # Exclude config.local.* files + targets: + - repo: "mongodb/docs" + branch: "main" + path_transform: "source/code-examples/go/${relative_path}" + commit_strategy: + type: "pull_request" + pr_title: "Update Go examples (no config files)" + pr_body: "Automated update of Go code examples, excluding configuration files" + + # Example 2: Exclude test files and build artifacts + - name: "java-server-production-only" + source_pattern: + type: "regex" + pattern: "^server/java-spring/(?P.+)$" + exclude_patterns: + - "/test/" # Exclude test directories + - "Test\\.java$" # Exclude test files + - "/target/" # Exclude Maven build directory + - "\\.class$" # Exclude compiled files + targets: + - repo: "mongodb/sample-app-java" + branch: "main" + path_transform: "server/${file}" + commit_strategy: + type: "pull_request" + pr_title: "Update Java server (production code only)" + pr_body: "Automated update excluding tests and build artifacts" + + # Example 3: Exclude dependencies and build artifacts from Node.js project + - name: "nodejs-app-source-only" + source_pattern: + type: "prefix" + pattern: "apps/nodejs/" + exclude_patterns: + - "node_modules/" # Exclude dependencies + - "/dist/" # Exclude build output + - "/build/" # Exclude build directory + - "\\.min\\.js$" # Exclude minified files + - "\\.min\\.css$" # Exclude minified CSS + - "\\.map$" # Exclude source maps + targets: + - repo: "mongodb/sample-app-nodejs" + branch: "main" + path_transform: "${relative_path}" + commit_strategy: + type: "pull_request" + pr_title: "Update Node.js app (source code only)" + pr_body: "Automated update excluding dependencies and build artifacts" + + # Example 4: Exclude all hidden files + - name: "python-examples-no-hidden" + source_pattern: + type: "glob" + pattern: "examples/python/**/*.py" + exclude_patterns: + - "/\\.[^/]+$" # Exclude files starting with . + - "/__pycache__/" # Exclude Python cache + - "\\.pyc$" # Exclude compiled Python files + targets: + - repo: "mongodb/docs" + branch: "main" + path_transform: "source/code-examples/python/${matched_pattern}" + commit_strategy: + type: "pull_request" + pr_title: "Update Python examples (no hidden files)" + pr_body: "Automated update excluding hidden files and cache" + + # Example 5: Exclude documentation from code examples + - name: "typescript-code-no-docs" + source_pattern: + type: "prefix" + pattern: "examples/typescript/" + exclude_patterns: + - "README\\.md$" # Exclude README files + - "\\.md$" # Exclude all markdown files + - "/docs/" # Exclude docs directory + - "CHANGELOG" # Exclude changelog files + targets: + - repo: "mongodb/typescript-examples" + branch: "main" + path_transform: "examples/${relative_path}" + commit_strategy: + type: "pull_request" + pr_title: "Update TypeScript examples (code only)" + pr_body: "Automated update excluding documentation files" + + # Example 6: Complex exclusions for a full-stack app + - name: "fullstack-app-clean" + source_pattern: + type: "prefix" + pattern: "apps/mflix/" + exclude_patterns: + # Config files + - "\.gitignore$" + - "\.env$" + - "\.env\\..*$" + # Dependencies + - "node_modules/" + - "vendor/" + - "__pycache__/" + # Build artifacts + - "/dist/" + - "/build/" + - "/target/" + - "\.min\.(js|css)$" + # Test files + - "/test/" + - "/tests/" + - "\.test\.(js|ts)$" + - "\.spec\.(js|ts)$" + # Documentation + - "README\\.md$" + - "/docs/" + # IDE files + - "\.vscode/" + - "\.idea/" + targets: + - repo: "mongodb/sample-app-mflix" + branch: "main" + path_transform: "${relative_path}" + commit_strategy: + type: "pull_request" + pr_title: "Update MFlix app (clean source)" + pr_body: | + Automated update of MFlix application + + Excludes: + - Configuration files + - Dependencies + - Build artifacts + - Test files + - Documentation + - IDE files + + # Example 7: Exclude specific file types + - name: "images-and-media-only" + source_pattern: + type: "glob" + pattern: "assets/**/*" + exclude_patterns: + - "\\.txt$" # Exclude text files + - "\\.md$" # Exclude markdown + - "\\.json$" # Exclude JSON + - "\\.xml$" # Exclude XML + - "\\.yaml$" # Exclude YAML + - "\\.yml$" # Exclude YML + targets: + - repo: "mongodb/docs-assets" + branch: "main" + path_transform: "images/${matched_pattern}" + commit_strategy: + type: "direct" + commit_message: "Update media assets (images only)" + + # Example 8: Exclude by directory depth + - name: "top-level-files-only" + source_pattern: + type: "regex" + pattern: "^examples/(?P[^/]+)/(?P[^/]+)$" # Only matches files directly in lang dir + # No subdirectories will match due to pattern, but we can add extra safety: + exclude_patterns: + - "/" # Exclude anything with a slash (subdirectories) + targets: + - repo: "mongodb/docs" + branch: "main" + path_transform: "examples/${lang}/${file}" + commit_strategy: + type: "pull_request" + pr_title: "Update top-level example files" + pr_body: "Automated update of top-level example files only" + diff --git a/examples-copier/configs/copier-config-examples/copier-config-pr-template-example.yaml b/examples-copier/configs/copier-config-examples/copier-config-pr-template-example.yaml new file mode 100644 index 0000000..726856d --- /dev/null +++ b/examples-copier/configs/copier-config-examples/copier-config-pr-template-example.yaml @@ -0,0 +1,213 @@ +# Example Configuration with PR Template Support +# This demonstrates how to use use_pr_template to fetch and merge PR templates from target repos +# +# When use_pr_template: true, the service will: +# 1. Fetch the PR template from the target repo (.github/pull_request_template.md) +# 2. Place the template content FIRST (checklists, review guidelines) +# 3. Add a separator (---) +# 4. Append your configured pr_body (automation info) +# +# This ensures reviewers see the target repo's guidelines prominently. + +source_repo: "mongodb/code-examples" +source_branch: "main" +batch_by_repo: false # Create separate PRs for each rule + +copy_rules: + # Example 1: Use PR template from target repo + - name: "go-examples-with-template" + source_pattern: + type: "prefix" + pattern: "examples/go/" + targets: + - repo: "mongodb/docs" + branch: "main" + path_transform: "source/code-examples/go/${relative_path}" + commit_strategy: + type: "pull_request" + pr_title: "Update Go examples" + pr_body: | + 🤖 **Automated Update of Go Examples** + + **Source Information:** + - Repository: ${source_repo} + - Branch: ${source_branch} + - PR: #${pr_number} + - Commit: ${commit_sha} + + **Changes:** + - Files updated: ${file_count} + - Target branch: ${target_branch} + use_pr_template: true # Fetch and merge PR template from mongodb/docs + auto_merge: false + + # Example 2: Different templates for different targets + - name: "shared-examples" + source_pattern: + type: "regex" + pattern: "^shared/(?P[^/]+)/(?P.+)$" + targets: + # Target 1: Main docs - use their PR template + - repo: "mongodb/docs" + branch: "main" + path_transform: "examples/${lang}/${file}" + commit_strategy: + type: "pull_request" + pr_title: "Update ${lang} examples" + pr_body: | + Automated update of ${lang} examples + + Files: ${file_count} + use_pr_template: true # Use mongodb/docs template + auto_merge: false + + # Target 2: Tutorials - use their different template + - repo: "mongodb/tutorials" + branch: "main" + path_transform: "code/${lang}/${file}" + commit_strategy: + type: "pull_request" + pr_title: "Update ${lang} tutorial code" + pr_body: | + Tutorial code update for ${lang} + + Source PR: #${pr_number} + use_pr_template: true # Use mongodb/tutorials template + auto_merge: false + + # Example 3: Conditional template usage + # Use template for production, skip for staging + - name: "python-examples-production" + source_pattern: + type: "prefix" + pattern: "examples/python/" + targets: + # Production: Use PR template for thorough review + - repo: "mongodb/docs" + branch: "main" + path_transform: "source/code/python/${relative_path}" + commit_strategy: + type: "pull_request" + pr_title: "Update Python examples (Production)" + pr_body: | + 🚀 **Production Update** + + Files: ${file_count} + Source: ${source_repo} + use_pr_template: true # Use template for production + auto_merge: false + + # Staging: Skip template for faster iteration + - repo: "mongodb/docs-staging" + branch: "main" + path_transform: "source/code/python/${relative_path}" + commit_strategy: + type: "pull_request" + pr_title: "Update Python examples (Staging)" + pr_body: | + 🧪 **Staging Update** + + Files: ${file_count} + use_pr_template: false # Skip template for staging + auto_merge: true # Auto-merge staging changes + + # Example 4: Rich PR body with template + - name: "java-examples-detailed" + source_pattern: + type: "regex" + pattern: "^examples/java/(?P[^/]+)/(?P.+)$" + targets: + - repo: "mongodb/java-driver" + branch: "main" + path_transform: "examples/${category}/${file}" + commit_strategy: + type: "pull_request" + pr_title: "Update ${category} examples" + pr_body: | + # Automated Example Update + + ## Summary + This PR updates Java code examples in the `${category}` category. + + ## Details + - **Category:** ${category} + - **Files Updated:** ${file_count} + - **Source Repository:** ${source_repo} + - **Source Branch:** ${source_branch} + - **Source PR:** #${pr_number} + - **Source Commit:** ${commit_sha} + - **Target Branch:** ${target_branch} + + ## Changes + These examples were automatically copied from the source repository. + All files have been validated and are ready for review. + + ## Testing + - [ ] Examples compile successfully + - [ ] Examples run without errors + - [ ] Documentation is up to date + + --- + + _This PR was automatically created by the Code Example Copier service._ + use_pr_template: true # Template appears first, then this content + auto_merge: false + + # Example 5: Minimal config with template + - name: "typescript-examples-simple" + source_pattern: + type: "prefix" + pattern: "examples/typescript/" + targets: + - repo: "mongodb/node-mongodb-native" + branch: "main" + path_transform: "examples/${relative_path}" + commit_strategy: + type: "pull_request" + pr_title: "Update TypeScript examples" + pr_body: "Automated update from ${source_repo} PR #${pr_number}" + use_pr_template: true # Template provides structure, this adds context + auto_merge: false + + # Example 6: No template (traditional approach) + - name: "rust-examples-no-template" + source_pattern: + type: "prefix" + pattern: "examples/rust/" + targets: + - repo: "mongodb/mongo-rust-driver" + branch: "main" + path_transform: "examples/${relative_path}" + commit_strategy: + type: "pull_request" + pr_title: "Update Rust examples" + pr_body: | + ## Automated Update + + This PR updates Rust code examples. + + **Details:** + - Files: ${file_count} + - Source: ${source_repo} + - PR: #${pr_number} + + **Review Checklist:** + - [ ] Examples compile + - [ ] Tests pass + - [ ] Documentation updated + use_pr_template: false # Use only this configured body + auto_merge: false + + # Example 7: Direct commit (template not applicable) + - name: "config-files-direct" + source_pattern: + type: "glob" + pattern: "config/**/*.yaml" + targets: + - repo: "mongodb/docs-config" + branch: "main" + path_transform: "examples/${matched_pattern}" + commit_strategy: + type: "direct" # Direct commits don't use PR templates + commit_message: "Update config files from ${source_repo}" + diff --git a/examples-copier/configs/copier-config.example.yaml b/examples-copier/configs/copier-config-examples/copier-config.example.yaml similarity index 100% rename from examples-copier/configs/copier-config.example.yaml rename to examples-copier/configs/copier-config-examples/copier-config.example.yaml diff --git a/examples-copier/configs/env.yaml.production b/examples-copier/configs/env.yaml.production index 9a75ff6..37a008a 100644 --- a/examples-copier/configs/env.yaml.production +++ b/examples-copier/configs/env.yaml.production @@ -1,61 +1,60 @@ -env_variables: - # ============================================================================= - # GitHub Configuration (Non-sensitive) - # ============================================================================= - GITHUB_APP_ID: "1166559" - INSTALLATION_ID: "62138132" - REPO_OWNER: "mongodb" - REPO_NAME: "docs-mongodb-internal" - SRC_BRANCH: "main" - - # ============================================================================= - # Secret Manager References (Sensitive Data - SECURE!) - # ============================================================================= - # GitHub App private key - loaded from Secret Manager - GITHUB_APP_PRIVATE_KEY_SECRET_NAME: "projects/1054147886816/secrets/CODE_COPIER_PEM/versions/latest" - - # Webhook secret - loaded from Secret Manager - WEBHOOK_SECRET_NAME: "projects/1054147886816/secrets/webhook-secret/versions/latest" - - # MongoDB URI - loaded from Secret Manager (for audit logging - OPTIONAL) - MONGO_URI_SECRET_NAME: "projects/1054147886816/secrets/mongo-uri/versions/latest" - - # ============================================================================= - # Application Settings (Non-sensitive) - # ============================================================================= - # PORT is automatically set by App Engine Flexible (do not override) - WEBSERVER_PATH: "/events" - CONFIG_FILE: "copier-config.yaml" - DEPRECATION_FILE: "deprecated_examples.json" - - # ============================================================================= - # Committer Information (Non-sensitive) - # ============================================================================= - COMMITTER_NAME: "GitHub Copier App" - COMMITTER_EMAIL: "bot@mongodb.com" - - # ============================================================================= - # Google Cloud Configuration (Non-sensitive) - # ============================================================================= - GOOGLE_CLOUD_PROJECT_ID: "github-copy-code-examples" - COPIER_LOG_NAME: "code-copier-log" - - # Logging Configuration (Optional - uncomment for debugging) - # LOG_LEVEL: "debug" # Enable verbose debug logging - # COPIER_DEBUG: "true" # Alternative debug flag - # COPIER_DISABLE_CLOUD_LOGGING: "true" # Disable GCP logging - - # ============================================================================= - # Feature Flags (Optional) - # ============================================================================= - AUDIT_ENABLED: "true" - METRICS_ENABLED: "true" - # DRY_RUN: "false" - - # ============================================================================= - # Default Behaviors (Optional) - # ============================================================================= - # DEFAULT_RECURSIVE_COPY: "true" - # DEFAULT_PR_MERGE: "false" - # DEFAULT_COMMIT_MESSAGE: "Automated PR with updated examples" +# ============================================================================= +# GitHub Configuration (Non-sensitive) +# ============================================================================= +GITHUB_APP_ID: "1166559" +INSTALLATION_ID: "62138132" +REPO_OWNER: "mongodb" +REPO_NAME: "docs-mongodb-internal" +SRC_BRANCH: "main" + +# ============================================================================= +# Secret Manager References (Sensitive Data - SECURE!) +# ============================================================================= +# GitHub App private key - loaded from Secret Manager +GITHUB_APP_PRIVATE_KEY_SECRET_NAME: "projects/1054147886816/secrets/CODE_COPIER_PEM/versions/latest" + +# Webhook secret - loaded from Secret Manager +WEBHOOK_SECRET_NAME: "projects/1054147886816/secrets/webhook-secret/versions/latest" + +# MongoDB URI - loaded from Secret Manager (for audit logging - OPTIONAL) +MONGO_URI_SECRET_NAME: "projects/1054147886816/secrets/mongo-uri/versions/latest" + +# ============================================================================= +# Application Settings (Non-sensitive) +# ============================================================================= +# PORT is automatically set by App Engine Flexible (do not override) +WEBSERVER_PATH: "/events" +CONFIG_FILE: "copier-config.yaml" +DEPRECATION_FILE: "deprecated_examples.json" + +# ============================================================================= +# Committer Information (Non-sensitive) +# ============================================================================= +COMMITTER_NAME: "GitHub Copier App" +COMMITTER_EMAIL: "bot@mongodb.com" + +# ============================================================================= +# Google Cloud Configuration (Non-sensitive) +# ============================================================================= +GOOGLE_CLOUD_PROJECT_ID: "github-copy-code-examples" +COPIER_LOG_NAME: "code-copier-log" + +# Logging Configuration (Optional - uncomment for debugging) +# LOG_LEVEL: "debug" # Enable verbose debug logging +# COPIER_DEBUG: "true" # Alternative debug flag +# COPIER_DISABLE_CLOUD_LOGGING: "true" # Disable GCP logging + +# ============================================================================= +# Feature Flags (Optional) +# ============================================================================= +AUDIT_ENABLED: "true" +METRICS_ENABLED: "true" +# DRY_RUN: "false" + +# ============================================================================= +# Default Behaviors (Optional) +# ============================================================================= +# DEFAULT_RECURSIVE_COPY: "true" +# DEFAULT_PR_MERGE: "false" +# DEFAULT_COMMIT_MESSAGE: "Automated PR with updated examples" diff --git a/examples-copier/docs/CONFIGURATION-GUIDE.md b/examples-copier/docs/CONFIGURATION-GUIDE.md index 7735c14..128f0c0 100644 --- a/examples-copier/docs/CONFIGURATION-GUIDE.md +++ b/examples-copier/docs/CONFIGURATION-GUIDE.md @@ -36,6 +36,7 @@ The examples-copier uses a YAML configuration file (default: `copier-config.yaml # Top-level configuration source_repo: "owner/source-repository" source_branch: "main" +batch_by_repo: false # Optional: batch all changes into one PR per target repo # Copy rules define what to copy and where copy_rules: @@ -70,7 +71,7 @@ source_repo: "mongodb/docs-code-examples" ### source_branch -**Type:** String (optional) +**Type:** String (optional) **Default:** `"main"` The branch to copy files from. @@ -79,9 +80,118 @@ The branch to copy files from. source_branch: "main" ``` +### batch_by_repo + +**Type:** Boolean (optional) +**Default:** `false` + +When `true`, all changes from a single source PR are batched into **one pull request per target repository**, regardless of how many copy rules match files. + +When `false` (default), each copy rule creates a **separate pull request** in the target repository. + +**Example - Separate PRs per rule (default):** +```yaml +batch_by_repo: false # or omit this field + +copy_rules: + - name: "copy-client" + # ... matches 5 files + - name: "copy-server" + # ... matches 3 files + - name: "copy-readme" + # ... matches 1 file + +# Result: 3 separate PRs in the target repo +``` + +**Example - Single batched PR:** +```yaml +batch_by_repo: true + +copy_rules: + - name: "copy-client" + # ... matches 5 files + - name: "copy-server" + # ... matches 3 files + - name: "copy-readme" + # ... matches 1 file + +# Result: 1 PR containing all 9 files in the target repo +``` + +**Use Cases:** +- ✅ **Use `batch_by_repo: true`** when you want all related changes in a single PR for easier review +- ✅ **Use `batch_by_repo: false`** when different rules need separate review processes or different reviewers + +**Note:** When batching is enabled, use `batch_pr_config` (see below) to customize PR metadata, or a generic title/body will be generated. + +### batch_pr_config + +**Type:** Object (optional) +**Used when:** `batch_by_repo: true` + +Defines PR metadata (title, body, commit message) for batched pull requests. This allows you to customize the PR with accurate file counts and custom messaging. + +**Fields:** +- `pr_title` - (optional) PR title template +- `pr_body` - (optional) PR body template +- `commit_message` - (optional) Commit message template +- `use_pr_template` - (optional) Fetch and merge PR template from target repo (default: false) + +**Available template variables:** +- `${source_repo}` - Source repository (e.g., "owner/repo") +- `${target_repo}` - Target repository +- `${source_branch}` - Source branch name +- `${target_branch}` - Target branch name +- `${file_count}` - **Accurate** total number of files in the batched PR +- `${pr_number}` - Source PR number +- `${commit_sha}` - Source commit SHA + +**Example:** +```yaml +source_repo: "mongodb/code-examples" +source_branch: "main" +batch_by_repo: true + +batch_pr_config: + pr_title: "Update code examples from ${source_repo}" + pr_body: | + 🤖 Automated update of code examples + + **Source Information:** + - Repository: ${source_repo} + - PR: #${pr_number} + - Commit: ${commit_sha} + + **Changes:** + - Total files: ${file_count} + - Target branch: ${target_branch} + commit_message: "Update examples from ${source_repo} PR #${pr_number}" + use_pr_template: true # Fetch PR template from target repos + +copy_rules: + - name: "copy-client" + # ... rule config + - name: "copy-server" + # ... rule config +``` + +**Default behavior (if `batch_pr_config` is not specified):** +```yaml +# Default PR title: +"Update files from owner/repo PR #123" + +# Default PR body: +"Automated update from owner/repo + +Source PR: #123 +Commit: abc1234 +Files: 42" +``` + ### copy_rules -**Type:** Array (required) +**Type:** Array (required) **Minimum:** 1 rule List of copy rules that define what files to copy and where. @@ -222,6 +332,164 @@ pattern: "^examples/v(?P[0-9]+)/(?P[^/]+)/(?P.+)$" pattern: "^examples/(?P[^/]+)(/(?P[^/]+))?/(?P[^/]+)$" ``` +### Excluding Files with `exclude_patterns` + +**Type:** Array of strings (optional) +**Format:** Go-compatible regex patterns + +You can exclude specific files from being matched by adding `exclude_patterns` to any source pattern. This is useful for filtering out files like `.gitignore`, `.env`, `node_modules`, build artifacts, etc. + +**Important:** Exclude patterns use **Go regex syntax** (no negative lookahead `(?!...)`). + +#### Basic Example + +```yaml +source_pattern: + type: "prefix" + pattern: "examples/" + exclude_patterns: + - "\.gitignore$" # Exclude .gitignore files + - "\.env$" # Exclude .env files + - "node_modules/" # Exclude node_modules directory +``` + +#### How It Works + +1. **Main pattern matches first** - The file must match the main pattern (`type` and `pattern`) +2. **Then exclusions are checked** - If the file matches any `exclude_patterns`, it's excluded +3. **Result** - File is only copied if it matches the main pattern AND doesn't match any exclusions + +#### Examples by Pattern Type + +**Prefix Pattern with Exclusions:** +```yaml +- name: "copy-examples-no-config" + source_pattern: + type: "prefix" + pattern: "examples/" + exclude_patterns: + - "\.gitignore$" + - "\.env$" + - "/node_modules/" + - "/dist/" + - "/build/" + targets: + - repo: "mongodb/docs" + branch: "main" + path_transform: "code-examples/${relative_path}" +``` + +**Regex Pattern with Exclusions:** +```yaml +- name: "java-server-no-tests" + source_pattern: + type: "regex" + pattern: "^mflix/server/java-spring/(?P.+)$" + exclude_patterns: + - "/test/" # Exclude test directories + - "Test\.java$" # Exclude test files + - "\.gitignore$" # Exclude .gitignore + targets: + - repo: "mongodb/sample-app-java" + branch: "main" + path_transform: "server/${file}" +``` + +**Glob Pattern with Exclusions:** +```yaml +- name: "js-files-no-minified" + source_pattern: + type: "glob" + pattern: "examples/**/*.js" + exclude_patterns: + - "\.min\.js$" # Exclude minified files + - "\.test\.js$" # Exclude test files + targets: + - repo: "mongodb/docs" + branch: "main" + path_transform: "code/${matched_pattern}" +``` + +#### Common Exclusion Patterns + +```yaml +# Exclude hidden files (starting with .) +exclude_patterns: + - "/\\.[^/]+$" + +# Exclude build artifacts +exclude_patterns: + - "/dist/" + - "/build/" + - "\.min\\.(js|css)$" + +# Exclude dependencies +exclude_patterns: + - "node_modules/" + - "vendor/" + - "__pycache__/" + +# Exclude config files +exclude_patterns: + - "\.gitignore$" + - "\.env$" + - "\.env\\..*$" + - "config\\.local\\." + +# Exclude test files +exclude_patterns: + - "/test/" + - "/tests/" + - "Test\\.java$" + - "_test\\.go$" + - "\\.test\\.(js|ts)$" + - "\\.spec\\.(js|ts)$" + +# Exclude documentation +exclude_patterns: + - "README\\.md$" + - "\\.md$" + - "/docs/" +``` + +#### Regex Syntax Notes + +**✅ Supported (Go regex):** +- Character classes: `[abc]`, `[a-z]`, `[^abc]` +- Quantifiers: `*`, `+`, `?`, `{n}`, `{n,}`, `{n,m}` +- Anchors: `^` (start), `$` (end) +- Alternation: `(js|ts|jsx|tsx)` +- Escaping: `\.`, `\(`, `\[`, etc. + +**❌ Not Supported:** +- Negative lookahead: `(?!...)` - Use multiple patterns instead +- Lookbehind: `(?<=...)`, `(?> .gitignore ``` -### env.yaml Structure +### env-cloudrun.yaml Structure **Important Notes:** -- Do NOT set `PORT` in `env.yaml` - App Engine Flexible automatically sets this +- Do NOT set `PORT` in `env-cloudrun.yaml` - Cloud Run automatically sets this - The application defaults to port 8080 for local development - Secret Manager references must include `/versions/latest` or a specific version number +- Format: Simple `KEY: value` pairs (not nested under `env_variables:`) ```yaml -env_variables: - # ============================================================================= - # GitHub Configuration (Non-sensitive) - # ============================================================================= - GITHUB_APP_ID: "YOUR_APP_ID" - INSTALLATION_ID: "YOUR_INSTALLATION_ID" - REPO_OWNER: "your-org" - REPO_NAME: "your-repo" - SRC_BRANCH: "main" - - # ============================================================================= - # Secret Manager References (Sensitive - SECURE!) - # ============================================================================= - GITHUB_APP_PRIVATE_KEY_SECRET_NAME: "projects/PROJECT_NUMBER/secrets/CODE_COPIER_PEM/versions/latest" - WEBHOOK_SECRET_NAME: "projects/PROJECT_NUMBER/secrets/webhook-secret/versions/latest" - MONGO_URI_SECRET_NAME: "projects/PROJECT_NUMBER/secrets/mongo-uri/versions/latest" - - # ============================================================================= - # Application Settings - # ============================================================================= - # PORT: "8080" # DO NOT SET - App Engine sets this automatically - WEBSERVER_PATH: "/events" - CONFIG_FILE: "copier-config.yaml" - DEPRECATION_FILE: "deprecated_examples.json" - - # ============================================================================= - # Committer Information - # ============================================================================= - COMMITTER_NAME: "GitHub Copier App" - COMMITTER_EMAIL: "bot@example.com" - - # ============================================================================= - # Google Cloud Configuration - # ============================================================================= - GOOGLE_PROJECT_ID: "your-project-id" - GOOGLE_LOG_NAME: "code-copier-log" - - # ============================================================================= - # Feature Flags - # ============================================================================= - AUDIT_ENABLED: "true" - METRICS_ENABLED: "true" - # DRY_RUN: "false" +# ============================================================================= +# GitHub Configuration (Non-sensitive) +# ============================================================================= +GITHUB_APP_ID: "YOUR_APP_ID" +INSTALLATION_ID: "YOUR_INSTALLATION_ID" +REPO_OWNER: "your-org" +REPO_NAME: "your-repo" +SRC_BRANCH: "main" + +# ============================================================================= +# Secret Manager References (Sensitive - SECURE!) +# ============================================================================= +GITHUB_APP_PRIVATE_KEY_SECRET_NAME: "projects/PROJECT_NUMBER/secrets/CODE_COPIER_PEM/versions/latest" +WEBHOOK_SECRET_NAME: "projects/PROJECT_NUMBER/secrets/webhook-secret/versions/latest" +MONGO_URI_SECRET_NAME: "projects/PROJECT_NUMBER/secrets/mongo-uri/versions/latest" + +# ============================================================================= +# Application Settings +# ============================================================================= +# PORT: "8080" # DO NOT SET - Cloud Run sets this automatically +WEBSERVER_PATH: "/events" +CONFIG_FILE: "copier-config.yaml" +DEPRECATION_FILE: "deprecated_examples.json" + +# ============================================================================= +# Committer Information +# ============================================================================= +COMMITTER_NAME: "GitHub Copier App" +COMMITTER_EMAIL: "bot@example.com" + +# ============================================================================= +# Google Cloud Configuration +# ============================================================================= +GOOGLE_CLOUD_PROJECT_ID: "your-project-id" +COPIER_LOG_NAME: "code-copier-log" + +# ============================================================================= +# Feature Flags +# ============================================================================= +AUDIT_ENABLED: "true" +METRICS_ENABLED: "true" +# DRY_RUN: "false" ``` ### Important Notes +**About env-cloudrun.yaml:** +- This file contains **environment variables only** (application configuration) +- It does **NOT** contain infrastructure settings (CPU, memory, timeout, etc.) +- Infrastructure settings must be specified via command-line flags or a `service.yaml` file +- Use the deployment script (`./scripts/deploy-cloudrun.sh`) to avoid typing all the flags + **✅ DO:** - Use Secret Manager references (`*_SECRET_NAME` variables) -- Keep `env.yaml` in `.gitignore` -- Use `env.yaml.production` as template +- Keep `env-cloudrun.yaml` in `.gitignore` +- Use simple `KEY: value` format (no `env_variables:` wrapper) **❌ DON'T:** -- Put actual secrets in `env.yaml` (use `*_SECRET_NAME` instead) -- Commit `env.yaml` to version control -- Share `env.yaml` via email/chat +- Put actual secrets in `env-cloudrun.yaml` (use `*_SECRET_NAME` instead) +- Commit `env-cloudrun.yaml` to version control +- Share `env-cloudrun.yaml` via email/chat ### How Secrets Are Loaded ``` Application Startup: -1. Load env.yaml → environment variables +1. Cloud Run loads env-cloudrun.yaml → environment variables 2. Read WEBHOOK_SECRET_NAME from env 3. Call Secret Manager API to get actual secret 4. Store in config.WebhookSecret @@ -315,42 +322,90 @@ services.LoadMongoURI(config) // Loads from Secret Manager ### Pre-Deployment Checklist - [ ] Secrets created in Secret Manager -- [ ] IAM permissions granted to App Engine -- [ ] `env.yaml` created and configured -- [ ] `env.yaml` in `.gitignore` -- [ ] `app.yaml` uses Flexible Environment +- [ ] IAM permissions granted to Cloud Run service account +- [ ] `env-cloudrun.yaml` created and configured +- [ ] `env-cloudrun.yaml` in `.gitignore` +- [ ] `Dockerfile` exists in project root + +### Deploy to Cloud Run -### Deploy to App Engine +#### Option 1: Use the deployment script (Recommended) + +The simplest way to deploy is using the provided script: ```bash cd examples-copier -# Deploy (env.yaml is included via 'includes' directive in app.yaml) -gcloud app deploy app.yaml +# Deploy to default region (us-central1) +./scripts/deploy-cloudrun.sh + +# Or specify a different region +./scripts/deploy-cloudrun.sh us-east1 +``` + +The script will: +- ✅ Check that `env-cloudrun.yaml` exists +- ✅ Verify your Google Cloud project is set +- ✅ Show configuration before deploying +- ✅ Deploy with all recommended settings +- ✅ Display the service URL and next steps + +#### Option 2: Manual deployment command + +If you prefer to run the command directly: + +```bash +cd examples-copier -# Or specify project -gcloud app deploy app.yaml --project=your-project-id +gcloud run deploy examples-copier \ + --source . \ + --region us-central1 \ + --env-vars-file=env-cloudrun.yaml \ + --allow-unauthenticated \ + --max-instances=10 \ + --cpu=1 \ + --memory=512Mi \ + --timeout=300s \ + --concurrency=80 \ + --port=8080 ``` +**Deployment options explained:** +- `--source .` - Build from Dockerfile in current directory +- `--region us-central1` - Deploy to US Central region +- `--env-vars-file` - Load environment variables from file +- `--allow-unauthenticated` - Allow public webhook access (required for GitHub webhooks) +- `--max-instances=10` - Limit concurrent instances (cost control) +- `--cpu=1` - 1 vCPU per instance +- `--memory=512Mi` - 512MB RAM per instance +- `--timeout=300s` - 5 minute timeout for webhook processing +- `--concurrency=80` - Handle up to 80 concurrent requests per instance +- `--port=8080` - Container port (matches Dockerfile EXPOSE) + ### Verify Deployment ```bash # Check deployment status -gcloud app versions list +gcloud run services list --region=us-central1 -# Get app URL -APP_URL=$(gcloud app describe --format="value(defaultHostname)") -echo "App URL: https://${APP_URL}" +# Get service URL +SERVICE_URL=$(gcloud run services describe examples-copier \ + --region=us-central1 \ + --format="value(status.url)") +echo "Service URL: ${SERVICE_URL}" # View logs -gcloud app logs tail -s default +gcloud run services logs read examples-copier --region=us-central1 --limit=50 + +# Or tail logs in real-time +gcloud run services logs tail examples-copier --region=us-central1 ``` ### Test Health Endpoint ```bash # Test health -curl https://${APP_URL}/health +curl ${SERVICE_URL}/health # Expected response: # { @@ -376,7 +431,7 @@ curl https://${APP_URL}/health - Go to: `https://github.com/YOUR_ORG/YOUR_REPO/settings/hooks` 2. **Add or edit webhook** - - **Payload URL:** `https://YOUR_APP.appspot.com/events` + - **Payload URL:** `https://examples-copier-XXXXXXXXXX-uc.a.run.app/events` (use your Cloud Run URL) - **Content type:** `application/json` - **Secret:** (the webhook secret from Secret Manager) - **Events:** Select "Pull requests" @@ -1007,6 +1062,5 @@ gcloud app deploy app.yaml **See also:** - [FAQ.md](FAQ.md) - Frequently asked questions -- [../WEBHOOK-SECRET-MANAGER-GUIDE.md](../WEBHOOK-SECRET-MANAGER-GUIDE.md) - Secret Manager details -- [../ENV-FILES-EXPLAINED.md](../ENV-FILES-EXPLAINED.md) - Environment file explanation +- [TROUBLESHOOTING.md](TROUBLESHOOTING.md) - Troubleshooting guide diff --git a/examples-copier/docs/FAQ.md b/examples-copier/docs/FAQ.md index 4d82a89..486a1f7 100644 --- a/examples-copier/docs/FAQ.md +++ b/examples-copier/docs/FAQ.md @@ -22,6 +22,9 @@ Examples-copier is a GitHub app that automatically copies code examples and file - Path transformations with variable substitution - Multiple target repositories - Flexible commit strategies (direct or PR) +- **Batch PRs** - Combine multiple rules into one PR per target repo +- **PR Template Integration** - Fetch and merge PR templates from target repos +- **File Exclusion** - Exclude patterns to filter out unwanted files - Deprecation tracking - MongoDB audit logging - Health and metrics endpoints @@ -334,6 +337,70 @@ commit_strategy: - `${commit_sha}` - Commit SHA - Plus any variables extracted from pattern matching +### How do I batch multiple rules into one PR? + +Use `batch_by_repo: true` to combine all changes into one PR per target repository: + +```yaml +batch_by_repo: true + +batch_pr_config: + pr_title: "Update from ${source_repo}" + pr_body: | + 🤖 Automated update + Files: ${file_count} # Accurate count across all rules + use_pr_template: true + commit_message: "Update from ${source_repo} PR #${pr_number}" +``` + +**Benefits:** +- Single PR per target repo instead of multiple PRs +- Accurate `${file_count}` across all matched rules +- Easier review for related changes + +### How do I use PR templates from target repos? + +Set `use_pr_template: true` in your commit strategy or batch config: + +```yaml +commit_strategy: + type: "pull_request" + pr_body: | + 🤖 Automated update + Files: ${file_count} + use_pr_template: true # Fetches .github/pull_request_template.md +``` + +The service will: +1. Fetch the PR template from the target repo +2. Place the template content first (checklists, guidelines) +3. Add a separator (`---`) +4. Append your configured content (automation info) + +This ensures reviewers see the target repo's review guidelines prominently. + +### How do I exclude files from being copied? + +Use `exclude_patterns` in your source pattern: + +```yaml +source_pattern: + type: "prefix" + pattern: "examples/" + exclude_patterns: + - "\.gitignore$" # Exclude .gitignore + - "node_modules/" # Exclude dependencies + - "\.env$" # Exclude .env files + - "/dist/" # Exclude build output + - "\.test\.(js|ts)$" # Exclude test files +``` + +**Common use cases:** +- Filter out configuration files (`.gitignore`, `.env`) +- Exclude dependencies (`node_modules/`, `vendor/`) +- Skip build artifacts (`/dist/`, `/build/`) +- Exclude test files (`*.test.js`, `*_test.go`) + ## Performance ### How many files can it handle? diff --git a/examples-copier/docs/WEBHOOK-EVENTS.md b/examples-copier/docs/WEBHOOK-EVENTS.md deleted file mode 100644 index 6fc2b60..0000000 --- a/examples-copier/docs/WEBHOOK-EVENTS.md +++ /dev/null @@ -1,232 +0,0 @@ -# Webhook Events Guide - -## Overview - -The examples-copier application receives GitHub webhook events and processes them to copy code examples between repositories. This document explains which events are processed and which are ignored. - -## Supported Events - -### Pull Request Events (`pull_request`) - -**Status:** ✅ **Processed** - -The application **only** processes `pull_request` events with the following criteria: - -- **Action:** `closed` -- **Merged:** `true` - -All other pull request actions are ignored: -- `opened` - PR created but not merged -- `synchronize` - PR updated with new commits -- `edited` - PR title/description changed -- `labeled` - Labels added/removed -- `review_requested` - Reviewers requested -- etc. - -**Example Log Output:** -``` -[INFO] PR event received | {"action":"closed","merged":true} -[INFO] processing merged PR | {"pr_number":123,"repo":"owner/repo","sha":"abc123"} -``` - -## Ignored Events - -The following GitHub webhook events are **intentionally ignored** and will not trigger any processing: - -### Common Ignored Events - -| Event Type | Description | Why Ignored | -|------------|-------------|-------------| -| `ping` | GitHub webhook test | Not a code change | -| `push` | Direct push to branch | Only process merged PRs | -| `installation` | App installed/uninstalled | Not relevant to copying | -| `installation_repositories` | Repos added/removed from app | Not relevant to copying | -| `repository` | Repository created/deleted | Not relevant to copying | -| `workflow_run` | GitHub Actions workflow | Not relevant to copying | -| `check_run` | CI check completed | Not relevant to copying | -| `status` | Commit status updated | Not relevant to copying | - -**Example Log Output:** -``` -[INFO] ignoring non-pull_request event | {"event_type":"ping","size_bytes":7233} -``` - -## Monitoring Webhook Events - -### Viewing Metrics - -Check the `/metrics` endpoint to see webhook event statistics: - -```bash -curl https://your-app.appspot.com/metrics | jq '.webhooks' -``` - -**Example Response:** -```json -{ - "received": 150, - "processed": 45, - "failed": 2, - "ignored": 103, - "event_types": { - "pull_request": 45, - "ping": 5, - "push": 50, - "workflow_run": 48 - }, - "success_rate": 95.74, - "processing_time": { - "avg_ms": 1250, - "min_ms": 450, - "max_ms": 3200, - "p50_ms": 1100, - "p95_ms": 2800, - "p99_ms": 3100 - } -} -``` - -### Understanding the Metrics - -- **`received`**: Total webhooks received (all event types) -- **`processed`**: Successfully processed merged PRs -- **`failed`**: Webhooks that encountered errors -- **`ignored`**: Non-PR events or non-merged PRs -- **`event_types`**: Breakdown by GitHub event type -- **`success_rate`**: Percentage of received webhooks successfully processed - -### Viewing Logs - -**Local Development:** -```bash -# Watch application logs -tail -f logs/app.log | grep "event_type" -``` - -**Google Cloud Platform:** -```bash -# View recent logs -gcloud app logs tail -s default | grep "event_type" - -# Filter for ignored events -gcloud app logs tail -s default | grep "ignoring non-pull_request" -``` - -## Configuring GitHub Webhooks - -### Recommended Configuration - -When setting up the GitHub webhook in your repository settings: - -1. **Payload URL:** `https://your-app.appspot.com/events` -2. **Content type:** `application/json` -3. **Secret:** (use your webhook secret) -4. **Events:** Select **"Pull requests"** only - -### Why Select Only "Pull requests"? - -While the application safely ignores other event types, selecting only "Pull requests" reduces unnecessary webhook traffic and makes monitoring clearer. - -**Benefits:** -- ✅ Reduces network traffic -- ✅ Reduces log noise -- ✅ Easier to monitor and debug -- ✅ Lower webhook delivery quota usage - -### If You Need Multiple Event Types - -If your webhook is shared with other systems that need different events, it's safe to enable additional event types. The examples-copier will simply ignore them. - -## Troubleshooting - -### High Number of Ignored Events - -**Symptom:** Metrics show many ignored events - -**Possible Causes:** -1. **Webhook configured for all events** - Reconfigure to only send `pull_request` events -2. **Multiple webhooks configured** - Check repository settings for duplicate webhooks -3. **Shared webhook** - Other systems may be using the same endpoint - -**Solution:** -```bash -# Check webhook configuration -# Go to: https://github.com/YOUR_ORG/YOUR_REPO/settings/hooks - -# Verify only "Pull requests" is selected -``` - -### No Events Being Processed - -**Symptom:** `processed` count is 0, but `ignored` count is high - -**Possible Causes:** -1. **PRs not being merged** - Only merged PRs are processed -2. **Wrong event type** - Verify webhook sends `pull_request` events -3. **Configuration error** - Check copier-config.yaml exists and is valid - -**Solution:** -```bash -# Check recent webhook deliveries in GitHub -# Go to: https://github.com/YOUR_ORG/YOUR_REPO/settings/hooks/WEBHOOK_ID - -# Look for: -# - Event type: pull_request -# - Action: closed -# - Merged: true -``` - -### Unexpected Event Types - -**Symptom:** Seeing event types you didn't expect - -**Common Scenarios:** -1. **`ping` events** - GitHub sends these when webhook is created/edited (normal) -2. **`push` events** - Someone may have enabled this in webhook settings -3. **`workflow_run` events** - GitHub Actions workflows triggering webhooks - -**Solution:** -Review and update webhook configuration to only send necessary events. - -## Best Practices - -### 1. Monitor Event Type Distribution - -Regularly check the `event_types` breakdown in metrics: - -```bash -curl https://your-app.appspot.com/metrics | jq '.webhooks.event_types' -``` - -**Expected Distribution:** -- Most events should be `pull_request` -- Occasional `ping` events are normal -- High numbers of other types suggest misconfiguration - -### 2. Set Up Alerts - -Configure alerts for: -- High `failed` count -- Low `success_rate` (< 90%) -- Unexpected event types appearing - -### 3. Regular Audits - -Periodically review: -- GitHub webhook configuration -- Application logs for ignored events -- Metrics trends over time - -## Related Documentation - -- [DEPLOYMENT.md](DEPLOYMENT.md) - Webhook configuration during deployment -- [WEBHOOK-TESTING.md](WEBHOOK-TESTING.md) - Testing webhook processing - -## Summary - -- ✅ **Only merged PRs are processed** -- ✅ **All other events are safely ignored** -- ✅ **Metrics track all event types** -- ✅ **Configure webhook to send only `pull_request` events for best results** -- ✅ **Monitor `/metrics` endpoint to understand webhook traffic** - diff --git a/examples-copier/project.toml b/examples-copier/project.toml new file mode 100644 index 0000000..2de4a02 --- /dev/null +++ b/examples-copier/project.toml @@ -0,0 +1,4 @@ +[[build.env]] +name = "GOOGLE_BUILDABLE" +value = "Dockerfile" + diff --git a/examples-copier/scripts/README.md b/examples-copier/scripts/README.md index 8b449c3..6233af8 100644 --- a/examples-copier/scripts/README.md +++ b/examples-copier/scripts/README.md @@ -142,6 +142,43 @@ Test 5: Sending deprecation notification... Check your Slack channel for 5 test notifications ``` +### convert-env-format.sh + +Convert between App Engine and Cloud Run environment file formats. + +**Usage:** +```bash +./scripts/convert-env-format.sh to-cloudrun +./scripts/convert-env-format.sh to-appengine +``` + +**What it does:** +- Converts between App Engine format (with `env_variables:` wrapper) and Cloud Run format (plain YAML) +- Handles indentation automatically +- Validates input file exists +- Prompts before overwriting existing files + +**Example:** +```bash +# Convert App Engine → Cloud Run +./scripts/convert-env-format.sh to-cloudrun env.yaml env-cloudrun.yaml + +# Convert Cloud Run → App Engine +./scripts/convert-env-format.sh to-appengine env-cloudrun.yaml env.yaml +``` + +**Format differences:** +```yaml +# App Engine format (env.yaml) +env_variables: + GITHUB_APP_ID: "123456" + REPO_OWNER: "mongodb" + +# Cloud Run format (env-cloudrun.yaml) +GITHUB_APP_ID: "123456" +REPO_OWNER: "mongodb" +``` + ### test-with-pr.sh Fetch real PR data from GitHub and send it to the webhook. diff --git a/examples-copier/scripts/check-installation-repos.sh b/examples-copier/scripts/check-installation-repos.sh new file mode 100755 index 0000000..a504c8d --- /dev/null +++ b/examples-copier/scripts/check-installation-repos.sh @@ -0,0 +1,154 @@ +#!/bin/bash + +# Script to check which repositories a GitHub App installation has access to +# Requires: curl, jq, gcloud + +set -e + +echo "🔍 Checking GitHub App Installation Repositories" +echo "================================================" +echo "" + +# Check if jq is installed +if ! command -v jq &> /dev/null; then + echo "❌ jq is required but not installed" + echo "Install: brew install jq" + exit 1 +fi + +# Get the installation ID from env.yaml +INSTALLATION_ID=$(grep "INSTALLATION_ID:" env.yaml | grep -v "#" | awk '{print $2}' | tr -d '"') + +if [ -z "$INSTALLATION_ID" ]; then + echo "❌ INSTALLATION_ID not found in env.yaml" + exit 1 +fi + +echo "Installation ID: $INSTALLATION_ID" +echo "" + +# Get the GitHub App private key from Secret Manager +echo "📥 Retrieving GitHub App private key from Secret Manager..." +PEM_KEY=$(gcloud secrets versions access latest --secret=CODE_COPIER_PEM) + +if [ -z "$PEM_KEY" ]; then + echo "❌ Failed to retrieve private key" + exit 1 +fi + +echo "✅ Private key retrieved" +echo "" + +# Get the GitHub App ID from env.yaml +APP_ID=$(grep "GITHUB_APP_ID:" env.yaml | awk '{print $2}' | tr -d '"') + +if [ -z "$APP_ID" ]; then + echo "❌ GITHUB_APP_ID not found in env.yaml" + exit 1 +fi + +echo "GitHub App ID: $APP_ID" +echo "" + +# Generate JWT token (simplified - requires ruby) +echo "🔐 Generating JWT token..." + +# Save PEM key to temp file +TMP_PEM=$(mktemp) +echo "$PEM_KEY" > "$TMP_PEM" + +# Generate JWT using ruby +JWT=$(ruby -rjwt -rjson -e " + private_key = OpenSSL::PKey::RSA.new(File.read('$TMP_PEM')) + payload = { + iat: Time.now.to_i - 60, + exp: Time.now.to_i + (10 * 60), + iss: '$APP_ID' + } + puts JWT.encode(payload, private_key, 'RS256') +" 2>/dev/null) + +# Clean up temp file +rm -f "$TMP_PEM" + +if [ -z "$JWT" ]; then + echo "❌ Failed to generate JWT token" + echo "Note: This script requires ruby with jwt gem installed" + echo "Install: gem install jwt" + exit 1 +fi + +echo "✅ JWT token generated" +echo "" + +# Get installation access token +echo "🔑 Getting installation access token..." +INSTALL_TOKEN_RESPONSE=$(curl -s -X POST \ + -H "Authorization: Bearer $JWT" \ + -H "Accept: application/vnd.github+json" \ + "https://api.github.com/app/installations/$INSTALLATION_ID/access_tokens") + +INSTALL_TOKEN=$(echo "$INSTALL_TOKEN_RESPONSE" | jq -r '.token') + +if [ "$INSTALL_TOKEN" == "null" ] || [ -z "$INSTALL_TOKEN" ]; then + echo "❌ Failed to get installation access token" + echo "Response:" + echo "$INSTALL_TOKEN_RESPONSE" | jq . + exit 1 +fi + +echo "✅ Installation access token obtained" +echo "" + +# Get installation details +echo "📋 Installation Details:" +echo "------------------------" +INSTALL_INFO=$(curl -s \ + -H "Authorization: Bearer $JWT" \ + -H "Accept: application/vnd.github+json" \ + "https://api.github.com/app/installations/$INSTALLATION_ID") + +ACCOUNT=$(echo "$INSTALL_INFO" | jq -r '.account.login') +ACCOUNT_TYPE=$(echo "$INSTALL_INFO" | jq -r '.account.type') +REPO_SELECTION=$(echo "$INSTALL_INFO" | jq -r '.repository_selection') + +echo "Account: $ACCOUNT ($ACCOUNT_TYPE)" +echo "Repository Selection: $REPO_SELECTION" +echo "" + +# Get list of repositories +echo "📚 Accessible Repositories:" +echo "---------------------------" + +if [ "$REPO_SELECTION" == "all" ]; then + echo "✅ Installation has access to ALL repositories in $ACCOUNT" + echo "" + echo "Fetching repository list..." + REPOS=$(curl -s \ + -H "Authorization: token $INSTALL_TOKEN" \ + -H "Accept: application/vnd.github+json" \ + "https://api.github.com/installation/repositories?per_page=100") + + echo "$REPOS" | jq -r '.repositories[] | " - \(.full_name)"' + + TOTAL=$(echo "$REPOS" | jq -r '.total_count') + echo "" + echo "Total: $TOTAL repositories" +else + echo "✅ Installation has access to SELECTED repositories" + echo "" + REPOS=$(curl -s \ + -H "Authorization: token $INSTALL_TOKEN" \ + -H "Accept: application/vnd.github+json" \ + "https://api.github.com/installation/repositories?per_page=100") + + echo "$REPOS" | jq -r '.repositories[] | " - \(.full_name)"' + + TOTAL=$(echo "$REPOS" | jq -r '.total_count') + echo "" + echo "Total: $TOTAL repositories" +fi + +echo "" +echo "✅ Done!" + diff --git a/examples-copier/scripts/convert-env-format.sh b/examples-copier/scripts/convert-env-format.sh new file mode 100755 index 0000000..44bf7b9 --- /dev/null +++ b/examples-copier/scripts/convert-env-format.sh @@ -0,0 +1,101 @@ +#!/bin/bash + +# Convert between App Engine (env.yaml) and Cloud Run (env-cloudrun.yaml) formats +# +# Usage: +# ./convert-env-format.sh to-cloudrun env.yaml env-cloudrun.yaml +# ./convert-env-format.sh to-appengine env-cloudrun.yaml env.yaml + +set -e + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +BLUE='\033[0;34m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# Print usage +usage() { + echo "Convert between App Engine and Cloud Run environment file formats" + echo "" + echo "Usage:" + echo " $0 to-cloudrun " + echo " $0 to-appengine " + echo "" + echo "Examples:" + echo " # Convert App Engine format to Cloud Run format" + echo " $0 to-cloudrun env.yaml env-cloudrun.yaml" + echo "" + echo " # Convert Cloud Run format to App Engine format" + echo " $0 to-appengine env-cloudrun.yaml env.yaml" + echo "" + echo "Formats:" + echo " App Engine: env_variables: wrapper with indented keys" + echo " Cloud Run: Plain YAML without wrapper" + exit 1 +} + +# Check arguments +if [ $# -ne 3 ]; then + usage +fi + +COMMAND=$1 +INPUT=$2 +OUTPUT=$3 + +# Validate command +if [ "$COMMAND" != "to-cloudrun" ] && [ "$COMMAND" != "to-appengine" ]; then + echo -e "${RED}Error: Invalid command '$COMMAND'${NC}" + echo "Must be 'to-cloudrun' or 'to-appengine'" + usage +fi + +# Check input file exists +if [ ! -f "$INPUT" ]; then + echo -e "${RED}Error: Input file '$INPUT' not found${NC}" + exit 1 +fi + +# Check if output file exists +if [ -f "$OUTPUT" ]; then + echo -e "${YELLOW}Warning: Output file '$OUTPUT' already exists${NC}" + read -p "Overwrite? (y/N) " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + echo "Aborted" + exit 1 + fi +fi + +# Convert to Cloud Run format (remove env_variables wrapper and unindent) +if [ "$COMMAND" = "to-cloudrun" ]; then + echo -e "${BLUE}Converting App Engine format to Cloud Run format...${NC}" + + # Remove 'env_variables:' line and unindent by 2 spaces + sed '/^env_variables:/d' "$INPUT" | sed 's/^ //' > "$OUTPUT" + + echo -e "${GREEN}✓ Converted to Cloud Run format: $OUTPUT${NC}" + echo "" + echo "Deploy with:" + echo " gcloud run deploy examples-copier --source . --env-vars-file=$OUTPUT" +fi + +# Convert to App Engine format (add env_variables wrapper and indent) +if [ "$COMMAND" = "to-appengine" ]; then + echo -e "${BLUE}Converting Cloud Run format to App Engine format...${NC}" + + # Add 'env_variables:' header and indent all lines by 2 spaces + echo "env_variables:" > "$OUTPUT" + sed 's/^/ /' "$INPUT" >> "$OUTPUT" + + echo -e "${GREEN}✓ Converted to App Engine format: $OUTPUT${NC}" + echo "" + echo "Deploy with:" + echo " gcloud app deploy app.yaml # Includes $OUTPUT automatically" +fi + +echo "" +echo -e "${YELLOW}Note: Review the output file before deploying!${NC}" + diff --git a/examples-copier/scripts/deploy-cloudrun.sh b/examples-copier/scripts/deploy-cloudrun.sh new file mode 100755 index 0000000..7052b96 --- /dev/null +++ b/examples-copier/scripts/deploy-cloudrun.sh @@ -0,0 +1,107 @@ +#!/bin/bash +# Deploy examples-copier to Google Cloud Run +# Usage: ./scripts/deploy-cloudrun.sh [region] + +set -e + +# Get the directory where this script is located +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# Get the project root (parent of scripts directory) +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +# Configuration +SERVICE_NAME="examples-copier" +REGION="${1:-us-central1}" +ENV_FILE="$PROJECT_ROOT/env-cloudrun.yaml" + +# Colors for output +GREEN='\033[0;32m' +BLUE='\033[0;34m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +echo -e "${BLUE}╔════════════════════════════════════════════════════════════════╗${NC}" +echo -e "${BLUE}║ Deploying examples-copier to Cloud Run ║${NC}" +echo -e "${BLUE}╚════════════════════════════════════════════════════════════════╝${NC}" +echo "" + +# Check if env-cloudrun.yaml exists +if [ ! -f "$ENV_FILE" ]; then + echo -e "${YELLOW}⚠️ Warning: $ENV_FILE not found${NC}" + echo "Create it from a template:" + echo " cp configs/env.yaml.production $ENV_FILE" + echo " # Edit with your values" + exit 1 +fi + +# Get current project +PROJECT=$(gcloud config get-value project 2>/dev/null) +if [ -z "$PROJECT" ]; then + echo -e "${YELLOW}⚠️ No Google Cloud project set${NC}" + echo "Set your project:" + echo " gcloud config set project YOUR_PROJECT_ID" + exit 1 +fi + +echo -e "${GREEN}📦 Configuration:${NC}" +echo " Service: $SERVICE_NAME" +echo " Region: $REGION" +echo " Project: $PROJECT" +echo " Env File: $ENV_FILE" +echo "" + +# Confirm deployment +read -p "Deploy to Cloud Run? (y/N) " -n 1 -r +echo +if [[ ! $REPLY =~ ^[Yy]$ ]]; then + echo "Deployment cancelled" + exit 0 +fi + +echo "" +echo -e "${BLUE}🚀 Deploying...${NC}" +echo "" + +# Change to project root for deployment +cd "$PROJECT_ROOT" + +# Deploy to Cloud Run using Dockerfile +# Note: Using --source with Dockerfile to ensure it uses Docker build, not buildpacks +gcloud run deploy "$SERVICE_NAME" \ + --source . \ + --region "$REGION" \ + --env-vars-file="$ENV_FILE" \ + --allow-unauthenticated \ + --max-instances=10 \ + --cpu=1 \ + --memory=512Mi \ + --timeout=300s \ + --concurrency=80 \ + --port=8080 \ + --platform=managed + +echo "" +echo -e "${GREEN}✅ Deployment complete!${NC}" +echo "" + +# Get service URL +SERVICE_URL=$(gcloud run services describe "$SERVICE_NAME" \ + --region="$REGION" \ + --format="value(status.url)" 2>/dev/null) + +if [ -n "$SERVICE_URL" ]; then + echo -e "${GREEN}🌐 Service URL:${NC}" + echo " $SERVICE_URL" + echo "" + echo -e "${BLUE}📋 Next steps:${NC}" + echo " 1. Test health endpoint:" + echo " curl $SERVICE_URL/health" + echo "" + echo " 2. View logs:" + echo " gcloud run services logs read $SERVICE_NAME --region=$REGION --limit=50" + echo "" + echo " 3. Configure GitHub webhook:" + echo " Payload URL: $SERVICE_URL/events" + echo " Secret: (from Secret Manager)" +fi + diff --git a/examples-copier/scripts/diagnose-github-auth.sh b/examples-copier/scripts/diagnose-github-auth.sh new file mode 100755 index 0000000..e6c89f9 --- /dev/null +++ b/examples-copier/scripts/diagnose-github-auth.sh @@ -0,0 +1,174 @@ +#!/bin/bash + +# Diagnostic script for GitHub App authentication issues +# This script helps diagnose 401 Bad credentials errors + +set -e + +echo "🔍 GitHub App Authentication Diagnostics" +echo "==========================================" +echo "" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# Check if gcloud is installed +if ! command -v gcloud &> /dev/null; then + echo -e "${RED}❌ gcloud CLI not found${NC}" + echo "Please install: https://cloud.google.com/sdk/docs/install" + exit 1 +fi + +echo -e "${GREEN}✅ gcloud CLI found${NC}" + +# Get project info +PROJECT_ID=$(gcloud config get-value project 2>/dev/null) +if [ -z "$PROJECT_ID" ]; then + echo -e "${RED}❌ No GCP project set${NC}" + echo "Run: gcloud config set project YOUR_PROJECT_ID" + exit 1 +fi + +echo -e "${GREEN}✅ GCP Project: $PROJECT_ID${NC}" + +PROJECT_NUMBER=$(gcloud projects describe "$PROJECT_ID" --format="value(projectNumber)") +echo -e "${GREEN}✅ Project Number: $PROJECT_NUMBER${NC}" + +SERVICE_ACCOUNT="${PROJECT_NUMBER}@appspot.gserviceaccount.com" +echo -e " Service Account: $SERVICE_ACCOUNT" +echo "" + +# Check Secret Manager API +echo "📦 Checking Secret Manager..." +if gcloud services list --enabled --filter="name:secretmanager.googleapis.com" --format="value(name)" | grep -q secretmanager; then + echo -e "${GREEN}✅ Secret Manager API enabled${NC}" +else + echo -e "${RED}❌ Secret Manager API not enabled${NC}" + echo "Run: gcloud services enable secretmanager.googleapis.com" + exit 1 +fi + +# Check if secrets exist +echo "" +echo "🔐 Checking Secrets..." + +check_secret() { + local secret_name=$1 + if gcloud secrets describe "$secret_name" &>/dev/null; then + echo -e "${GREEN}✅ Secret exists: $secret_name${NC}" + + # Check IAM permissions + if gcloud secrets get-iam-policy "$secret_name" --format="value(bindings.members)" | grep -q "$SERVICE_ACCOUNT"; then + echo -e "${GREEN} ✅ Service account has access${NC}" + else + echo -e "${RED} ❌ Service account does NOT have access${NC}" + echo -e "${YELLOW} Fix: gcloud secrets add-iam-policy-binding $secret_name --member=\"serviceAccount:${SERVICE_ACCOUNT}\" --role=\"roles/secretmanager.secretAccessor\"${NC}" + fi + else + echo -e "${RED}❌ Secret NOT found: $secret_name${NC}" + fi +} + +check_secret "CODE_COPIER_PEM" +check_secret "webhook-secret" + +# Check if we can access the PEM key +echo "" +echo "🔑 Checking GitHub App Private Key..." +if gcloud secrets versions access latest --secret=CODE_COPIER_PEM &>/dev/null; then + PEM_FIRST_LINE=$(gcloud secrets versions access latest --secret=CODE_COPIER_PEM | head -n 1) + if [[ "$PEM_FIRST_LINE" == "-----BEGIN RSA PRIVATE KEY-----" ]] || [[ "$PEM_FIRST_LINE" == "-----BEGIN PRIVATE KEY-----" ]]; then + echo -e "${GREEN}✅ Private key format looks correct${NC}" + else + echo -e "${RED}❌ Private key format looks incorrect${NC}" + echo " First line: $PEM_FIRST_LINE" + fi +else + echo -e "${RED}❌ Cannot access private key${NC}" +fi + +# Check env.yaml +echo "" +echo "⚙️ Checking env.yaml configuration..." +if [ -f "env.yaml" ]; then + echo -e "${GREEN}✅ env.yaml found${NC}" + + # Extract values + GITHUB_APP_ID=$(grep "GITHUB_APP_ID:" env.yaml | awk '{print $2}' | tr -d '"') + INSTALLATION_ID=$(grep "INSTALLATION_ID:" env.yaml | grep -v "#" | awk '{print $2}' | tr -d '"') + REPO_OWNER=$(grep "REPO_OWNER:" env.yaml | grep -v "#" | awk '{print $2}' | tr -d '"') + REPO_NAME=$(grep "REPO_NAME:" env.yaml | grep -v "#" | awk '{print $2}' | tr -d '"') + + echo " GitHub App ID: $GITHUB_APP_ID" + echo " Installation ID: $INSTALLATION_ID" + echo " Repository: $REPO_OWNER/$REPO_NAME" + + if [ -z "$GITHUB_APP_ID" ] || [ -z "$INSTALLATION_ID" ] || [ -z "$REPO_OWNER" ] || [ -z "$REPO_NAME" ]; then + echo -e "${RED}❌ Missing required configuration${NC}" + else + echo -e "${GREEN}✅ Configuration looks complete${NC}" + fi +else + echo -e "${RED}❌ env.yaml not found${NC}" +fi + +# Check App Engine deployment +echo "" +echo "🚀 Checking App Engine deployment..." +if gcloud app describe &>/dev/null; then + APP_URL=$(gcloud app describe --format="value(defaultHostname)") + echo -e "${GREEN}✅ App Engine app exists${NC}" + echo " URL: https://$APP_URL" + + # Try to hit health endpoint + echo "" + echo "🏥 Checking health endpoint..." + if curl -s -f "https://$APP_URL/health" &>/dev/null; then + echo -e "${GREEN}✅ Health endpoint responding${NC}" + curl -s "https://$APP_URL/health" | python3 -m json.tool 2>/dev/null || echo "" + else + echo -e "${RED}❌ Health endpoint not responding${NC}" + fi +else + echo -e "${YELLOW}⚠️ No App Engine app deployed yet${NC}" +fi + +# Summary +echo "" +echo "📋 Summary & Next Steps" +echo "=======================" +echo "" + +# Check for common issues +ISSUES_FOUND=0 + +if ! gcloud secrets get-iam-policy CODE_COPIER_PEM --format="value(bindings.members)" | grep -q "$SERVICE_ACCOUNT"; then + echo -e "${RED}❌ Issue: Service account doesn't have access to CODE_COPIER_PEM${NC}" + echo " Fix: Run ./scripts/grant-secret-access.sh" + ISSUES_FOUND=$((ISSUES_FOUND + 1)) +fi + +if [ ! -f "env.yaml" ]; then + echo -e "${RED}❌ Issue: env.yaml not found${NC}" + echo " Fix: cp configs/env.yaml.example env.yaml && nano env.yaml" + ISSUES_FOUND=$((ISSUES_FOUND + 1)) +fi + +if [ $ISSUES_FOUND -eq 0 ]; then + echo -e "${GREEN}✅ No obvious issues found${NC}" + echo "" + echo "If you're still seeing 401 errors, check:" + echo "1. GitHub App is installed on the repository: https://github.com/settings/installations" + echo "2. Installation ID matches the repository" + echo "3. Private key in Secret Manager matches the GitHub App" + echo "4. GitHub App has 'Contents' read permission" + echo "" + echo "View logs: gcloud app logs tail -s default" +else + echo "" + echo -e "${YELLOW}Found $ISSUES_FOUND issue(s) - please fix them and try again${NC}" +fi + diff --git a/examples-copier/scripts/test-github-access.sh b/examples-copier/scripts/test-github-access.sh new file mode 100755 index 0000000..9b8a7d7 --- /dev/null +++ b/examples-copier/scripts/test-github-access.sh @@ -0,0 +1,76 @@ +#!/bin/bash + +# Test if the GitHub App can access the configured repository +# This script checks the recent logs for 401 errors + +set -e + +echo "🔍 Testing GitHub Repository Access" +echo "====================================" +echo "" + +# Get configuration from env.yaml +REPO_OWNER=$(grep "REPO_OWNER:" env.yaml | grep -v "#" | awk '{print $2}' | tr -d '"') +REPO_NAME=$(grep "REPO_NAME:" env.yaml | grep -v "#" | awk '{print $2}' | tr -d '"') +INSTALLATION_ID=$(grep "INSTALLATION_ID:" env.yaml | grep -v "#" | awk '{print $2}' | tr -d '"') + +echo "Configuration:" +echo " Repository: $REPO_OWNER/$REPO_NAME" +echo " Installation ID: $INSTALLATION_ID" +echo "" + +# Check health endpoint +echo "📊 Checking application health..." +HEALTH=$(curl -s https://github-copy-code-examples.ue.r.appspot.com/health) +AUTH_STATUS=$(echo "$HEALTH" | python3 -c "import sys, json; print(json.load(sys.stdin)['github']['authenticated'])") + +if [ "$AUTH_STATUS" == "True" ]; then + echo "✅ GitHub authentication is working" +else + echo "❌ GitHub authentication is NOT working" + exit 1 +fi +echo "" + +# Check recent logs for 401 errors +echo "🔍 Checking recent logs for 401 errors..." +RECENT_ERRORS=$(gcloud logging read "resource.type=gae_app AND severity>=ERROR AND textPayload=~'401 Bad credentials'" --limit=5 --format="value(timestamp,textPayload)" --freshness=30m 2>/dev/null) + +if [ -z "$RECENT_ERRORS" ]; then + echo "✅ No recent 401 errors found!" + echo "" + echo "🎉 GitHub App can successfully access the repository!" +else + echo "❌ Found recent 401 errors:" + echo "" + echo "$RECENT_ERRORS" + echo "" + echo "This means the GitHub App cannot access one or more repositories." + echo "" + echo "Possible causes:" + echo "1. GitHub App is not installed on the repository" + echo "2. Installation ID doesn't match the repository" + echo "3. GitHub App doesn't have 'Contents' read permission" + echo "" + echo "To fix:" + echo "1. Go to: https://github.com/settings/installations" + echo "2. Find your GitHub App installation" + echo "3. Make sure $REPO_OWNER/$REPO_NAME is in the list of accessible repositories" + echo "4. If not, click 'Configure' and add it" +fi + +echo "" +echo "📋 Summary" +echo "==========" +echo "Repository: $REPO_OWNER/$REPO_NAME" +echo "Installation ID: $INSTALLATION_ID" +echo "Authentication: $AUTH_STATUS" + +if [ -z "$RECENT_ERRORS" ]; then + echo "Status: ✅ WORKING" + exit 0 +else + echo "Status: ❌ NEEDS ATTENTION" + exit 1 +fi + diff --git a/examples-copier/scripts/validate-config-detailed.py b/examples-copier/scripts/validate-config-detailed.py new file mode 100755 index 0000000..be7d415 --- /dev/null +++ b/examples-copier/scripts/validate-config-detailed.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +""" +Detailed validation script for copier-config.yaml files +""" + +import sys +import yaml +import re + +def validate_config(file_path): + """Validate a copier-config.yaml file and report all issues""" + + issues = [] + warnings = [] + + # Try to load the YAML + try: + with open(file_path, 'r') as f: + config = yaml.safe_load(f) + except yaml.YAMLError as e: + print(f"❌ YAML Parsing Error:") + print(f" {e}") + return False + except Exception as e: + print(f"❌ Error reading file: {e}") + return False + + print("✅ YAML syntax is valid") + print() + + # Validate structure + if not isinstance(config, dict): + issues.append("Config must be a dictionary") + return False + + # Check required fields + if 'source_repo' not in config: + issues.append("Missing required field: source_repo") + + if 'copy_rules' not in config: + issues.append("Missing required field: copy_rules") + + if issues: + print("❌ Structural Issues:") + for issue in issues: + print(f" - {issue}") + return False + + print(f"📋 Config Summary:") + print(f" Source: {config.get('source_repo')}") + print(f" Branch: {config.get('source_branch', 'main')}") + print(f" Rules: {len(config.get('copy_rules', []))}") + print() + + # Validate each rule + rules = config.get('copy_rules', []) + for i, rule in enumerate(rules, 1): + rule_name = rule.get('name', f'Rule {i}') + print(f"🔍 Validating Rule {i}: {rule_name}") + + # Check rule structure + if 'source_pattern' not in rule: + issues.append(f"Rule '{rule_name}': Missing source_pattern") + continue + + if 'targets' not in rule: + issues.append(f"Rule '{rule_name}': Missing targets") + continue + + # Validate source_pattern + pattern = rule['source_pattern'] + if not isinstance(pattern, dict): + issues.append(f"Rule '{rule_name}': source_pattern must be a dictionary") + continue + + pattern_type = pattern.get('type') + pattern_str = pattern.get('pattern') + + if not pattern_type: + issues.append(f"Rule '{rule_name}': Missing pattern type") + elif pattern_type not in ['prefix', 'glob', 'regex']: + issues.append(f"Rule '{rule_name}': Invalid pattern type '{pattern_type}' (must be prefix, glob, or regex)") + + if not pattern_str: + issues.append(f"Rule '{rule_name}': Missing pattern string") + else: + # Check for type/pattern mismatch + has_regex_syntax = bool(re.search(r'\(\?P<\w+>', pattern_str)) + + if pattern_type == 'prefix' and has_regex_syntax: + issues.append(f"Rule '{rule_name}': Pattern type is 'prefix' but pattern contains regex syntax '(?P<...>)'") + warnings.append(f"Rule '{rule_name}': Should use type: 'regex' instead of 'prefix'") + + # Validate regex patterns + if pattern_type == 'regex': + try: + re.compile(pattern_str) + except re.error as e: + issues.append(f"Rule '{rule_name}': Invalid regex pattern: {e}") + + # Validate targets + targets = rule.get('targets', []) + if not isinstance(targets, list): + issues.append(f"Rule '{rule_name}': targets must be a list") + continue + + if len(targets) == 0: + warnings.append(f"Rule '{rule_name}': No targets defined") + + for j, target in enumerate(targets, 1): + if not isinstance(target, dict): + issues.append(f"Rule '{rule_name}', Target {j}: Must be a dictionary") + continue + + # Check required target fields + if 'repo' not in target: + issues.append(f"Rule '{rule_name}', Target {j}: Missing 'repo' field") + + if 'branch' not in target: + warnings.append(f"Rule '{rule_name}', Target {j}: Missing 'branch' field (will use default)") + + if 'path_transform' not in target: + warnings.append(f"Rule '{rule_name}', Target {j}: Missing 'path_transform' field") + + # Validate commit_strategy + if 'commit_strategy' in target: + strategy = target['commit_strategy'] + if not isinstance(strategy, dict): + issues.append(f"Rule '{rule_name}', Target {j}: commit_strategy must be a dictionary") + else: + strategy_type = strategy.get('type') + if strategy_type and strategy_type not in ['direct', 'pull_request']: + issues.append(f"Rule '{rule_name}', Target {j}: Invalid commit_strategy type '{strategy_type}'") + + print(f" ✓ Rule validated") + + print() + + # Print summary + if issues: + print("❌ VALIDATION FAILED") + print() + print("Issues found:") + for issue in issues: + print(f" ❌ {issue}") + print() + + if warnings: + print("⚠️ Warnings:") + for warning in warnings: + print(f" ⚠️ {warning}") + print() + + if not issues and not warnings: + print("✅ Configuration is valid with no issues!") + return True + elif not issues: + print("✅ Configuration is valid (with warnings)") + return True + else: + return False + +if __name__ == '__main__': + if len(sys.argv) != 2: + print("Usage: validate-config-detailed.py ") + sys.exit(1) + + file_path = sys.argv[1] + success = validate_config(file_path) + sys.exit(0 if success else 1) + diff --git a/examples-copier/service.yaml.example b/examples-copier/service.yaml.example new file mode 100644 index 0000000..e757792 --- /dev/null +++ b/examples-copier/service.yaml.example @@ -0,0 +1,136 @@ +apiVersion: serving.knative.dev/v1 +kind: Service +metadata: + annotations: + run.googleapis.com/build-enable-automatic-updates: 'false' + run.googleapis.com/build-id: b757980d-a5c4-4c4e-8080-e592075e3d98 + run.googleapis.com/build-image-uri: us-central1-docker.pkg.dev/github-copy-code-examples/cloud-run-source-deploy/examples-copier + run.googleapis.com/build-name: projects/1054147886816/locations/us-central1/builds/b757980d-a5c4-4c4e-8080-e592075e3d98 + run.googleapis.com/build-source-location: gs://run-sources-github-copy-code-examples-us-central1/services/examples-copier/1761840094.638811-19107807ada84a65915888b814b6e6dd.zip#1761840095211493 + run.googleapis.com/client-name: gcloud + run.googleapis.com/client-version: 542.0.0 + run.googleapis.com/ingress: all + run.googleapis.com/ingress-status: all + run.googleapis.com/operation-id: 5b7f8ffb-c7c1-42f7-a0c5-8ec5532bc0c4 + run.googleapis.com/urls: '["https://examples-copier-1054147886816.us-central1.run.app","https://examples-copier-7c5nckqo6a-uc.a.run.app"]' + serving.knative.dev/creator: cory.bullinger@gcp.corp.mongodb.com + serving.knative.dev/lastModifier: cory.bullinger@gcp.corp.mongodb.com + creationTimestamp: '2025-10-06T18:32:15.311946Z' + generation: 24 + labels: + cloud.googleapis.com/location: us-central1 + name: examples-copier + namespace: '1054147886816' + resourceVersion: AAZCYmPd434 + selfLink: /apis/serving.knative.dev/v1/namespaces/1054147886816/services/examples-copier + uid: c8854b49-365c-4168-a33a-f8a58682a348 +spec: + template: + metadata: + annotations: + autoscaling.knative.dev/maxScale: '10' + run.googleapis.com/client-name: gcloud + run.googleapis.com/client-version: 542.0.0 + run.googleapis.com/startup-cpu-boost: 'true' + labels: + client.knative.dev/nonce: xoklrkvxac + run.googleapis.com/startupProbeType: Default + spec: + containerConcurrency: 80 + containers: + - env: + - name: WEBHOOK_SECRET + valueFrom: + secretKeyRef: + key: latest + name: webhook-secret + - name: MONGO_URI + valueFrom: + secretKeyRef: + key: latest + name: mongo-uri + - name: ADMIN_TOKEN + valueFrom: + secretKeyRef: + key: latest + name: admin-token + - name: GITHUB_APP_ID + value: '1166559' + - name: INSTALLATION_ID + value: '62138132' + - name: REPO_OWNER + value: mongodb + - name: REPO_NAME + value: docs-sample-apps + - name: SRC_BRANCH + value: main + - name: GITHUB_APP_PRIVATE_KEY_SECRET_NAME + value: projects/1054147886816/secrets/CODE_COPIER_PEM/versions/latest + - name: WEBHOOK_SECRET_NAME + value: projects/1054147886816/secrets/webhook-secret/versions/latest + - name: MONGO_URI_SECRET_NAME + value: projects/1054147886816/secrets/mongo-uri/versions/latest + - name: WEBSERVER_PATH + value: /events + - name: CONFIG_FILE + value: copier-config.yaml + - name: DEPRECATION_FILE + value: deprecated_examples.json + - name: COMMITTER_NAME + value: GitHub Copier App + - name: COMMITTER_EMAIL + value: bot@mongodb.com + - name: GOOGLE_CLOUD_PROJECT_ID + value: github-copy-code-examples + - name: COPIER_LOG_NAME + value: code-copier-log + - name: LOG_LEVEL + value: debug + - name: COPIER_DEBUG + value: 'true' + - name: AUDIT_ENABLED + value: 'false' + - name: METRICS_ENABLED + value: 'true' + - name: DRY_RUN + value: 'false' + image: us-central1-docker.pkg.dev/github-copy-code-examples/cloud-run-source-deploy/examples-copier@sha256:d61a9184ff45bea59d3dceba098f99c0bbce2242898607dac65009b8f9f0eae7 + ports: + - containerPort: 8080 + name: http1 + resources: + limits: + cpu: '1' + memory: 512Mi + startupProbe: + failureThreshold: 1 + periodSeconds: 240 + tcpSocket: + port: 8080 + timeoutSeconds: 240 + serviceAccountName: 1054147886816-compute@developer.gserviceaccount.com + timeoutSeconds: 300 + traffic: + - latestRevision: true + percent: 100 +status: + address: + url: https://examples-copier-7c5nckqo6a-uc.a.run.app + conditions: + - lastTransitionTime: '2025-10-30T16:03:29.978238Z' + status: 'True' + type: Ready + - lastTransitionTime: '2025-10-30T16:03:25.908981Z' + status: 'True' + type: ConfigurationsReady + - lastTransitionTime: '2025-10-30T16:03:29.928929Z' + status: 'True' + type: RoutesReady + latestCreatedRevisionName: examples-copier-00024-h29 + latestReadyRevisionName: examples-copier-00024-h29 + observedGeneration: 24 + traffic: + - latestRevision: true + percent: 100 + revisionName: examples-copier-00024-h29 + url: https://examples-copier-7c5nckqo6a-uc.a.run.app diff --git a/examples-copier/services/batch_pr_config_test.go b/examples-copier/services/batch_pr_config_test.go new file mode 100644 index 0000000..19340cc --- /dev/null +++ b/examples-copier/services/batch_pr_config_test.go @@ -0,0 +1,127 @@ +package services_test + +import ( + "testing" + + "github.com/mongodb/code-example-tooling/code-copier/services" + "github.com/mongodb/code-example-tooling/code-copier/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBatchPRConfig_LoadsCorrectly(t *testing.T) { + loader := services.NewConfigLoader() + + yamlContent := ` +source_repo: "org/source-repo" +source_branch: "main" +batch_by_repo: true + +batch_pr_config: + pr_title: "Custom batch PR title" + pr_body: "Custom batch PR body with ${file_count} files" + commit_message: "Batch commit from ${source_repo}" + +copy_rules: + - name: "test-rule" + source_pattern: + type: "prefix" + pattern: "examples/" + targets: + - repo: "org/target-repo" + branch: "main" + path_transform: "docs/${relative_path}" + commit_strategy: + type: "pull_request" +` + + config, err := loader.LoadConfigFromContent(yamlContent, "config.yaml") + require.NoError(t, err) + require.NotNil(t, config) + + assert.True(t, config.BatchByRepo) + require.NotNil(t, config.BatchPRConfig) + assert.Equal(t, "Custom batch PR title", config.BatchPRConfig.PRTitle) + assert.Equal(t, "Custom batch PR body with ${file_count} files", config.BatchPRConfig.PRBody) + assert.Equal(t, "Batch commit from ${source_repo}", config.BatchPRConfig.CommitMessage) +} + +func TestBatchPRConfig_OptionalField(t *testing.T) { + loader := services.NewConfigLoader() + + yamlContent := ` +source_repo: "org/source-repo" +source_branch: "main" +batch_by_repo: true + +copy_rules: + - name: "test-rule" + source_pattern: + type: "prefix" + pattern: "examples/" + targets: + - repo: "org/target-repo" + branch: "main" + path_transform: "docs/${relative_path}" + commit_strategy: + type: "pull_request" +` + + config, err := loader.LoadConfigFromContent(yamlContent, "config.yaml") + require.NoError(t, err) + require.NotNil(t, config) + + assert.True(t, config.BatchByRepo) + assert.Nil(t, config.BatchPRConfig) // Should be nil when not specified +} + +func TestBatchPRConfig_StructureValidation(t *testing.T) { + // Test that the BatchPRConfig struct is properly defined + yamlConfig := &types.YAMLConfig{ + SourceRepo: "owner/source-repo", + SourceBranch: "main", + BatchByRepo: true, + BatchPRConfig: &types.BatchPRConfig{ + PRTitle: "Batch update from ${source_repo}", + PRBody: "Updated ${file_count} files from PR #${pr_number}", + CommitMessage: "Batch commit", + }, + } + + // Verify the config structure + assert.NotNil(t, yamlConfig.BatchPRConfig) + assert.Equal(t, "Batch update from ${source_repo}", yamlConfig.BatchPRConfig.PRTitle) + assert.Equal(t, "Updated ${file_count} files from PR #${pr_number}", yamlConfig.BatchPRConfig.PRBody) + assert.Equal(t, "Batch commit", yamlConfig.BatchPRConfig.CommitMessage) +} + +func TestMessageTemplater_RendersFileCount(t *testing.T) { + templater := services.NewMessageTemplater() + + ctx := types.NewMessageContext() + ctx.SourceRepo = "owner/source-repo" + ctx.FileCount = 42 + ctx.PRNumber = 123 + + template := "Updated ${file_count} files from ${source_repo} PR #${pr_number}" + result := templater.RenderPRBody(template, ctx) + + assert.Equal(t, "Updated 42 files from owner/source-repo PR #123", result) +} + +func TestMessageTemplater_DefaultBatchPRBody(t *testing.T) { + templater := services.NewMessageTemplater() + + ctx := types.NewMessageContext() + ctx.SourceRepo = "owner/source-repo" + ctx.FileCount = 15 + ctx.PRNumber = 456 + + // Empty template should use default + result := templater.RenderPRBody("", ctx) + + assert.Contains(t, result, "15 file(s)") + assert.Contains(t, result, "owner/source-repo") + assert.Contains(t, result, "#456") +} + diff --git a/examples-copier/services/exclude_patterns_test.go b/examples-copier/services/exclude_patterns_test.go new file mode 100644 index 0000000..6c9c49e --- /dev/null +++ b/examples-copier/services/exclude_patterns_test.go @@ -0,0 +1,321 @@ +package services + +import ( + "testing" + + "github.com/mongodb/code-example-tooling/code-copier/types" +) + +func TestExcludePatterns_PrefixPattern(t *testing.T) { + matcher := NewPatternMatcher() + + tests := []struct { + name string + filePath string + pattern string + excludePatterns []string + shouldMatch bool + }{ + { + name: "No exclusions - should match", + filePath: "examples/test.js", + pattern: "examples/", + excludePatterns: nil, + shouldMatch: true, + }, + { + name: "Exclude .gitignore - should not match", + filePath: "examples/.gitignore", + pattern: "examples/", + excludePatterns: []string{`\.gitignore$`}, + shouldMatch: false, + }, + { + name: "Exclude .gitignore - other file should match", + filePath: "examples/test.js", + pattern: "examples/", + excludePatterns: []string{`\.gitignore$`}, + shouldMatch: true, + }, + { + name: "Exclude multiple patterns - .env excluded", + filePath: "examples/.env", + pattern: "examples/", + excludePatterns: []string{`\.gitignore$`, `\.env$`}, + shouldMatch: false, + }, + { + name: "Exclude multiple patterns - .gitignore excluded", + filePath: "examples/.gitignore", + pattern: "examples/", + excludePatterns: []string{`\.gitignore$`, `\.env$`}, + shouldMatch: false, + }, + { + name: "Exclude multiple patterns - normal file matches", + filePath: "examples/test.js", + pattern: "examples/", + excludePatterns: []string{`\.gitignore$`, `\.env$`}, + shouldMatch: true, + }, + { + name: "Exclude all .md files", + filePath: "examples/README.md", + pattern: "examples/", + excludePatterns: []string{`\.md$`}, + shouldMatch: false, + }, + { + name: "Exclude node_modules directory", + filePath: "examples/node_modules/package.json", + pattern: "examples/", + excludePatterns: []string{`node_modules/`}, + shouldMatch: false, + }, + { + name: "Exclude build artifacts", + filePath: "examples/dist/bundle.js", + pattern: "examples/", + excludePatterns: []string{`/dist/`, `/build/`, `\.min\.js$`}, + shouldMatch: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sourcePattern := types.SourcePattern{ + Type: types.PatternTypePrefix, + Pattern: tt.pattern, + ExcludePatterns: tt.excludePatterns, + } + + result := matcher.Match(tt.filePath, sourcePattern) + + if result.Matched != tt.shouldMatch { + t.Errorf("Expected match=%v, got match=%v for file=%s with excludes=%v", + tt.shouldMatch, result.Matched, tt.filePath, tt.excludePatterns) + } + }) + } +} + +func TestExcludePatterns_RegexPattern(t *testing.T) { + matcher := NewPatternMatcher() + + tests := []struct { + name string + filePath string + pattern string + excludePatterns []string + shouldMatch bool + }{ + { + name: "Regex match with no exclusions", + filePath: "mflix/server/java-spring/src/Main.java", + pattern: `^mflix/server/java-spring/(?P.+)$`, + excludePatterns: nil, + shouldMatch: true, + }, + { + name: "Regex match - exclude .gitignore", + filePath: "mflix/server/java-spring/.gitignore", + pattern: `^mflix/server/java-spring/(?P.+)$`, + excludePatterns: []string{`\.gitignore$`}, + shouldMatch: false, + }, + { + name: "Regex match - exclude test files", + filePath: "mflix/server/java-spring/src/test/TestMain.java", + pattern: `^mflix/server/java-spring/(?P.+)$`, + excludePatterns: []string{`/test/`}, + shouldMatch: false, + }, + { + name: "Regex match - normal file passes exclusion", + filePath: "mflix/server/java-spring/src/Main.java", + pattern: `^mflix/server/java-spring/(?P.+)$`, + excludePatterns: []string{`/test/`, `\.gitignore$`}, + shouldMatch: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sourcePattern := types.SourcePattern{ + Type: types.PatternTypeRegex, + Pattern: tt.pattern, + ExcludePatterns: tt.excludePatterns, + } + + result := matcher.Match(tt.filePath, sourcePattern) + + if result.Matched != tt.shouldMatch { + t.Errorf("Expected match=%v, got match=%v for file=%s with excludes=%v", + tt.shouldMatch, result.Matched, tt.filePath, tt.excludePatterns) + } + }) + } +} + +func TestExcludePatterns_GlobPattern(t *testing.T) { + matcher := NewPatternMatcher() + + tests := []struct { + name string + filePath string + pattern string + excludePatterns []string + shouldMatch bool + }{ + { + name: "Glob match with no exclusions", + filePath: "examples/test.js", + pattern: "examples/*.js", + excludePatterns: nil, + shouldMatch: true, + }, + { + name: "Glob match - exclude .min.js files", + filePath: "examples/bundle.min.js", + pattern: "examples/*.js", + excludePatterns: []string{`\.min\.js$`}, + shouldMatch: false, + }, + { + name: "Glob match - normal file passes exclusion", + filePath: "examples/app.js", + pattern: "examples/*.js", + excludePatterns: []string{`\.min\.js$`}, + shouldMatch: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sourcePattern := types.SourcePattern{ + Type: types.PatternTypeGlob, + Pattern: tt.pattern, + ExcludePatterns: tt.excludePatterns, + } + + result := matcher.Match(tt.filePath, sourcePattern) + + if result.Matched != tt.shouldMatch { + t.Errorf("Expected match=%v, got match=%v for file=%s with excludes=%v", + tt.shouldMatch, result.Matched, tt.filePath, tt.excludePatterns) + } + }) + } +} + +func TestExcludePatterns_Validation(t *testing.T) { + tests := []struct { + name string + excludePatterns []string + shouldError bool + }{ + { + name: "Valid regex patterns", + excludePatterns: []string{`\.gitignore$`, `\.env$`, `/node_modules/`}, + shouldError: false, + }, + { + name: "Empty pattern - should error", + excludePatterns: []string{""}, + shouldError: true, + }, + { + name: "Invalid regex - should error", + excludePatterns: []string{`[invalid`}, + shouldError: true, + }, + { + name: "Mix of valid and invalid - should error", + excludePatterns: []string{`\.gitignore$`, `[invalid`}, + shouldError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sourcePattern := types.SourcePattern{ + Type: types.PatternTypePrefix, + Pattern: "examples/", + ExcludePatterns: tt.excludePatterns, + } + + err := sourcePattern.Validate() + + if tt.shouldError && err == nil { + t.Errorf("Expected validation error for patterns=%v, but got none", tt.excludePatterns) + } + if !tt.shouldError && err != nil { + t.Errorf("Expected no validation error for patterns=%v, but got: %v", tt.excludePatterns, err) + } + }) + } +} + +func TestExcludePatterns_ComplexScenarios(t *testing.T) { + matcher := NewPatternMatcher() + + tests := []struct { + name string + filePath string + pattern string + excludePatterns []string + shouldMatch bool + description string + }{ + { + name: "Exclude all hidden files", + filePath: "examples/.hidden", + pattern: "examples/", + excludePatterns: []string{`/\.[^/]+$`}, + shouldMatch: false, + description: "Files starting with . should be excluded", + }, + { + name: "Exclude all hidden files - normal file matches", + filePath: "examples/visible.txt", + pattern: "examples/", + excludePatterns: []string{`/\.[^/]+$`}, + shouldMatch: true, + description: "Normal files should match", + }, + { + name: "Exclude build artifacts and dependencies", + filePath: "examples/node_modules/package.json", + pattern: "examples/", + excludePatterns: []string{`node_modules/`, `dist/`, `build/`, `\.min\.(js|css)$`}, + shouldMatch: false, + description: "node_modules should be excluded", + }, + { + name: "Exclude build artifacts - source file matches", + filePath: "examples/src/app.js", + pattern: "examples/", + excludePatterns: []string{`node_modules/`, `dist/`, `build/`, `\.min\.(js|css)$`}, + shouldMatch: true, + description: "Source files should match", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sourcePattern := types.SourcePattern{ + Type: types.PatternTypePrefix, + Pattern: tt.pattern, + ExcludePatterns: tt.excludePatterns, + } + + result := matcher.Match(tt.filePath, sourcePattern) + + if result.Matched != tt.shouldMatch { + t.Errorf("%s: Expected match=%v, got match=%v for file=%s", + tt.description, tt.shouldMatch, result.Matched, tt.filePath) + } + }) + } +} + diff --git a/examples-copier/services/github_read.go b/examples-copier/services/github_read.go index c9eccd5..ae46272 100644 --- a/examples-copier/services/github_read.go +++ b/examples-copier/services/github_read.go @@ -44,7 +44,31 @@ func GetFilesChangedInPr(pr_number int) ([]ChangedFile, error) { Status: string(edge.Node.ChangeType), }) } + LogInfo(fmt.Sprintf("PR has %d changed files.", len(changedFiles))) + + // Log all files for debugging (especially to see if server files are included) + LogInfo("=== ALL FILES FROM GRAPHQL API ===") + for i, file := range changedFiles { + LogInfo(fmt.Sprintf(" [%d] %s (status: %s)", i, file.Path, file.Status)) + } + LogInfo("=== END FILE LIST ===") + + // Count files by directory for debugging + clientCount := 0 + serverCount := 0 + otherCount := 0 + for _, file := range changedFiles { + if len(file.Path) >= 13 && file.Path[:13] == "mflix/client/" { + clientCount++ + } else if len(file.Path) >= 13 && file.Path[:13] == "mflix/server/" { + serverCount++ + } else { + otherCount++ + } + } + LogInfo(fmt.Sprintf("File breakdown: client=%d, server=%d, other=%d", clientCount, serverCount, otherCount)) + return changedFiles, nil } diff --git a/examples-copier/services/github_write_to_target.go b/examples-copier/services/github_write_to_target.go index e80a35e..d38ba17 100644 --- a/examples-copier/services/github_write_to_target.go +++ b/examples-copier/services/github_write_to_target.go @@ -51,6 +51,13 @@ func normalizeRepoName(repoName string) string { // AddFilesToTargetRepoBranch uploads files to the target repository branch // using the specified commit strategy (direct or via pull request). func AddFilesToTargetRepoBranch() { + AddFilesToTargetRepoBranchWithFetcher(nil) +} + +// AddFilesToTargetRepoBranchWithFetcher uploads files to the target repository branch +// using the specified commit strategy (direct or via pull request). +// If prTemplateFetcher is provided, it will be used to fetch PR templates when use_pr_template is true. +func AddFilesToTargetRepoBranchWithFetcher(prTemplateFetcher PRTemplateFetcher) { ctx := context.Background() for key, value := range FilesToUpload { @@ -88,6 +95,19 @@ func AddFilesToTargetRepoBranch() { // Get PR body from value prBody := value.PRBody + // Fetch and merge PR template if requested + if value.UsePRTemplate && prTemplateFetcher != nil && strategy != "direct" { + targetBranch := strings.TrimPrefix(key.BranchPath, "refs/heads/") + template, err := prTemplateFetcher.FetchPRTemplate(ctx, client, key.RepoName, targetBranch) + if err != nil { + LogWarning(fmt.Sprintf("Failed to fetch PR template for %s: %v", key.RepoName, err)) + } else if template != "" { + // Merge configured body with template + prBody = MergePRBodyWithTemplate(prBody, template) + LogInfo(fmt.Sprintf("Merged PR template for %s", key.RepoName)) + } + } + // Get auto-merge setting from value mergeWithoutReview := value.AutoMergePR diff --git a/examples-copier/services/pattern_matcher.go b/examples-copier/services/pattern_matcher.go index 8bf5fb8..b49ad40 100644 --- a/examples-copier/services/pattern_matcher.go +++ b/examples-copier/services/pattern_matcher.go @@ -25,16 +25,31 @@ func NewPatternMatcher() PatternMatcher { // Match matches a file path against a source pattern func (pm *DefaultPatternMatcher) Match(filePath string, pattern types.SourcePattern) types.MatchResult { + // First, check if the main pattern matches + var result types.MatchResult switch pattern.Type { case types.PatternTypePrefix: - return pm.matchPrefix(filePath, pattern.Pattern) + result = pm.matchPrefix(filePath, pattern.Pattern) case types.PatternTypeGlob: - return pm.matchGlob(filePath, pattern.Pattern) + result = pm.matchGlob(filePath, pattern.Pattern) case types.PatternTypeRegex: - return pm.matchRegex(filePath, pattern.Pattern) + result = pm.matchRegex(filePath, pattern.Pattern) default: return types.NewMatchResult(false, nil) } + + // If the main pattern didn't match, return early + if !result.Matched { + return result + } + + // Check if the file should be excluded + if pm.shouldExclude(filePath, pattern.ExcludePatterns) { + return types.NewMatchResult(false, nil) + } + + // Main pattern matched and file is not excluded + return result } // matchPrefix matches using simple prefix matching @@ -86,14 +101,19 @@ func (pm *DefaultPatternMatcher) matchGlob(filePath, pattern string) types.Match func (pm *DefaultPatternMatcher) matchRegex(filePath, pattern string) types.MatchResult { re, err := regexp.Compile(pattern) if err != nil { + LogInfo(fmt.Sprintf("REGEX COMPILE ERROR: pattern=%s, error=%v", pattern, err)) return types.NewMatchResult(false, nil) } - + match := re.FindStringSubmatch(filePath) if match == nil { + // Log server file pattern attempts for debugging + if strings.Contains(pattern, "server/") && strings.Contains(filePath, "server/") { + LogInfo(fmt.Sprintf("REGEX NO MATCH: file=%s, pattern=%s", filePath, pattern)) + } return types.NewMatchResult(false, nil) } - + // Extract named capture groups variables := make(map[string]string) for i, name := range re.SubexpNames() { @@ -101,7 +121,12 @@ func (pm *DefaultPatternMatcher) matchRegex(filePath, pattern string) types.Matc variables[name] = match[i] } } - + + // Log server file matches for debugging + if strings.Contains(pattern, "server/") { + LogInfo(fmt.Sprintf("REGEX MATCH SUCCESS: file=%s, pattern=%s, variables=%v", filePath, pattern, variables)) + } + return types.NewMatchResult(true, variables) } @@ -226,23 +251,46 @@ func (mt *DefaultMessageTemplater) render(template string, ctx *types.MessageCon return result } +// shouldExclude checks if a file path matches any of the exclude patterns +func (pm *DefaultPatternMatcher) shouldExclude(filePath string, excludePatterns []string) bool { + if len(excludePatterns) == 0 { + return false + } + + for _, excludePattern := range excludePatterns { + // Compile and match the exclude pattern + re, err := regexp.Compile(excludePattern) + if err != nil { + // If the pattern is invalid, log and skip it + // (validation should have caught this earlier) + continue + } + + if re.MatchString(filePath) { + return true + } + } + + return false +} + // MatchAndTransform is a helper that combines pattern matching and path transformation func MatchAndTransform(filePath string, rule types.CopyRule, target types.TargetConfig) (string, map[string]string, bool) { matcher := NewPatternMatcher() transformer := NewPathTransformer() - + // Match the file against the pattern matchResult := matcher.Match(filePath, rule.SourcePattern) if !matchResult.Matched { return "", nil, false } - + // Transform the path targetPath, err := transformer.Transform(filePath, target.PathTransform, matchResult.Variables) if err != nil { return "", nil, false } - + return targetPath, matchResult.Variables, true } diff --git a/examples-copier/services/pr_template_fetcher.go b/examples-copier/services/pr_template_fetcher.go new file mode 100644 index 0000000..2501056 --- /dev/null +++ b/examples-copier/services/pr_template_fetcher.go @@ -0,0 +1,104 @@ +package services + +import ( + "context" + "fmt" + "strings" + + "github.com/google/go-github/v48/github" +) + +// PRTemplateFetcher defines the interface for fetching PR templates from repositories +type PRTemplateFetcher interface { + // FetchPRTemplate fetches the PR template from a target repository + // Returns the template content, or empty string if not found + FetchPRTemplate(ctx context.Context, client *github.Client, repoFullName string, branch string) (string, error) +} + +// DefaultPRTemplateFetcher implements PRTemplateFetcher +type DefaultPRTemplateFetcher struct{} + +// NewPRTemplateFetcher creates a new PR template fetcher +func NewPRTemplateFetcher() PRTemplateFetcher { + return &DefaultPRTemplateFetcher{} +} + +// FetchPRTemplate fetches the PR template from a target repository +// It checks multiple common locations for PR templates: +// 1. .github/pull_request_template.md +// 2. .github/PULL_REQUEST_TEMPLATE.md +// 3. docs/pull_request_template.md +// 4. PULL_REQUEST_TEMPLATE.md +func (f *DefaultPRTemplateFetcher) FetchPRTemplate(ctx context.Context, client *github.Client, repoFullName string, branch string) (string, error) { + // Parse repo owner and name + parts := strings.Split(repoFullName, "/") + if len(parts) != 2 { + return "", fmt.Errorf("invalid repo format: %s (expected owner/repo)", repoFullName) + } + owner := parts[0] + repo := parts[1] + + // Common PR template locations (in order of preference) + templatePaths := []string{ + ".github/pull_request_template.md", + ".github/PULL_REQUEST_TEMPLATE.md", + "docs/pull_request_template.md", + "PULL_REQUEST_TEMPLATE.md", + "pull_request_template.md", + } + + // Try each location + for _, path := range templatePaths { + content, err := f.fetchFileContent(ctx, client, owner, repo, path, branch) + if err == nil && content != "" { + LogInfo(fmt.Sprintf("Found PR template in %s/%s at %s", owner, repo, path)) + return content, nil + } + // Continue to next location if not found + } + + // No template found + LogDebug(fmt.Sprintf("No PR template found in %s/%s (checked %d locations)", owner, repo, len(templatePaths))) + return "", nil +} + +// fetchFileContent fetches the content of a file from a repository +func (f *DefaultPRTemplateFetcher) fetchFileContent(ctx context.Context, client *github.Client, owner, repo, path, branch string) (string, error) { + opts := &github.RepositoryContentGetOptions{ + Ref: branch, + } + + fileContent, _, _, err := client.Repositories.GetContents(ctx, owner, repo, path, opts) + if err != nil { + // File not found or other error - return empty + return "", err + } + + if fileContent == nil { + return "", fmt.Errorf("file content is nil") + } + + // Decode the content + content, err := fileContent.GetContent() + if err != nil { + return "", fmt.Errorf("failed to decode content: %w", err) + } + + return content, nil +} + +// MergePRBodyWithTemplate merges a configured PR body with a PR template +// The template is placed first, then the configured body is appended, separated by a horizontal rule +func MergePRBodyWithTemplate(configuredBody, template string) string { + if template == "" { + return configuredBody + } + + if configuredBody == "" { + return template + } + + // Merge: template first, then separator, then configured body + return fmt.Sprintf("%s\n\n---\n\n%s", template, configuredBody) +} + diff --git a/examples-copier/services/pr_template_fetcher_test.go b/examples-copier/services/pr_template_fetcher_test.go new file mode 100644 index 0000000..ab4e24a --- /dev/null +++ b/examples-copier/services/pr_template_fetcher_test.go @@ -0,0 +1,259 @@ +package services + +import ( + "context" + "testing" + + "github.com/google/go-github/v48/github" + "github.com/jarcoal/httpmock" + "github.com/stretchr/testify/require" +) + +func TestFetchPRTemplate_Found(t *testing.T) { + httpmock.Activate() + defer httpmock.DeactivateAndReset() + + owner := "testowner" + repo := "testrepo" + branch := "main" + + // Mock the first location (.github/pull_request_template.md) + // Note: GitHub API returns base64-encoded content with encoding field + httpmock.RegisterResponder("GET", + "https://api.github.com/repos/testowner/testrepo/contents/.github/pull_request_template.md", + httpmock.NewJsonResponderOrPanic(200, map[string]interface{}{ + "name": "pull_request_template.md", + "path": ".github/pull_request_template.md", + "type": "file", + "encoding": "base64", + "content": "IyBQdWxsIFJlcXVlc3QgVGVtcGxhdGUKCiMjIERlc2NyaXB0aW9uCgpQbGVhc2UgZGVzY3JpYmUgeW91ciBjaGFuZ2VzLg==", // base64 encoded + }), + ) + + client := github.NewClient(nil) + fetcher := NewPRTemplateFetcher() + + template, err := fetcher.FetchPRTemplate(context.Background(), client, owner+"/"+repo, branch) + + require.NoError(t, err) + require.NotEmpty(t, template) + require.Contains(t, template, "Pull Request Template") +} + +func TestFetchPRTemplate_NotFound(t *testing.T) { + httpmock.Activate() + defer httpmock.DeactivateAndReset() + + owner := "testowner" + repo := "testrepo" + branch := "main" + + // Mock all locations as not found + locations := []string{ + ".github/pull_request_template.md", + ".github/PULL_REQUEST_TEMPLATE.md", + "docs/pull_request_template.md", + "PULL_REQUEST_TEMPLATE.md", + "pull_request_template.md", + } + + for _, location := range locations { + httpmock.RegisterResponder("GET", + "https://api.github.com/repos/testowner/testrepo/contents/"+location, + httpmock.NewStringResponder(404, `{"message": "Not Found"}`), + ) + } + + client := github.NewClient(nil) + fetcher := NewPRTemplateFetcher() + + template, err := fetcher.FetchPRTemplate(context.Background(), client, owner+"/"+repo, branch) + + require.NoError(t, err) + require.Empty(t, template) +} + +func TestFetchPRTemplate_SecondLocation(t *testing.T) { + httpmock.Activate() + defer httpmock.DeactivateAndReset() + + owner := "testowner" + repo := "testrepo" + branch := "main" + + // First location not found + httpmock.RegisterResponder("GET", + "https://api.github.com/repos/testowner/testrepo/contents/.github/pull_request_template.md", + httpmock.NewStringResponder(404, `{"message": "Not Found"}`), + ) + + // Second location found (.github/PULL_REQUEST_TEMPLATE.md) + httpmock.RegisterResponder("GET", + "https://api.github.com/repos/testowner/testrepo/contents/.github/PULL_REQUEST_TEMPLATE.md", + httpmock.NewJsonResponderOrPanic(200, map[string]interface{}{ + "name": "PULL_REQUEST_TEMPLATE.md", + "path": ".github/PULL_REQUEST_TEMPLATE.md", + "type": "file", + "encoding": "base64", + "content": "IyBQUiBUZW1wbGF0ZQoKU2Vjb25kIGxvY2F0aW9u", // base64 encoded + }), + ) + + client := github.NewClient(nil) + fetcher := NewPRTemplateFetcher() + + template, err := fetcher.FetchPRTemplate(context.Background(), client, owner+"/"+repo, branch) + + require.NoError(t, err) + require.NotEmpty(t, template) + require.Contains(t, template, "PR Template") + require.Contains(t, template, "Second location") +} + +func TestFetchPRTemplate_InvalidRepoFormat(t *testing.T) { + client := github.NewClient(nil) + fetcher := NewPRTemplateFetcher() + + template, err := fetcher.FetchPRTemplate(context.Background(), client, "invalid-repo-format", "main") + + require.Error(t, err) + require.Empty(t, template) + require.Contains(t, err.Error(), "invalid repo format") +} + +func TestMergePRBodyWithTemplate_BothPresent(t *testing.T) { + configuredBody := "🤖 Automated update\n\nFiles: 10" + template := "## Checklist\n\n- [ ] Tests added\n- [ ] Docs updated" + + merged := MergePRBodyWithTemplate(configuredBody, template) + + require.Contains(t, merged, configuredBody) + require.Contains(t, merged, template) + require.Contains(t, merged, "---") // Separator + // Template should come first, then configured body + require.Less(t, 0, len(merged)) + templateIndex := len(template) + configuredIndex := len(merged) - len(configuredBody) + require.Less(t, templateIndex, configuredIndex) +} + +func TestMergePRBodyWithTemplate_OnlyConfigured(t *testing.T) { + configuredBody := "🤖 Automated update" + template := "" + + merged := MergePRBodyWithTemplate(configuredBody, template) + + require.Equal(t, configuredBody, merged) + require.NotContains(t, merged, "---") +} + +func TestMergePRBodyWithTemplate_OnlyTemplate(t *testing.T) { + configuredBody := "" + template := "## Checklist\n\n- [ ] Tests added" + + merged := MergePRBodyWithTemplate(configuredBody, template) + + require.Equal(t, template, merged) + require.NotContains(t, merged, "---") +} + +func TestMergePRBodyWithTemplate_BothEmpty(t *testing.T) { + configuredBody := "" + template := "" + + merged := MergePRBodyWithTemplate(configuredBody, template) + + require.Empty(t, merged) +} + +func TestPRTemplateFetcher_ChecksMultipleLocations(t *testing.T) { + httpmock.Activate() + defer httpmock.DeactivateAndReset() + + owner := "testowner" + repo := "testrepo" + branch := "main" + + // Track which locations were checked + checkedLocations := []string{} + + locations := []string{ + ".github/pull_request_template.md", + ".github/PULL_REQUEST_TEMPLATE.md", + "docs/pull_request_template.md", + "PULL_REQUEST_TEMPLATE.md", + "pull_request_template.md", + } + + for _, location := range locations { + loc := location // capture for closure + httpmock.RegisterResponder("GET", + "https://api.github.com/repos/testowner/testrepo/contents/"+location, + httpmock.NewStringResponder(404, `{"message": "Not Found"}`), + ) + // Track that this location was checked by registering a callback + checkedLocations = append(checkedLocations, loc) + } + + client := github.NewClient(nil) + fetcher := NewPRTemplateFetcher() + + _, _ = fetcher.FetchPRTemplate(context.Background(), client, owner+"/"+repo, branch) + + // Should have registered all locations + require.Len(t, checkedLocations, len(locations)) +} + +func TestPRTemplateFetcher_StopsAtFirstMatch(t *testing.T) { + httpmock.Activate() + defer httpmock.DeactivateAndReset() + + owner := "testowner" + repo := "testrepo" + branch := "main" + + // First location found + httpmock.RegisterResponder("GET", + "https://api.github.com/repos/testowner/testrepo/contents/.github/pull_request_template.md", + httpmock.NewJsonResponderOrPanic(200, map[string]interface{}{ + "name": "pull_request_template.md", + "path": ".github/pull_request_template.md", + "type": "file", + "encoding": "base64", + "content": "VGVtcGxhdGU=", // base64 "Template" + }), + ) + + // Other locations should not be checked + otherLocations := []string{ + ".github/PULL_REQUEST_TEMPLATE.md", + "docs/pull_request_template.md", + "PULL_REQUEST_TEMPLATE.md", + "pull_request_template.md", + } + + for _, location := range otherLocations { + httpmock.RegisterResponder("GET", + "https://api.github.com/repos/testowner/testrepo/contents/"+location, + httpmock.NewStringResponder(404, `{"message": "Not Found"}`), + ) + } + + client := github.NewClient(nil) + fetcher := NewPRTemplateFetcher() + + template, err := fetcher.FetchPRTemplate(context.Background(), client, owner+"/"+repo, branch) + + require.NoError(t, err) + require.NotEmpty(t, template) + + // Verify the first location was called + info := httpmock.GetCallCountInfo() + require.Equal(t, 1, info["GET https://api.github.com/repos/testowner/testrepo/contents/.github/pull_request_template.md"]) + + // Verify other locations were not called + for _, location := range otherLocations { + require.Equal(t, 0, info["GET https://api.github.com/repos/testowner/testrepo/contents/"+location]) + } +} + diff --git a/examples-copier/services/service_container.go b/examples-copier/services/service_container.go index cbfe1db..90fed06 100644 --- a/examples-copier/services/service_container.go +++ b/examples-copier/services/service_container.go @@ -18,6 +18,7 @@ type ServiceContainer struct { PatternMatcher PatternMatcher PathTransformer PathTransformer MessageTemplater MessageTemplater + PRTemplateFetcher PRTemplateFetcher AuditLogger AuditLogger MetricsCollector *MetricsCollector SlackNotifier SlackNotifier @@ -36,6 +37,7 @@ func NewServiceContainer(config *configs.Config) (*ServiceContainer, error) { patternMatcher := NewPatternMatcher() pathTransformer := NewPathTransformer() messageTemplater := NewMessageTemplater() + prTemplateFetcher := NewPRTemplateFetcher() metricsCollector := NewMetricsCollector() // Initialize Slack notifier @@ -60,16 +62,17 @@ func NewServiceContainer(config *configs.Config) (*ServiceContainer, error) { } return &ServiceContainer{ - Config: config, - FileStateService: fileStateService, - ConfigLoader: configLoader, - PatternMatcher: patternMatcher, - PathTransformer: pathTransformer, - MessageTemplater: messageTemplater, - AuditLogger: auditLogger, - MetricsCollector: metricsCollector, - SlackNotifier: slackNotifier, - StartTime: time.Now(), + Config: config, + FileStateService: fileStateService, + ConfigLoader: configLoader, + PatternMatcher: patternMatcher, + PathTransformer: pathTransformer, + MessageTemplater: messageTemplater, + PRTemplateFetcher: prTemplateFetcher, + AuditLogger: auditLogger, + MetricsCollector: metricsCollector, + SlackNotifier: slackNotifier, + StartTime: time.Now(), }, nil } diff --git a/examples-copier/services/webhook_handler_new.go b/examples-copier/services/webhook_handler_new.go index 11333a9..dba0318 100644 --- a/examples-copier/services/webhook_handler_new.go +++ b/examples-copier/services/webhook_handler_new.go @@ -273,9 +273,14 @@ func handleMergedPRWithContainer(ctx context.Context, prNumber int, sourceCommit // Process files with new pattern matching processFilesWithPatternMatching(ctx, prNumber, sourceCommitSHA, changedFiles, yamlConfig, config, container) + // Finalize PR metadata for batched PRs with accurate file counts + if yamlConfig.BatchByRepo { + finalizeBatchPRMetadata(yamlConfig, config, prNumber, sourceCommitSHA, container) + } + // Upload queued files FilesToUpload = container.FileStateService.GetFilesToUpload() - AddFilesToTargetRepoBranch() + AddFilesToTargetRepoBranchWithFetcher(container.PRTemplateFetcher) container.FileStateService.ClearFilesToUpload() // Update deprecation file - copy from FileStateService to global map for legacy function @@ -364,7 +369,7 @@ func processFilesWithPatternMatching(ctx context.Context, prNumber int, sourceCo // Process each target for _, target := range rule.Targets { - processFileForTarget(ctx, prNumber, sourceCommitSHA, file, rule, target, matchResult.Variables, yamlConfig.SourceBranch, config, container) + processFileForTarget(ctx, prNumber, sourceCommitSHA, file, rule, target, matchResult.Variables, yamlConfig, config, container) } } } @@ -372,7 +377,7 @@ func processFilesWithPatternMatching(ctx context.Context, prNumber int, sourceCo // processFileForTarget processes a single file for a specific target func processFileForTarget(ctx context.Context, prNumber int, sourceCommitSHA string, file types.ChangedFile, - rule types.CopyRule, target types.TargetConfig, variables map[string]string, sourceBranch string, config *configs.Config, container *ServiceContainer) { + rule types.CopyRule, target types.TargetConfig, variables map[string]string, yamlConfig *types.YAMLConfig, config *configs.Config, container *ServiceContainer) { // Transform path targetPath, err := container.PathTransformer.Transform(file.Path, target.PathTransform, variables) @@ -393,7 +398,7 @@ func processFileForTarget(ctx context.Context, prNumber int, sourceCommitSHA str "status": file.Status, "target": targetPath, }) - handleFileDeprecation(ctx, prNumber, sourceCommitSHA, file, rule, target, targetPath, sourceBranch, config, container) + handleFileDeprecation(ctx, prNumber, sourceCommitSHA, file, rule, target, targetPath, yamlConfig.SourceBranch, config, container) return } @@ -403,12 +408,12 @@ func processFileForTarget(ctx context.Context, prNumber int, sourceCommitSHA str "status": file.Status, "target": targetPath, }) - handleFileCopyWithAudit(ctx, prNumber, sourceCommitSHA, file, rule, target, targetPath, variables, sourceBranch, config, container) + handleFileCopyWithAudit(ctx, prNumber, sourceCommitSHA, file, rule, target, targetPath, variables, yamlConfig, config, container) } // handleFileCopyWithAudit handles file copying with audit logging func handleFileCopyWithAudit(ctx context.Context, prNumber int, sourceCommitSHA string, file types.ChangedFile, - rule types.CopyRule, target types.TargetConfig, targetPath string, variables map[string]string, sourceBranch string, + rule types.CopyRule, target types.TargetConfig, targetPath string, variables map[string]string, yamlConfig *types.YAMLConfig, config *configs.Config, container *ServiceContainer) { startTime := time.Now() @@ -440,7 +445,7 @@ func handleFileCopyWithAudit(ctx context.Context, prNumber int, sourceCommitSHA fc.Name = github.String(targetPath) // Queue file for upload - queueFileForUploadWithStrategy(target, *fc, rule, variables, prNumber, sourceCommitSHA, sourceBranch, config, container) + queueFileForUploadWithStrategy(target, *fc, rule, variables, prNumber, sourceCommitSHA, yamlConfig, config, container) // Log successful copy event fileSize := int64(0) @@ -510,22 +515,28 @@ func handleFileDeprecation(ctx context.Context, prNumber int, sourceCommitSHA st // queueFileForUploadWithStrategy queues a file for upload with the appropriate strategy func queueFileForUploadWithStrategy(target types.TargetConfig, file github.RepositoryContent, - rule types.CopyRule, variables map[string]string, prNumber int, sourceCommitSHA string, sourceBranch string, config *configs.Config, container *ServiceContainer) { + rule types.CopyRule, variables map[string]string, prNumber int, sourceCommitSHA string, yamlConfig *types.YAMLConfig, config *configs.Config, container *ServiceContainer) { - // Include rule name and commit strategy in the key to allow multiple rules - // targeting the same repo/branch with different strategies + // Determine commit strategy commitStrategy := string(target.CommitStrategy.Type) if commitStrategy == "" { commitStrategy = "direct" // default } + // Create upload key + // If batch_by_repo is true, exclude rule name to batch all changes into one PR per repo + // Otherwise, include rule name to create separate PRs per rule key := types.UploadKey{ RepoName: target.Repo, BranchPath: "refs/heads/" + target.Branch, - RuleName: rule.Name, CommitStrategy: commitStrategy, } + if !yamlConfig.BatchByRepo { + // Include rule name to create separate PRs per rule (default behavior) + key.RuleName = rule.Name + } + // Get existing entry or create new filesToUpload := container.FileStateService.GetFilesToUpload() entry, exists := filesToUpload[key] @@ -538,6 +549,7 @@ func queueFileForUploadWithStrategy(target types.TargetConfig, file github.Repos // Set commit strategy entry.CommitStrategy = types.CommitStrategy(target.CommitStrategy.Type) entry.AutoMergePR = target.CommitStrategy.AutoMerge + entry.UsePRTemplate = target.CommitStrategy.UsePRTemplate // Add file to content first so we can get accurate file count entry.Content = append(entry.Content, file) @@ -546,7 +558,7 @@ func queueFileForUploadWithStrategy(target types.TargetConfig, file github.Repos msgCtx := types.NewMessageContext() msgCtx.RuleName = rule.Name msgCtx.SourceRepo = fmt.Sprintf("%s/%s", config.RepoOwner, config.RepoName) - msgCtx.SourceBranch = sourceBranch + msgCtx.SourceBranch = yamlConfig.SourceBranch msgCtx.TargetRepo = target.Repo msgCtx.TargetBranch = target.Branch msgCtx.FileCount = len(entry.Content) @@ -554,14 +566,26 @@ func queueFileForUploadWithStrategy(target types.TargetConfig, file github.Repos msgCtx.CommitSHA = sourceCommitSHA msgCtx.Variables = variables - if target.CommitStrategy.CommitMessage != "" { - entry.CommitMessage = container.MessageTemplater.RenderCommitMessage(target.CommitStrategy.CommitMessage, msgCtx) - } - if target.CommitStrategy.PRTitle != "" { - entry.PRTitle = container.MessageTemplater.RenderPRTitle(target.CommitStrategy.PRTitle, msgCtx) - } - if target.CommitStrategy.PRBody != "" { - entry.PRBody = container.MessageTemplater.RenderPRBody(target.CommitStrategy.PRBody, msgCtx) + // For batched PRs, skip setting PR metadata here - it will be set later with accurate file counts + // For non-batched PRs, always update with current rule's messages + if yamlConfig.BatchByRepo { + // Batching by repo - PR metadata will be set in finalizeBatchPRMetadata() + // Only set commit message if not already set + if entry.CommitMessage == "" && target.CommitStrategy.CommitMessage != "" { + entry.CommitMessage = container.MessageTemplater.RenderCommitMessage(target.CommitStrategy.CommitMessage, msgCtx) + } + // Leave PRTitle and PRBody empty - will be set with accurate file count later + } else { + // Not batching - update messages for each rule (last one wins) + if target.CommitStrategy.CommitMessage != "" { + entry.CommitMessage = container.MessageTemplater.RenderCommitMessage(target.CommitStrategy.CommitMessage, msgCtx) + } + if target.CommitStrategy.PRTitle != "" { + entry.PRTitle = container.MessageTemplater.RenderPRTitle(target.CommitStrategy.PRTitle, msgCtx) + } + if target.CommitStrategy.PRBody != "" { + entry.PRBody = container.MessageTemplater.RenderPRBody(target.CommitStrategy.PRBody, msgCtx) + } } container.FileStateService.AddFileToUpload(key, entry) @@ -580,3 +604,60 @@ func addToDeprecationMapForTarget(targetPath string, target types.TargetConfig, key := target.Repo + ":" + targetPath fileStateService.AddFileToDeprecate(key, entry) } + +// finalizeBatchPRMetadata sets PR metadata for batched PRs with accurate file counts +// This is called after all files have been collected +func finalizeBatchPRMetadata(yamlConfig *types.YAMLConfig, config *configs.Config, prNumber int, sourceCommitSHA string, container *ServiceContainer) { + filesToUpload := container.FileStateService.GetFilesToUpload() + + for key, entry := range filesToUpload { + // Create message context with accurate file count + msgCtx := types.NewMessageContext() + msgCtx.SourceRepo = fmt.Sprintf("%s/%s", config.RepoOwner, config.RepoName) + msgCtx.SourceBranch = yamlConfig.SourceBranch + msgCtx.TargetRepo = key.RepoName + msgCtx.TargetBranch = entry.TargetBranch + msgCtx.FileCount = len(entry.Content) // Accurate file count! + msgCtx.PRNumber = prNumber + msgCtx.CommitSHA = sourceCommitSHA + + // Use batch_pr_config if available, otherwise use defaults + if yamlConfig.BatchPRConfig != nil { + // Use dedicated batch PR config + if yamlConfig.BatchPRConfig.PRTitle != "" { + entry.PRTitle = container.MessageTemplater.RenderPRTitle(yamlConfig.BatchPRConfig.PRTitle, msgCtx) + } else { + // Default title + entry.PRTitle = fmt.Sprintf("Update files from %s PR #%d", msgCtx.SourceRepo, prNumber) + } + + if yamlConfig.BatchPRConfig.PRBody != "" { + entry.PRBody = container.MessageTemplater.RenderPRBody(yamlConfig.BatchPRConfig.PRBody, msgCtx) + } else { + // Default body + entry.PRBody = fmt.Sprintf("Automated update from %s\n\nSource PR: #%d\nCommit: %s\nFiles: %d", + msgCtx.SourceRepo, prNumber, sourceCommitSHA[:7], len(entry.Content)) + } + + // Override commit message if specified in batch config + if yamlConfig.BatchPRConfig.CommitMessage != "" && entry.CommitMessage == "" { + entry.CommitMessage = container.MessageTemplater.RenderCommitMessage(yamlConfig.BatchPRConfig.CommitMessage, msgCtx) + } + + // Set UsePRTemplate from batch config + entry.UsePRTemplate = yamlConfig.BatchPRConfig.UsePRTemplate + } else { + // No batch_pr_config - use generic defaults + if entry.PRTitle == "" { + entry.PRTitle = fmt.Sprintf("Update files from %s PR #%d", msgCtx.SourceRepo, prNumber) + } + if entry.PRBody == "" { + entry.PRBody = fmt.Sprintf("Automated update from %s\n\nSource PR: #%d\nCommit: %s\nFiles: %d", + msgCtx.SourceRepo, prNumber, sourceCommitSHA[:7], len(entry.Content)) + } + } + + // Update the entry in the map + container.FileStateService.AddFileToUpload(key, entry) + } +} diff --git a/examples-copier/types/config.go b/examples-copier/types/config.go index ce8da64..cae1b80 100644 --- a/examples-copier/types/config.go +++ b/examples-copier/types/config.go @@ -2,6 +2,7 @@ package types import ( "fmt" + "regexp" "strings" ) @@ -26,9 +27,19 @@ func (p PatternType) String() string { // YAMLConfig represents the new YAML-based configuration structure type YAMLConfig struct { - SourceRepo string `yaml:"source_repo" json:"source_repo"` - SourceBranch string `yaml:"source_branch" json:"source_branch"` - CopyRules []CopyRule `yaml:"copy_rules" json:"copy_rules"` + SourceRepo string `yaml:"source_repo" json:"source_repo"` + SourceBranch string `yaml:"source_branch" json:"source_branch"` + BatchByRepo bool `yaml:"batch_by_repo,omitempty" json:"batch_by_repo,omitempty"` // If true, batch all changes into one PR per target repo + BatchPRConfig *BatchPRConfig `yaml:"batch_pr_config,omitempty" json:"batch_pr_config,omitempty"` // PR config used when batch_by_repo is true + CopyRules []CopyRule `yaml:"copy_rules" json:"copy_rules"` +} + +// BatchPRConfig defines PR metadata for batched PRs +type BatchPRConfig struct { + PRTitle string `yaml:"pr_title,omitempty" json:"pr_title,omitempty"` + PRBody string `yaml:"pr_body,omitempty" json:"pr_body,omitempty"` + CommitMessage string `yaml:"commit_message,omitempty" json:"commit_message,omitempty"` + UsePRTemplate bool `yaml:"use_pr_template,omitempty" json:"use_pr_template,omitempty"` } // CopyRule defines a single rule for copying files with pattern matching @@ -40,8 +51,9 @@ type CopyRule struct { // SourcePattern defines how to match source files type SourcePattern struct { - Type PatternType `yaml:"type" json:"type"` - Pattern string `yaml:"pattern" json:"pattern"` + Type PatternType `yaml:"type" json:"type"` + Pattern string `yaml:"pattern" json:"pattern"` + ExcludePatterns []string `yaml:"exclude_patterns,omitempty" json:"exclude_patterns,omitempty"` // Optional: regex patterns to exclude from matches } // TargetConfig defines where and how to copy matched files @@ -59,6 +71,7 @@ type CommitStrategyConfig struct { CommitMessage string `yaml:"commit_message,omitempty" json:"commit_message,omitempty"` PRTitle string `yaml:"pr_title,omitempty" json:"pr_title,omitempty"` PRBody string `yaml:"pr_body,omitempty" json:"pr_body,omitempty"` + UsePRTemplate bool `yaml:"use_pr_template,omitempty" json:"use_pr_template,omitempty"` // If true, fetch and use PR template from target repo AutoMerge bool `yaml:"auto_merge,omitempty" json:"auto_merge,omitempty"` BatchSize int `yaml:"batch_size,omitempty" json:"batch_size,omitempty"` } @@ -119,6 +132,20 @@ func (p *SourcePattern) Validate() error { if p.Pattern == "" { return fmt.Errorf("pattern is required") } + + // Validate exclude patterns if provided + if len(p.ExcludePatterns) > 0 { + for i, excludePattern := range p.ExcludePatterns { + if excludePattern == "" { + return fmt.Errorf("exclude_patterns[%d] is empty", i) + } + // Validate that it's a valid regex pattern + if _, err := regexp.Compile(excludePattern); err != nil { + return fmt.Errorf("exclude_patterns[%d] is not a valid regex: %w", i, err) + } + } + } + return nil } diff --git a/examples-copier/types/types.go b/examples-copier/types/types.go index 1331bb8..9fffc5c 100644 --- a/examples-copier/types/types.go +++ b/examples-copier/types/types.go @@ -110,6 +110,7 @@ type UploadFileContent struct { CommitMessage string `json:"commit_message,omitempty"` PRTitle string `json:"pr_title,omitempty"` PRBody string `json:"pr_body,omitempty"` + UsePRTemplate bool `json:"use_pr_template,omitempty"` // If true, fetch and merge PR template from target repo AutoMergePR bool `json:"auto_merge_pr,omitempty"` }