diff --git a/examples-copier/Makefile b/examples-copier/Makefile new file mode 100644 index 0000000..3ea26a1 --- /dev/null +++ b/examples-copier/Makefile @@ -0,0 +1,163 @@ +.PHONY: help build test test-unit test-webhook clean run run-dry install + +# Default target +help: + @echo "Examples Copier - Makefile" + @echo "" + @echo "Available targets:" + @echo " make build - Build all binaries" + @echo " make test - Run all tests" + @echo " make test-unit - Run unit tests only" + @echo " make test-webhook - Build webhook test tool" + @echo " make run - Run application" + @echo " make run-dry - Run in dry-run mode" + @echo " make run-local - Run in local dev mode (recommended)" + @echo " make run-local-quick - Quick local run (no cloud logging)" + @echo " make install - Install all tools to \$$GOPATH/bin" + @echo " make clean - Remove built binaries" + @echo "" + @echo "Testing with webhooks:" + @echo " make test-webhook-example - Test with example payload" + @echo " make test-webhook-pr PR=123 OWNER=org REPO=repo - Test with real PR" + @echo "" + @echo "Quick start for local testing:" + @echo " make run-local-quick # Start app (Terminal 1)" + @echo " make test-webhook-example # Send test webhook (Terminal 2)" + @echo "" + +# Build all binaries +build: + @echo "Building examples-copier..." + @go build -o examples-copier . + @echo "Building config-validator..." + @go build -o config-validator ./cmd/config-validator + @echo "Building test-webhook..." + @go build -o test-webhook ./cmd/test-webhook + @echo "✓ All binaries built successfully" + +# Run all tests +test: test-unit + @echo "✓ All tests passed" + +# Run unit tests +test-unit: + @echo "Running unit tests..." + @go test ./services -v + +# Run unit tests with coverage +test-coverage: + @echo "Running tests with coverage..." + @go test ./services -coverprofile=coverage.out + @go tool cover -html=coverage.out -o coverage.html + @echo "✓ Coverage report generated: coverage.html" + +# Build test-webhook tool +test-webhook: + @echo "Building test-webhook..." + @go build -o test-webhook ./cmd/test-webhook + @echo "✓ test-webhook built" + +# Test with example payload +test-webhook-example: test-webhook + @echo "Testing with example payload..." + @./test-webhook -payload test-payloads/example-pr-merged.json + +# Test with real PR (requires PR, OWNER, REPO variables) +test-webhook-pr: test-webhook + @if [ -z "$(PR)" ] || [ -z "$(OWNER)" ] || [ -z "$(REPO)" ]; then \ + echo "Error: PR, OWNER, and REPO must be set"; \ + echo "Usage: make test-webhook-pr PR=123 OWNER=myorg REPO=myrepo"; \ + exit 1; \ + fi + @./test-webhook -pr $(PR) -owner $(OWNER) -repo $(REPO) + +# Test with real PR using helper script +test-pr: + @if [ -z "$(PR)" ]; then \ + echo "Error: PR must be set"; \ + echo "Usage: make test-pr PR=123"; \ + exit 1; \ + fi + @./scripts/test-with-pr.sh $(PR) + +# Run application +run: build + @echo "Starting examples-copier..." + @./examples-copier + +# Run in dry-run mode +run-dry: build + @echo "Starting examples-copier in dry-run mode..." + @DRY_RUN=true ./examples-copier + +# Run in local development mode (recommended) +run-local: build + @echo "Starting examples-copier in local development mode..." + @./scripts/run-local.sh + +# Run with cloud logging disabled (quick local testing) +run-local-quick: build + @echo "Starting examples-copier (local, no cloud logging)..." + @COPIER_DISABLE_CLOUD_LOGGING=true DRY_RUN=true ./examples-copier + +# Validate configuration +validate: build + @echo "Validating configuration..." + @./examples-copier -validate + +# Install binaries to $GOPATH/bin +install: + @echo "Installing binaries..." + @go install . + @go install ./cmd/config-validator + @go install ./cmd/test-webhook + @echo "✓ Binaries installed to \$$GOPATH/bin" + +# Clean built binaries +clean: + @echo "Cleaning built binaries..." + @rm -f examples-copier config-validator test-webhook + @rm -f coverage.out coverage.html + @echo "✓ Clean complete" + +# Format code +fmt: + @echo "Formatting code..." + @go fmt ./... + @echo "✓ Code formatted" + +# Run linter +lint: + @echo "Running linter..." + @golangci-lint run ./... + @echo "✓ Linting complete" + +# Download dependencies +deps: + @echo "Downloading dependencies..." + @go mod download + @go mod tidy + @echo "✓ Dependencies updated" + +# Show version info +version: + @echo "Go version:" + @go version + @echo "" + @echo "Module info:" + @go list -m + +# Development setup +dev-setup: deps build + @echo "Setting up development environment..." + @chmod +x scripts/test-with-pr.sh + @echo "✓ Development environment ready" + +# Quick test cycle +quick-test: build test-unit + @echo "✓ Quick test cycle complete" + +# Full test cycle with webhook testing +full-test: build test-unit test-webhook-example + @echo "✓ Full test cycle complete" + diff --git a/examples-copier/QUICK-REFERENCE.md b/examples-copier/QUICK-REFERENCE.md new file mode 100644 index 0000000..783b80c --- /dev/null +++ b/examples-copier/QUICK-REFERENCE.md @@ -0,0 +1,420 @@ +# Quick Reference Guide + +## Command Line + +### Application + +```bash +# Run with default settings +./examples-copier + +# Run with custom environment +./examples-copier -env ./configs/.env.production + +# Dry-run mode (no actual commits) +./examples-copier -dry-run + +# Validate configuration only +./examples-copier -validate + +# Show help +./examples-copier -help +``` + +### CLI Validator + +```bash +# Validate config +./config-validator validate -config copier-config.yaml -v + +# Test pattern +./config-validator test-pattern -type regex -pattern "^examples/(?P[^/]+)/.*$" -file "examples/go/main.go" + +# Test transformation +./config-validator test-transform -template "docs/${lang}/${file}" -file "examples/go/main.go" -pattern "^examples/(?P[^/]+)/(?P.+)$" + +# Initialize new config +./config-validator init -output copier-config.yaml + +# Convert formats +./config-validator convert -input config.json -output copier-config.yaml +``` + +## Configuration Patterns + +### Prefix Pattern +```yaml +source_pattern: + type: "prefix" + pattern: "examples/go/" +``` + +### Glob Pattern +```yaml +source_pattern: + type: "glob" + pattern: "examples/*/main.go" +``` + +### Regex Pattern +```yaml +source_pattern: + type: "regex" + pattern: "^examples/(?P[^/]+)/(?P.+)$" +``` + +## Path Transformations + +### Built-in Variables +- `${path}` - Full source path +- `${filename}` - File name only +- `${dir}` - Directory path +- `${ext}` - File extension + +### Examples +```yaml +# Keep same path +path_transform: "${path}" + +# Change directory +path_transform: "docs/${path}" + +# Reorganize structure +path_transform: "docs/${lang}/${category}/${filename}" + +# Change extension +path_transform: "${dir}/${filename}.md" +``` + +## Commit Strategies + +### Direct Commit +```yaml +commit_strategy: + type: "direct" + commit_message: "Update examples" +``` + +### Pull Request +```yaml +commit_strategy: + type: "pull_request" + commit_message: "Update examples" + pr_title: "Update code examples" + pr_body: "Automated update" + auto_merge: true +``` + +## Message Templates + +### Available Variables +- `${rule_name}` - Copy rule name +- `${source_repo}` - Source repository +- `${target_repo}` - Target repository +- `${source_branch}` - Source branch +- `${target_branch}` - Target branch +- `${file_count}` - Number of files +- Custom variables from regex patterns + +### Examples +```yaml +commit_message: "Update ${category} examples from ${lang}" +pr_title: "Update ${category} examples" +pr_body: "Copying ${file_count} files from ${source_repo}" +``` + +## API Endpoints + +### Health Check +```bash +curl http://localhost:8080/health +``` + +### Metrics +```bash +curl http://localhost:8080/metrics +``` + +### Webhook +```bash +curl -X POST http://localhost:8080/webhook \ + -H "Content-Type: application/json" \ + -H "X-Hub-Signature-256: sha256=..." \ + -d @webhook-payload.json +``` + +## Environment Variables + +### Required +```bash +REPO_OWNER=your-org +REPO_NAME=your-repo +GITHUB_APP_ID=123456 +GITHUB_INSTALLATION_ID=789012 +GCP_PROJECT_ID=your-project +PEM_KEY_NAME=projects/123/secrets/KEY/versions/latest +``` + +### Optional +```bash +# Application +PORT=8080 +CONFIG_FILE=copier-config.yaml +DEPRECATION_FILE=deprecated_examples.json +DRY_RUN=false + +# Logging +LOG_LEVEL=info +COPIER_DEBUG=false +COPIER_DISABLE_CLOUD_LOGGING=false + +# Audit +AUDIT_ENABLED=true +MONGO_URI=mongodb+srv://... +AUDIT_DATABASE=code_copier +AUDIT_COLLECTION=audit_events + +# Metrics +METRICS_ENABLED=true + +# Webhook +WEBHOOK_SECRET=your-secret +``` + +## MongoDB Queries + +### Recent Events +```javascript +db.audit_events.find().sort({timestamp: -1}).limit(10) +``` + +### Failed Operations +```javascript +db.audit_events.find({success: false}).sort({timestamp: -1}) +``` + +### Events by Rule +```javascript +db.audit_events.find({rule_name: "Copy Go examples"}) +``` + +### Statistics +```javascript +db.audit_events.aggregate([ + {$match: {event_type: "copy"}}, + {$group: { + _id: "$rule_name", + count: {$sum: 1}, + avg_duration: {$avg: "$duration_ms"} + }} +]) +``` + +### Success Rate +```javascript +db.audit_events.aggregate([ + {$group: { + _id: "$success", + count: {$sum: 1} + }} +]) +``` + +## Testing + +### Run Unit Tests +```bash +# All tests +go test ./services -v + +# Specific test +go test ./services -v -run TestPatternMatcher + +# With coverage +go test ./services -cover +``` + +### Test with Webhooks + +#### Option 1: Use Example Payload +```bash +# Build test tool +go build -o test-webhook ./cmd/test-webhook + +# Send example payload +./test-webhook -payload test-payloads/example-pr-merged.json + +# Dry-run (see payload without sending) +./test-webhook -payload test-payloads/example-pr-merged.json -dry-run +``` + +#### Option 2: Use Real PR Data +```bash +# Set GitHub token +export GITHUB_TOKEN=ghp_your_token_here + +# Fetch and send real PR data +./test-webhook -pr 123 -owner myorg -repo myrepo + +# Test against production +./test-webhook -pr 123 -owner myorg -repo myrepo \ + -url https://myapp.appspot.com/webhook \ + -secret "my-webhook-secret" +``` + +#### Option 3: Use Helper Script (Interactive) +```bash +# Make executable +chmod +x scripts/test-with-pr.sh + +# Run interactive test +./scripts/test-with-pr.sh 123 myorg myrepo +``` + +### Test in Dry-Run Mode +```bash +# Start app in dry-run mode +DRY_RUN=true ./examples-copier & + +# Send test webhook +./test-webhook -pr 123 -owner myorg -repo myrepo + +# Check logs (no actual commits made) +``` + +### Build +```bash +# Main application +go build -o examples-copier . + +# CLI validator +go build -o config-validator ./cmd/config-validator + +# Test webhook tool +go build -o test-webhook ./cmd/test-webhook + +# All tools +go build -o examples-copier . && \ +go build -o config-validator ./cmd/config-validator && \ +go build -o test-webhook ./cmd/test-webhook +``` + +## Common Patterns + +### Copy All Go Files +```yaml +source_pattern: + type: "regex" + pattern: "^examples/.*\\.go$" +targets: + - repo: "org/docs" + path_transform: "code/${path}" +``` + +### Organize by Language +```yaml +source_pattern: + type: "regex" + pattern: "^examples/(?P[^/]+)/(?P.+)$" +targets: + - repo: "org/docs" + path_transform: "languages/${lang}/${rest}" +``` + +### Multiple Targets with Different Transforms +```yaml +source_pattern: + type: "prefix" + pattern: "examples/" +targets: + - repo: "org/docs-v1" + path_transform: "examples/${path}" + - repo: "org/docs-v2" + path_transform: "code-samples/${path}" +``` + +### Conditional Copying (by file type) +```yaml +source_pattern: + type: "regex" + pattern: "^examples/.*\\.(?Pgo|py|js)$" +targets: + - repo: "org/docs" + path_transform: "code/${ext}/${filename}" +``` + +## Troubleshooting + +### Check Logs +```bash +# Application logs +gcloud app logs tail -s default + +# Local logs +LOG_LEVEL=debug ./examples-copier +``` + +### Validate Config +```bash +./config-validator validate -config copier-config.yaml -v +``` + +### Test Pattern Matching +```bash +./config-validator test-pattern \ + -type regex \ + -pattern "your-pattern" \ + -file "test/file.go" +``` + +### Dry Run +```bash +DRY_RUN=true ./examples-copier +``` + +### Check Health +```bash +curl http://localhost:8080/health +``` + +### Check Metrics +```bash +curl http://localhost:8080/metrics | jq +``` + +## File Locations + +``` +examples-copier/ +├── README.md # Main documentation +├── MIGRATION-GUIDE.md # Migration from legacy +├── QUICK-REFERENCE.md # This file +├── REFACTORING-SUMMARY.md # Feature details +├── DEPLOYMENT-GUIDE.md # Deployment instructions +├── TESTING-SUMMARY.md # Test documentation +├── configs/ +│ ├── .env # Environment config +│ ├── .env.example.new # Environment template +│ └── config.example.yaml # Config template +└── cmd/ + └── config-validator/ # CLI tool +``` + +## Quick Start Checklist + +- [ ] Clone repository +- [ ] Copy `.env.example.new` to `.env` +- [ ] Set required environment variables +- [ ] Create `copier-config.yaml` in source repo +- [ ] Validate config: `./config-validator validate -config copier-config.yaml` +- [ ] Test in dry-run: `DRY_RUN=true ./examples-copier` +- [ ] Deploy: `./examples-copier` +- [ ] Configure GitHub webhook +- [ ] Monitor: `curl http://localhost:8080/health` + +## Support + +- **Documentation**: [README.md](README.md) +- **Migration**: [MIGRATION-GUIDE.md](./docs/MIGRATION-GUIDE.md) +- **Deployment**: [DEPLOYMENT-GUIDE.md](./docs/DEPLOYMENT-GUIDE.md) + diff --git a/examples-copier/README.md b/examples-copier/README.md index c0a38f7..54b0ae9 100644 --- a/examples-copier/README.md +++ b/examples-copier/README.md @@ -1,214 +1,466 @@ # GitHub Docs Code Example Copier -A GitHub app that listens for PR events from a source repository. Driven by a -single `config.json` file in the source repo, the app copies files -(currently, generated code snippets and examples) to one or more target -repositories upon PR merge in source repo. - -## Behavior - -On PR merge in source repo, the app identifies changed files from the PR payload. -1. Read files from a specific path the source repo, with optional recursion (default: recursive copy) -2. Copy changed files to target repos/branches as defined in the config (default: "main"); deletions are recorded to a deprecation file. -3. Writes can be direct commits or through PRs (optional auto-merge), with clear handling for conflicts and mergeability. - -### Configuration defaults -- Strategy priority: per-config `copier_commit_strategy` > `COPIER_COMMIT_STRATEGY` env > default "direct" for testing. -- Messages: per-config `commit_message` > `DEFAULT_COMMIT_MESSAGE` env > system default ("Automated PR with updated examples"). -- PR title: falls back to the resolved commit message when `pr_title` is empty. -- Conflicts: - - Direct commits surface non-fast-forward (HTTP 422) with a clear error suggesting PR strategy. - - PR auto-merge path polls GitHub for mergeability; if not mergeable, it leaves the PR open for manual resolution. -- Useful environment settings: - - `COPIER_COMMIT_STRATEGY` ("direct"|"pr"), `DEFAULT_COMMIT_MESSAGE`, `DEFAULT_RECURSIVE_COPY`, `DEFAULT_PR_MERGE`. - - Logging: `COPIER_DISABLE_CLOUD_LOGGING`, `LOG_LEVEL=debug` or `COPIER_DEBUG=true`. - -## Project Structure - -| Directory Name | Use Case | -|----------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `configs` | The configuration files (.env.*) are here, as is the `environment.go` file, which creates globals for those settings. No other part of the code should read the .env files. | -| `services` | The handlers for the services we're interacting with: GitHub and Webhook handlers. For better organization, GitHub handlers have been separated into Auth, Upload, and Download services. | -| `types` | All the `type StructName struct` should be here. These are the structs needs to map webhook json to objects we can work with. | - -## Logic Flow - -At its core, this is a simple Go web server that listens for messages and handles them -as they come in. Handling the messages means: -- Determining if it's a message we care about -- Pulling out the changed file list from the message -- Reading the config file, and if a file is in a config setting, -- Copy/replace the file at the target repo. - -Basic flow: - -1. Configure GitHub permissions (`services/GitHub_auth.go`) -2. Listen for PR payloads from the GitHub webhook. (`services/web_server.go`) -3. Is the PR closed and merged? If no, ignore. (`services/webhook_handler.go`) -4. Parse the payload to get the list of changed files. (`services/GitHub_download.go`) -5. Read the config file from the source repo. -6. If the path to a changed file is defined in the config file, and it is not a - "DELETE" action, copy the file to the specified target repos. (`services/GitHub_upload.go`) -7. If the path to a changed file is defined in the config file, and it *is* a "DELETE" - action, add the deleted file's name and path to the `deprecated_examples.json` file. - (`services/GitHub_download.go`) -8. Sit idle until the next payload arrives. Rinse and repeat. - -## Install the App on a Target Repo -To install the app on a new target repository: -1. [Give the App repo access](# Give the app repo access) -2. [Install the App in the new source repo](# Install the App on a new Source Repo) - -### Give the app repo access -1. Go to the [App's Configuration page](https://github.com/apps/docs-examples-copier/installations/62138132). - You'll need to authorize your GitHub account first. You should then see the following screen: - !["gui request tag"](./readme_files/configure_app.png) -2. In the `Select repositories` dropdown, select the new target repo. Then click - **Update access**. -> **NOTE:** You must be a repo owner to complete the next steps. If you are not a repo owner, -you will not see the `Select repositories` dropdown, and you will not be able -> to select the new target repository. -> !["gui request tag"](./readme_files/request.png) - -### Confirm the new target repository -1. In the new target repository's settings, go to the - [GitHub Apps section](https://GitHub.com/mongodb/stitch-tutorial-todo-backend/settings/installations). - Scroll down and confirm that `Docs Examples Copier` is installed. - -## Install the App on a new Source Repo -In the source repo, do the following: -1. [Set up a webhook](# Set Up A Webhook) -2. [Update the .env file](# Update the env file) -3. [Add config.json and deprecated_examples.json files](# Add config.json and deprecated_examples.json files) -4. [Configure Permissions for the Web App](# Configure Permissions for the Web App) - -### Set Up A Webhook -Go to the source repo's -[webhooks settings page](https://GitHub.com/mongodb/docs-code-examples/settings/hooks/). -- Add the new `Payload URL`. -- Set the `Content Type` to `application/json` -- Enable SSL Verification -- Choose `Let me select individual events` - - Choose *only* `Pull Requests`. Do **not** choose any other "Pull Request"-related - options! -- At the bottom of the page, make sure `Active` is checked, and then save your changes. - -At this point, with PR-related activity, the payload will be sent to the app. -The app ignores all PR activity except for when a PR is closed and merged. - -### Update the env file -The .env file specifies settings for the source repo. - -Copy the `.env.example` and update the main values for your repo: - -```dotenv -GITHUB_APP_ID="GitHub App ID (numeric) for your GitHub App" -# Optional (not required for JWT auth): -# GITHUB_APP_CLIENT_ID="OAuth client ID of your GitHub App (starts with Iv)" -INSTALLATION_ID="When you install the app, you get an installation ID, something like 73438188" - -GOOGLE_CLOUD_PROJECT_ID="The Google Cloud Project (GCP) ID" -COPIER_LOG_NAME="The name of the log in Google Cloud Logging" - -REPO_NAME="The name of the *source* repo" -REPO_OWNER="The owner of the *source* repo" -REF="The *source* branch to monitor for changes - e.g. 'main' or 'master'" - -COMMITER_EMAIL="The email you want to appear as the committer of the changes, e.g. 'foo@example.com'" -COMMITER_NAME="The name you want to appear as the committer of the changes, e.g. 'GitHub Copier App'" - -PORT="leave empty for the default server port, or specify a port, like 8080" -WEBSERVER_PATH="/events" - -DEPRECATION_FILE="The path to the deprecation file, e.g. deprecated_examples.json" -CONFIG_FILE="The path to the config file, e.g. config.json" -``` -Update any of the optional settings in the `.env.example` file as needed. - -### Add config and deprecation json files - -Create the config file to hold config settings and an empty `.json` file to hold deprecated file paths. -See the [config.example.json](configs/config.example.json) for reference. +A GitHub app that automatically copies code examples and files from a source repository to one or more target repositories when pull requests are merged. Features advanced pattern matching, path transformations, audit logging, and comprehensive monitoring. + +## Features + +### Core Functionality +- **Automated File Copying** - Copies files from source to target repos on PR merge +- **Advanced Pattern Matching** - Prefix, glob, and regex patterns with variable extraction +- **Path Transformations** - Template-based path transformations with variable substitution +- **Multiple Targets** - Copy files to multiple repositories and branches +- **Flexible Commit Strategies** - Direct commits or pull requests with auto-merge +- **Deprecation Tracking** - Automatic tracking of deleted files + +### Enhanced Features +- **YAML Configuration** - Modern YAML config with JSON backward compatibility +- **Message Templating** - Template-ized commit messages and PR titles +- **Audit Logging** - MongoDB-based event tracking for all operations +- **Health & Metrics** - `/health` and `/metrics` endpoints for monitoring +- **Development Tools** - Dry-run mode, CLI validation, enhanced logging +- **Thread-Safe** - Concurrent webhook processing with proper state management + +## 🚀 Quick Start + +### Prerequisites + +- Go 1.23.4+ +- GitHub App credentials +- Google Cloud project (for Secret Manager and logging) +- MongoDB Atlas (optional, for audit logging) + +### Installation + +```bash +# Clone the repository +git clone https://github.com/your-org/code-example-tooling.git +cd code-example-tooling/examples-copier + +# Install dependencies +go mod download + +# Build the application +go build -o examples-copier . + +# Build CLI tools +go build -o config-validator ./cmd/config-validator +``` + +### Configuration + +1. **Copy .env example file** + +```bash +cp configs/.env.example.new configs/.env +``` + +2. **Set required environment variables** + +```bash +# GitHub Configuration +REPO_OWNER=your-org +REPO_NAME=your-repo +SRC_BRANCH=main +GITHUB_APP_ID=123456 +GITHUB_INSTALLATION_ID=789012 + +# Google Cloud +GCP_PROJECT_ID=your-project +PEM_KEY_NAME=projects/123/secrets/CODE_COPIER_PEM/versions/latest + +# Application Settings +PORT=8080 +CONFIG_FILE=copier-config.yaml +DEPRECATION_FILE=deprecated_examples.json + +# Optional: MongoDB Audit Logging +AUDIT_ENABLED=true +MONGO_URI=mongodb+srv://user:pass@cluster.mongodb.net +AUDIT_DATABASE=code_copier +AUDIT_COLLECTION=audit_events + +# Optional: Development Features +DRY_RUN=false +METRICS_ENABLED=true +``` + +3. **Create configuration file** + +Create `copier-config.yaml` in your source repository: + +```yaml +source_repo: "your-org/source-repo" +source_branch: "main" + +copy_rules: + - name: "Copy Go examples" + source_pattern: + type: "regex" + pattern: "^examples/(?P[^/]+)/(?P[^/]+)/(?P.+)$" + targets: + - repo: "your-org/target-repo" + branch: "main" + path_transform: "docs/examples/${lang}/${category}/${file}" + commit_strategy: + type: "pull_request" + commit_message: "Update ${category} examples from ${lang}" + pr_title: "Update ${category} examples" + auto_merge: false + deprecation_check: + enabled: true + file: "deprecated_examples.json" +``` + +### Running the Application + +```bash +# Run with default settings +./examples-copier + +# Run with custom environment file +./examples-copier -env ./configs/.env.production + +# Run in dry-run mode (no actual commits) +./examples-copier -dry-run + +# Validate configuration only +./examples-copier -validate +``` + +## Configuration + +### Pattern Types + +#### Prefix Pattern +Simple string prefix matching: + +```yaml +source_pattern: + type: "prefix" + pattern: "examples/go/" +``` + +Matches: `examples/go/main.go`, `examples/go/database/connect.go` + +#### Glob Pattern +Wildcard matching with `*` and `?`: + +```yaml +source_pattern: + type: "glob" + pattern: "examples/*/main.go" +``` + +Matches: `examples/go/main.go`, `examples/python/main.go` + +#### Regex Pattern +Full regex with named capture groups: + +```yaml +source_pattern: + type: "regex" + pattern: "^examples/(?P[^/]+)/(?P.+)$" +``` + +Matches: `examples/go/main.go` (extracts `lang=go`, `file=main.go`) + +### Path Transformations + +Transform source paths to target paths using variables: + +```yaml +path_transform: "docs/${lang}/${category}/${file}" +``` + +**Built-in Variables:** +- `${path}` - Full source path +- `${filename}` - File name only +- `${dir}` - Directory path +- `${ext}` - File extension + +**Custom Variables:** +- Any named groups from regex patterns +- Example: `(?P[^/]+)` creates `${lang}` + +### Commit Strategies + +#### Direct Commit +```yaml +commit_strategy: + type: "direct" + commit_message: "Update examples from ${source_repo}" +``` + +#### Pull Request +```yaml +commit_strategy: + type: "pull_request" + commit_message: "Update examples" + pr_title: "Update ${category} examples" + pr_body: "Automated update from ${source_repo}" + auto_merge: true +``` + +### Message Templates + +Use variables in commit messages and PR titles: + +```yaml +commit_message: "Update ${category} examples from ${lang}" +pr_title: "Update ${category} examples" +``` + +**Available Variables:** +- `${rule_name}` - Name of the copy rule +- `${source_repo}` - Source repository +- `${target_repo}` - Target repository +- `${source_branch}` - Source branch +- `${target_branch}` - Target branch +- `${file_count}` - Number of files being copied +- Any custom variables from pattern matching + +## CLI Tools + +### Config Validator + +Validate and test configurations before deployment: + +```bash +# Validate config file +./config-validator validate -config copier-config.yaml -v + +# Test pattern matching +./config-validator test-pattern \ + -type regex \ + -pattern "^examples/(?P[^/]+)/(?P.+)$" \ + -file "examples/go/main.go" + +# Test path transformation +./config-validator test-transform \ + -template "docs/${lang}/${file}" \ + -file "examples/go/main.go" \ + -pattern "^examples/(?P[^/]+)/(?P.+)$" + +# Initialize new config from template +./config-validator init -output copier-config.yaml + +# Convert between formats +./config-validator convert -input config.json -output copier-config.yaml +``` + +## Monitoring + +### Health Endpoint + +Check application health: + +```bash +curl http://localhost:8080/health +``` + +Response: ```json -[ - { - "source_directory": "path/to/source/directory", - "target_repo": "example-repo", - "target_branch": "main", - "target_directory": "path/to/target/directory", - "recursive_copy": true, - "copier_commit_strategy": "Optional commit strategy for this config entry ('pr' or 'direct')", - "pr_title": "Optional PR title for this config entry", - "commit_message": "Optional commit message for this config entry", - "merge_without_review": false - } -] +{ + "status": "healthy", + "started": true, + "github": { + "status": "healthy", + "authenticated": true + }, + "queues": { + "upload_count": 0, + "deprecation_count": 0 + }, + "uptime": "1h23m45s" +} +``` + +### Metrics Endpoint + +Get performance metrics: + +```bash +curl http://localhost:8080/metrics ``` -Leave the deprecation file an empty array: +Response: ```json -[ -] -``` - -### Configure Permissions for the Web App -To configure the app in the source repo, go to the repo's list of web apps. -You should see the Docs Examples Copier listed: -!["list of web apps"](./readme_files/webapps.png) - -## Hosting -This app is hosted in a Google Cloud App Engine, in the organization owned by MongoDB. -The PEM file needed for GitHub Authentication is stored as a secret in the Google Secrets Manager. -For testing locally, you will need to download the auth file from gcloud and store it locally. -See the [Google Cloud documentation](https://cloud.google.com/docs/authentication/application-default-credentials#GAC) -for more information. - -### Change Where the App is Hosted -If you deploy this app to a new host/server, you will need to create a new webhook -in the source repo. See [Set Up A Webhook](# Set Up A Webhook) - - -## How to Modify and Test -To make changes to this app: -1. Clone this repo. -2. Make the necessary changes outlined earlier. -3. Change the `.env.test` to match your environment needs, or create a new `.env` file and reference - it in the next step. -4. Test by running `go run app.go -env ./configs/.env.test` - -5. Note: you **do not need to change the GitHub app installation**. Why? I think -because it is entirely self-contained within this Go app. - -### Testing notes -As of this writing, the source repo (https://GitHub.com/mongodb/docs-code-examples) has -two webhooks configured: one points to the production version of this application, and -the other points to a [smee.io proxy](https://smee.io/5Zchxoa1xH7WfYo). - -#### What is smee.io? -Smee.io provides a simple way to point a public endpoint to localhost on your computer. -It requires the smee cli, which is very lightweight. You run the proxy with a single -command (`smee -u https://smee.io/5Zchxoa1xH7WfYo`) and any webhooks that go to that -url will be directed to http://localhost:3000/. This is entirely optional, and there are -probably other solutions for testing. I just found this dead simple. - -**Note** The current production deployment looks for messages on the default -port and the `/events` route, while your testing hooks (like smee) might only send -messages on a specific port and/or the default path. This is why you can change the -port and route in the `.env.*` files. - -## Future Work - -- ~~BUG/SECURITY: Move .pem to google secret.~~ -- ~~Where do we view the log for the app when it hits a snag?~~ - Fixed in 112c8953cbb54d3743b25744fe01f6649f783faa. Added Google - Logging and centralized logging service for terminal logging. -- ~~Currently each write is a separate commit. Bad. Fix.~~ - Fixed in f91ccfce74edff56eb305068357a069d12a2020f -- Slack integrations: - - notifies a channel when the - `deprecated_examples.json` file changes, so writers can find those deprecated examples - in the docs and update/remove them accordingly. See this - [Slack API page](https://api.slack.com/messaging/webhooks). - - posts log updates (e.g. PR created and ready for review) -- Automate further with hook to Audit DB to get doc files with literal includes & iocode blocks (???) -- ~~Mock tests~~ \ No newline at end of file +{ + "webhooks": { + "received": 42, + "processed": 40, + "failed": 2, + "success_rate": 95.24, + "processing_time": { + "avg_ms": 234.5, + "p50_ms": 200, + "p95_ms": 450, + "p99_ms": 890 + } + }, + "files": { + "matched": 150, + "uploaded": 145, + "upload_failed": 5, + "deprecated": 3, + "upload_success_rate": 96.67 + } +} +``` + +## Audit Logging + +When enabled, all operations are logged to MongoDB: + +```javascript +// Query recent copy events +db.audit_events.find({ + event_type: "copy", + success: true +}).sort({timestamp: -1}).limit(10) + +// Find failed operations +db.audit_events.find({ + success: false +}).sort({timestamp: -1}) + +// Statistics by rule +db.audit_events.aggregate([ + {$match: {event_type: "copy"}}, + {$group: { + _id: "$rule_name", + count: {$sum: 1}, + avg_duration: {$avg: "$duration_ms"} + }} +]) +``` + +## Testing + +### Run Unit Tests + +```bash +# Run all tests +go test ./services -v + +# Run specific test suite +go test ./services -v -run TestPatternMatcher + +# Run with coverage +go test ./services -cover +go test ./services -coverprofile=coverage.out +go tool cover -html=coverage.out +``` + +## Development + +### Dry-Run Mode + +Test without making actual changes: + +```bash +DRY_RUN=true ./examples-copier +``` + +In dry-run mode: +- Webhooks are processed +- Files are matched and transformed +- Audit events are logged +- **NO actual commits or PRs are created** + +### Enhanced Logging + +Enable detailed logging: + +```bash +LOG_LEVEL=debug ./examples-copier +# or +COPIER_DEBUG=true ./examples-copier +``` + +## Architecture + +### Project Structure + +``` +examples-copier/ +├── app.go # Main application entry point +├── cmd/ +│ └── config-validator/ # CLI validation tool +├── configs/ +│ ├── environment.go # Environment configuration +│ ├── .env.example.new # Environment template +│ └── config.example.yaml # Config template +├── services/ +│ ├── pattern_matcher.go # Pattern matching engine +│ ├── config_loader.go # Config loading & validation +│ ├── audit_logger.go # MongoDB audit logging +│ ├── health_metrics.go # Health & metrics endpoints +│ ├── file_state_service.go # Thread-safe state management +│ ├── service_container.go # Dependency injection +│ └── webhook_handler_new.go # New webhook handler +└── types/ + ├── config.go # Configuration types + └── types.go # Core types +``` + +### Service Container + +The application uses dependency injection for clean architecture: + +```go +container := NewServiceContainer(config) +// All services initialized and wired together +``` + +## Deployment + +See [DEPLOYMENT-GUIDE.md](DEPLOYMENT-GUIDE.md) for complete deployment instructions. + +### Google Cloud App Engine + +```bash +gcloud app deploy +``` + +### Docker + +```bash +docker build -t examples-copier . +docker run -p 8080:8080 --env-file .env examples-copier +``` + +## Security + +- **Webhook Signature Verification** - HMAC-SHA256 validation +- **Secret Management** - Google Cloud Secret Manager +- **Least Privilege** - Minimal GitHub App permissions +- **Audit Trail** - Complete operation logging + +## Documentation + +### Getting Started + +- **[Configuration Guide](docs/CONFIGURATION-GUIDE.md)** - Complete configuration reference ⭐ NEW +- **[Pattern Matching Guide](docs/PATTERN-MATCHING-GUIDE.md)** - Pattern matching with examples +- **[Local Testing](docs/LOCAL-TESTING.md)** - Test locally before deploying +- **[Deployment Guide](docs/DEPLOYMENT-GUIDE.md)** - Deploy to production + +### Reference + +- **[Pattern Matching Cheat Sheet](docs/PATTERN-MATCHING-CHEATSHEET.md)** - Quick pattern syntax reference +- **[Architecture](docs/ARCHITECTURE.md)** - System design and components +- **[Migration Guide](docs/MIGRATION-GUIDE.md)** - Migrate from legacy JSON config +- **[Troubleshooting](docs/TROUBLESHOOTING.md)** - Common issues and solutions +- **[FAQ](docs/FAQ.md)** - Frequently asked questions + +### Features + +- **[Slack Notifications](docs/SLACK-NOTIFICATIONS.md)** - Slack integration guide +- **[Webhook Testing](docs/WEBHOOK-TESTING.md)** - Test with real PR data + +### Tools + +- **[config-validator](cmd/config-validator/README.md)** - Validate and test configurations +- **[test-webhook](cmd/test-webhook/README.md)** - Test webhook processing +- **[Scripts](scripts/README.md)** - Helper scripts diff --git a/examples-copier/app.go b/examples-copier/app.go index 326cb47..37b1d7d 100644 --- a/examples-copier/app.go +++ b/examples-copier/app.go @@ -1,29 +1,208 @@ package main import ( + "context" "flag" "fmt" + "log" + "net/http" + "os" + "os/signal" + "syscall" + "time" "github.com/mongodb/code-example-tooling/code-copier/configs" "github.com/mongodb/code-example-tooling/code-copier/services" ) func main() { - // Take the environment file path from command line arguments or default to "./configs/.env" + // Parse command line flags var envFile string - flag.StringVar(&envFile, "env", "./configs/.env", "env file") - help := flag.Bool("help", false, "show help") + var dryRun bool + var validateOnly bool + + flag.StringVar(&envFile, "env", "./configs/.env", "Path to environment file") + flag.BoolVar(&dryRun, "dry-run", false, "Enable dry-run mode (no actual changes)") + flag.BoolVar(&validateOnly, "validate", false, "Validate configuration and exit") + help := flag.Bool("help", false, "Show help") flag.Parse() - if help != nil && *help == true { - fmt.Println("Usage: go run app -env [path to env file]") + + if *help { + printHelp() return } - _, err := configs.LoadEnvironment(envFile) + // Load environment configuration + config, err := configs.LoadEnvironment(envFile) if err != nil { - fmt.Printf("Error loading environment: %v\n", err) + fmt.Printf("❌ Error loading environment: %v\n", err) + os.Exit(1) + } + + // Override dry-run from command line + if dryRun { + config.DryRun = true + } + + // Initialize services + container, err := services.NewServiceContainer(config) + if err != nil { + fmt.Printf("❌ Failed to initialize services: %v\n", err) + os.Exit(1) + } + defer container.Close(context.Background()) + + // If validate-only mode, validate config and exit + if validateOnly { + if err := validateConfiguration(container); err != nil { + fmt.Printf("❌ Configuration validation failed: %v\n", err) + os.Exit(1) + } + fmt.Println("✅ Configuration is valid") return } + + // Initialize Google Cloud logging + services.InitializeGoogleLogger() + defer services.CloseGoogleLogger() + + // Configure GitHub permissions services.ConfigurePermissions() - services.SetupWebServerAndListen() + + // Print startup banner + printBanner(config, container) + + // Start web server + if err := startWebServer(config, container); err != nil { + fmt.Printf("❌ Failed to start web server: %v\n", err) + os.Exit(1) + } +} + +func printHelp() { + fmt.Println("GitHub Code Example Copier") + fmt.Println() + fmt.Println("Usage:") + fmt.Println(" app [options]") + fmt.Println() + fmt.Println("Options:") + fmt.Println(" -env string Path to environment file (default: ./configs/.env)") + fmt.Println(" -dry-run Enable dry-run mode (no actual changes)") + fmt.Println(" -validate Validate configuration and exit") + fmt.Println(" -help Show this help message") + fmt.Println() + fmt.Println("Examples:") + fmt.Println(" app -env ./configs/.env.test") + fmt.Println(" app -dry-run") + fmt.Println(" app -validate -env ./configs/.env.prod") +} + +func printBanner(config *configs.Config, container *services.ServiceContainer) { + fmt.Println("╔════════════════════════════════════════════════════════════════╗") + fmt.Println("║ GitHub Code Example Copier ║") + fmt.Println("╠════════════════════════════════════════════════════════════════╣") + fmt.Printf("║ Port: %-48s║\n", config.Port) + fmt.Printf("║ Webhook Path: %-48s║\n", config.WebserverPath) + fmt.Printf("║ Config File: %-48s║\n", config.ConfigFile) + fmt.Printf("║ Dry Run: %-48v║\n", config.DryRun) + fmt.Printf("║ Audit Log: %-48v║\n", config.AuditEnabled) + fmt.Printf("║ Metrics: %-48v║\n", config.MetricsEnabled) + fmt.Println("╚════════════════════════════════════════════════════════════════╝") + fmt.Println() +} + +func validateConfiguration(container *services.ServiceContainer) error { + ctx := context.Background() + _, err := container.ConfigLoader.LoadConfig(ctx, container.Config) + return err +} + +func startWebServer(config *configs.Config, container *services.ServiceContainer) error { + // Create HTTP handler with all routes + mux := http.NewServeMux() + + // Webhook endpoint + mux.HandleFunc(config.WebserverPath, func(w http.ResponseWriter, r *http.Request) { + handleWebhook(w, r, config, container) + }) + + // Health endpoint + mux.HandleFunc("/health", services.HealthHandler(container.FileStateService, container.StartTime)) + + // Metrics endpoint (if enabled) + if config.MetricsEnabled { + mux.HandleFunc("/metrics", services.MetricsHandler(container.MetricsCollector, container.FileStateService)) + } + + // Info endpoint + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/" { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "text/plain") + fmt.Fprintf(w, "GitHub Code Example Copier\n") + fmt.Fprintf(w, "Webhook endpoint: %s\n", config.WebserverPath) + fmt.Fprintf(w, "Health check: /health\n") + if config.MetricsEnabled { + fmt.Fprintf(w, "Metrics: /metrics\n") + } + }) + + // Create server + port := ":" + config.Port + server := &http.Server{ + Addr: port, + Handler: mux, + ReadTimeout: 30 * time.Second, + WriteTimeout: 30 * time.Second, + IdleTimeout: 120 * time.Second, + } + + // Handle graceful shutdown + go func() { + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) + <-sigChan + + log.Println("Shutting down server...") + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + if err := server.Shutdown(ctx); err != nil { + log.Printf("Server shutdown error: %v\n", err) + } + }() + + // Start server + services.LogInfo(fmt.Sprintf("Starting web server on port %s", port)) + if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { + return fmt.Errorf("server error: %w", err) + } + + return nil +} + +func handleWebhook(w http.ResponseWriter, r *http.Request, config *configs.Config, container *services.ServiceContainer) { + // Record webhook received + container.MetricsCollector.RecordWebhookReceived() + startTime := time.Now() + + // Create context with timeout + baseCtx, rid := services.WithRequestID(r) + timeout := time.Duration(60) * time.Second + ctx, cancel := context.WithTimeout(baseCtx, timeout) + defer cancel() + + r = r.WithContext(ctx) + + services.LogWebhookOperation(ctx, "receive", "webhook received", nil, map[string]interface{}{ + "request_id": rid, + }) + + // Handle webhook with new pattern matching + services.HandleWebhookWithContainer(w, r, config, container) + + // Record processing time + container.MetricsCollector.RecordWebhookProcessed(time.Since(startTime)) } diff --git a/examples-copier/cmd/config-validator/README.md b/examples-copier/cmd/config-validator/README.md new file mode 100644 index 0000000..fed9044 --- /dev/null +++ b/examples-copier/cmd/config-validator/README.md @@ -0,0 +1,345 @@ +# config-validator + +Command-line tool for validating and testing examples-copier configurations. + +## Overview + +The `config-validator` tool helps you: +- Validate configuration files +- Test pattern matching +- Test path transformations +- Convert legacy JSON configs to YAML +- Debug configuration issues + +## Installation + +```bash +cd examples-copier +go build -o config-validator ./cmd/config-validator +``` + +## Commands + +### validate + +Validate a configuration file. + +**Usage:** +```bash +./config-validator validate -config [-v] +``` + +**Options:** +- `-config` - Path to configuration file (required) +- `-v` - Verbose output (optional) + +**Examples:** + +```bash +# Validate YAML config +./config-validator validate -config copier-config.yaml + +# Validate with verbose output +./config-validator validate -config copier-config.yaml -v + +# Validate JSON config +./config-validator validate -config config.json +``` + +**Output:** +``` +✅ Configuration is valid! + +Summary: + Source: mongodb/docs-code-examples + Branch: main + Rules: 2 + +Rule 1: Copy generated examples + Pattern: regex - ^generated-examples/(?P[^/]+)/(?P.+)$ + Targets: 1 + +Rule 2: Copy all generated examples + Pattern: prefix - generated-examples/ + Targets: 1 +``` + +### test-pattern + +Test if a pattern matches a file path and see what variables are extracted. + +**Usage:** +```bash +./config-validator test-pattern -type -pattern -file +``` + +**Options:** +- `-type` - Pattern type: `prefix`, `glob`, or `regex` (required) +- `-pattern` - Pattern to test (required) +- `-file` - File path to test against (required) + +**Examples:** + +```bash +# Test regex pattern +./config-validator test-pattern \ + -type regex \ + -pattern "^examples/(?P[^/]+)/(?P.+)$" \ + -file "examples/go/main.go" + +# Test prefix pattern +./config-validator test-pattern \ + -type prefix \ + -pattern "examples/" \ + -file "examples/go/main.go" + +# Test glob pattern +./config-validator test-pattern \ + -type glob \ + -pattern "**/*.go" \ + -file "examples/go/main.go" +``` + +**Output (regex):** +``` +✅ Pattern matched! + +Extracted variables: + lang = go + file = main.go +``` + +**Output (prefix):** +``` +✅ Pattern matched! + +Extracted variables: + matched_prefix = examples + relative_path = go/main.go +``` + +**Output (no match):** +``` +❌ Pattern did not match +``` + +### test-transform + +Test path transformation with variables. + +**Usage:** +```bash +./config-validator test-transform -source -template