Merge pull request #1393 from microsoftgraph/dev

Fix broken OpenAPI generation

Author: MIchaelMainer

.azure-pipelines/generation-pipeline.yml

@@ -3,8 +3,15 @@
 trigger: none
 pr: none
 schedules:
-- cron: '0 10 * * TUE'
-  displayName: Tuesday generation (PST 2am, EST 5am, EAT 3pm)
+
+# Minute: 0 → At the start of the hour.
+# Hour: 10 → 10:00 AM UTC.
+# Day of Month: 1-7,15-21 → On the 1st–7th and 15th–21st.
+# Month: * → Every month.
+# Weekday: TUE → Only on Tuesdays.
+# This cron will run bi-weekly unless the month has 5 Tuesdays, in which case it will skip the 5th Tuesday (won't be bi-weekly that month).
+- cron: '0 10 1-7,15-21 * TUE'
+  displayName: Bi-weekly Tuesday generation (1st and 3rd Tuesday, PST 2am, EST 5am, EAT 3pm)
   branches:
     include:
     - main
@@ -291,10 +298,45 @@ extends:
           outputPath: $(cleanOpenAPIFileBetaOutputPath)
           cleanMetadataFolder: $(cleanOpenAPIFolderBeta)
 
+    # Approval stage for v1 OpenAPI generation. All subsequent stages depending on OpenAPI generation
+    # will be blocked until approval is granted. Any new stages depending on OpenAPI generation
+    # should also depend on this approval stage.
+    - stage: open_api_v1_approval
+      dependsOn:
+      - stage_v1_openapi
+      condition: eq(dependencies.stage_v1_openapi.result, 'Succeeded')
+      jobs:
+      - deployment: OpenApiApproval
+        displayName: 'OpenAPI v1.0 Generation Review'
+        environment: 'openapi-generation-review'
+        strategy:
+          runOnce:
+            deploy:
+              steps:
+              - script: echo "OpenAPI generation approved."
+
+    # Approval stage for beta OpenAPI generation. All subsequent stages depending on OpenAPI generation
+    # will be blocked until approval is granted. Any new stages depending on OpenAPI generation
+    # should also depend on this approval stage.
+    - stage: open_api_beta_approval
+      dependsOn:
+      - stage_beta_openapi
+      condition: eq(dependencies.stage_beta_openapi.result, 'Succeeded')
+      jobs:
+      - deployment: OpenApiApproval
+        displayName: 'OpenAPI Beta Generation Review'
+        environment: 'openapi-generation-review'
+        strategy:
+          runOnce:
+            deploy:
+              steps:
+              - script: echo "OpenAPI generation approved."              
+
     - stage: stage_csharp_v1_kiota
       dependsOn:
       - stage_build_and_publish_kiota
       - stage_v1_openapi
+      - open_api_v1_approval
       condition: |
         and
         (
@@ -336,6 +378,7 @@ extends:
       dependsOn:
       - stage_build_and_publish_kiota
       - stage_beta_openapi
+      - open_api_beta_approval
       condition: |
         and
         (
@@ -377,6 +420,7 @@ extends:
       dependsOn:
       - stage_build_and_publish_kiota
       - stage_v1_openapi
+      - open_api_v1_approval
       condition: |
         and
         (
@@ -416,6 +460,7 @@ extends:
       dependsOn:
       - stage_build_and_publish_kiota
       - stage_beta_openapi
+      - open_api_beta_approval
       condition: |
         and
         (
@@ -455,6 +500,7 @@ extends:
       dependsOn:
       - stage_build_and_publish_kiota
       - stage_v1_openapi
+      - open_api_v1_approval
       condition: |
         and
         (
@@ -496,6 +542,7 @@ extends:
       dependsOn:
       - stage_build_and_publish_kiota
       - stage_beta_openapi
+      - open_api_beta_approval
       condition: |
         and
         (
@@ -537,6 +584,7 @@ extends:
       dependsOn:
       - stage_build_and_publish_kiota
       - stage_beta_openapi
+      - open_api_beta_approval
       condition: |
         and
         (
@@ -578,6 +626,7 @@ extends:
       dependsOn:
       - stage_build_and_publish_kiota
       - stage_v1_openapi
+      - open_api_v1_approval
       condition: |
         and
         (
@@ -693,6 +742,7 @@ extends:
       dependsOn:
       - stage_build_and_publish_kiota
       - stage_v1_openapi
+      - open_api_v1_approval
       condition: |
         and
         (
@@ -735,6 +785,7 @@ extends:
       dependsOn:
       - stage_build_and_publish_kiota
       - stage_beta_openapi
+      - open_api_beta_approval
       condition: |
         and
         (
@@ -777,6 +828,7 @@ extends:
       dependsOn:
       - stage_build_and_publish_kiota
       - stage_v1_openapi
+      - open_api_v1_approval
       condition: |
         and
         (
@@ -817,6 +869,7 @@ extends:
       dependsOn:
       - stage_build_and_publish_kiota
       - stage_beta_openapi
+      - open_api_beta_approval
       condition: |
         and
         (

.azure-pipelines/generation-templates/capture-metadata.yml

@@ -166,9 +166,9 @@ steps:
 
 # Checkin clean metadata into metadata repo or make it an artifact.
 - pwsh: '$(scriptsDirectory)/git-push-cleanmetadata.ps1'
-
-  displayName: push clean ${{ parameters.endpoint }} metadata to msgraph-metadata repo
+  displayName: push clean ${{ parameters.endpoint }} CSDL metadata to msgraph-metadata repo
   env:
+    CreateOpenAPIPR: False
     EndpointVersion: ${{ parameters.endpoint }}
     PublishChanges: $(publishChanges)
   workingDirectory: '$(Build.SourcesDirectory)/msgraph-metadata'

.azure-pipelines/generation-templates/capture-openapi.yml

@@ -1,3 +1,4 @@
+# capture-openapi.yml
 # Copyright (c) Microsoft Corporation. All Rights Reserved. Licensed under the MIT License. See License in the project root for license information.
 # The following template creates multiple artifacts from a matrix, this is the intended behavior to follow the same workflow before the governance migration
 
@@ -58,13 +59,12 @@ jobs:
     persistCredentials: true
 
   - template: /.azure-pipelines/generation-templates/checkout-metadata.yml@self
-
   # required for the hidi to run
   - template: /.azure-pipelines/generation-templates/use-dotnet-sdk.yml@self
     parameters:
       version: '8.x'
 
-# required for the hidi installation validation
+  # required for the hidi installation validation
   - template: /.azure-pipelines/generation-templates/use-dotnet-sdk.yml@self
     parameters:
       version: '9.x'
@@ -78,8 +78,13 @@ jobs:
     displayName: ensure the generation happens from master latest
     workingDirectory: $(Build.SourcesDirectory)/msgraph-metadata
 
-  - pwsh: '$(scriptsDirectory)/generate-open-api.ps1 -endpointVersion ${{ parameters.endpoint }} -settings "$(conversionSettingsDirectory)/$(File)" -platformName "$(Name)"'
-    displayName: 'update ${{ parameters.endpoint }} open API description'
+  - pwsh: |
+      Write-Host "`ngit status before generation:"
+      git status
+      $(scriptsDirectory)/generate-open-api.ps1 -endpointVersion ${{ parameters.endpoint }} -settings "$(conversionSettingsDirectory)/$(File)" -platformName "$(Name)"
+      Write-Host "`ngit status after generation:"
+      git status
+    displayName: 'generate ${{ parameters.endpoint }} open API description'
     workingDirectory: $(Build.SourcesDirectory)/msgraph-metadata
 
   # publish metadata as an artifact
@@ -88,16 +93,16 @@ jobs:
       sourceFolder: ${{ parameters.outputPath }}
       contents: '**/$(Name).yaml'
       targetFolder: '$(Build.ArtifactStagingDirectory)/$(Name)'
-    displayName: Copy generated metadata
+    displayName: Copy generated OpenAPI yaml file to ArtifactStagingDirectory
+
   - pwsh: |
       ./scripts/run-openapi-validation.ps1 -repoDirectory (Get-Location).Path -version "${{ parameters.endpoint }}" -platformName "$(Name)"
-    displayName: ensure that OpenAPI docs can be parsed
+    displayName: Validate that OpenAPI docs can be parsed
     workingDirectory: $(Build.SourcesDirectory)/msgraph-metadata
 
-
 - job: publish_openapi
   dependsOn: convert_openapi
-  displayName: Publish
+  displayName: Publish OpenAPI files
   ## If there's new settings added please add them here too
   templateContext:
     inputs:
@@ -123,35 +128,61 @@ jobs:
     displayName: checkout generator
     fetchDepth: 1
     persistCredentials: true
-    # Copy files from the maxtrix artifacts to a single folder
+
+  #  Note that msgraph-metadata repo has detached HEAD
+  - template: /.azure-pipelines/generation-templates/checkout-metadata.yml@self    
+
+  - template: /.azure-pipelines/generation-templates/set-user-config.yml@self  
+
+  # Copy OpenAPI files from the matrix artifacts to the ArtifactStagingDirectory
+  # This is used for library generation later
   - task: CopyFiles@2
     inputs:
       sourceFolder: '$(Build.SourcesDirectory)/artifacts'
       contents: '**/*.yaml'
       targetFolder: '$(Build.ArtifactStagingDirectory)'
-      displayName: Copy artifact metadata
-    # Copy files from the target path where artifacts should be downloaded
+    displayName: Copy OpenAPI input artifacts to artifact staging directory
+    
+  # Copy OpenAPI files from ArtifactStagingDirectory to local msgraph-metadata repo
   - task: CopyFiles@2
     inputs:
       sourceFolder: '$(Build.ArtifactStagingDirectory)'
       contents: '**/*.yaml'
       targetFolder: '$(Build.SourcesDirectory)/msgraph-metadata/openapi/${{ parameters.endpoint }}'
-    displayName: Copy downloaded metadata
+      overwrite: true
+    displayName: Copy OpenAPI files to local msgraph-metadata repo
 
-    # publish metadata as an artifact
-  - task: CopyFiles@2
-    inputs:
-      sourceFolder: ${{ parameters.outputPath }}
-      contents: '**/*.yaml'
-      targetFolder: '$(Build.ArtifactStagingDirectory)'
-    displayName: Copy downloaded metadata
-  - template: /.azure-pipelines/generation-templates/checkout-metadata.yml@self
-  - template: /.azure-pipelines/generation-templates/set-user-config.yml@self
+  # Push changes to msgraph-metadata repo
   - pwsh: '$(scriptsDirectory)/git-push-cleanmetadata.ps1'
-
-    displayName: push clean ${{ parameters.endpoint }} OpenAPI description to msgraph-metadata repo
+    displayName: Publish ${{ parameters.endpoint }} OpenAPI description to msgraph-metadata repo
     env:
+      CreateOpenAPIPR: True
       EndpointVersion: ${{ parameters.endpoint }}
       PublishChanges: $(publishChanges)
     workingDirectory: '$(Build.SourcesDirectory)/msgraph-metadata'
     enabled: true
+    
+
+  # Create PR - note that this PR is not used for gating. It's just for discovery purposes. 
+  # Library generation PRs will still be created based on the OpenAPI files.
+
+  - task: AzureKeyVault@2
+    displayName: "Azure Key Vault: Get Secrets"
+    inputs:
+      azureSubscription: "Federated AKV Managed Identity Connection"
+      KeyVaultName: akv-prod-eastus
+      SecretsFilter: "microsoft-graph-devx-bot-appid,microsoft-graph-devx-bot-privatekey"
+
+  - pwsh: '$(scriptsDirectory)/create-pull-request.ps1'
+    displayName: 'Create Pull Request for the generated OpenAPI files for msgraph-metadata'
+    env:
+      BaseBranch: master
+      GeneratePullRequest: true
+      GhAppId: $(microsoft-graph-devx-bot-appid)
+      GhAppKey: $(microsoft-graph-devx-bot-privatekey)
+      OverrideSkipCI: false
+      RepoName: 'microsoftgraph/msgraph-metadata'
+      ScriptsDirectory: $(scriptsDirectory)
+      # Version is intentionally left empty for OpenAPI PRs as versioning is not applicable in this context.
+      Version: ''
+    workingDirectory: '$(Build.SourcesDirectory)/msgraph-metadata'

.azure-pipelines/generation-templates/use-dotnet-sdk.yml

@@ -5,7 +5,7 @@ parameters:
 
 steps:
 - task: UseDotNet@2
-  displayName: 'Use .NET SDK'
+  displayName: 'Use .NET SDK ${{ parameters.version }}'
   inputs:
     packageType: sdk
     version: ${{ parameters.version }}

.github/workflows/codeql.yml

@@ -52,7 +52,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v3
+      uses: github/codeql-action/init@v4
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -66,7 +66,7 @@ jobs:
     # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v3
+      uses: github/codeql-action/autobuild@v4
       with:
         working-directory: ${{ github.workspace }}/src/Typewriter
 
@@ -81,6 +81,6 @@ jobs:
     #     ./location_of_script_within_repo/buildscript.sh
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v3
+      uses: github/codeql-action/analyze@v4
       with:
         category: "/language:${{matrix.language}}"

scripts/create-pull-request.ps1

@@ -1,24 +1,27 @@
 if (($env:OverrideSkipCI -eq $False) -and ($env:BUILD_REASON -eq 'Manual')) # Skip CI if manually running this pipeline.
 {
-    Write-Host "Skipping pull request creation due Skip CI." -ForegroundColor Green
+    Write-Host "Skipping pull request creation due Skip CI."
     return;
 }
 
 if (($env:GeneratePullRequest -eq $False)) { # Skip CI if manually running this pipeline.
-    Write-Host "Skipping pull request creation due this repository being disabled" -ForegroundColor Green
+    Write-Host "Skipping pull request creation due this repository being disabled"
     return;
 }
 
-# Special case for beta typings as it uses a non-conforming preview versioning.
+# Special case for beta typings as it uses a non-conforming preview versioning. Helps with triggering Release Please.
 if ($env:RepoName.Contains("msgraph-beta-typescript-typings"))
 {
-    $title = "feat: generated $version models and request builders"    
+    $title = "feat: generated $env:Version models and request builders"    
+}
+elseif ($env:RepoName.Contains("msgraph-metadata")) # we are only generating OpenAPI PRs for the metadata repo
+{
+    $title = "Generated $env:Version OpenAPI descriptions"
 }
 else {
-    $title = "Generated $version models and request builders"
+    $title = "Generated $env:Version models and request builders"
 }
 
-$version = $env:Version
 $body = ":bangbang:**_Important_**:bangbang: <br> Check for unexpected deletions or changes in this PR and ensure relevant CI checks are passing. <br><br> **Note:** This pull request was automatically created by Azure pipelines."
 $baseBranchParameter = ""
 
@@ -30,10 +33,10 @@ if (![string]::IsNullOrEmpty($env:BaseBranch))
 # The installed application is required to have the following permissions: read/write on pull requests/
 $tokenGenerationScript = "$env:ScriptsDirectory\Generate-Github-Token.ps1"
 $env:GITHUB_TOKEN = & $tokenGenerationScript -AppClientId $env:GhAppId -AppPrivateKeyContents $env:GhAppKey -Repository $env:RepoName
-Write-Host "Fetched Github Token for PR generation and set as environment variable." -ForegroundColor Green
+Write-Host "Fetched Github Token for PR generation and set as environment variable."
 
 # No need to specify reviewers as code owners should be added automatically.
 Invoke-Expression "gh auth login" # login to GitHub
 Invoke-Expression "gh pr create -t ""$title"" -b ""$body"" $baseBranchParameter | Write-Host"
 
-Write-Host "Pull Request Created successfully." -ForegroundColor Green
+Write-Host "Pull Request Created successfully."