Skip to content

Commit 89b0d7e

Browse files
shyamnamboodiripadstephentoubpeterwald
authored
Add MEAI.Evaluation libraries (#5873)
Co-authored-by: Stephen Toub <[email protected]> Co-authored-by: Peter Waldschmidt <[email protected]>
1 parent c086106 commit 89b0d7e

File tree

162 files changed

+20144
-3
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

162 files changed

+20144
-3
lines changed

eng/Versions.props

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,7 @@
138138
Win-x64 is used here because we have picked an arbitrary runtime identifier to flow the version of the latest NETCore.App runtime.
139139
All Runtime.$rid packages should have the same version.
140140
-->
141+
<MicrosoftMLTokenizersVersion>1.0.1</MicrosoftMLTokenizersVersion>
141142
<MicrosoftNETCoreAppRuntimeVersion>$(MicrosoftNETCoreAppRuntimewinx64Version)</MicrosoftNETCoreAppRuntimeVersion>
142143
<!-- Compatibility with VS 17.8/.NET SDK 8.0.1xx -->
143144
<MicrosoftCodeAnalysisVersion>4.8.0</MicrosoftCodeAnalysisVersion>

eng/packages/General-LTS.props

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
<PackageVersion Include="Microsoft.Extensions.Http" Version="$(MicrosoftExtensionsHttpLTSVersion)" />
2424
<PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="$(MicrosoftExtensionsLoggingAbstractionsLTSVersion)" />
2525
<PackageVersion Include="Microsoft.Extensions.Logging.Configuration" Version="$(MicrosoftExtensionsLoggingConfigurationLTSVersion)" />
26+
<PackageVersion Include="Microsoft.Extensions.Logging.Console" Version="$(MicrosoftExtensionsLoggingConsoleLTSVersion)" />
2627
<PackageVersion Include="Microsoft.Extensions.Logging" Version="$(MicrosoftExtensionsLoggingLTSVersion)" />
2728
<PackageVersion Include="Microsoft.Extensions.ObjectPool" Version="$(MicrosoftExtensionsObjectPoolLTSVersion)" />
2829
<PackageVersion Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="$(MicrosoftExtensionsOptionsConfigurationExtensionsLTSVersion)" />

eng/packages/General-net9.props

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
<PackageVersion Include="Microsoft.Extensions.Http" Version="$(MicrosoftExtensionsHttpVersion)" />
2424
<PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="$(MicrosoftExtensionsLoggingAbstractionsVersion)" />
2525
<PackageVersion Include="Microsoft.Extensions.Logging.Configuration" Version="$(MicrosoftExtensionsLoggingConfigurationVersion)" />
26+
<PackageVersion Include="Microsoft.Extensions.Logging.Console" Version="$(MicrosoftExtensionsLoggingConsoleVersion)" />
2627
<PackageVersion Include="Microsoft.Extensions.Logging" Version="$(MicrosoftExtensionsLoggingVersion)" />
2728
<PackageVersion Include="Microsoft.Extensions.ObjectPool" Version="$(MicrosoftExtensionsObjectPoolVersion)" />
2829
<PackageVersion Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="$(MicrosoftExtensionsOptionsConfigurationExtensionsVersion)" />

eng/packages/General.props

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
<?xml version="1.0" encoding="utf-8"?>
22
<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
33
<ItemGroup>
4+
<PackageVersion Include="Azure.Storage.Files.DataLake" Version="12.21.0" />
45
<PackageVersion Include="Azure.AI.Inference" Version="1.0.0-beta.2" />
56
<PackageVersion Include="ICSharpCode.Decompiler" Version="8.2.0.7535" />
67
<PackageVersion Include="Microsoft.Bcl.HashCode" Version="1.1.1" />
@@ -10,6 +11,7 @@
1011
<PackageVersion Include="Microsoft.CodeAnalysis.CSharp" Version="$(MicrosoftCodeAnalysisVersion)" />
1112
<PackageVersion Include="Microsoft.CodeAnalysis" Version="$(MicrosoftCodeAnalysisVersion)" />
1213
<PackageVersion Include="Microsoft.IO.RecyclableMemoryStream" Version="3.0.0" />
14+
<PackageVersion Include="Microsoft.ML.Tokenizers" Version="$(MicrosoftMLTokenizersVersion)" />
1315
<PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
1416
<PackageVersion Include="OpenAI" Version="2.2.0-beta.1" />
1517
<PackageVersion Include="Polly" Version="8.4.2" />

eng/packages/TestOnly.props

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
<ItemGroup>
44
<PackageVersion Include="AutoFixture.AutoMoq" Version="4.17.0" />
55
<PackageVersion Include="Azure.AI.OpenAI" Version="2.2.0-beta.1" />
6+
<PackageVersion Include="Azure.Identity" Version="1.13.2" />
67
<PackageVersion Include="autofixture" Version="4.17.0" />
78
<PackageVersion Include="BenchmarkDotNet" Version="0.13.5" />
89
<PackageVersion Include="FluentAssertions" Version="6.11.0" />
@@ -12,7 +13,7 @@
1213
<PackageVersion Include="Microsoft.Diagnostics.Tracing.TraceEvent" Version="3.1.3" />
1314
<PackageVersion Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="9.0.0" />
1415
<PackageVersion Include="Microsoft.Extensions.Configuration.UserSecrets" Version="9.0.0" />
15-
<PackageVersion Include="Microsoft.ML.Tokenizers" Version="0.22.0-preview.24378.1" />
16+
<PackageVersion Include="Microsoft.ML.Tokenizers.Data.O200kBase" Version="$(MicrosoftMLTokenizersVersion)" />
1617
<PackageVersion Include="Moq.AutoMock" Version="3.1.0" />
1718
<PackageVersion Include="Moq" Version="4.18.4" />
1819
<PackageVersion Include="OpenTelemetry.Exporter.InMemory" Version="1.9.0" />

eng/pipelines/templates/BuildAndTest.yml

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,12 @@ parameters:
2323
default: false
2424

2525
steps:
26+
- task: NodeTool@0
27+
displayName: Add NodeJS/npm
28+
inputs:
29+
versionSpec: "20.x"
30+
checkLatest: true
31+
2632
- script: ${{ parameters.buildScript }}
2733
-restore
2834
/bl:${{ parameters.repoLogPath }}/restore.binlog
@@ -45,6 +51,11 @@ steps:
4551
$(_OfficialBuildIdArgs)
4652
displayName: Build
4753

54+
- ${{ if eq(parameters.isWindows, 'true') }}:
55+
- pwsh: |
56+
$(Build.SourcesDirectory)/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/azure-devops-report/build.ps1 -OutputPath $(Build.ArtifactStagingDirectory)\VSIX
57+
displayName: Build Azure DevOps plugin
58+
4859
- ${{ if ne(parameters.skipTests, 'true') }}:
4960
- script: $(Build.SourcesDirectory)/.dotnet/dotnet dotnet-coverage collect
5061
--settings $(Build.SourcesDirectory)/eng/CodeCoverage.config

eng/xunit.runner.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
{
22
"diagnosticMessages": true,
3-
"longRunningTestSeconds": 300
3+
"longRunningTestSeconds": 300,
4+
"shadowCopy": false
45
}
Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
#!/usr/bin/env pwsh
2+
3+
<#
4+
.SYNOPSIS
5+
Configures local repo for online evaluation tests, which use external resources.
6+
7+
.DESCRIPTION
8+
This script copies appsettings files from a location on the developer's machine to the test
9+
project directories so that the tests are configured to connect to external resources. The online
10+
configuration files are gitignore'd and not checked in to the repo.
11+
12+
.PARAMETER Configure
13+
Configure this repo for online evaluation tests, by copying appsettings files from the developer's
14+
machine to this repo.
15+
.PARAMETER Unconfigure
16+
Unconfigure this repo for online evaluation tests, by removing the appsettings files from this repo.
17+
.PARAMETER ConfigRoot
18+
ConfigRoot specifies where to copy the configuration files from. The default is $HOME/.config/dotnet-extensions.
19+
#>
20+
21+
param (
22+
[switch]$Configure=$False,
23+
[switch]$Unconfigure=$False,
24+
[string]$ConfigRoot=$Null
25+
)
26+
27+
Write-Host "$PSScriptRoot"
28+
29+
if ($Configure -and $Unconfigure) {
30+
Write-Error -Message "Cannot specify both -Configure and -Unconfigure"
31+
Exit 1
32+
}
33+
34+
if (!(Test-Path $ConfigRoot)) {
35+
$ConfigRoot = "$HOME/.config/dotnet-extensions"
36+
}
37+
38+
$ProjectRoot = Resolve-Path "$PSScriptRoot/../test/Libraries"
39+
$ReportingConfig = "Microsoft.Extensions.AI.Evaluation.Reporting.Tests/appsettings.local.json"
40+
$IntegrationConfig = "Microsoft.Extensions.AI.Evaluation.Integration.Tests/appsettings.local.json"
41+
42+
if ($Configure) {
43+
if (!(Test-Path -Path "$ConfigRoot/$ReportingConfig")) {
44+
Write-Host "No configuration found at $ConfigRoot/$ReportingConfig"
45+
Exit 0
46+
}
47+
if (!(Test-Path -Path "$ConfigRoot/$IntegrationConfig")) {
48+
Write-Host "No configuration found at $ConfigRoot/$IntegrationConfig"
49+
Exit 0
50+
}
51+
52+
Copy-Item -Path "$ConfigRoot/$ReportingConfig" -Destination "$ProjectRoot/$ReportingConfig" -Force
53+
Copy-Item -Path "$ConfigRoot/$IntegrationConfig" -Destination "$ProjectRoot/$IntegrationConfig" -Force
54+
55+
Write-Host "Test configured to use external resources"
56+
} elseif ($Unconfigure) {
57+
Remove-Item -Path "$ProjectRoot/$ReportingConfig" -Force
58+
Remove-Item -Path "$ProjectRoot/$IntegrationConfig" -Force
59+
60+
Write-Host "Test unconfigured from using external resources"
61+
} else {
62+
Write-Error -Message "Must specify either -Configure or -Unconfigure"
63+
Exit 1
64+
}
65+

src/Libraries/Directory.Build.props

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111
<InjectTrimAttributesOnLegacy Condition="'$(InjectTrimAttributesOnLegacy)' == ''">true</InjectTrimAttributesOnLegacy>
1212
<InjectSkipLocalsInitAttributeOnLegacy Condition="'$(InjectSkipLocalsInitAttributeOnLegacy)' == ''">true</InjectSkipLocalsInitAttributeOnLegacy>
1313
<InjectCallerAttributesOnLegacy Condition="'$(InjectCallerAttributesOnLegacy)' == ''">true</InjectCallerAttributesOnLegacy>
14-
<InjectDiagnosticAttributesOnLegacy Condition="'$(InjectDiagnosticAttributesOnLegacy)' == ''">true</InjectDiagnosticAttributesOnLegacy>
1514
<InjectSharedThrow Condition="'$(InjectSharedThrow)' == ''">true</InjectSharedThrow>
1615
</PropertyGroup>
1716
</Project>
Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
4+
using System.IO;
5+
using System.Threading;
6+
using System.Threading.Tasks;
7+
using Microsoft.Extensions.AI.Evaluation.Console.Utilities;
8+
using Microsoft.Extensions.AI.Evaluation.Reporting.Storage;
9+
using Microsoft.Extensions.Logging;
10+
11+
namespace Microsoft.Extensions.AI.Evaluation.Console.Commands;
12+
13+
internal sealed class CleanCacheCommand(ILogger logger)
14+
{
15+
internal async Task<int> InvokeAsync(DirectoryInfo storageRootDir, CancellationToken cancellationToken = default)
16+
{
17+
string storageRootPath = storageRootDir.FullName;
18+
logger.LogInformation("Storage root path: {storageRootPath}", storageRootPath);
19+
logger.LogInformation("Deleting expired cache entries...");
20+
21+
var cacheProvider = new DiskBasedResponseCacheProvider(storageRootPath);
22+
23+
await logger.ExecuteWithCatchAsync(
24+
() => cacheProvider.DeleteExpiredCacheEntriesAsync(cancellationToken)).ConfigureAwait(false);
25+
26+
return 0;
27+
}
28+
}

0 commit comments

Comments
 (0)