Skip to content

Commit d5fe389

Browse files
haofan-mschristinalau0
haofan-ms
and
christinalau0
authored
chore: new Windows signed scripts and package version (#176)
* windows scripts changes * new scripts version * update windows provision script package version * clean up * address comments * additional windows scripts changes * additional * missed quote * Update generated code --------- Co-authored-by: christinalau0 <[email protected]> Co-authored-by: Christina Lau <[email protected]>
1 parent 3d5c791 commit d5fe389

18 files changed

+294
-673
lines changed

‎packer.mk‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ run-packer-windows: az-login
1717
@packer version && set -o pipefail && ($(MAKE) init-packer | tee packer-output) && ($(MAKE) build-packer-windows | tee -a packer-output)
1818

1919
az-copy: az-login
20-
azcopy-preview copy "${OS_DISK_SAS}" "${SA_CONTAINER_URL}?${SA_TOKEN}"
20+
azcopy-preview copy "${OS_DISK_SAS}" "${SA_CONTAINER_URL}?${SA_TOKEN}" --overwrite=false
2121

2222
delete-sa: az-login
2323
az storage account delete -n ${PACKER_TEMP_SA} -g ${PACKER_TEMP_GROUP} --yes

‎parts/k8s/kuberneteswindowsfunctions.ps1‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -200,7 +200,7 @@ function Get-LogCollectionScripts {
200200
DownloadFileOverHttp -Url 'https://github.com/microsoft/SDN/raw/master/Kubernetes/windows/debug/stoppacketcapture.cmd' -DestinationPath 'c:\k\debug\stoppacketcapture.cmd'
201201
DownloadFileOverHttp -Url 'https://github.com/Microsoft/SDN/raw/master/Kubernetes/windows/debug/VFP.psm1' -DestinationPath 'c:\k\debug\VFP.psm1'
202202
DownloadFileOverHttp -Url 'https://github.com/microsoft/SDN/raw/master/Kubernetes/windows/helper.psm1' -DestinationPath 'c:\k\debug\helper.psm1'
203-
DownloadFileOverHttp -Url 'https://github.com/microsoft/SDN/raw/master/Kubernetes/windows/hns.psm1' -DestinationPath 'c:\k\debug\hns.psm1'
203+
DownloadFileOverHttp -Url 'https://github.com/microsoft/SDN/raw/master/Kubernetes/windows/hns.v2.psm1' -DestinationPath 'c:\k\debug\hns.v2.psm1'
204204
}
205205

206206
function Register-LogsCleanupScriptTask {
@@ -235,7 +235,7 @@ function Write-KubeClusterConfig {
235235
$Global:ClusterConfiguration = [PSCustomObject]@{ }
236236

237237
$Global:ClusterConfiguration | Add-Member -MemberType NoteProperty -Name Cri -Value @{
238-
Name = $global:ContainerRuntime;
238+
Name = "containerd";
239239
Images = @{
240240
# e.g. "mcr.microsoft.com/oss/kubernetes/pause:3.8"
241241
"Pause" = $global:WindowsPauseImageURL
@@ -298,10 +298,8 @@ function Update-DefenderPreferences {
298298
Add-MpPreference -ExclusionProcess "c:\k\kubelet.exe"
299299

300300
if ($global:EnableCsiProxy) {
301-
Add-MpPreference -ExclusionProcess "c:\k\csi-proxy-server.exe"
301+
Add-MpPreference -ExclusionProcess "c:\k\csi-proxy.exe"
302302
}
303303

304-
if ($global:ContainerRuntime -eq 'containerd') {
305-
Add-MpPreference -ExclusionProcess "c:\program files\containerd\containerd.exe"
306-
}
304+
Add-MpPreference -ExclusionProcess "c:\program files\containerd\containerd.exe"
307305
}

‎parts/k8s/kuberneteswindowssetup.ps1‎

Lines changed: 19 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ $global:ExcludeMasterFromStandardLB = "{{WrapAsVariable "excludeMasterFromStanda
115115
# Windows defaults, not changed by aks-engine
116116
$global:CacheDir = "c:\akse-cache"
117117
$global:KubeDir = "c:\k"
118-
$global:HNSModule = [Io.path]::Combine("$global:KubeDir", "hns.psm1")
118+
$global:HNSModule = [Io.path]::Combine("$global:KubeDir", "hns.v2.psm1")
119119

120120
$global:KubeDnsSearchPath = "svc.cluster.local"
121121

@@ -173,7 +173,6 @@ Expand-Archive scripts.zip -DestinationPath "C:\\AzureData\\"
173173
. c:\AzureData\k8s\windowscontainerdfunc.ps1
174174
. c:\AzureData\k8s\windowshostsconfigagentfunc.ps1
175175

176-
$useContainerD = ($global:ContainerRuntime -eq "containerd")
177176
$global:KubeClusterConfigPath = "c:\k\kubeclusterconfig.json"
178177

179178
try
@@ -277,27 +276,17 @@ try
277276
Get-KubeBinaries -KubeBinariesURL $global:WindowsKubeBinariesURL
278277
}
279278

280-
if ($useContainerD) {
281-
Write-Log "Installing ContainerD"
282-
$containerdTimer = [System.Diagnostics.Stopwatch]::StartNew()
283-
$cniBinPath = $global:AzureCNIBinDir
284-
$cniConfigPath = $global:AzureCNIConfDir
285-
if ($global:NetworkPlugin -eq "kubenet") {
286-
$cniBinPath = $global:CNIPath
287-
$cniConfigPath = $global:CNIConfigPath
288-
}
289-
Install-Containerd -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath -KubeDir $global:KubeDir
290-
$containerdTimer.Stop()
291-
$global:AppInsightsClient.TrackMetric("Install-ContainerD", $containerdTimer.Elapsed.TotalSeconds)
292-
# TODO: disable/uninstall Docker later
293-
} else {
294-
Write-Log "Install docker"
295-
$dockerTimer = [System.Diagnostics.Stopwatch]::StartNew()
296-
Install-Docker -DockerVersion $global:DockerVersion
297-
Set-DockerLogFileOptions
298-
$dockerTimer.Stop()
299-
$global:AppInsightsClient.TrackMetric("Install-Docker", $dockerTimer.Elapsed.TotalSeconds)
279+
Write-Log "Installing ContainerD"
280+
$containerdTimer = [System.Diagnostics.Stopwatch]::StartNew()
281+
$cniBinPath = $global:AzureCNIBinDir
282+
$cniConfigPath = $global:AzureCNIConfDir
283+
if ($global:NetworkPlugin -eq "kubenet") {
284+
$cniBinPath = $global:CNIPath
285+
$cniConfigPath = $global:CNIConfigPath
300286
}
287+
Install-Containerd -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath -KubeDir $global:KubeDir
288+
$containerdTimer.Stop()
289+
$global:AppInsightsClient.TrackMetric("Install-ContainerD", $containerdTimer.Elapsed.TotalSeconds)
301290

302291
Write-Log "Write Azure cloud provider config"
303292
Write-AzureConfig `
@@ -351,19 +340,14 @@ try
351340

352341
Write-Log "Create the Pause Container kubletwin/pause"
353342
$infraContainerTimer = [System.Diagnostics.Stopwatch]::StartNew()
354-
New-InfraContainer -KubeDir $global:KubeDir -ContainerRuntime $global:ContainerRuntime
343+
New-InfraContainer -KubeDir $global:KubeDir
355344
$infraContainerTimer.Stop()
356345
$global:AppInsightsClient.TrackMetric("New-InfraContainer", $infraContainerTimer.Elapsed.TotalSeconds)
357346

358-
if (-not (Test-ContainerImageExists -Image "kubletwin/pause" -ContainerRuntime $global:ContainerRuntime)) {
347+
if (-not (Test-ContainerImageExists -Image "kubletwin/pause")) {
359348
Write-Log "Could not find container with name kubletwin/pause"
360-
if ($useContainerD) {
361-
$o = ctr -n k8s.io image list
362-
Write-Log $o
363-
} else {
364-
$o = docker image list
365-
Write-Log $o
366-
}
349+
$o = ctr -n k8s.io image list
350+
Write-Log $o
367351
throw "kubletwin/pause container does not exist!"
368352
}
369353

@@ -404,19 +388,14 @@ try
404388
}
405389
elseif ($global:NetworkPlugin -eq "kubenet") {
406390
Write-Log "Fetching additional files needed for kubenet"
407-
if ($useContainerD) {
408-
# TODO: CNI may need to move to c:\program files\containerd\cni\bin with ContainerD
409-
Install-SdnBridge -Url $global:ContainerdSdnPluginUrl -CNIPath $global:CNIPath
410-
} else {
411-
Update-WinCNI -CNIPath $global:CNIPath
412-
}
391+
# TODO: CNI may need to move to c:\program files\containerd\cni\bin with ContainerD
392+
Install-SdnBridge -Url $global:ContainerdSdnPluginUrl -CNIPath $global:CNIPath
413393
}
414394

415395
New-ExternalHnsNetwork -IsDualStackEnabled $global:IsDualStackEnabled
416396

417397
Install-KubernetesServices `
418-
-KubeDir $global:KubeDir `
419-
-ContainerRuntime $global:ContainerRuntime
398+
-KubeDir $global:KubeDir
420399

421400
Get-LogCollectionScripts
422401

@@ -441,7 +420,7 @@ try
441420
PREPROVISION_EXTENSION
442421

443422
Write-Log "Update service failure actions"
444-
Update-ServiceFailureActions -ContainerRuntime $global:ContainerRuntime
423+
Update-ServiceFailureActions
445424

446425
Adjust-DynamicPortRange
447426
Register-LogsCleanupScriptTask

‎parts/k8s/windowsazurecnifunc.ps1‎

Lines changed: 31 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,4 @@
1-
2-
3-
# TODO: remove - dead code?
4-
function
5-
Set-VnetPluginMode()
6-
{
7-
Param(
8-
[Parameter(Mandatory=$true)][string]
9-
$AzureCNIConfDir,
10-
[Parameter(Mandatory=$true)][string]
11-
$Mode
12-
)
13-
# Sets Azure VNET CNI plugin operational mode.
14-
$fileName = [Io.path]::Combine("$AzureCNIConfDir", "10-azure.conflist")
15-
(Get-Content $fileName) | %{$_ -replace "`"mode`":.*", "`"mode`": `"$Mode`","} | Out-File -encoding ASCII -filepath $fileName
16-
}
17-
18-
19-
function
20-
Install-VnetPlugins
1+
function Install-VnetPlugins
212
{
223
Param(
234
[Parameter(Mandatory=$true)][string]
@@ -44,16 +25,7 @@ Install-VnetPlugins
4425
move $AzureCNIBinDir/*.conflist $AzureCNIConfDir
4526
}
4627

47-
# TODO: remove - dead code?
48-
function
49-
Set-AzureNetworkPlugin()
50-
{
51-
# Azure VNET network policy requires tunnel (hairpin) mode because policy is enforced in the host.
52-
Set-VnetPluginMode "tunnel"
53-
}
54-
55-
function
56-
Set-AzureCNIConfig
28+
function Set-AzureCNIConfig
5729
{
5830
Param(
5931
[Parameter(Mandatory=$true)][string]
@@ -98,7 +70,35 @@ Set-AzureCNIConfig
9870
$configJson.plugins.AdditionalArgs[0].Value.ExceptionList = $processedExceptions
9971
}
10072
else {
101-
$configJson.plugins.AdditionalArgs[0].Value.ExceptionList = $exceptionAddresses
73+
if ($IsDualStackEnabled) {
74+
$ipv4Cidrs = @()
75+
$ipv6Cidrs = @()
76+
foreach ($cidr in $exceptionAddresses) {
77+
# this is the pwsh way of strings.Count(s, ":") >= 2
78+
if (($cidr -split ":").Count -ge 3) {
79+
$ipv6Cidrs += $cidr
80+
} else {
81+
$ipv4Cidrs += $cidr
82+
}
83+
}
84+
85+
# we just assume the first entry in additional Args is the exception
86+
# list for IPv4 and then append a new EnpointPolicy for IPv6. We
87+
# probably shouldn't hard code the first one like this and just build
88+
# 2 EndpointPolicies and append to the AdditionalArgs.
89+
$configJson.plugins.AdditionalArgs[0].Value.ExceptionList = $ipv4Cidrs
90+
91+
$outboundException = [PSCustomObject]@{
92+
Name = 'EndpointPolicy'
93+
Value = [PSCustomObject]@{
94+
Type = 'OutBoundNAT'
95+
ExceptionList = $ipv6Cidrs
96+
}
97+
}
98+
$configJson.plugins[0].AdditionalArgs += $outboundException
99+
} else {
100+
$configJson.plugins.AdditionalArgs[0].Value.ExceptionList = $exceptionAddresses
101+
}
102102
}
103103

104104
if ($IsDualStackEnabled){

‎parts/k8s/windowscnifunc.ps1‎

Lines changed: 1 addition & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2,26 +2,13 @@ function Get-HnsPsm1
22
{
33
Param(
44
[string]
5-
$HnsUrl = "https://github.com/Microsoft/SDN/raw/master/Kubernetes/windows/hns.psm1",
5+
$HnsUrl = "https://github.com/Microsoft/SDN/raw/master/Kubernetes/windows/hns.v2.psm1",
66
[Parameter(Mandatory=$true)][string]
77
$HNSModule
88
)
99
DownloadFileOverHttp -Url $HnsUrl -DestinationPath "$HNSModule"
1010
}
1111

12-
function Update-WinCNI
13-
{
14-
Param(
15-
[string]
16-
$WinCniUrl = "https://github.com/Microsoft/SDN/raw/master/Kubernetes/flannel/l2bridge/cni/win-bridge.exe",
17-
[Parameter(Mandatory=$true)][string]
18-
$CNIPath
19-
)
20-
$wincni = "win-bridge.exe"
21-
$wincniFile = [Io.path]::Combine($CNIPath, $wincni)
22-
DownloadFileOverHttp -Url $WinCniUrl -DestinationPath $wincniFile
23-
}
24-
2512
function Install-SdnBridge
2613
{
2714
Param(

‎parts/k8s/windowsconfigfunc.ps1‎

Lines changed: 1 addition & 94 deletions
Original file line numberDiff line numberDiff line change
@@ -40,95 +40,6 @@ function Set-Explorer
4040
New-ItemProperty -Path HKLM:"\\SOFTWARE\\Policies\\Microsoft\\Internet Explorer\\Main" -Name "Start Page" -Type String -Value http://bing.com
4141
}
4242

43-
function Install-Docker
44-
{
45-
Param(
46-
[Parameter(Mandatory=$true)][string]
47-
$DockerVersion
48-
)
49-
50-
# DOCKER_API_VERSION needs to be set for Docker versions older than 18.09.0 EE
51-
# due to https://github.com/kubernetes/kubernetes/issues/69996
52-
# this issue was fixed by https://github.com/kubernetes/kubernetes/issues/69996#issuecomment-438499024
53-
# Note: to get a list of all versions, use this snippet
54-
# $versions = (curl.exe -L "https://go.microsoft.com/fwlink/?LinkID=825636&clcid=0x409" | ConvertFrom-Json).Versions | Get-Member -Type NoteProperty | Select-Object Name
55-
# Docker version to API version decoder: https://docs.docker.com/develop/sdk/#api-version-matrix
56-
57-
switch ($DockerVersion.Substring(0,5))
58-
{
59-
"17.06" {
60-
Write-Log "Docker 17.06 found, setting DOCKER_API_VERSION to 1.30"
61-
[System.Environment]::SetEnvironmentVariable('DOCKER_API_VERSION', '1.30', [System.EnvironmentVariableTarget]::Machine)
62-
}
63-
64-
"18.03" {
65-
Write-Log "Docker 18.03 found, setting DOCKER_API_VERSION to 1.37"
66-
[System.Environment]::SetEnvironmentVariable('DOCKER_API_VERSION', '1.37', [System.EnvironmentVariableTarget]::Machine)
67-
}
68-
69-
default {
70-
Write-Log "Docker version $DockerVersion found, clearing DOCKER_API_VERSION"
71-
[System.Environment]::SetEnvironmentVariable('DOCKER_API_VERSION', $null, [System.EnvironmentVariableTarget]::Machine)
72-
}
73-
}
74-
75-
try {
76-
$installDocker = $true
77-
$dockerService = Get-Service | ? Name -like 'docker'
78-
if ($dockerService.Count -eq 0) {
79-
Write-Log "Docker is not installed. Install docker version($DockerVersion)."
80-
}
81-
else {
82-
$dockerServerVersion = docker version --format '{{.Server.Version}}'
83-
Write-Log "Docker service is installed with docker version($dockerServerVersion)."
84-
if ($dockerServerVersion -eq $DockerVersion) {
85-
$installDocker = $false
86-
Write-Log "Same version docker installed will skip installing docker version($dockerServerVersion)."
87-
}
88-
else {
89-
Write-Log "Same version docker is not installed. Will install docker version($DockerVersion)."
90-
}
91-
}
92-
93-
if ($installDocker) {
94-
Find-Package -Name Docker -ProviderName DockerMsftProvider -RequiredVersion $DockerVersion -ErrorAction Stop
95-
Write-Log "Found version $DockerVersion. Installing..."
96-
Install-Package -Name Docker -ProviderName DockerMsftProvider -Update -Force -RequiredVersion $DockerVersion
97-
net start docker
98-
Write-Log "Installed version $DockerVersion"
99-
}
100-
} catch {
101-
Write-Log "Error while installing package: $_.Exception.Message"
102-
$currentDockerVersion = (Get-Package -Name Docker -ProviderName DockerMsftProvider).Version
103-
Write-Log "Not able to install docker version. Using default version $currentDockerVersion"
104-
}
105-
}
106-
107-
function Set-DockerLogFileOptions {
108-
Write-Log "Updating log file options in docker config"
109-
$dockerConfigPath = "C:\ProgramData\docker\config\daemon.json"
110-
111-
if (-not (Test-Path $dockerConfigPath)) {
112-
"{}" | Out-File $dockerConfigPath
113-
}
114-
115-
$dockerConfig = Get-Content $dockerConfigPath | ConvertFrom-Json
116-
$dockerConfig | Add-Member -Name "log-driver" -Value "json-file" -MemberType NoteProperty
117-
$logOpts = @{ "max-size" = "50m"; "max-file" = "5" }
118-
$dockerConfig | Add-Member -Name "log-opts" -Value $logOpts -MemberType NoteProperty
119-
$dockerConfig = $dockerConfig | ConvertTo-Json -Depth 10
120-
121-
Write-Log "New docker config:"
122-
Write-Log $dockerConfig
123-
124-
# daemon.json MUST be encoded as UTF8-no-BOM!
125-
Remove-Item $dockerConfigPath
126-
$fileEncoding = New-Object System.Text.UTF8Encoding $false
127-
[IO.File]::WriteAllLInes($dockerConfigPath, $dockerConfig, $fileEncoding)
128-
129-
Restart-Service docker
130-
}
131-
13243
# Pagefile adjustments
13344
function Adjust-PageFileSize()
13445
{
@@ -153,13 +64,9 @@ function Adjust-DynamicPortRange()
15364
# Service start actions. These should be split up later and included in each install step
15465
function Update-ServiceFailureActions
15566
{
156-
Param(
157-
[Parameter(Mandatory = $true)][string]
158-
$ContainerRuntime
159-
)
16067
sc.exe failure "kubelet" actions= restart/60000/restart/60000/restart/60000 reset= 900
16168
sc.exe failure "kubeproxy" actions= restart/60000/restart/60000/restart/60000 reset= 900
162-
sc.exe failure $ContainerRuntime actions= restart/60000/restart/60000/restart/60000 reset= 900
69+
sc.exe failure "containerd" actions= restart/60000/restart/60000/restart/60000 reset= 900
16370
}
16471

16572
function Add-SystemPathEntry

‎parts/k8s/windowscontainerdfunc.ps1‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,27 @@ function RegisterContainerDService {
3333
& "$KubeDir\nssm.exe" set containerd AppRotateSeconds 86400 | RemoveNulls
3434
& "$KubeDir\nssm.exe" set containerd AppRotateBytes 10485760 | RemoveNulls
3535

36-
$svc = Get-Service -Name "containerd" -ErrorAction SilentlyContinue
37-
if ($svc.Status -ne "Running") {
36+
$retryCount=0
37+
$retryInterval=10
38+
$maxRetryCount=6 # 1 minutes
39+
40+
do {
41+
$svc = Get-Service -Name "containerd" -ErrorAction SilentlyContinue
42+
if ($null -eq $svc) {
43+
throw "Error: containerd.exe did not get installed as a service correctly."
44+
}
45+
if ($svc.Status -eq "Running") {
46+
break
47+
}
48+
Write-Log "Starting containerd, current status: $svc.Status"
3849
Start-Service containerd
50+
$retryCount++
51+
Write-Log "Retry $retryCount : Sleep $retryInterval and check containerd status"
52+
Sleep $retryInterval
53+
} while ($retryCount -lt $maxRetryCount)
54+
55+
if ($svc.Status -ne "Running") {
56+
throw "Eror: containerd service is not running"
3957
}
4058
}
4159

0 commit comments

Comments
 (0)