Skip to content

Commit 2209ec1

Browse files
saheerbsaheerb
committed
github actions for mbed-os-env docker management
1 parent 862a942 commit 2209ec1

File tree

10 files changed

+1075
-0
lines changed

10 files changed

+1075
-0
lines changed

‎.github/workflows/ci_scripts/ghcr_utils.py‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
#!/usr/bin/env python
2+
3+
"""
4+
Copyright (c) 2017-2021 ARM Limited. All rights reserved.
5+
SPDX-License-Identifier: Apache-2.0
6+
Licensed under the Apache License, Version 2.0 (the "License");
7+
you may not use this file except in compliance with the License.
8+
You may obtain a copy of the License at
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations
15+
"""
16+
import click
17+
import requests
18+
import logging
19+
import sys
20+
import time
21+
import json
22+
import subprocess
23+
24+
"""
25+
This file contains ghcr utlity wrapper used for:
26+
- retrieving digest of docker image
27+
- deleting images in ghcr
28+
"""
29+
@click.command()
30+
@click.pass_context
31+
@click.option("-r", "--repository", required=True)
32+
@click.option("-t", "--tag", required=True)
33+
@click.option("-p", "--platform", required=False)
34+
def get_digest(ctx, repository, tag, platform=None):
35+
command = f"docker run quay.io/skopeo/stable --creds={ctx.obj['username']}:{ctx.obj['passwd']} inspect docker://ghcr.io/{ctx.obj['username']}/{repository}:{tag} --raw"
36+
output = subprocess.run(command.split(), stdout=subprocess.PIPE).stdout.decode(
37+
"utf-8"
38+
)
39+
output = json.loads(output)
40+
# logging.info(output)
41+
42+
images = output["manifests"]
43+
digest = ""
44+
if len(images) > 1 and platform == None:
45+
logging.error(
46+
"This tag has more than one platform associated to it, please input a platform"
47+
)
48+
sys.exit(1)
49+
50+
for image in images:
51+
if platform != None:
52+
if (platform.split("/")[0] == image["platform"]["os"]) and (
53+
platform.split("/")[1] == image["platform"]["architecture"]
54+
):
55+
digest = image["digest"]
56+
else:
57+
digest = image["digest"]
58+
59+
print(digest)
60+
61+
62+
# delete images older than provided argument (number_of_days)
63+
@click.command()
64+
@click.pass_context
65+
@click.option("-r", "--repository", required=True)
66+
@click.option(
67+
"-n",
68+
"--number_of_days",
69+
default="10",
70+
help="number of days since image was created",
71+
required=False,
72+
)
73+
def delete_old_images(ctx, repository, number_of_days):
74+
s = requests.Session()
75+
github_api_accept = "application/vnd.github.v3+json"
76+
s.headers.update(
77+
{"Authorization": f'token {ctx.obj["passwd"]}', "Accept": github_api_accept}
78+
)
79+
r = s.get(f"https://api.github.com/user/packages/container/{repository}/versions")
80+
versions = r.json()
81+
version_id = None
82+
pattern = "%d.%m.%Y %H:%M:%S"
83+
pattern = "%Y-%m-%dT%H:%M:%SZ"
84+
current_time = time.time()
85+
for version in versions:
86+
logging.info (version)
87+
epoch = int(time.mktime(time.strptime(version["updated_at"], pattern)))
88+
89+
if (current_time - epoch) / (24 * 60 * 60) > int(number_of_days):
90+
version_id = version["id"]
91+
logging.debug(f"deleteing image with version id {version_id}")
92+
93+
url = f"https://api.github.com/user/packages/container/{repository}/versions/{version_id}"
94+
resp = s.delete(url)
95+
resp.raise_for_status()
96+
97+
98+
@click.group()
99+
@click.pass_context
100+
@click.option("-u", "--username", required=False)
101+
@click.option("-p", "--passwd", required=False)
102+
@click.option("-v", "--verbose", is_flag=True, default=False)
103+
def main(ctx, username, passwd, verbose):
104+
ctx.obj = {"username": username, "passwd": passwd}
105+
106+
if verbose:
107+
logging.basicConfig(
108+
stream=sys.stdout,
109+
format="%(levelname)s %(asctime)s %(message)s",
110+
datefmt="%m/%d/%Y %I:%M:%S %p",
111+
)
112+
logging.getLogger().setLevel(logging.DEBUG)
113+
else:
114+
logging.basicConfig(
115+
format="%(levelname)s %(asctime)s %(message)s",
116+
datefmt="%m/%d/%Y %I:%M:%S %p",
117+
)
118+
logging.getLogger().setLevel(logging.INFO)
119+
120+
121+
if __name__ == "__main__":
122+
main.add_command(get_digest)
123+
main.add_command(delete_old_images)
124+
main()

‎.github/workflows/docker_management.branch.yml‎

Lines changed: 245 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,245 @@
1+
name: Publish or Update docker image for head of branch
2+
# This work flow is disabled for forked repositories.
3+
# If you need to enable it, find references for github.repository_owner in this file
4+
# and replace with ARMmbed with your organisation/account name
5+
# Read more details in https://github.com/ARMmbed/mbed-os/blob/master/docs/design-documents/docker_management
6+
7+
8+
9+
on:
10+
11+
# passive update once a week
12+
schedule:
13+
- cron: '15 4 * * 6'
14+
15+
# build on master branch when there is changes for active update
16+
push:
17+
branches:
18+
- master
19+
20+
paths:
21+
- requirements.txt
22+
- docker_images/mbed-os-env/**
23+
- .github/workflows/docker_management.branch.yml
24+
25+
26+
# manual trigger when needed
27+
workflow_dispatch:
28+
29+
30+
jobs:
31+
prepare-tags:
32+
if: github.repository_owner == 'ARMmbed'
33+
runs-on: ubuntu-latest
34+
35+
steps:
36+
-
37+
name: Extract branch name
38+
shell: bash
39+
run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
40+
id: extract_branch
41+
42+
-
43+
name: Checkout
44+
uses: actions/checkout@v2
45+
with:
46+
fetch-depth: 0
47+
48+
-
49+
name: Set UUID
50+
id: generate-uuid
51+
uses: filipstefansson/uuid-action@v1
52+
53+
# set docker tags we are building, and intending to publish
54+
# dev-tag is temporary for testing purpose. This should be considered as unstable.
55+
# dated-tag is created for versioning purpose
56+
# prod-tag-latest could be used by customers, CI etc for keeping up to date
57+
-
58+
name: Get build information
59+
shell: bash
60+
run: |
61+
mkdir -p build_info
62+
date=$(date +"%Y.%m.%dT%H.%M.%S")
63+
echo dev-${{ steps.extract_branch.outputs.branch }}-${date}-${{ steps.generate-uuid.outputs.uuid }} > build_info/dev_tag
64+
echo ${{ steps.extract_branch.outputs.branch }}-${date} > build_info/prod_tag_dated
65+
echo ${{ steps.extract_branch.outputs.branch }}-latest > build_info/prod_tag_latest
66+
echo ${{ steps.extract_branch.outputs.branch }} > build_info/mbed_os_version
67+
68+
-
69+
name: Archive information
70+
uses: actions/upload-artifact@v2
71+
with:
72+
name: build-info
73+
path: build_info
74+
75+
76+
build-container:
77+
runs-on: ubuntu-latest
78+
needs: prepare-tags
79+
outputs:
80+
DEV_DIGEST: ${{ steps.docker_info_dev.outputs.DIGEST }}
81+
PROD_DIGEST: ${{ steps.docker_info_prod.outputs.DIGEST }}
82+
83+
steps:
84+
-
85+
name: unarchive artefacts
86+
uses: actions/download-artifact@v2
87+
with:
88+
name: build-info
89+
90+
-
91+
name: Get build info from archive
92+
shell: bash
93+
id: build_info
94+
run: |
95+
value=`cat dev_tag`
96+
echo "DEV TAG is $value"
97+
echo "::set-output name=DOCKER_DEV_TAG::$value"
98+
value=`cat prod_tag_dated`
99+
echo "PROD TAG DATED is $value"
100+
echo "::set-output name=DOCKER_PROD_TAG_DATED::$value"
101+
value=`cat prod_tag_latest`
102+
echo "::set-output name=DOCKER_PROD_TAG_LATEST::$value"
103+
echo "PROD TAG is $value"
104+
105+
-
106+
name: Set up Docker Buildx
107+
uses: docker/setup-buildx-action@v1
108+
109+
-
110+
name: Set up QEMU
111+
uses: docker/setup-qemu-action@v1
112+
113+
-
114+
name: Login to DockerHub
115+
uses: docker/login-action@v1
116+
with:
117+
registry: ghcr.io
118+
username: ${{ github.actor }}
119+
password: ${{ secrets.GITHUB_TOKEN }}
120+
121+
-
122+
name: Checkout
123+
uses: actions/checkout@v2
124+
125+
-
126+
name: Build docker containers
127+
uses: docker/build-push-action@v2
128+
id: docker_build_dev
129+
with:
130+
context: .
131+
platforms: linux/amd64,linux/arm64
132+
push: true
133+
file: ./docker_images/mbed-os-env/Dockerfile
134+
tags: ghcr.io/${{ github.actor }}/mbed-os-env-tmp:${{ steps.build_info.outputs.DOCKER_DEV_TAG }}
135+
136+
test-container:
137+
runs-on: ubuntu-latest
138+
needs: build-container
139+
strategy:
140+
matrix:
141+
platform: [linux/amd64, linux/arm64]
142+
143+
steps:
144+
-
145+
name: unarchive artefacts
146+
uses: actions/download-artifact@v2
147+
with:
148+
name: build-info
149+
150+
-
151+
name: Get build info from archive
152+
shell: bash
153+
id: build_info
154+
run: |
155+
value=`cat dev_tag`
156+
echo "TAG is $value"
157+
echo "::set-output name=DOCKER_DEV_TAG::$value"
158+
value=`cat prod_tag_dated`
159+
echo "TAG is $value"
160+
echo "::set-output name=DOCKER_PROD_TAG_DATED::$value"
161+
value=`cat prod_tag_latest`
162+
echo "::set-output name=DOCKER_PROD_TAG_LATEST::$value"
163+
value=`cat mbed_os_version`
164+
echo "::set-output name=MBED_OS_VERSION::$value"
165+
166+
-
167+
name: Checkout
168+
uses: actions/checkout@v2
169+
170+
-
171+
name: Find DEV DOCKER DIGEST
172+
id: docker_info_dev
173+
run: |
174+
DIGEST=$(python ./.github/workflows/ci_scripts/ghcr_utils.py -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} get-digest -r mbed-os-env-tmp -t ${{ steps.build_info.outputs.DOCKER_DEV_TAG }} -p ${{ matrix.platform }} )
175+
echo "::set-output name=DIGEST::$DIGEST"
176+
echo "Docker DIGEST: $DIGEST"
177+
178+
# as the dev images are created only for master branch, run test against
179+
# development branch of blinky
180+
-
181+
name: Checkout
182+
uses: actions/checkout@v2
183+
with:
184+
repository: ARMmbed/mbed-os-example-blinky
185+
path: mbed-os-example-blinky
186+
ref: development
187+
-
188+
name: Set up QEMU
189+
uses: docker/setup-qemu-action@v1
190+
191+
-
192+
name: test the container
193+
id: test
194+
uses: addnab/docker-run-action@v3
195+
with:
196+
username: ${{ github.actor }}
197+
password: ${{ secrets.GITHUB_TOKEN }}
198+
registry: ghcr.io
199+
options: -v ${{ github.workspace }}:/work -w=/work
200+
image: ghcr.io/${{ github.actor }}/mbed-os-env-tmp@${{ steps.docker_info_dev.outputs.DIGEST }}
201+
shell: bash
202+
203+
run: |
204+
uname -m
205+
ls
206+
#./mbed-os/docker_images/mbed-os-env/test.sh DEVELOPMENT ${{ steps.build_info.outputs.MBED_OS_VERSION }}
207+
cd mbed-os-example-blinky
208+
mbed deploy
209+
# build using CLI1
210+
mbed compile -m K64F -t GCC_ARM
211+
212+
# build using CLI2
213+
mbed-tools compile -m K64F -t GCC_ARM
214+
215+
216+
deploy-container:
217+
runs-on: ubuntu-latest
218+
needs: test-container
219+
220+
steps:
221+
-
222+
name: unarchive artefacts
223+
uses: actions/download-artifact@v2
224+
with:
225+
name: build-info
226+
227+
-
228+
name: Get build info from archive
229+
shell: bash
230+
id: build_info
231+
run: |
232+
value=`cat dev_tag`
233+
echo "TAG is $value"
234+
echo "::set-output name=DOCKER_DEV_TAG::$value"
235+
value=`cat prod_tag_dated`
236+
echo "TAG is $value"
237+
echo "::set-output name=DOCKER_PROD_TAG_DATED::$value"
238+
value=`cat prod_tag_latest`
239+
echo "::set-output name=DOCKER_PROD_TAG_LATEST::$value"
240+
241+
-
242+
name: copy dev tag to prod
243+
run: |
244+
docker run quay.io/skopeo/stable --src-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} --dest-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} copy --all docker://ghcr.io/${{ github.actor }}/mbed-os-env-tmp:${{ steps.build_info.outputs.DOCKER_DEV_TAG }} docker://ghcr.io/${{ github.actor }}/mbed-os-env:${{ steps.build_info.outputs.DOCKER_PROD_TAG_LATEST }}
245+
docker run quay.io/skopeo/stable --src-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} --dest-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} copy --all docker://ghcr.io/${{ github.actor }}/mbed-os-env-tmp:${{ steps.build_info.outputs.DOCKER_DEV_TAG }} docker://ghcr.io/${{ github.actor }}/mbed-os-env:${{ steps.build_info.outputs.DOCKER_PROD_TAG_DATED }}

0 commit comments

Comments
 (0)