From ce4e87550a83a211fe7939c46e4fb50eddd114c6 Mon Sep 17 00:00:00 2001 From: David Fridrich Date: Mon, 18 Aug 2025 14:13:08 +0200 Subject: [PATCH] fix: remote python pack deployment --- cmd/func-util/main.go | 9 +- pkg/pipelines/tekton/pipelines_provider.go | 126 ++++++++++++++++++++- pkg/pipelines/tekton/tasks.go | 26 +++-- pkg/pipelines/tekton/templates.go | 42 ++++--- pkg/pipelines/tekton/templates_pack.go | 9 +- 5 files changed, 181 insertions(+), 31 deletions(-) diff --git a/cmd/func-util/main.go b/cmd/func-util/main.go index c7b2bb4e8f..4d1830ca47 100644 --- a/cmd/func-util/main.go +++ b/cmd/func-util/main.go @@ -87,7 +87,12 @@ func scaffold(ctx context.Context) error { } if f.Runtime != "go" && f.Runtime != "python" { - // Scaffolding is for now supported/needed only for Go. + // Scaffolding is for now supported/needed only for Go/Python + return nil + } + + // special case for python-pack + if f.Build.Builder == "pack" && f.Runtime == "python" { return nil } @@ -147,7 +152,7 @@ func deploy(ctx context.Context) error { return fmt.Errorf("cannot determine working directory: %w", err) } } - + fmt.Printf("root: %v\n", root) f, err := fn.NewFunction(root) if err != nil { return fmt.Errorf("cannot load function: %w", err) diff --git a/pkg/pipelines/tekton/pipelines_provider.go b/pkg/pipelines/tekton/pipelines_provider.go index 470eebca15..fc0726736e 100644 --- a/pkg/pipelines/tekton/pipelines_provider.go +++ b/pkg/pipelines/tekton/pipelines_provider.go @@ -272,6 +272,20 @@ func sourcesAsTarStream(f fn.Function) *io.PipeReader { } } + // Hack for python+pack where the function needs to be in a subdirectory + // and the actual main (scaffolding) needs to be in the source. + // So main in "root" and function in "root/subdir" otherwise pack builder + // determines that current function is JAVA or something because we currently + // build in the .s2i directory (needs fixing) and it contains a 'bin/' dir. + // buildpacks determines its somehow java. + // + // You can see this in the python scaffolding injector made for local builds + // for buildpacks builder & python runtime. + hName := "source" + usePyInjector := f.Build.Builder == "pack" && f.Runtime == "python" + if usePyInjector { + hName = "source/fn" + } pr, pw := io.Pipe() const nobodyID = 65534 @@ -282,7 +296,7 @@ func sourcesAsTarStream(f fn.Function) *io.PipeReader { err := tw.WriteHeader(&tar.Header{ Typeflag: tar.TypeDir, - Name: "source/", + Name: hName, Mode: 0777, Uid: nobodyID, Gid: nobodyID, @@ -341,7 +355,8 @@ func sourcesAsTarStream(f fn.Function) *io.PipeReader { return fmt.Errorf("cannot create a tar header: %w", err) } // "source" is expected path in workspace pvc - hdr.Name = path.Join("source", filepath.ToSlash(relp)) + // current Hack: python + pack builder needs subdir ('source/fn') + hdr.Name = path.Join(hName, filepath.ToSlash(relp)) err = tw.WriteHeader(hdr) if err != nil { @@ -365,6 +380,13 @@ func sourcesAsTarStream(f fn.Function) *io.PipeReader { if err != nil { _ = pw.CloseWithError(fmt.Errorf("error while creating tar stream from sources: %w", err)) } else { + // python injector hack for python+pack builder + if usePyInjector { + err = pythonInjector(tw, f.Invoke) + if err != nil { + _ = pw.CloseWithError(fmt.Errorf("cannot inject python main into tar stream: %w", err)) + } + } _ = tw.Close() _ = pw.Close() } @@ -372,6 +394,47 @@ func sourcesAsTarStream(f fn.Function) *io.PipeReader { return pr } +// inject python main etc. into the tar file at the root dir instead of the usual +// subdir for python + pack builder +func pythonInjector(tw *tar.Writer, invoke string) (err error) { + if invoke == "" { + invoke = "http" + } + // the function files were all written in "source/fn" therefore new header + // for the actual "source" is neeeded + for _, f := range []struct { + path string + content string + }{ + { + path: "service/main.py", + content: mainContent(invoke), + }, + { + path: "pyproject.toml", + content: tomlContent, + }, + { + path: "service/__init__.py", + content: "", + }, + } { + err := tw.WriteHeader(&tar.Header{ + Name: "/source/" + f.path, + Size: int64(len(f.content)), + Mode: 0644, + }) + if err != nil { + return err + } + _, err = tw.Write([]byte(f.content)) + if err != nil { + return err + } + } + return nil +} + // Remove tries to remove all resources that are present on the cluster and belongs to the input function and it's pipelines func (pp *PipelinesProvider) Remove(ctx context.Context, f fn.Function) error { return pp.removeClusterResources(ctx, f) @@ -574,3 +637,62 @@ func createPipelinePersistentVolumeClaim(ctx context.Context, f fn.Function, nam } return nil } + +func mainContent(invoke string) string { + template := `""" +This code is glue between a user's Function and the middleware which will +expose it as a network service. This code is written on-demand when a +Function is being built, deployed or run. This will be included in the +final container. +""" +import logging +from func_python.%s import serve + +logging.basicConfig(level=logging.INFO) + +try: + from function import new as handler # type: ignore[import] +except ImportError: + try: + from function import handle as handler # type: ignore[import] + except ImportError: + logging.error("Function must export either 'new' or 'handle'") + raise + +def main(): + logging.info("Functions middleware invoking user function") + serve(handler) + +if __name__ == "__main__": + main() +` + return fmt.Sprintf(template, invoke) +} + +const tomlContent = `[project] +name = "service" +description = "an autogenerated service which runs a Function" +version = "0.1.0" +requires-python = ">=3.9" +license = "MIT" +dependencies = [ + "func-python", + "function @ ./fn" +] +authors = [ + { name="The Knative Authors", email="knative-dev@googlegroups.com"}, +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.metadata] +allow-direct-references = true + +[tool.poetry.dependencies] +python = ">=3.9,<4.0" + +[tool.poetry.scripts] +script = "service.main:main" +` diff --git a/pkg/pipelines/tekton/tasks.go b/pkg/pipelines/tekton/tasks.go index 4a9c584904..f480692e39 100644 --- a/pkg/pipelines/tekton/tasks.go +++ b/pkg/pipelines/tekton/tasks.go @@ -62,8 +62,11 @@ spec: description: The registry associated with the function image. - name: BUILDER_IMAGE description: The image on which builds will run (must include lifecycle and compatible buildpacks). - - name: SOURCE_SUBPATH - description: A subpath within the "source" input where the source to build is located. + - name: contextDir + description: context directory for the function project + default: "" + - name: BUILD_TARGET + description: A directory to where to build from. (passed to pack builder) default: "" - name: ENV_VARS type: array @@ -157,10 +160,10 @@ spec: ############################################ func_file="$(workspaces.source.path)/func.yaml" - if [ "$(params.SOURCE_SUBPATH)" != "" ]; then - func_file="$(workspaces.source.path)/$(params.SOURCE_SUBPATH)/func.yaml" + if [ "$(params.contextDir)" != "" ]; then + func_file="$(workspaces.source.path)/$(params.contextDir)/func.yaml" fi - echo "--> Saving 'func.yaml'" + echo "--> Saving 'func.yaml' from '$func_file'" cp $func_file /emptyDir/func.yaml ############################################ @@ -183,7 +186,7 @@ spec: - name: DOCKER_CONFIG value: $(workspaces.dockerconfig.path) args: - - "-app=$(workspaces.source.path)/$(params.SOURCE_SUBPATH)" + - "-app=$(workspaces.source.path)/$(params.BUILD_TARGET)" - "-cache-dir=$(workspaces.cache.path)" - "-cache-image=$(params.CACHE_IMAGE)" - "-uid=$(params.USER_ID)" @@ -220,13 +223,13 @@ spec: digest=$(cat $(results.IMAGE_DIGEST.path)) func_file="$(workspaces.source.path)/func.yaml" - if [ "$(params.SOURCE_SUBPATH)" != "" ]; then - func_file="$(workspaces.source.path)/$(params.SOURCE_SUBPATH)/func.yaml" + if [ "$(params.contextDir)" != "" ]; then + func_file="$(workspaces.source.path)/$(params.contextDir)/func.yaml" fi if [[ ! -f "$func_file" ]]; then echo "--> Restoring 'func.yaml'" - mkdir -p "$(workspaces.source.path)/$(params.SOURCE_SUBPATH)" + mkdir -p "$(workspaces.source.path)/$(params.contextDir)" cp /emptyDir/func.yaml $func_file fi @@ -413,13 +416,16 @@ spec: - name: image description: Container image to be deployed default: "" + - name: subpath + default: "" + description: Optional Subpath to where func.yaml is workspaces: - name: source description: The workspace containing the function project steps: - name: func-deploy image: "%s" - command: ["deploy", "$(params.path)", "$(params.image)"] + command: ["deploy", "$(params.path)", "$(params.image)", "$(params.subpath)"] `, DeployerImage) } diff --git a/pkg/pipelines/tekton/templates.go b/pkg/pipelines/tekton/templates.go index 3d22cdeb4b..e833f67415 100644 --- a/pkg/pipelines/tekton/templates.go +++ b/pkg/pipelines/tekton/templates.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path" + "path/filepath" "strings" "text/template" @@ -77,14 +78,15 @@ const ( ) type templateData struct { - FunctionName string - Annotations map[string]string - Labels map[string]string - ContextDir string - FunctionImage string - Registry string - BuilderImage string - BuildEnvs []string + FunctionName string + Annotations map[string]string + Labels map[string]string + ContextDir string + FunctionImage string + SubPathOverride string + Registry string + BuilderImage string + BuildEnvs []string PipelineName string PipelineRunName string @@ -387,14 +389,15 @@ func createAndApplyPipelineRunTemplate(f fn.Function, namespace string, labels m } data := templateData{ - FunctionName: f.Name, - Annotations: f.Deploy.Annotations, - Labels: labels, - ContextDir: contextDir, - FunctionImage: f.Deploy.Image, - Registry: f.Registry, - BuilderImage: getBuilderImage(f), - BuildEnvs: buildEnvs, + FunctionName: f.Name, + Annotations: f.Deploy.Annotations, + Labels: labels, + ContextDir: contextDir, + SubPathOverride: contextDir, + FunctionImage: f.Deploy.Image, + Registry: f.Registry, + BuilderImage: getBuilderImage(f), + BuildEnvs: buildEnvs, PipelineName: getPipelineName(f), PipelineRunName: getPipelineRunGenerateName(f), @@ -407,6 +410,13 @@ func createAndApplyPipelineRunTemplate(f fn.Function, namespace string, labels m Revision: pipelinesTargetBranch, } + // this is for current impl. of python+pack hack + // its for pack builder which needs the path of where to deploy and its + // different from where func.yaml is + if f.Runtime == "python" && f.Build.Builder == "pack" { + data.ContextDir = filepath.Join(data.ContextDir, "fn") + } + var template string switch f.Build.Builder { case builders.Pack: diff --git a/pkg/pipelines/tekton/templates_pack.go b/pkg/pipelines/tekton/templates_pack.go index e9ab0e3953..050e3c61f4 100644 --- a/pkg/pipelines/tekton/templates_pack.go +++ b/pkg/pipelines/tekton/templates_pack.go @@ -28,6 +28,9 @@ spec: description: Path where the function project is name: contextDir type: string + - default: '' + name: subPathOverride + description: Dir to build from (might not be where func.yaml is) - description: Function image name name: imageName type: string @@ -57,8 +60,10 @@ spec: value: $(params.imageName) - name: REGISTRY value: $(params.registry) - - name: SOURCE_SUBPATH + - name: contextDir value: $(params.contextDir) + - name: SOURCE_SUBPATH + value: $(params.subPathOverride) - name: BUILDER_IMAGE value: $(params.builderImage) - name: ENV_VARS @@ -120,6 +125,8 @@ spec: value: {{.Revision}} - name: contextDir value: "{{.ContextDir}}" + - name: subPathOverride + value: "{{.SubPathOverride}}" - name: imageName value: {{.FunctionImage}} - name: registry