使用Bazel为AWS Lambda创建Python Zip

Question

我有一个包含一组python aws lambdas的monorepo，我正在使用 bazel> bazel 用于建造和包装Lambdas 。我现在正在尝试使用Bazel创建一个遵循预期 aws lambdas包装我可以上传到lambda。想知道与巴泽尔一起做这件事的最佳方法是什么？

以下是我迄今为止尝试过的几件不同的事情：

尝试1：py_binary

build.bazel

py_binary(
name = "main_binary",
srcs = glob(["*.py"]),
main = "main.py",
visibility = ["//appcode/api/transaction_details:__subpackages__"],
deps = [
        requirement("Faker"),
    ],
)

问题：

这会生成以下内容：

• main_binary（ Python可执行文件）
• main_binary.runfiles
• main_binary.runfiles_manifest

lambda希望处理程序的格式为lambda_function.lambda_handler。由于main_binary是可执行文件和python文件，因此它不会公开实际的处理程序方法，而lambda则炸毁了，因为它找不到它。我尝试更新处理程序配置，以简单地指向main_binary，但它会炸毁，因为它期望两个参数（即lambda_function.lambda_handler）。

尝试2：py_library + pkg_zip

build.bazel

py_library(
name = "main",
srcs = glob(["*.py"]),
visibility = ["//appcode/api/transaction_details:__subpackages__"],
deps = [
        requirement("Faker"),
    ],
)

pkg_zip(
name = "main_zip",
srcs =["//appcode/api/transaction_details/src:main" ],
)

问题：

这将生成一个zip文件，并使用：

• main.py
• __ INT __ INT__。py

zip文件现在包括main.py，但其运行时依赖性都不是。因此，Lambda炸毁了，因为它找不到faker。

其他尝试：

我也尝试使用-build_python_zip flag以及@bazel_tools // tools/zip：拉链具有通用规则，但它们都与以前的两次尝试相似。

Answer 1

我们使用@bazel_tools //工具/zip：zipper带有自定义规则。我们还将serverless拉动rules_nodejs并通过Bazel运行，这会导致软件包构建在运行SLS部署之前发生。

我们使用pip_parse来自rules_python。我不确定下面的_SHORT_PATH功能是否适用于pip_install或其他机制。

尽管尴尬，但支持文件过滤。理想情况下，拉链生成将通过单独的二进制（即，一个python脚本）来处理，该脚本将允许使用正则表达式/globs/等进行过滤。 Bazel不支持Starlark中的正则表达式，因此我们使用自己的东西。

我包括了一个摘录：

lambda.bzl

"""
Support for serverless deployments.
"""

def contains(pattern):
    return "contains:" + pattern

def startswith(pattern):
    return "startswith:" + pattern

def endswith(pattern):
    return "endswith:" + pattern

def _is_ignored(path, patterns):
    for p in patterns:
        if p.startswith("contains:"):
            if p[len("contains:"):] in path:
                return True
        elif p.startswith("startswith:"):
            if path.startswith(p[len("startswith:"):]):
                return True
        elif p.startswith("endswith:"):
            if path.endswith(p[len("endswith:"):]):
                return True
        else:
            fail("Invalid pattern: " + p)

    return False

def _short_path(file_):
    # Remove prefixes for external and generated files.
    # E.g.,
    #   ../py_deps_pypi__pydantic/pydantic/__init__.py -> pydantic/__init__.py
    short_path = file_.short_path
    if short_path.startswith("../"):
        second_slash = short_path.index("/", 3)
        short_path = short_path[second_slash + 1:]
    return short_path

def _py_lambda_zip_impl(ctx):
    deps = ctx.attr.target[DefaultInfo].default_runfiles.files

    f = ctx.outputs.output

    args = []
    for dep in deps.to_list():
        short_path = _short_path(dep)

        # Skip ignored patterns
        if _is_ignored(short_path, ctx.attr.ignore):
            continue

        args.append(short_path + "=" + dep.path)

    ctx.actions.run(
        outputs = [f],
        inputs = deps,
        executable = ctx.executable._zipper,
        arguments = ["cC", f.path] + args,
        progress_message = "Creating archive...",
        mnemonic = "archiver",
    )

    out = depset(direct = [f])
    return [
        DefaultInfo(
            files = out,
        ),
        OutputGroupInfo(
            all_files = out,
        ),
    ]

_py_lambda_zip = rule(
    implementation = _py_lambda_zip_impl,
    attrs = {
        "target": attr.label(),
        "ignore": attr.string_list(),
        "_zipper": attr.label(
            default = Label("@bazel_tools//tools/zip:zipper"),
            cfg = "host",
            executable = True,
        ),
        "output": attr.output(),
    },
    executable = False,
    test = False,
)

def py_lambda_zip(name, target, ignore, **kwargs):
    _py_lambda_zip(
        name = name,
        target = target,
        ignore = ignore,
        output = name + ".zip",
        **kwargs
    )

build.bazel

load("@npm_serverless//serverless:index.bzl", "serverless")
load(":lambda.bzl", "contains", "endswith", "py_lambda_zip", "startswith")

py_binary(
    name = "my_lambda_app",
    ...
)

py_lambda_zip(
    name = "lambda_archive",
    ignore = [
        contains("/__pycache__/"),
        endswith(".pyc"),
        endswith(".pyo"),
        
        # Ignore boto since it's provided by Lambda.
        startswith("boto3/"),
        startswith("botocore/"),

        # With the move to hermetic toolchains, the zip gets a lib/ directory containing the
        # python runtime. We don't need that.
        startswith("lib/"),
    ],
    target = ":my_lambda_app",

    # Only allow building on linux, since we don't want to upload a lambda zip file
    # with e.g. macos compiled binaries.
    target_compatible_with = [
        "@platforms//os:linux",
    ],
)

# The sls command requires that serverless.yml be in its working directory, and that the yaml file
# NOT be a symlink. So this target builds a directory containing a copy of serverless.yml, and also 
# symlinks the generated lambda_archive.zip in the same directory.
#
# It also generates a chdir.js script that we instruct node to execute to change to the proper working directory.
genrule(
    name = "sls_files",
    srcs = [
        "lambda_archive.zip",
        "serverless.yml",
    ],
    outs = [
        "sls_files/lambda_archive.zip",
        "sls_files/serverless.yml",
        "sls_files/chdir.js",
    ],
    cmd = """
        mkdir -p $(@D)/sls_files
        cp $(location serverless.yml) $(@D)/sls_files/serverless.yml
        cp -P $(location lambda_archive.zip) $(@D)/sls_files/lambda_archive.zip

        echo "const fs = require('fs');" \
             "const path = require('path');" \
             "process.chdir(path.dirname(fs.realpathSync(__filename)));" > $(@D)/sls_files/chdir.js
    """,
)

# Usage:
#   bazel run //:sls -- deploy <more args>
serverless(
    name = "sls",
    args = ["""--node_options=--require=./$(location sls_files/chdir.js)"""],
    data = [
        "sls_files/chdir.js",
        "sls_files/serverless.yml",
        "sls_files/lambda_archive.zip",
    ],
)

server> serverless.yml

service: my-app

package:
  artifact: lambda_archive.zip

# ... other config ...

Answer 2

以下是我对上一个答案的更改以生成lambda zip。感谢@jvolkman的原始建议。

生成sup者来

load("@rules_python//python:pip.bzl", "compile_pip_requirements")

compile_pip_requirements(
    name = "requirements",
    extra_args = ["--allow-unsafe"],
    requirements_in = "requirements.txt",
    requirements_txt = "requirements_lock.txt",
)

project/build.bazel： 添加了从project> project> project/unignts.txt insuert_lost> project/workspace.bazel：与pip_parse

workspace(name = "mdc-eligibility")

load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
http_archive(
    name = "rules_python",
    sha256 = "9fcf91dbcc31fde6d1edb15f117246d912c33c36f44cf681976bd886538deba6",
    strip_prefix = "rules_python-0.8.0",
    url = "https://github.com/bazelbuild/rules_python/archive/refs/tags/0.8.0.tar.gz",
)

load("@rules_python//python:repositories.bzl", "python_register_toolchains")
python_register_toolchains(
    name = "python3_9",
    python_version = "3.9",
)

load("@rules_python//python:pip.bzl", "pip_parse")
load("@python3_9//:defs.bzl", "interpreter")
pip_parse(
   name = "mndc-eligibility-deps",
   requirements_lock = "//:requirements_lock.txt",
   python_interpreter_target = interpreter,
   quiet = False
)
load("@mndc-eligibility-deps//:requirements.bzl", "install_deps")
install_deps()

proges/build_rules/lambda_packaging/lambda.bzl交换pip_install： 由修改的自定义规则由@jvolkman将源代码包含在生成的邮政编码中。

def contains(pattern):
    return "contains:" + pattern

def startswith(pattern):
    return "startswith:" + pattern

def endswith(pattern):
    return "endswith:" + pattern

def _is_ignored(path, patterns):
    for p in patterns:
        if p.startswith("contains:"):
            if p[len("contains:"):] in path:
                return True
        elif p.startswith("startswith:"):
            if path.startswith(p[len("startswith:"):]):
                return True
        elif p.startswith("endswith:"):
            if path.endswith(p[len("endswith:"):]):
                return True
        else:
            fail("Invalid pattern: " + p)

    return False

def _short_path(file_):
    # Remove prefixes for external and generated files.
    # E.g.,
    #   ../py_deps_pypi__pydantic/pydantic/__init__.py -> pydantic/__init__.py
    short_path = file_.short_path
    if short_path.startswith("../"):
        second_slash = short_path.index("/", 3)
        short_path = short_path[second_slash + 1:]
    return short_path

# steven chambers

def _py_lambda_zip_impl(ctx):
    deps = ctx.attr.target[DefaultInfo].default_runfiles.files

    f = ctx.outputs.output

    args = []
    for dep in deps.to_list():
        short_path = _short_path(dep)

        # Skip ignored patterns
        if _is_ignored(short_path, ctx.attr.ignore):
            continue

        args.append(short_path + "=" + dep.path)

    # MODIFICATION: Added source files to the map of files to zip
    source_files = ctx.attr.target[DefaultInfo].files
    for source_file in source_files.to_list():
        args.append(source_file.basename+"="+source_file.path)

    ctx.actions.run(
        outputs = [f],
        inputs = deps,
        executable = ctx.executable._zipper,
        arguments = ["cC", f.path] + args,
        progress_message = "Creating archive...",
        mnemonic = "archiver",
    )

    out = depset(direct = [f])
    return [
        DefaultInfo(
            files = out,
        ),
        OutputGroupInfo(
            all_files = out,
        ),
    ]

_py_lambda_zip = rule(
    implementation = _py_lambda_zip_impl,
    attrs = {
        "target": attr.label(),
        "ignore": attr.string_list(),
        "_zipper": attr.label(
            default = Label("@bazel_tools//tools/zip:zipper"),
            cfg = "host",
            executable = True,
        ),
        "output": attr.output(),
    },
    executable = False,
    test = False,
)

def py_lambda_zip(name, target, ignore, **kwargs):
    _py_lambda_zip(
        name = name,
        target = target,
        ignore = ignore,
        output = name + ".zip",
        **kwargs
    )

project/appcode/api/transaction_details/src/build.bazel ：使用自定义py_lambda_zip py_library py_lambda_zip

load("@mndc-eligibility-deps//:requirements.bzl", "requirement")
load("@python3_9//:defs.bzl", "interpreter")
load("//build_rules/lambda_packaging:lambda.bzl", "contains", "endswith", "py_lambda_zip", "startswith")

py_library(
    name = "main",
    srcs = glob(["*.py"]),
    visibility = ["//appcode/api/transaction_details:__subpackages__"],
    deps = [
            requirement("Faker"),
        ],
)

py_lambda_zip(
    name = "lambda_archive",
    ignore = [
        contains("/__pycache__/"),
        endswith(".pyc"),
        endswith(".pyo"),

        # Ignore boto since it's provided by Lambda.
        startswith("boto3/"),
        startswith("botocore/"),

        # With the move to hermetic toolchains, the zip gets a lib/ directory containing the
        # python runtime. We don't need that.
        startswith("lib/"),
    ],
    target = ":main",
)