Lambda runtime parity by joe4dev · Pull Request #7824 · localstack/localstack · GitHub
Skip to content
This repository was archived by the owner on Mar 23, 2026. It is now read-only.

Lambda runtime parity #7824

Merged
joe4dev merged 6 commits into from
Mar 10, 2023
Merged

Lambda runtime parity #7824

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
9ff89aa
Switch lambda runtime to new RIE pre-release
joe4dev Mar 8, 2023
7f41b1d
Add lambda init user config
joe4dev Mar 8, 2023
90b4517
Test runtime path introspection
joe4dev Mar 8, 2023
90b95f2
Improve environment variables runtime parity
joe4dev Mar 8, 2023
f636088
Skip user parity validation in old provider
joe4dev Mar 9, 2023
8a3301b
Clarify conditional handler env var
joe4dev Mar 9, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions localstack/config.py
View file
Open in desktop
54 changes: 39 additions & 15 deletions localstack/services/awslambda/invocation/runtime_environment.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,38 +87,62 @@ def get_environment_variables(self) -> Dict[str, str]:
"""
credentials = self.get_credentials()
env_vars = {
# Runtime API specifics
"LOCALSTACK_RUNTIME_ID": self.id,
"LOCALSTACK_RUNTIME_ENDPOINT": self.runtime_executor.get_runtime_endpoint(),
# General Lambda Environment Variables
"AWS_LAMBDA_LOG_GROUP_NAME": self.get_log_group_name(),
"AWS_LAMBDA_LOG_STREAM_NAME": self.get_log_stream_name(),
"AWS_LAMBDA_FUNCTION_NAME": self.function_version.id.function_name,
"AWS_LAMBDA_FUNCTION_TIMEOUT": self.function_version.config.timeout,
"AWS_LAMBDA_FUNCTION_MEMORY_SIZE": self.function_version.config.memory_size, # TODO use correct memory size
"AWS_LAMBDA_FUNCTION_VERSION": self.function_version.id.qualifier,
# 1) Public AWS defined runtime environment variables (in same order):
# https://docs.aws.amazon.com/lambda/latest/dg/configuration-envvars.html
# a) Reserved environment variables
# _HANDLER conditionally added below
# TODO: _X_AMZN_TRACE_ID
"AWS_DEFAULT_REGION": self.function_version.id.region,
"AWS_REGION": self.function_version.id.region,
"TASK_ROOT": "/var/task", # TODO custom runtimes?
"RUNTIME_ROOT": "/var/runtime", # TODO custom runtimes?
# AWS_EXECUTION_ENV conditionally added below
"AWS_LAMBDA_FUNCTION_NAME": self.function_version.id.function_name,
"AWS_LAMBDA_FUNCTION_MEMORY_SIZE": self.function_version.config.memory_size,
"AWS_LAMBDA_FUNCTION_VERSION": self.function_version.id.qualifier,
"AWS_LAMBDA_INITIALIZATION_TYPE": self.initialization_type,
"TZ": ":UTC", # TODO does this have to match local system time? format?
"AWS_LAMBDA_LOG_GROUP_NAME": self.get_log_group_name(),
"AWS_LAMBDA_LOG_STREAM_NAME": self.get_log_stream_name(),
# Access IDs for role
"AWS_ACCESS_KEY_ID": credentials["AccessKeyId"],
"AWS_SECRET_ACCESS_KEY": credentials["SecretAccessKey"],
"AWS_SESSION_TOKEN": credentials["SessionToken"],
# TODO xray
# LocalStack endpoint specifics
# AWS_LAMBDA_RUNTIME_API is set in the runtime interface emulator (RIE)
"LAMBDA_TASK_ROOT": "/var/task",
"LAMBDA_RUNTIME_DIR": "/var/runtime",
# b) Unreserved environment variables
# LANG
# LD_LIBRARY_PATH
# NODE_PATH
# PYTHONPATH
# GEM_PATH
"AWS_XRAY_CONTEXT_MISSING": "LOG_ERROR",
# TODO: AWS_XRAY_DAEMON_ADDRESS
# AWS_LAMBDA_DOTNET_PREJIT
"TZ": ":UTC",
# 2) Public AWS RIE interface: https://github.com/aws/aws-lambda-runtime-interface-emulator
"AWS_LAMBDA_FUNCTION_TIMEOUT": self.function_version.config.timeout,
# 3) Public LocalStack endpoint
"LOCALSTACK_HOSTNAME": self.runtime_executor.get_endpoint_from_executor(),
"EDGE_PORT": str(config.EDGE_PORT),
"AWS_ENDPOINT_URL": f"http://{self.runtime_executor.get_endpoint_from_executor()}:{config.EDGE_PORT}",
# 4) Internal LocalStack runtime API
"LOCALSTACK_RUNTIME_ID": self.id,
"LOCALSTACK_RUNTIME_ENDPOINT": self.runtime_executor.get_runtime_endpoint(),
# LOCALSTACK_USER conditionally added below
}
# Conditionally added environment variables
# config.handler is None for image lambdas and will be populated at runtime (e.g., by RIE)
if self.function_version.config.handler:
env_vars["_HANDLER"] = self.function_version.config.handler
# Not defined for custom runtimes (e.g., provided, provided.al2)
if self.function_version.config.runtime:
env_vars["AWS_EXECUTION_ENV"] = f"Aws_Lambda_{self.function_version.config.runtime}"
if self.function_version.config.environment:
env_vars.update(self.function_version.config.environment)
if config.LAMBDA_INIT_DEBUG:
# Disable dropping privileges because it breaks debugging
env_vars["LOCALSTACK_USER"] = ""
if config.LAMBDA_INIT_USER:
env_vars["LOCALSTACK_USER"] = config.LAMBDA_INIT_USER
return env_vars

# Lifecycle methods
Expand Down
2 changes: 1 addition & 1 deletion localstack/services/awslambda/packages.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

LAMBDA_RUNTIME_INIT_URL = "https://github.com/localstack/lambda-runtime-init/releases/download/{version}/aws-lambda-rie-{arch}"

LAMBDA_RUNTIME_DEFAULT_VERSION = "v0.1.11-pre"
LAMBDA_RUNTIME_DEFAULT_VERSION = "v0.1.12-pre"

@joe4dev joe4dev Mar 8, 2023 •
edited
Loading

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is the key change adopting the new RIE release including https://github.com/localstack/lambda-runtime-init/pull/13/files


# GO Lambda runtime
GO_RUNTIME_VERSION = "0.4.0"
Expand Down
2 changes: 1 addition & 1 deletion localstack/testing/aws/lambda_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ def get_invoke_init_type(
) -> Literal["on-demand", "provisioned-concurrency"]:
"""check the environment in the lambda for AWS_LAMBDA_INITIALIZATION_TYPE indicating ondemand/provisioned"""
invoke_result = client.invoke(FunctionName=function_name, Qualifier=qualifier)
return json.loads(to_str(invoke_result["Payload"].read()))["env"][
return json.loads(to_str(invoke_result["Payload"].read()))["environment"][
"AWS_LAMBDA_INITIALIZATION_TYPE"
]

Expand Down
18 changes: 16 additions & 2 deletions tests/integration/awslambda/functions/lambda_introspect.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,30 @@
import getpass
import os
import platform
import re
import stat
import subprocess
import time
from pathlib import Path


def handler(event, context):

if event.get("wait"):
time.sleep(event["wait"])

paths = ["/var/task", "/opt", "/tmp", "/lambda-entrypoint.sh"]
path_details = {}
for p in paths:
path_label = re.sub("/", "_", p)
path = Path(p)
path_details[f"{path_label}_mode"] = stat.filemode(path.stat().st_mode)
path_details[f"{path_label}_uid"] = path.stat().st_uid
path_details[f"{path_label}_owner"] = path.owner()
path_details[f"{path_label}_gid"] = path.stat().st_gid
# Raises KeyError "'getgrgid(): gid not found: 995'"
# path_details[f"{path_label}_group"] = path.group()

return {
# Tested in tests/integration/awslambda/test_lambda_common.py
# "environment": dict(os.environ),
Expand All @@ -20,6 +34,6 @@ def handler(event, context):
"user_whoami": subprocess.getoutput("whoami"),
"platform_system": platform.system(),
"platform_machine": platform.machine(),
"pwd_filemode": stat.filemode(os.stat(".").st_mode),
"opt_filemode": stat.filemode(os.stat("/opt").st_mode),
"pwd": os.getcwd(),
"paths": path_details,
}
40 changes: 28 additions & 12 deletions tests/integration/awslambda/test_lambda.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -306,11 +306,6 @@ def _assert_invocations():


class TestLambdaBehavior:
@pytest.mark.aws_validated
@pytest.mark.skip_snapshot_verify(
# TODO: run lambdas as user `sbx_user1051`
paths=["$..Payload.user_login_name", "$..Payload.user_whoami"]
)
@pytest.mark.skip_snapshot_verify(
condition=is_old_provider,
paths=[
Expand All @@ -320,41 +315,62 @@ class TestLambdaBehavior:
"$..Payload.errorMessage",
"$..Payload.errorType",
"$..Payload.event",
"$..Payload.opt_filemode",
"$..Payload.platform_machine",
"$..Payload.platform_system",
"$..Payload.pwd_filemode",
"$..Payload.stackTrace",
"$..Payload.paths",
"$..Payload.pwd",
"$..Payload.user_login_name",
"$..Payload.user_whoami",
],
)
@pytest.mark.skip_snapshot_verify(
paths=[
# fixable by setting /tmp permissions to 700
"$..Payload.paths._tmp_mode",
# requires creating a new user `slicer` and chown /var/task
"$..Payload.paths._var_task_gid",
"$..Payload.paths._var_task_owner",
"$..Payload.paths._var_task_uid",
],
)
@pytest.mark.aws_validated
def test_runtime_introspection_x86(self, lambda_client, create_lambda_function, snapshot):
func_name = f"test_lambda_x86_{short_uid()}"
create_lambda_function(
func_name=func_name,
handler_file=TEST_LAMBDA_INTROSPECT_PYTHON,
runtime=Runtime.python3_9,
timeout=9,
Architectures=[Architecture.x86_64],
)

invoke_result = lambda_client.invoke(FunctionName=func_name)
snapshot.match("invoke_runtime_x86_introspection", invoke_result)

@pytest.mark.aws_validated
@pytest.mark.skip_snapshot_verify(
# TODO: run lambdas as user `sbx_user1051`
paths=["$..Payload.user_login_name", "$..Payload.user_whoami"]
)
@pytest.mark.skipif(is_old_provider(), reason="unsupported in old provider")
@pytest.mark.skipif(
not is_arm_compatible() and not is_aws(),
reason="ARM architecture not supported on this host",
)
@pytest.mark.skip_snapshot_verify(
paths=[
# fixable by setting /tmp permissions to 700
"$..Payload.paths._tmp_mode",
# requires creating a new user `slicer` and chown /var/task
"$..Payload.paths._var_task_gid",
"$..Payload.paths._var_task_owner",
"$..Payload.paths._var_task_uid",
],
)
@pytest.mark.aws_validated
def test_runtime_introspection_arm(self, lambda_client, create_lambda_function, snapshot):
func_name = f"test_lambda_arm_{short_uid()}"
create_lambda_function(
func_name=func_name,
handler_file=TEST_LAMBDA_INTROSPECT_PYTHON,
runtime=Runtime.python3_9,
timeout=9,
Architectures=[Architecture.arm64],
)

Expand Down
46 changes: 40 additions & 6 deletions tests/integration/awslambda/test_lambda.snapshot.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2582,7 +2582,7 @@
}
},
"tests/integration/awslambda/test_lambda.py::TestLambdaBehavior::test_runtime_introspection_arm": {
"recorded-date": "17-02-2023, 14:00:04",
"recorded-date": "08-03-2023, 14:18:06",
"recorded-content": {
"invoke_runtime_arm_introspection": {
"ExecutedVersion": "$LATEST",
Expand All @@ -2592,8 +2592,25 @@
"user_whoami": "sbx_user1051",
"platform_system": "Linux",
"platform_machine": "aarch64",
"pwd_filemode": "drwxr-xr-x",
"opt_filemode": "drwxr-xr-x"
"pwd": "/var/task",
"paths": {
"_var_task_mode": "drwxr-xr-x",
"_var_task_uid": 998,
"_var_task_owner": "slicer",
"_var_task_gid": 995,
"_opt_mode": "drwxr-xr-x",
"_opt_uid": 0,
"_opt_owner": "root",
"_opt_gid": 0,
"_tmp_mode": "drwx------",
"_tmp_uid": 993,
"_tmp_owner": "sbx_user1051",
"_tmp_gid": 990,
"_lambda-entrypoint.sh_mode": "-rwxr-xr-x",
"_lambda-entrypoint.sh_uid": 0,
"_lambda-entrypoint.sh_owner": "root",
"_lambda-entrypoint.sh_gid": 0
}
},
"StatusCode": 200,
"ResponseMetadata": {
Expand All @@ -2604,7 +2621,7 @@
}
},
"tests/integration/awslambda/test_lambda.py::TestLambdaBehavior::test_runtime_introspection_x86": {
"recorded-date": "17-02-2023, 14:00:01",
"recorded-date": "08-03-2023, 14:16:54",
"recorded-content": {
"invoke_runtime_x86_introspection": {
"ExecutedVersion": "$LATEST",
Expand All @@ -2614,8 +2631,25 @@
"user_whoami": "sbx_user1051",
"platform_system": "Linux",
"platform_machine": "x86_64",
"pwd_filemode": "drwxr-xr-x",
"opt_filemode": "drwxr-xr-x"
"pwd": "/var/task",
"paths": {
"_var_task_mode": "drwxr-xr-x",
"_var_task_uid": 998,
"_var_task_owner": "slicer",
"_var_task_gid": 995,
"_opt_mode": "drwxr-xr-x",
"_opt_uid": 0,
"_opt_owner": "root",
"_opt_gid": 0,
"_tmp_mode": "drwx------",
"_tmp_uid": 993,
"_tmp_owner": "sbx_user1051",
"_tmp_gid": 990,
"_lambda-entrypoint.sh_mode": "-rwxr-xr-x",
"_lambda-entrypoint.sh_uid": 0,
"_lambda-entrypoint.sh_owner": "root",
"_lambda-entrypoint.sh_gid": 0
}
},
"StatusCode": 200,
"ResponseMetadata": {
Expand Down
26 changes: 13 additions & 13 deletions tests/integration/awslambda/test_lambda_common.py
View file
Open in desktop