diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 041d76c..f347299 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,6 +29,9 @@ jobs: - name: Install Hatch run: | python -m pip install hatch==1.15.0 + - name: Install specific version of Virtual Env due to bug with hatch + run: | + python -m pip install virtualenv==20.39.0 - uses: webfactory/ssh-agent@v0.9.1 with: ssh-private-key: ${{ secrets.SDK_KEY }} diff --git a/.github/workflows/create-emulator-pr.yml b/.github/workflows/create-emulator-pr.yml deleted file mode 100644 index ff53a88..0000000 --- a/.github/workflows/create-emulator-pr.yml +++ /dev/null @@ -1,189 +0,0 @@ -name: Create Emulator PR - -on: - pull_request: - branches: [ main ] - types: [opened, synchronize, closed] - -permissions: - contents: read - pull-requests: write - issues: write - -jobs: - cleanup-emulator-pr: - if: github.event.action == 'closed' - runs-on: ubuntu-latest - steps: - - uses: webfactory/ssh-agent@v0.9.1 - with: - ssh-private-key: ${{ secrets.EMULATOR_KEY }} - - - name: Delete emulator branch - run: | - PR_NUMBER="${{ github.event.pull_request.number }}" - EMULATOR_BRANCH="testing-sdk-pr-${PR_NUMBER}-sync" - - git clone git@github.com:aws/aws-durable-execution-emulator.git - cd aws-durable-execution-emulator - git push origin --delete "$EMULATOR_BRANCH" || echo "Branch may not exist" - - create-emulator-pr: - if: github.event.action == 'opened' || github.event.action == 'synchronize' - runs-on: ubuntu-latest - steps: - - name: Checkout testing SDK repo - uses: actions/checkout@v5 - with: - path: testing-sdk - - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: "3.13" - - - name: Install uv - uses: astral-sh/setup-uv@v4 - - - uses: webfactory/ssh-agent@v0.9.1 - with: - ssh-private-key: | - ${{ secrets.EMULATOR_PRIVATE_KEY }} - ${{ secrets.SDK_KEY }} - - - name: Checkout emulator repo - run: | - git clone git@github.com:aws/aws-durable-execution-emulator.git emulator - - - name: Create branch and update uv.lock - working-directory: emulator - run: | - # Configure git - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - - # Get PR info - BRANCH_NAME="${{ github.event.pull_request.head.ref }}" - PR_NUMBER="${{ github.event.pull_request.number }}" - EMULATOR_BRANCH="testing-sdk-pr-${PR_NUMBER}-sync" - - # Create or update branch - git fetch origin - if git show-ref --verify --quiet refs/remotes/origin/"$EMULATOR_BRANCH"; then - git checkout "$EMULATOR_BRANCH" - git reset --hard origin/main - else - git checkout -b "$EMULATOR_BRANCH" - fi - - # Update pyproject.toml to use local testing SDK (temporary, not committed) - TESTING_SDK_PATH="$(realpath ../testing-sdk)" - sed -i.bak "s|aws-durable-execution-sdk-python-testing @ git+ssh://git@github.com/aws/aws-durable-execution-sdk-python-testing.git|aws-durable-execution-sdk-python-testing @ file://${TESTING_SDK_PATH}|" pyproject.toml - rm pyproject.toml.bak - - # Generate new uv.lock with the specific testing SDK commit - uv lock - - # Show what changed - echo "=== Changes to be committed ===" - git diff --name-status - git diff uv.lock || echo "uv.lock is a new file" - - # Restore original pyproject.toml (don't commit the temporary change) - git checkout pyproject.toml - - # Commit and push only the uv.lock file - git add uv.lock - if git commit -m "Lock testing SDK branch: $BRANCH_NAME (PR #$PR_NUMBER)"; then - echo "Changes committed successfully" - git push --force-with-lease origin "$EMULATOR_BRANCH" - echo "Branch pushed successfully" - else - echo "No changes to commit" - # Still need to push the branch even if no changes - git push --force-with-lease origin "$EMULATOR_BRANCH" || git push origin "$EMULATOR_BRANCH" - fi - - - name: Create or update PR in emulator repo - uses: actions/github-script@v7 - with: - github-token: ${{ secrets.EMULATOR_REPO_TOKEN }} - script: | - const fs = require('fs'); - const pr = context.payload.pull_request; - const branch_name = pr.head.ref; - const emulator_branch = `testing-sdk-pr-${pr.number}-sync`; - - // Wait a moment for branch to be available - await new Promise(resolve => setTimeout(resolve, 2000)); - - // Read and populate PR template - const template = fs.readFileSync('testing-sdk/.github/workflows/emulator-pr-template.md', 'utf8'); - const pr_body = template - .replace(/{{PR_NUMBER}}/g, pr.number) - .replace(/{{BRANCH_NAME}}/g, branch_name); - - try { - // Check if PR already exists - let existingPR = null; - try { - const prs = await github.rest.pulls.list({ - owner: 'aws', - repo: 'aws-durable-execution-emulator', - head: `aws:${emulator_branch}`, - state: 'open' - }); - existingPR = prs.data[0]; - } catch (e) { - console.log('No existing PR found'); - } - - if (existingPR) { - // Update existing PR - await github.rest.pulls.update({ - owner: 'aws', - repo: 'aws-durable-execution-emulator', - pull_number: existingPR.number, - title: `Lock testing SDK branch: ${branch_name} (PR #${pr.number})`, - body: pr_body - }); - - console.log(`Updated emulator PR: ${existingPR.html_url}`); - - // Comment on original PR about update - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: pr.number, - body: `🔄 **Emulator PR Updated**\n\nThe emulator PR has been updated with locked dependencies:\n\n➡️ ${existingPR.html_url}` - }); - } else { - // Create new PR - console.log("Creating an emulator PR") - const response = await github.rest.pulls.create({ - owner: 'aws', - repo: 'aws-durable-execution-emulator', - title: `Lock testing SDK branch: ${branch_name} (PR #${pr.number})`, - head: emulator_branch, - base: 'main', - body: pr_body, - draft: true - }); - - console.log(`Created emulator PR: ${response.data.html_url}`); - - // Comment on original PR - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: pr.number, - body: `🤖 **Emulator PR Created**\n\nA draft PR has been created with locked dependencies:\n\n➡️ ${response.data.html_url}\n\nThe emulator will build binaries using the exact testing SDK commit locked in uv.lock.` - }); - } - - } catch (error) { - console.log(`Error managing PR: ${error.message}`); - console.log(`Error status: ${error.status}`); - console.log(`Error response: ${JSON.stringify(error.response?.data)}`); - core.setFailed(`Failed to manage emulator PR: ${error.message}`); - } diff --git a/.github/workflows/ecr-release.yml b/.github/workflows/ecr-release.yml new file mode 100644 index 0000000..c1ccad2 --- /dev/null +++ b/.github/workflows/ecr-release.yml @@ -0,0 +1,155 @@ +name: ecr-release.yml +on: + release: + types: [published] + +permissions: + contents: read + id-token: write # This is required for requesting the JWT + +env: + path_to_dockerfile: "emulator/DockerFile" + docker_build_dir: "emulator/" + aws_region: "us-east-1" + ecr_repository_name: "o4w4w0v6/aws-durable-execution-emulator" + +jobs: + build-and-upload-image-to-ecr: + runs-on: ubuntu-latest + outputs: + full_image_arm64: ${{ steps.build-publish.outputs.full_image_arm64 }} + full_image_x86_64: ${{ steps.build-publish.outputs.full_image_x86_64 }} + ecr_registry_repository: ${{ steps.build-publish.outputs.ecr_registry_repository }} + strategy: + matrix: + include: + - arch: x86_64 + - arch: arm64 + steps: + - name: Grab version from generate-version job + id: version + env: + VERSION: $${{ github.event.release.name }} + run: | + echo "$VERSION" + - uses: actions/checkout@v6 + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.13" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install hatch + - name: Set up QEMU for multi-platform builds + if: matrix.arch == 'arm64' + uses: docker/setup-qemu-action@v3 + with: + platforms: arm64 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.ECR_UPLOAD_IAM_ROLE_ARN }} + aws-region: ${{ env.aws_region }} + - name: Login to Amazon ECR + id: login-ecr-public + uses: aws-actions/amazon-ecr-login@v2 + with: + registry-type: public + - name: Build, tag, and push image to Amazon ECR + id: build-publish + shell: bash + env: + ECR_REGISTRY: ${{ steps.login-ecr-public.outputs.registry }} + ECR_REPOSITORY: ${{ env.ecr_repository_name }} + IMAGE_TAG: "${{ env.image_tag }}${{ github.event.release.name }}" + PER_ARCH_IMAGE_TAG: "${{ matrix.arch }}${{ github.event.release.name }}" + run: | + if [ "${{ matrix.arch }}" = "x86_64" ]; then + docker build --platform linux/amd64 --provenance false "${{ env.docker_build_dir }}" -f "${{ env.path_to_dockerfile }}" -t "$ECR_REGISTRY/$ECR_REPOSITORY:$PER_ARCH_IMAGE_TAG" + else + docker build --platform linux/arm64 --provenance false "${{ env.docker_build_dir }}" -f "${{ env.path_to_dockerfile }}" -t "$ECR_REGISTRY/$ECR_REPOSITORY:$PER_ARCH_IMAGE_TAG" + fi + docker push "$ECR_REGISTRY/$ECR_REPOSITORY:$PER_ARCH_IMAGE_TAG" + echo "IMAGE $PER_ARCH_IMAGE_TAG is pushed to $ECR_REGISTRY/$ECR_REPOSITORY" + echo "image_tag=$PER_ARCH_IMAGE_TAG" + echo "full_image=$ECR_REGISTRY/$ECR_REPOSITORY:$PER_ARCH_IMAGE_TAG" + echo "ecr_registry_repository=$ECR_REGISTRY/$ECR_REPOSITORY" >> $GITHUB_OUTPUT + echo "full_image_${{ matrix.arch }}=$ECR_REGISTRY/$ECR_REPOSITORY:$PER_ARCH_IMAGE_TAG" >> $GITHUB_OUTPUT + create-ecr-manifest-per-arch: + runs-on: ubuntu-latest + needs: [build-and-upload-image-to-ecr] + steps: + - name: Grab image and registry/repository name from previous steps + id: ecr_names + env: + ECR_REGISTRY_REPOSITORY: ${{ needs.build-and-upload-image-to-ecr.outputs.ecr_registry_repository }} + FULL_IMAGE_ARM64: ${{ needs.build-and-upload-image-to-ecr.outputs.full_image_arm64 }} + FULL_IMAGE_X86_64: ${{ needs.build-and-upload-image-to-ecr.outputs.full_image_x86_64 }} + run: | + echo "full_image_arm64=$FULL_IMAGE_ARM64" + echo "ecr_registry_repository=$ECR_REGISTRY_REPOSITORY" + echo "full_image_x86_64=$FULL_IMAGE_X86_64" + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.ECR_UPLOAD_IAM_ROLE_ARN }} + aws-region: ${{ env.aws_region }} + - name: Login to Amazon ECR + id: login-ecr-public + uses: aws-actions/amazon-ecr-login@v2 + with: + registry-type: public + - name: Create ECR manifest with explicit tag + if: github.event.release.name != '' + id: create-ecr-manifest-explicit + run: | + docker manifest create "${{ needs.build-and-upload-image-to-ecr.outputs.ecr_registry_repository }}:${{ github.event.release.name }}" \ + "${{ needs.build-and-upload-image-to-ecr.outputs.full_image_x86_64 }}" \ + "${{ needs.build-and-upload-image-to-ecr.outputs.full_image_arm64 }}" + - name: Annotate ECR manifest with explicit arm64 tag + if: github.event.release.name != '' + id: annotate-ecr-manifest-explicit-arm64 + run: | + docker manifest annotate "${{ needs.build-and-upload-image-to-ecr.outputs.ecr_registry_repository }}:${{ github.event.release.name }}" \ + "${{ needs.build-and-upload-image-to-ecr.outputs.full_image_arm64 }}" \ + --arch arm64 \ + --os linux + - name: Annotate ECR manifest with explicit amd64 tag + if: github.event.release.name != '' + id: annotate-ecr-manifest-explicit-amd64 + run: | + docker manifest annotate "${{ needs.build-and-upload-image-to-ecr.outputs.ecr_registry_repository }}:${{ github.event.release.name }}" \ + "${{ needs.build-and-upload-image-to-ecr.outputs.full_image_x86_64 }}" \ + --arch amd64 \ + --os linux + - name: Push ECR manifest with explicit version + if: github.event.release.name != '' + id: push-ecr-manifest-explicit + run: | + docker manifest push "${{ needs.build-and-upload-image-to-ecr.outputs.ecr_registry_repository }}:$${{ github.event.release.name }}" + - name: Create ECR manifest with latest tag + id: create-ecr-manifest-latest + run: | + docker manifest create "${{ needs.build-and-upload-image-to-ecr.outputs.ecr_registry_repository }}" \ + "${{ needs.build-and-upload-image-to-ecr.outputs.full_image_arm64 }}" \ + "${{ needs.build-and-upload-image-to-ecr.outputs.full_image_x86_64 }}" + - name: Annotate ECR manifest with latest tag arm64 + id: annotate-ecr-manifest-latest-arm64 + run: | + docker manifest annotate "${{ needs.build-and-upload-image-to-ecr.outputs.ecr_registry_repository }}" \ + "${{ needs.build-and-upload-image-to-ecr.outputs.full_image_arm64 }}" \ + --arch arm64 \ + --os linux + - name: Annotate ECR manifest with latest tag amd64 + id: annotate-ecr-manifest-latest-amd64 + run: | + docker manifest annotate "${{ needs.build-and-upload-image-to-ecr.outputs.ecr_registry_repository }}" \ + "${{ needs.build-and-upload-image-to-ecr.outputs.full_image_x86_64 }}" \ + --arch amd64 \ + --os linux + - name: Push ECR manifest with latest + id: push-ecr-manifest-latest + run: | + docker manifest push "${{ needs.build-and-upload-image-to-ecr.outputs.ecr_registry_repository }}" \ No newline at end of file diff --git a/.github/workflows/emulator-pr-template.md b/.github/workflows/emulator-pr-template.md deleted file mode 100644 index 96fd09f..0000000 --- a/.github/workflows/emulator-pr-template.md +++ /dev/null @@ -1,11 +0,0 @@ -*Issue #, if available:* Related to aws/aws-durable-execution-sdk-python-testing#{{PR_NUMBER}} - -*Description of changes:* Testing changes from testing SDK branch `{{BRANCH_NAME}}` using locked dependencies in uv.lock - -## Dependencies -This PR locks the testing SDK to a specific commit from branch `{{BRANCH_NAME}}` using uv.lock for reproducible builds. - -PYTHON_LANGUAGE_SDK_BRANCH: main -PYTHON_TESTING_SDK_BRANCH: {{BRANCH_NAME}} - -By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice. diff --git a/.gitignore b/.gitignore index ea7c0c4..5fb723c 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,6 @@ dist/ examples/build/* examples/*.zip + +durable-executions.db* +.coverage diff --git a/emulator/DockerFile b/emulator/DockerFile new file mode 100644 index 0000000..87146ad --- /dev/null +++ b/emulator/DockerFile @@ -0,0 +1,21 @@ +FROM python:3.13-slim + +WORKDIR /emulator + +# Copy project files +COPY . . + +# check that correct files are copied over +RUN ls + +# Update the package lists and install Git +# The commands are combined to reduce the final image size +# uncomment if we need to depend directly on main branch +# RUN apt-get update && apt-get upgrade && apt-get install -y --no-install-recommends git && apt-get purge -y --auto-remove && rm -rf /var/lib/apt/lists/* + +# install emulator +RUN pip install --no-cache-dir -e . + +ENTRYPOINT ["durable-functions-emulator"] + +CMD ["--host", "0.0.0.0", "--port", "9014"] \ No newline at end of file diff --git a/emulator/README.md b/emulator/README.md new file mode 100644 index 0000000..40c8048 --- /dev/null +++ b/emulator/README.md @@ -0,0 +1,125 @@ +# AWS Durable Execution Emulator + +A local emulator for AWS Lambda durable functions that enables local development and testing of durable function applications. Powered by the AWS Durable Execution Testing SDK for Python. + +## Overview + +The AWS Lambda Durable Execution Emulator provides a local development environment for building and testing durable function applications before deploying to AWS Lambda. It uses the AWS Durable Execution Testing SDK for Python as its execution engine, providing robust durable execution capabilities with full AWS API compatibility. + +## Features + +- **Local Development**: Run durable functions locally without AWS infrastructure +- **API Compatibility**: Compatible with AWS Lambda Durable Functions APIs +- **Health Check Endpoint**: Built-in health monitoring +- **Logging**: Configurable logging levels for debugging +- **Testing Support**: Built-in test framework support + +## Installation + +### From source + +```bash +git clone https://github.com/aws/aws-lambda-durable-functions-emulator.git +cd aws-lambda-durable-functions-emulator +pip install --no-cache-dir -e . +``` + +## Usage + +### Starting the Emulator + +```bash +# Using the installed command + +durable-functions-emulator + +# With custom host and port +durable-functions-emulator --host 0.0.0.0 --port 8080 +``` + +### Environment Variables + +- `HOST`: Server host (default: 0.0.0.0) +- `PORT`: Server port (default: 5000) +- `LOG`: Logging level (default: INFO) +- `STORAGE_DIR`: Directory for persistent storage +- `EXECUTION_STORE_TYPE`: Type of execution store (default: sqlite) + - `filesystem`: File-based storage + - `sqlite`: SQLite database storage (default) +- `LAMBDA_ENDPOINT`: Lambda endpoint URL for testing +- `LOCAL_RUNNER_ENDPOINT`: Local runner endpoint URL +- `LOCAL_RUNNER_REGION`: AWS region for local runner +- `LOCAL_RUNNER_MODE`: Runner mode (default: local) + +### Health Check + +The emulator provides a health check endpoint: + +```bash +curl http://localhost:5000/health +``` + +## Development + +### Prerequisites + +- Python 3.13+ +- [Hatch](https://hatch.pypa.io/) for project management + +### Setup + +```bash +git clone https://github.com/aws/aws-lambda-durable-functions-emulator.git +cd aws-lambda-durable-functions-emulator +hatch run pip install -e . +``` + +### Running Tests + +```bash +# Run all tests +hatch run test + +# Run with coverage +hatch run test:cov + +# Type checking +hatch run types:check +``` + +### Building + +```bash +hatch build +``` + +## API Reference + +### Health Check + +- **GET** `/health` - Returns emulator status + +### Durable Execution APIs + +- **POST** `/2025-12-01/durable-execution-state//checkpoint` - Checkpoint execution state +- **GET** `/2025-12-01/durable-execution-state//getState` - Get execution state +- **GET** `/2025-12-01/durable-executions/` - Get execution details +- **GET** `/2025-12-01/durable-executions//history` - Get execution history + +### Callback APIs + +- **POST** `/2025-12-01/durable-execution-callbacks//succeed` - Send success callback +- **POST** `/2025-12-01/durable-execution-callbacks//fail` - Send failure callback +- **POST** `/2025-12-01/durable-execution-callbacks//heartbeat` - Send heartbeat callback + +## Contributing + +We welcome contributions! Please see [CONTRIBUTING.md](CONTRIBUTING.md) for details. + +## License + +This project is licensed under the Apache License 2.0 - see the [LICENSE](LICENSE) file for details. + +## Security + +See [CONTRIBUTING.md](CONTRIBUTING.md#security-issue-notifications) for more information. diff --git a/emulator/pyproject.toml b/emulator/pyproject.toml new file mode 100644 index 0000000..acb8d50 --- /dev/null +++ b/emulator/pyproject.toml @@ -0,0 +1,125 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "aws-lambda-durable-functions-emulator" +dynamic = ["version"] +description = "Local emulator for AWS Lambda Durable Functions" +readme = "README.md" +requires-python = ">=3.13" +license = "Apache-2.0" +keywords = ["aws", "lambda", "durable", "functions", "emulator"] +authors = [ + { name = "AWS Lambda Team" }, +] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.13", +] +dependencies = [ + "aws-durable-execution-sdk-python-testing~=1.1.1", + "aws_durable_execution_sdk_python~=1.3.0", + "requests>=2.31.0,<3.0.0", + "boto3>=1.34.0,<2.0.0", +] + +[project.urls] +Documentation = "https://github.com/aws/aws-lambda-durable-functions-emulator#readme" +Issues = "https://github.com/aws/aws-lambda-durable-functions-emulator/issues" +Source = "https://github.com/aws/aws-lambda-durable-functions-emulator" + +[project.scripts] +durable-functions-emulator = "aws_lambda_durable_functions_emulator.server:main" + +[tool.hatch.build.targets.sdist] +packages = ["src/aws_lambda_durable_functions_emulator"] + +[tool.hatch.build.targets.wheel] +packages = ["src/aws_lambda_durable_functions_emulator"] + +[tool.hatch.metadata] +allow-direct-references = true + +[tool.hatch.version] +path = "src/aws_lambda_durable_functions_emulator/__about__.py" + +[tool.hatch.envs.default] +dependencies = [ + "pytest", +] +dev-mode = true + +[tool.hatch.envs.default.scripts] +test = "pytest {args:tests}" +dev = "python -m aws_lambda_durable_functions_emulator.server --host 127.0.0.1 --port 5000" +emulator = "python -m aws_lambda_durable_functions_emulator.server {args}" + +[tool.hatch.envs.test] +dependencies = [ + "coverage[toml]", + "pytest", + "pytest-cov", +] + +[tool.hatch.envs.test.scripts] +test = "pytest {args:tests}" +cov = "pytest --cov-report=term-missing --cov-config=pyproject.toml --cov=src/aws_lambda_durable_functions_emulator --cov=tests --cov-fail-under=60" + +[tool.hatch.envs.types] +extra-dependencies = [ + "mypy>=1.0.0", + "pytest", + "types-boto3", + "boto3-stubs[essential]", +] +[tool.hatch.envs.types.scripts] +check = "mypy --install-types --non-interactive {args:src/aws_lambda_durable_functions_emulator tests}" + +[tool.hatch.envs.lint] +extra-dependencies = [ + "ruff>=0.1.0", +] +[tool.hatch.envs.lint.scripts] +check = "ruff check {args:src tests}" +format = "ruff format {args:src tests}" +fix = "ruff check --fix {args:src tests}" + +[tool.ruff] +target-version = "py313" +line-length = 120 + +[tool.ruff.lint.isort] +known-first-party = ["aws_lambda_durable_functions_emulator"] + +[tool.ruff.lint.flake8-tidy-imports] +ban-relative-imports = "all" + +[tool.ruff.lint.per-file-ignores] +# Tests can use magic values, assertions, and relative imports +"tests/**/*" = ["PLR2004", "S101", "TID252", "S104"] +# Allow binding to all interfaces for server configuration +"src/aws_lambda_durable_functions_emulator/config.py" = ["S104"] + +[tool.coverage.run] +source_pkgs = ["aws_lambda_durable_functions_emulator", "tests"] +branch = true +parallel = true +omit = [ + "src/aws_lambda_durable_functions_emulator/__about__.py", +] + +[tool.coverage.paths] +aws_lambda_durable_functions_emulator = ["src/aws_lambda_durable_functions_emulator", "*/aws-durable-execution-sdk-python-testing/emulator/src/aws_lambda_durable_functions_emulator"] +tests = ["tests", "*/aws-durable-execution-sdk-python-testing/emulator/tests"] + +[tool.coverage.report] +exclude_lines = [ + "no cov", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", +] \ No newline at end of file diff --git a/emulator/src/aws_lambda_durable_functions_emulator/__about__.py b/emulator/src/aws_lambda_durable_functions_emulator/__about__.py new file mode 100644 index 0000000..1c35361 --- /dev/null +++ b/emulator/src/aws_lambda_durable_functions_emulator/__about__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: 2025-present Amazon.com, Inc. or its affiliates. +# +# SPDX-License-Identifier: Apache-2.0 +__version__ = "0.1.0" diff --git a/emulator/src/aws_lambda_durable_functions_emulator/__init__.py b/emulator/src/aws_lambda_durable_functions_emulator/__init__.py new file mode 100644 index 0000000..27d6498 --- /dev/null +++ b/emulator/src/aws_lambda_durable_functions_emulator/__init__.py @@ -0,0 +1 @@ +"""AWS Lambda Durable Functions Emulator""" diff --git a/emulator/src/aws_lambda_durable_functions_emulator/__main__.py b/emulator/src/aws_lambda_durable_functions_emulator/__main__.py new file mode 100644 index 0000000..63ac5b1 --- /dev/null +++ b/emulator/src/aws_lambda_durable_functions_emulator/__main__.py @@ -0,0 +1,6 @@ +"""CLI entry point for the durable functions emulator""" + +from aws_lambda_durable_functions_emulator.server import main + +if __name__ == "__main__": + main() diff --git a/emulator/src/aws_lambda_durable_functions_emulator/config.py b/emulator/src/aws_lambda_durable_functions_emulator/config.py new file mode 100644 index 0000000..49b6232 --- /dev/null +++ b/emulator/src/aws_lambda_durable_functions_emulator/config.py @@ -0,0 +1,138 @@ +import logging +import os +from dataclasses import dataclass, field +from pathlib import Path +from urllib.parse import urlparse + +from aws_durable_execution_sdk_python_testing.stores.base import StoreType +from aws_durable_execution_sdk_python_testing.web.server import WebServiceConfig + +# Constants +MAX_PORT = 65535 + + +def get_host() -> str: + """Get the server host from environment variable or default.""" + return os.environ.get("HOST", "0.0.0.0") + + +def get_port() -> int: + """Get the server port from environment variable or default.""" + return int(os.environ.get("PORT", "5000")) + + +def get_log_level() -> int: + """Get the logging level from environment variable or default.""" + log_level_str = os.environ.get("LOG", "INFO").upper() + return getattr(logging, log_level_str, logging.INFO) + + +def get_lambda_endpoint() -> str: + """Get the Lambda endpoint from environment variable.""" + return os.environ.get("LAMBDA_ENDPOINT", "http://localhost:3001") + + +def get_storage_dir() -> str | None: + """Get the storage directory from environment variable.""" + return os.environ.get("STORAGE_DIR") + + +def get_execution_store_type() -> str: + """Get the execution store type from environment variable.""" + return os.environ.get("EXECUTION_STORE_TYPE", StoreType.SQLITE.value).lower() + + +@dataclass +class EmulatorConfig: + """Configuration for the AWS Lambda Durable Functions Emulator.""" + + host: str = field(default_factory=get_host) + port: int = field(default_factory=get_port) + log_level: int = field(default_factory=get_log_level) + lambda_endpoint: str = field(default_factory=get_lambda_endpoint) + storage_dir: str | None = field(default_factory=get_storage_dir) + execution_store_type: str = field(default_factory=get_execution_store_type) + + def __post_init__(self): + """Validate configuration after initialization.""" + self._validate_config() + + def to_web_service_config(self): + """Convert to testing library web service config.""" + return WebServiceConfig( + host=self.host, port=self.port, log_level=self.log_level + ) + + def _validate_config(self): + """Validate all configuration parameters.""" + + # Validate Lambda endpoint URL + def _raise_invalid_endpoint( + endpoint: str, cause: Exception | None = None + ) -> None: + msg = f"Invalid Lambda endpoint URL: {endpoint}" + raise ValueError(msg) from cause + + try: + parsed = urlparse(self.lambda_endpoint) + if not parsed.scheme or not parsed.netloc: + _raise_invalid_endpoint(self.lambda_endpoint) + except (ValueError, TypeError) as e: + _raise_invalid_endpoint(self.lambda_endpoint, e) + + # Validate storage directory if specified + if self.storage_dir: + + def _raise_storage_error( + storage_dir: str, cause: Exception | None = None + ) -> None: + msg = f"Storage directory is not writable: {storage_dir}" + raise ValueError(msg) from cause + + def _raise_access_error( + storage_dir: str, cause: Exception | None = None + ) -> None: + msg = f"Cannot access storage directory: {storage_dir}" + raise ValueError(msg) from cause + + try: + storage_path = Path(self.storage_dir) + if storage_path.exists(): + if not storage_path.is_dir(): + msg = f"Storage path is not a directory: {self.storage_dir}" + raise ValueError(msg) + # Test write permissions + test_file = storage_path / ".write_test" + try: + test_file.write_text("test") + test_file.unlink() + except (OSError, PermissionError) as e: + _raise_storage_error(self.storage_dir, e) + else: + # Try to create the directory + storage_path.mkdir(parents=True, exist_ok=True) + except (OSError, PermissionError) as e: + _raise_access_error(self.storage_dir, e) + + # Validate log level + valid_log_levels = [ + logging.DEBUG, + logging.INFO, + logging.WARNING, + logging.ERROR, + logging.CRITICAL, + ] + if self.log_level not in valid_log_levels: + msg = f"Invalid log level: {self.log_level}. Must be one of {valid_log_levels}" + raise ValueError(msg) + + # Validate port range + if not (1 <= self.port <= MAX_PORT): + msg = f"Invalid port: {self.port}. Must be between 1 and {MAX_PORT}" + raise ValueError(msg) + + # Validate execution store type + valid_store_types = [StoreType.FILESYSTEM.value, StoreType.SQLITE.value] + if self.execution_store_type not in valid_store_types: + msg = f"Invalid execution store type: {self.execution_store_type}. Must be one of {valid_store_types}" + raise ValueError(msg) diff --git a/emulator/src/aws_lambda_durable_functions_emulator/factory.py b/emulator/src/aws_lambda_durable_functions_emulator/factory.py new file mode 100644 index 0000000..1d29912 --- /dev/null +++ b/emulator/src/aws_lambda_durable_functions_emulator/factory.py @@ -0,0 +1,97 @@ +"""Factory for creating testing library components with emulator configuration.""" + +import logging +import os +from pathlib import Path +from typing import TYPE_CHECKING + +import aws_durable_execution_sdk_python +import botocore.loaders +from aws_durable_execution_sdk_python_testing.checkpoint.processor import ( + CheckpointProcessor, +) +from aws_durable_execution_sdk_python_testing.executor import Executor +from aws_durable_execution_sdk_python_testing.invoker import LambdaInvoker +from aws_durable_execution_sdk_python_testing.scheduler import Scheduler +from aws_durable_execution_sdk_python_testing.stores.base import ( + StoreType, +) +from aws_durable_execution_sdk_python_testing.stores.filesystem import ( + FileSystemExecutionStore, +) +from aws_durable_execution_sdk_python_testing.stores.sqlite import ( + SQLiteExecutionStore, +) + +if TYPE_CHECKING: + from aws_lambda_durable_functions_emulator.config import EmulatorConfig + +logger = logging.getLogger(__name__) + + +class TestingLibraryComponentFactory: + """Factory for creating testing library components.""" + + @staticmethod + def create_store(config: "EmulatorConfig"): + """Create execution store based on emulator configuration.""" + store_type = config.execution_store_type + + if store_type == StoreType.SQLITE.value: + logger.info("Creating SQLite execution store") + if config.storage_dir: + db_path = Path(config.storage_dir) / "durable-executions.db" + else: + db_path = Path("durable-executions.db") + return SQLiteExecutionStore.create_and_initialize(db_path) + + logger.info("Creating file-system execution store") + return FileSystemExecutionStore.create(config.storage_dir or ".") + + @staticmethod + def create_scheduler(): + """Create scheduler for timer and event management.""" + logger.info("Creating scheduler") + scheduler = Scheduler() + logger.info("Starting scheduler") + scheduler.start() + return scheduler + + @staticmethod + def create_invoker(config: "EmulatorConfig"): + """Create Lambda invoker with emulator configuration.""" + logger.info("Creating Lambda invoker with endpoint: %s", config.lambda_endpoint) + + # Load lambdainternal service model + package_path = os.path.dirname(aws_durable_execution_sdk_python.__file__) + data_path = f"{package_path}/botocore/data" + os.environ["AWS_DATA_PATH"] = data_path + + loader = botocore.loaders.Loader() + loader.search_paths.append(data_path) + + return LambdaInvoker.create(config.lambda_endpoint, "us-east-1") + + @staticmethod + def create_checkpoint_processor(store, scheduler): + logger.info("Creating checkpoint processor") + checkpoint_processor = CheckpointProcessor(store, scheduler) + logger.info("Created checkpoint processor") + return checkpoint_processor + + @staticmethod + def create_executor(config: "EmulatorConfig"): + """Create complete executor with all components.""" + logger.info("Creating executor with all components") + + store = TestingLibraryComponentFactory.create_store(config) + scheduler = TestingLibraryComponentFactory.create_scheduler() + invoker = TestingLibraryComponentFactory.create_invoker(config) + checkpoint_processor = ( + TestingLibraryComponentFactory.create_checkpoint_processor(store, scheduler) + ) + + executor = Executor(store, scheduler, invoker, checkpoint_processor) + checkpoint_processor.add_execution_observer(executor) + + return executor diff --git a/emulator/src/aws_lambda_durable_functions_emulator/server.py b/emulator/src/aws_lambda_durable_functions_emulator/server.py new file mode 100644 index 0000000..0c8df4f --- /dev/null +++ b/emulator/src/aws_lambda_durable_functions_emulator/server.py @@ -0,0 +1,145 @@ +"""Main server for the AWS Lambda Durable Functions Emulator using testing library""" + +import argparse +import logging +import sys +from typing import TYPE_CHECKING + +from aws_durable_execution_sdk_python_testing.web.server import WebServer + +from aws_lambda_durable_functions_emulator.config import ( + EmulatorConfig, + get_host, + get_lambda_endpoint, + get_log_level, + get_port, + get_storage_dir, +) +from aws_lambda_durable_functions_emulator.factory import TestingLibraryComponentFactory + +if TYPE_CHECKING: + from aws_durable_execution_sdk_python_testing.executor import Executor + +# Configure logging +log_level = get_log_level() +logging.basicConfig( + level=log_level, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s" +) +logger = logging.getLogger("durable_functions_emulator") +logger.setLevel(log_level) + +# Suppress third-party debug logging +logging.getLogger("botocore").setLevel(logging.WARNING) +logging.getLogger("boto3").setLevel(logging.WARNING) +logging.getLogger("urllib3").setLevel(logging.WARNING) + +logger.info("Logging level set to: %s", logging.getLevelName(log_level)) + + +class EmulatorServer: + """Emulator server that wraps the testing library's WebServer.""" + + def __init__(self, config: EmulatorConfig) -> None: + self.config = config + + logger.info("Configuration validation passed") + + # Create testing library components using emulator config + self.executor = self._create_executor(config) + + # Convert emulator config to testing library config + web_config = self._create_web_config(config) + + # Use testing library's WebServer directly + self.web_server: WebServer = WebServer(web_config, self.executor) + + logger.info("EmulatorServer initialized on %s:%s", config.host, config.port) + + def _create_executor(self, config: EmulatorConfig) -> "Executor": + """Create executor with emulator configuration using factory.""" + return TestingLibraryComponentFactory.create_executor(config) + + def _create_web_config(self, config: EmulatorConfig): + """Convert emulator config to testing library config.""" + return config.to_web_service_config() + + def start(self): + """Start the emulator server.""" + try: + logger.info("Starting emulator server...") + with self.web_server: + logger.info( + "Server listening on %s:%s", self.config.host, self.config.port + ) + self.web_server.serve_forever() + except KeyboardInterrupt: + logger.info("Server shutdown requested by user") + except Exception: + logger.exception("Server error") + raise + finally: + logger.info("Server shutdown complete") + + +def main(): + """Main entry point for the emulator server.""" + parser = argparse.ArgumentParser( + description="AWS Lambda Durable Functions Emulator (powered by testing library)" + ) + parser.add_argument( + "--host", + type=str, + help="Host to bind to (default: from HOST env var or 0.0.0.0)", + ) + parser.add_argument( + "--port", type=int, help="Port to bind to (default: from PORT env var or 5000)" + ) + args = parser.parse_args() + + try: + # Create emulator configuration + config = EmulatorConfig( + host=args.host or get_host(), port=args.port or get_port() + ) + + # Create and start emulator server + logger.info( + "Starting AWS Lambda Durable Functions Emulator on %s:%s", + config.host, + config.port, + ) + server = EmulatorServer(config) + server.start() + + except ValueError: + logger.exception("Configuration error") + logger.info("Please check your configuration and try again.") + logger.info("Environment variables:") + logger.info(" HOST=%s", config.host if "config" in locals() else get_host()) + logger.info(" PORT=%s", config.port if "config" in locals() else get_port()) + logger.info(" LAMBDA_ENDPOINT=%s", get_lambda_endpoint()) + logger.info(" STORAGE_DIR=%s", get_storage_dir()) + sys.exit(1) + + except ImportError: + logger.exception("Missing dependency") + logger.info( + "Please install the aws-durable-execution-sdk-python-testing package:" + ) + logger.info(" pip install aws-durable-execution-sdk-python-testing") + sys.exit(1) + + except OSError: + logger.exception("Network error") + logger.info("Failed to bind to %s:%s", config.host, config.port) + logger.info("Please check that the port is not already in use and try again.") + sys.exit(1) + + except Exception: + logger.exception("Unexpected error during startup") + logger.exception("Full error details:") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/emulator/tests/__init__.py b/emulator/tests/__init__.py new file mode 100644 index 0000000..fb3c937 --- /dev/null +++ b/emulator/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for the durable functions emulator""" diff --git a/emulator/tests/test_config.py b/emulator/tests/test_config.py new file mode 100644 index 0000000..cd4648d --- /dev/null +++ b/emulator/tests/test_config.py @@ -0,0 +1,73 @@ +"""Tests for the emulator configuration""" + +import os + +import pytest +from aws_durable_execution_sdk_python_testing.stores.base import StoreType + +from aws_lambda_durable_functions_emulator.config import EmulatorConfig + + +def test_execution_store_type_default(): + """Test that default execution store type is sqlite""" + # Clean up any existing env vars + if "EXECUTION_STORE_TYPE" in os.environ: + del os.environ["EXECUTION_STORE_TYPE"] + + config = EmulatorConfig() + assert config.execution_store_type == StoreType.SQLITE.value + + +def test_execution_store_type_filesystem(): + """Test filesystem execution store type""" + os.environ["EXECUTION_STORE_TYPE"] = StoreType.FILESYSTEM.value + + config = EmulatorConfig() + assert config.execution_store_type == StoreType.FILESYSTEM.value + + # Clean up + del os.environ["EXECUTION_STORE_TYPE"] + + +def test_execution_store_type_sqlite(): + """Test SQLite execution store type""" + os.environ["EXECUTION_STORE_TYPE"] = StoreType.SQLITE.value + + config = EmulatorConfig() + assert config.execution_store_type == StoreType.SQLITE.value + + # Clean up + del os.environ["EXECUTION_STORE_TYPE"] + + +def test_execution_store_type_case_insensitive(): + """Test that execution store type is case insensitive""" + os.environ["EXECUTION_STORE_TYPE"] = "SQLITE" + + config = EmulatorConfig() + assert config.execution_store_type == StoreType.SQLITE.value + + # Clean up + del os.environ["EXECUTION_STORE_TYPE"] + + +def test_execution_store_type_invalid(): + """Test that invalid execution store type raises ValueError""" + os.environ["EXECUTION_STORE_TYPE"] = "invalid" + + with pytest.raises(ValueError, match="Invalid execution store type"): + EmulatorConfig() + + # Clean up + del os.environ["EXECUTION_STORE_TYPE"] + + +def test_execution_store_type_validation(): + """Test that only valid store types are accepted""" + valid_types = [StoreType.FILESYSTEM.value, StoreType.SQLITE.value] + + for store_type in valid_types: + os.environ["EXECUTION_STORE_TYPE"] = store_type + config = EmulatorConfig() + assert config.execution_store_type == store_type + del os.environ["EXECUTION_STORE_TYPE"] diff --git a/emulator/tests/test_factory.py b/emulator/tests/test_factory.py new file mode 100644 index 0000000..6bc51bd --- /dev/null +++ b/emulator/tests/test_factory.py @@ -0,0 +1,82 @@ +"""Tests for the component factory""" + +import os +import tempfile +from pathlib import Path + +from aws_durable_execution_sdk_python_testing.stores.base import StoreType + +from aws_lambda_durable_functions_emulator.config import EmulatorConfig +from aws_lambda_durable_functions_emulator.factory import TestingLibraryComponentFactory + + +def test_create_store_filesystem(): + """Test that filesystem store can be created""" + with tempfile.TemporaryDirectory() as temp_dir: + os.environ["EXECUTION_STORE_TYPE"] = StoreType.FILESYSTEM.value + os.environ["STORAGE_DIR"] = temp_dir + + config = EmulatorConfig() + store = TestingLibraryComponentFactory.create_store(config) + + assert store is not None + assert "FileSystemExecutionStore" in str(type(store)) + + # Clean up + del os.environ["EXECUTION_STORE_TYPE"] + del os.environ["STORAGE_DIR"] + + +def test_create_store_sqlite(): + """Test that SQLite store can be created""" + with tempfile.TemporaryDirectory() as temp_dir: + os.environ["EXECUTION_STORE_TYPE"] = StoreType.SQLITE.value + os.environ["STORAGE_DIR"] = temp_dir + + config = EmulatorConfig() + store = TestingLibraryComponentFactory.create_store(config) + + assert store is not None + assert "SQLiteExecutionStore" in str(type(store)) + + # Verify database file was created + db_path = Path(temp_dir) / "durable-executions.db" + assert db_path.exists() + + # Clean up + del os.environ["EXECUTION_STORE_TYPE"] + del os.environ["STORAGE_DIR"] + + +def test_create_store_default(): + """Test that default store type is sqlite""" + # Clean up any existing env vars + for key in ["EXECUTION_STORE_TYPE", "STORAGE_DIR"]: + if key in os.environ: + del os.environ[key] + + config = EmulatorConfig() + store = TestingLibraryComponentFactory.create_store(config) + + assert store is not None + assert "SQLiteExecutionStore" in str(type(store)) + + +def test_create_scheduler(): + """Test that scheduler can be created""" + scheduler = TestingLibraryComponentFactory.create_scheduler() + assert scheduler is not None + + +def test_create_invoker(): + """Test that invoker can be created""" + config = EmulatorConfig() + invoker = TestingLibraryComponentFactory.create_invoker(config) + assert invoker is not None + + +def test_create_executor(): + """Test that executor can be created with all components""" + config = EmulatorConfig() + executor = TestingLibraryComponentFactory.create_executor(config) + assert executor is not None diff --git a/emulator/tests/test_server.py b/emulator/tests/test_server.py new file mode 100644 index 0000000..a9fbe5f --- /dev/null +++ b/emulator/tests/test_server.py @@ -0,0 +1,35 @@ +"""Tests for the emulator server""" + +import pytest + +from aws_lambda_durable_functions_emulator.config import EmulatorConfig +from aws_lambda_durable_functions_emulator.server import EmulatorServer + + +def test_emulator_config_creation(): + """Test that EmulatorConfig can be created with defaults""" + config = EmulatorConfig() + assert config.host == "0.0.0.0" + assert config.port == 5000 + assert config.lambda_endpoint == "http://localhost:3001" + assert config.storage_dir is None + + +def test_emulator_config_validation(): + """Test that EmulatorConfig validates parameters""" + # Test invalid port + with pytest.raises(ValueError, match="Invalid port"): + EmulatorConfig(port=0) + + # Test invalid Lambda endpoint + with pytest.raises(ValueError, match="Invalid Lambda endpoint URL"): + EmulatorConfig(lambda_endpoint="not-a-url") + + +def test_emulator_server_creation(): + """Test that EmulatorServer can be created""" + config = EmulatorConfig() + server = EmulatorServer(config) + assert server.config == config + assert server.executor is not None + assert server.web_server is not None