Feature/s3 ElasticSearch composite

.github/workflows/inferno.yml

@@ -12,7 +12,7 @@ env:
   # Remote Docker host (set via GitHub repository secrets or variables; leave unset for local Docker)
   DOCKER_HOST: ${{ secrets.DOCKER_HOST }}
   DOCKER_HOST_IP: ${{ secrets.DOCKER_HOST_IP }}
-  # S3 backend configuration (optional; S3 tests are skipped when not set)
+  # S3+Elasticsearch backend configuration (optional; s3-elasticsearch tests are skipped when not set)
   HFS_S3_BUCKET: ${{ secrets.HFS_S3_BUCKET || vars.HFS_S3_BUCKET }}
 
 jobs:
@@ -50,9 +50,12 @@ jobs:
     name: Inferno US Core ${{ matrix.version_label }} Tests (${{ matrix.backend }})
     needs: build
     runs-on: [self-hosted, Linux]
+    permissions:
+      id-token: write
+      contents: read
     strategy:
       fail-fast: false
-      max-parallel: 3
+      max-parallel: 2
       matrix:
         suite_id:
           [
@@ -63,7 +66,7 @@ jobs:
             us_core_v700,
             us_core_v800,
           ]
-        backend: [sqlite, sqlite-elasticsearch, postgres, s3]
+        backend: [sqlite, sqlite-elasticsearch, postgres, s3-elasticsearch]
         include:
           - { suite_id: us_core_v311, version_label: "v3.1.1" }
           - { suite_id: us_core_v400, version_label: "v4.0.0" }
@@ -140,7 +143,7 @@ jobs:
           echo "OMITTED_TESTS=[${OMITTED}]" >> $GITHUB_ENV
 
       - name: Start Elasticsearch
-        if: matrix.backend == 'sqlite-elasticsearch'
+        if: matrix.backend == 'sqlite-elasticsearch' || matrix.backend == 's3-elasticsearch'
         run: |
           ES_CONTAINER="es-${{ matrix.suite_id }}-${{ matrix.backend }}"
           docker rm -f $ES_CONTAINER 2>/dev/null || true
@@ -186,9 +189,12 @@ jobs:
           for i in {1..30}; do
             if docker exec $PG_CONTAINER pg_isready -U helios > /dev/null 2>&1; then
               PG_PORT=$(docker port $PG_CONTAINER 5432 | head -1 | sed 's/.*://')
-              echo "PostgreSQL is ready on port $PG_PORT"
-              echo "PG_PORT=$PG_PORT" >> $GITHUB_ENV
-              exit 0
+              # Verify external TCP connectivity to the mapped port, not just internal readiness
+              if timeout 2 bash -c "cat < /dev/null > /dev/tcp/$DOCKER_HOST_IP/$PG_PORT" 2>/dev/null; then
+                echo "PostgreSQL is ready on port $PG_PORT"
+                echo "PG_PORT=$PG_PORT" >> $GITHUB_ENV
+                exit 0
+              fi
             fi
             echo "Attempt $i/30: PostgreSQL not ready yet..."
             sleep 2
@@ -197,8 +203,38 @@ jobs:
           docker logs $PG_CONTAINER
           exit 1
 
+      - name: Configure AWS credentials
+        if: matrix.backend == 's3-elasticsearch' && env.HFS_S3_BUCKET != ''
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.AWS_ROLE_ARN || vars.AWS_ROLE_ARN }}
+          aws-region: ${{ secrets.AWS_REGION || vars.AWS_REGION || 'us-east-1' }}
+
+      - name: Install AWS CLI
+        if: matrix.backend == 's3-elasticsearch' && env.HFS_S3_BUCKET != ''
+        run: |
+          if ! command -v aws &>/dev/null; then
+            if ! /tmp/aws-bin/aws --version &>/dev/null; then
+              curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o /tmp/awscliv2.zip
+              unzip -qo /tmp/awscliv2.zip -d /tmp
+              /tmp/aws/install --install-dir /tmp/aws-cli --bin-dir /tmp/aws-bin
+              rm -rf /tmp/awscliv2.zip /tmp/aws
+            fi
+            echo "/tmp/aws-bin" >> $GITHUB_PATH
+          fi
+          aws --version 2>/dev/null || /tmp/aws-bin/aws --version
+
+      - name: Empty S3 prefix
+        if: matrix.backend == 's3-elasticsearch' && env.HFS_S3_BUCKET != ''
+        run: |
+          BUCKET="${{ secrets.HFS_S3_BUCKET || vars.HFS_S3_BUCKET }}"
+          PREFIX="ci/${{ matrix.suite_id }}/"
+          echo "Emptying s3://$BUCKET/$PREFIX"
+          aws s3 rm "s3://$BUCKET/$PREFIX" --recursive
+          echo "Prefix emptied"
+
       - name: Start HFS server
-        if: matrix.backend != 's3' || env.HFS_S3_BUCKET != ''
+        if: matrix.backend != 's3-elasticsearch' || env.HFS_S3_BUCKET != ''
         run: |
           HFS_LOG="/tmp/hfs-${{ matrix.suite_id }}-${{ matrix.backend }}.log"
           echo "HFS_LOG=$HFS_LOG" >> $GITHUB_ENV
@@ -215,11 +251,12 @@ jobs:
             HFS_PG_USER=helios \
             HFS_PG_PASSWORD=helios \
             ./target/debug/hfs --log-level info --port $HFS_PORT --host 0.0.0.0 > "$HFS_LOG" 2>&1 &
-          elif [ "${{ matrix.backend }}" = "s3" ]; then
-            HFS_STORAGE_BACKEND=s3 \
+          elif [ "${{ matrix.backend }}" = "s3-elasticsearch" ]; then
+            HFS_STORAGE_BACKEND=s3-elasticsearch \
             HFS_S3_BUCKET=${{ secrets.HFS_S3_BUCKET || vars.HFS_S3_BUCKET }} \
+            HFS_S3_PREFIX=ci/${{ matrix.suite_id }}/ \
             HFS_S3_VALIDATE_BUCKETS=false \
-            AWS_REGION=${{ secrets.AWS_REGION || vars.AWS_REGION || 'us-east-1' }} \
+            HFS_ELASTICSEARCH_NODES=http://$DOCKER_HOST_IP:$ES_PORT \
             ./target/debug/hfs --log-level info --port $HFS_PORT --host 0.0.0.0 > "$HFS_LOG" 2>&1 &
           else
             ./target/debug/hfs --database-url :memory: --log-level info --port $HFS_PORT --host 0.0.0.0 > "$HFS_LOG" 2>&1 &
@@ -228,6 +265,7 @@ jobs:
           echo "HFS_PID=$(cat /tmp/hfs.pid)" >> $GITHUB_ENV
 
       - name: Wait for HFS to be ready
+        if: matrix.backend != 's3-elasticsearch' || env.HFS_S3_BUCKET != ''
         run: |
           echo "Waiting for HFS to start..."
           for i in {1..30}; do
@@ -253,10 +291,17 @@ jobs:
           exit 1
 
       - name: Load Inferno test data into HFS
+        if: matrix.backend != 's3-elasticsearch' || env.HFS_S3_BUCKET != ''
         run: |
-          ./crates/hfs/tests/inferno/install.sh
+          ./crates/hfs/tests/inferno/install.sh || {
+            echo "=== HFS log output ==="
+            cat "$HFS_LOG"
+            echo "=== end of log ==="
+            exit 1
+          }
 
       - name: Wait for Inferno to be ready
+        if: matrix.backend != 's3-elasticsearch' || env.HFS_S3_BUCKET != ''
         run: |
           echo "Waiting for persistent Inferno container to respond..."
           for i in {1..30}; do
@@ -271,9 +316,11 @@ jobs:
           exit 1
 
       - name: Create results directory
+        if: matrix.backend != 's3-elasticsearch' || env.HFS_S3_BUCKET != ''
         run: mkdir -p inferno-results
 
       - name: Run Inferno tests
+        if: matrix.backend != 's3-elasticsearch' || env.HFS_S3_BUCKET != ''
         run: |
           # Create a test session for US Core ${{ matrix.version_label }}
           # Retry to handle Inferno's SQLite "database is locked" errors under concurrent load
@@ -387,6 +434,7 @@ jobs:
           done
 
       - name: Check test results
+        if: matrix.backend != 's3-elasticsearch' || env.HFS_S3_BUCKET != ''
         run: |
           if [ ! -f inferno-results/results.json ]; then
             echo "No results file found"

CLAUDE.md

@@ -194,6 +194,9 @@ HFS_STORAGE_BACKEND=postgres HFS_DATABASE_URL="postgresql://user:pass@localhost/
 # With SQLite + Elasticsearch
 HFS_STORAGE_BACKEND=sqlite-es HFS_ELASTICSEARCH_NODES="http://localhost:9200" cargo run --bin hfs
 
+# With S3 (requires --features s3)
+HFS_STORAGE_BACKEND=s3 HFS_S3_BUCKET=my-bucket cargo run --bin hfs --features s3
+
 # With environment overrides
 HFS_SERVER_PORT=3000 HFS_LOG_LEVEL=debug cargo run --bin hfs
 ```
@@ -260,6 +263,23 @@ HFS_SERVER_PORT=3000 HFS_LOG_LEVEL=debug cargo run --bin hfs
 | SQLite + Elasticsearch | `sqlite-elasticsearch` or `sqlite-es` | SQLite for CRUD, ES for search |
 | PostgreSQL | `postgres` or `pg` or `postgresql` | PostgreSQL only |
 | PostgreSQL + Elasticsearch | `postgres-elasticsearch` or `pg-es` | PG for CRUD, ES for search |
+| S3 | `s3` | AWS S3 object storage for CRUD, versioning, history, bulk ops (no search) |
+| S3 + Elasticsearch | `s3-elasticsearch` or `s3-es` | S3 for CRUD, ES for search |
+
+#### S3 Backend
+Requires building with the `s3` feature:
+```bash
+cargo build -p helios-hfs --features s3
+HFS_STORAGE_BACKEND=s3 HFS_S3_BUCKET=my-bucket HFS_S3_REGION=us-east-1 cargo run --bin hfs --features s3
+```
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `HFS_S3_BUCKET` | `hfs` | S3 bucket name (prefix-per-tenant mode) |
+| `HFS_S3_REGION` | (AWS chain) | AWS region override |
+| `HFS_S3_VALIDATE_BUCKETS` | `true` | Validate bucket existence on startup |
+
+Standard AWS credential chain applies (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, instance profiles, etc.). For S3-compatible endpoints (e.g., MinIO), configure `S3BackendConfig` directly with `endpoint_url` and `force_path_style`.
 
 ### Multi-tenancy
 ```bash

README.md

@@ -172,6 +172,7 @@ The Helios FHIR Server supports multiple storage backend configurations. Choose
 | **PostgreSQL** | Built-in full-text search (tsvector/tsquery) | Production OLTP deployments |
 | **PostgreSQL + Elasticsearch** | Elasticsearch-powered search with PostgreSQL CRUD | Production deployments needing RDBMS + robust search |
 | **S3** | Object storage for CRUD, versioning, history, and bulk operations (no search) | Archival, bulk analytics, cost-effective storage |
+| **S3 + Elasticsearch** | Elasticsearch-powered search with S3 CRUD | Large-scale storage with full FHIR search |
 
 ### Running the Server
 
@@ -200,6 +201,14 @@ HFS_ELASTICSEARCH_NODES=http://localhost:9200 \
 HFS_STORAGE_BACKEND=s3 \
 HFS_S3_BUCKET=my-fhir-bucket \
 AWS_PROFILE=your-aws-profile \
+AWS_REGION=us-east-1 \
+  ./hfs
+
+# S3 + Elasticsearch
+HFS_STORAGE_BACKEND=s3-elasticsearch \
+HFS_S3_BUCKET=my-fhir-bucket \
+HFS_ELASTICSEARCH_NODES=http://localhost:9200 \
+AWS_PROFILE=your-aws-profile \
 AWS_REGION=us-east-1 \
   ./hfs
 ```
@@ -208,7 +217,7 @@ AWS_REGION=us-east-1 \
 
 | Variable | Default | Description |
 |---|---|---|
-| `HFS_STORAGE_BACKEND` | `sqlite` | Backend mode: `sqlite`, `sqlite-elasticsearch`, `postgres`, `postgres-elasticsearch`, or `s3` |
+| `HFS_STORAGE_BACKEND` | `sqlite` | Backend mode: `sqlite`, `sqlite-elasticsearch`, `postgres`, `postgres-elasticsearch`, `s3`, or `s3-elasticsearch` |
 | `HFS_SERVER_PORT` | `8080` | Server port |
 | `HFS_SERVER_HOST` | `127.0.0.1` | Host to bind |
 | `HFS_DATABASE_URL` | `fhir.db` | Database URL (SQLite path or PostgreSQL connection string) |
@@ -220,6 +229,7 @@ AWS_REGION=us-east-1 \
 | `HFS_ELASTICSEARCH_PASSWORD` | *(none)* | ES basic auth password |
 | `HFS_S3_BUCKET` | `hfs` | S3 bucket name (prefix-per-tenant mode) |
 | `HFS_S3_REGION` | *(AWS provider chain)* | AWS region override |
+| `HFS_S3_PREFIX` | *(none)* | Optional key prefix prepended to all S3 object keys |
 | `HFS_S3_VALIDATE_BUCKETS` | `true` | Validate bucket access on startup |
 
 For detailed backend setup instructions (building from source, Docker commands, and search offloading architecture), see the [persistence crate documentation](crates/persistence/README.md#building--running-storage-backends).

crates/hfs/Cargo.toml

@@ -53,6 +53,9 @@ clap = { version = "4.0", features = ["derive", "env"] }
 # Logging
 tracing = "0.1"
 
+# Sync primitives (used by build_search_registry for the elasticsearch composite backend)
+parking_lot = "0.12"
+
 # Error handling
 anyhow = "1.0"