Use 427 tokenizer

.github/workflows/dev-build.yml

@@ -1,7 +1,7 @@
-### This is the Terraform-generated dev-build.yml workflow for the         ### 
+### This is the Terraform-generated dev-build.yml workflow for the         ###
 ### timdex-semantic-builder-dev app repository.                            ###
-### If this is a Lambda repo, uncomment the FUNCTION line at the end of    ### 
-### the document. If the container requires any additional pre-build       ### 
+### If this is a Lambda repo, uncomment the FUNCTION line at the end of    ###
+### the document. If the container requires any additional pre-build       ###
 ### commands, uncomment and edit the PREBUILD line at the end of the       ###
 ### document.                                                              ###
 
@@ -12,7 +12,7 @@ on:
     branches:
       - main
     paths-ignore:
-      - '.github/**'
+      - ".github/**"
 
 permissions:
   id-token: write
@@ -22,7 +22,7 @@ jobs:
   prep:
     name: Prep for Build
     runs-on: ubuntu-latest
-    outputs: 
+    outputs:
       cpuarch: ${{ steps.setarch.outputs.cpuarch }}
     steps:
       - name: Checkout
@@ -57,4 +57,4 @@ jobs:
       ECR: "timdex-semantic-builder-dev"
       CPU_ARCH: ${{ needs.prep.outputs.cpuarch }}
       FUNCTION: "timdex-semantic-builder-dev"
-      # PREBUILD: 
+      PREBUILD: echo "hello from PREBUILD! (testing workflow. Will remove later.)"

.pre-commit-config.yaml

@@ -1,29 +1,28 @@
 default_language_version:
-  python: python3.14 # set for project python version
+  python: python3.14
+default_stages:
+  - pre-push
 repos:
   - repo: local
     hooks:
-      - id: black-apply
-        name: black-apply
-        entry: uv run black
+      - id: ruff-format
+        name: ruff-format
+        entry: uv run ruff format --diff
         language: system
         pass_filenames: true
-        types: ["python"]
+        types: [ "python" ]
+
       - id: mypy
         name: mypy
         entry: uv run mypy
         language: system
         pass_filenames: true
-        types: ["python"]
-        exclude: "tests/"
-      - id: ruff-apply
-        name: ruff-apply
-        entry: uv run ruff check --fix
+        types: [ "python" ]
+        exclude: "(tests/|output/|migrations/)"
+
+      - id: ruff-check
+        name: ruff-check
+        entry: uv run ruff check
         language: system
         pass_filenames: true
-        types: ["python"]
-      # - id: pip-audit
-      #   name: pip-audit
-      #   entry: uv run pip-audit
-      #   language: system
-      #   pass_filenames: false
+        types: [ "python" ]

Dockerfile

@@ -16,4 +16,4 @@ FROM public.ecr.aws/lambda/python:3.14
 
 COPY --from=builder ${LAMBDA_TASK_ROOT} ${LAMBDA_TASK_ROOT}
 
-CMD ["lambdas.my_function.lambda_handler"]
+CMD ["lambdas.tokenizer_handler.lambda_handler"]

LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+   Copyright 2026 MIT Libraries
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

Makefile

@@ -15,22 +15,26 @@ help: # Preview Makefile commands
 /^[-_[:alpha:]]+:.?*#/ { printf "  %-15s%s\n", $$1, $$2 }' $(MAKEFILE_LIST)
 
 # ensure OS binaries aren't called if naming conflict with Make recipes
-.PHONY: help venv install update test coveralls lint black mypy ruff safety lint-apply black-apply ruff-apply check-arch dist-dev publish-dev update-lambda-dev docker-clean
+.PHONY: help venv install update test coveralls lint lint-fix security sam-build sam-invoke console check-arch dist-dev publish-dev update-lambda-dev docker-clean
 
 ##############################################
 # Python Environment and Dependency commands
 ##############################################
 
-install: .venv .git/hooks/pre-commit # Install Python dependencies, hooks, and create virtual environment if not exists
+install: .venv .git/hooks/pre-commit .git/hooks/pre-push # Install Python dependencies and create virtual environment if not exists
 	uv sync --dev
 
 .venv: # Creates virtual environment if not found
 	@echo "Creating virtual environment at .venv..."
 	uv venv .venv
 
-.git/hooks/pre-commit: # Sets up pre-commit hook if not setup
-	@echo "Installing pre-commit hooks..."
-	uv run pre-commit install
+.git/hooks/pre-commit: # Sets up pre-commit commit hooks if not setup
+	@echo "Installing pre-commit commit hooks..."
+	uv run pre-commit install --hook-type pre-commit
+
+.git/hooks/pre-push: # Sets up pre-commit push hooks if not setup
+	@echo "Installing pre-commit push hooks..."
+	uv run pre-commit install --hook-type pre-push
 
 venv: .venv # Create the Python virtual environment
 
@@ -50,44 +54,36 @@ coveralls: test # Write coverage data to an LCOV report
 	uv run coverage lcov -o ./coverage/lcov.info
 
 ####################################
-# Code quality and safety commands
+# Code linting and formatting
 ####################################
 
-lint: black mypy ruff # Run linters
-
-black: # Run 'black' linter and print a preview of suggested changes
-	uv run black --check --diff .
-
-mypy: # Run 'mypy' linter
+lint: # Run linters
+	uv run ruff format --diff
 	uv run mypy .
-
-ruff: # Run 'ruff' linter and print a preview of errors
 	uv run ruff check .
 
-safety: # Check for security vulnerabilities
-	uv run pip-audit
-
-lint-apply: black-apply ruff-apply # Apply changes with 'black' and resolve 'fixable errors' with 'ruff'
-
-black-apply: # Apply changes with 'black'
-	uv run black .
-
-ruff-apply: # Resolve 'fixable errors' with 'ruff'
+lint-fix: # Run linting, auto fix behaviors where supported
+	uv run ruff format .
 	uv run ruff check --fix .
 
+security: # Check for security vulnerabilities
+	uv run pip-audit
+
 ####################################
 # SAM Lambda
 ####################################
-sam-build: # Build SAM image for running Lambda locally
-	sam build --template tests/sam/template.yaml
+sam-build: # Build Docker image for running Lambda locally
+	sam build --template tests/sam/template.yaml --debug
 
-sam-http-run: # Run lambda locally as an HTTP server
-	sam local start-api --template tests/sam/template.yaml --env-vars tests/sam/env.json
+sam-invoke: # Invoke lambda locally with tests/sam/event.json
+	sam local invoke Tokenizer --template tests/sam/template.yaml --env-vars tests/sam/env.json --event tests/sam/event.json
 
-sam-http-ping: # Send curl command to SAM HTTP server
-	curl --location 'http://localhost:3000/myapp' \
-	--header 'Content-Type: application/json' \
-	--data '{"msg":"in a bottle"}'
+####################################
+# Convenience commands
+####################################
+console: # Start a Python REPL with the project environment loaded
+	@echo "'exit' to quit console"
+	uv run ipython
 
 ####################################
 # Deployment to ECR
@@ -126,7 +122,7 @@ publish-dev: dist-dev ## Build, tag and push (intended for developer-based manua
     docker image prune -f --filter "dangling=true"
 
 ## If this is a Lambda repo, uncomment the two lines below
-update-lambda-dev: ## Updates the lambda with whatever is the most recent image in the ecr (intended for developer-based manual update)
+update-lambda-dev: check-arch ## Updates the lambda with whatever is the most recent image in the ecr (intended for developer-based manual update)
 	@ARCH_TAG=$$(cat .arch_tag); \
 	aws lambda update-function-code \
 		--region us-east-1 \