diff --git a/.data/.private b/.data/.private new file mode 100644 index 0000000..e69de29 diff --git a/.env.example b/.env.example index 50f95b1..6e4ea80 100644 --- a/.env.example +++ b/.env.example @@ -1,8 +1,23 @@ -PORT="3000" -API_TOKEN="..." -APPLICATION_ID="..." -CONTRACT_LABEL="M3ters" -CHIRPSTACK_HOST="localhost" -MAINNET_RPC="https://sepolia.drpc.org" -PREFERRED_PROVER_NODE="http://prover.m3ter.ing" +# Server Configuration +PORT=3000 + +# Module Configuration +BACKEND_MODULES="core/arweave,core/is_on,core/prover,core/streamr" +UI_MODULES="streamr:core/streamr/ui" + +# ChirpStack Configuration +API_TOKEN=... +APPLICATION_ID=... +CHIRPSTACK_HOST=localhost + +# Contract & Network Configuration +CONTRACT_LABEL=M3ters +MAINNET_RPC=https://sepolia.drpc.org ETHEREUM_PRIVATE_KEY="..." + +# Streamr Configuration +STREAMR_STREAM_ID="0x567853282663b601bfdb9203819b1fbb3fe18926/m3tering/test" +STREAMR_CRONSCHEDULE="0 * * * *" # Every hour + +# Optional: Prover Node (disabled publishing to prover if not set) +# PREFERRED_PROVER_NODE="http://prover.m3ter.ing" \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..c6456be --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,151 @@ +name: M3tering Console Test and Publish Docker Image + +on: + workflow_dispatch: + push: + pull_request: + branches: + - main + +env: + REGISTRY: ghcr.io + +permissions: + contents: read + packages: write + attestations: write + id-token: write + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v2 + with: + node-version: "20" + + - name: Install dependencies + run: npm install + + - name: Run tests + run: npm test + + build-platform: + needs: [test] + strategy: + matrix: + include: + - platform: linux/amd64 + runner: ubuntu-latest + arch: amd64 + - platform: linux/arm64 + runner: ubuntu-24.04-arm + arch: arm64 + + runs-on: ${{ matrix.runner }} + + steps: + - name: Checkout repository + uses: actions/checkout@v5 + + - name: Normalize image names + run: | + echo "CONSOLE_IMAGE_NAME=${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV + + - name: Free disk space + uses: jlumbroso/free-disk-space@main + with: + tool-cache: true + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: true + swap-storage: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Console image - push by digest with repository name + - name: Build Console image (single platform) + id: build-console + uses: docker/build-push-action@v6 + with: + context: . + platforms: ${{ matrix.platform }} + push: ${{ github.event_name != 'pull_request' }} + outputs: type=image,name=${{ env.REGISTRY }}/${{ env.CONSOLE_IMAGE_NAME }},push-by-digest=true + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.CONSOLE_IMAGE_NAME }}:buildcache-${{ matrix.arch }} + cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.CONSOLE_IMAGE_NAME }}:buildcache-${{ matrix.arch }},mode=max + + - name: Save digests to files + if: github.event_name != 'pull_request' + run: | + echo ${{ steps.build-console.outputs.digest }} > digests-${{ matrix.arch }}-console-digest.txt + + - name: Upload digests artifact + if: github.event_name != 'pull_request' + uses: actions/upload-artifact@v4 + with: + name: digests-${{ matrix.arch }} + path: | + digests-${{ matrix.arch }}-console-digest.txt + + create-manifests: + needs: build-platform + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + + steps: + - name: Normalize image names + run: | + echo "CONSOLE_IMAGE_NAME=${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV + + - name: Download all digests + uses: actions/download-artifact@v4 + with: + pattern: digests-* + merge-multiple: true + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Extract metadata for Console image + id: meta-console + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.CONSOLE_IMAGE_NAME }} + + - name: Create multi-arch manifest for Console + run: | + AMD64_DIGEST=$(cat digests-amd64-console-digest.txt) + ARM64_DIGEST=$(cat digests-arm64-console-digest.txt) + echo AMD64_DIGEST=$AMD64_DIGEST + echo ARM64_DIGEST=$ARM64_DIGEST + TAGS=(${{ steps.meta-console.outputs.tags }}) + TAG_ARGS="" + for tag in "${TAGS[@]}"; do + TAG_ARGS="$TAG_ARGS --tag $tag" + done + docker buildx imagetools create $TAG_ARGS \ + ${{ env.REGISTRY }}/${{ env.CONSOLE_IMAGE_NAME }}@$AMD64_DIGEST \ + ${{ env.REGISTRY }}/${{ env.CONSOLE_IMAGE_NAME }}@$ARM64_DIGEST diff --git a/.gitignore b/.gitignore index 0351bbb..195e462 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,3 @@ -# console configuration file -# console.config.json - #test scripts emulate.ts test-database.ts diff --git a/.vscode/settings.json b/.vscode/settings.json index 7d1e594..a6fb97c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,9 +2,10 @@ "cSpell.words": [ "ardrive", "arweave", - "m3ters", - "Emmo00", "ccip", - "Mauchly", + "CRONSCHEDULE", + "Emmo00", + "m3ters", + "Mauchly" ] } \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 8c45d16..6a78d91 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ FROM node:20-alpine # Create working directory WORKDIR /opt/app -RUN apk add --no-cache cmake make g++ python3 openssl-dev py3-setuptools +RUN apk add --no-cache cmake make g++ python3 openssl-dev py3-setuptools git # Copy and install dependencies COPY package*.json ./ @@ -14,7 +14,6 @@ RUN npm install --include=dev && npm cache clean --force COPY babel.config.js . COPY tsconfig.json . COPY src ./src -COPY console.config.json . # Build project RUN npm run build @@ -22,5 +21,10 @@ RUN npm run build # Expose application port EXPOSE 3000 +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] + # Start app CMD [ "npm", "start" ] diff --git a/README.md b/README.md index a04dea3..bbbe6a3 100644 --- a/README.md +++ b/README.md @@ -22,14 +22,29 @@ A modular, extensible service console for providers on the M3tering protocol. Fe Create `.env` file: ``` + # Server Configuration PORT=3000 + + # Module Configuration + BACKEND_MODULES="core/arweave,core/is_on,core/prover,core/streamr" + UI_MODULES="streamr:core/streamr/ui" + + # ChirpStack Configuration API_TOKEN=... APPLICATION_ID=... - CONTRACT_LABEL=M3ters CHIRPSTACK_HOST=localhost + + # Contract & Network Configuration + CONTRACT_LABEL=M3ters MAINNET_RPC=https://sepolia.drpc.org - PREFERRED_PROVER_NODE=http://34.244.149.153 ETHEREUM_PRIVATE_KEY="..." + + # Streamr Configuration + STREAMR_STREAM_ID="0x567853282663b601bfdb9203819b1fbb3fe18926/m3tering/test" + STREAMR_CRONSCHEDULE="0 * * * *" # Every hour + + # Optional: Prover Node (defaults to automatic selection) + # PREFERRED_PROVER_NODE="http://prover.m3ter.ing" ``` 3. **Docker Build and Run** @@ -65,36 +80,36 @@ The M3tering Console provides two complementary extension systems: 1. **Backend Hooks** - Hook into the console lifecycle (MQTT, database, message processing) 2. **UI Hooks** - Add custom icons, panels, and actions to the web interface -Both systems use a config-driven approach where modules are loaded dynamically from paths specified in `console.config.json`. +Both systems use an environment-driven approach where modules are loaded dynamically from paths specified in your `.env` file. ## Configuration -```json -{ - "modules": [ - "core/arweave", - "core/prover", - "core/streamr", - "core/is_on", - "core/prune_sync" - ], - "uiModules": { - "streamr": "core/streamr/ui" - }, - "streamr": { - "streamId": [ - "0x567853282663b601bfdb9203819b1fbb3fe18926/m3tering/test" - ], - "cronSchedule": "0 * * * *" - }, - "prune_sync": { - "cronSchedule": "0 * * * *" - } -} +Modules are configured via environment variables. Modules are automatically pulled from GitHub repositories when the container starts up. + +```bash +# Backend Modules (comma-separated GitHub repositories) +BACKEND_MODULES="core/arweave,core/is_on,core/prover,core/streamr,username/my-custom-module" + +# UI Modules (colon-separated format: moduleId:github_repo) +UI_MODULES="streamr:core/streamr/ui,my-module:username/my-custom-module" + +# Module-specific configuration +STREAMR_STREAM_ID="0x567853282663b601bfdb9203819b1fbb3fe18926/m3tering/test" +STREAMR_CRONSCHEDULE="0 * * * *" # Every hour ``` -- **`modules`**: Array of paths to backend hook modules (relative to `src/lib/`) -- **`uiModules`**: Object mapping module IDs to UI module paths (relative to `src/lib/`) +- **`BACKEND_MODULES`**: Comma-separated list of GitHub repositories in the format `/` or built-in paths like `core/arweave` +- **`UI_MODULES`**: Comma-separated list of `moduleId:/` pairs +- **Module-specific variables**: Each module can have its own configuration variables (e.g., `STREAMR_STREAM_ID`) + +### Publishing Custom Modules + +To use your own extensions: + +1. Publish your module code to a GitHub repository +2. Reference it in your `.env` file using the format `/` +3. The extension code is automatically cloned from GitHub when the Docker container starts up +4. For specific versions, append `#` or `#` (e.g., `username/my-module#v1.0.0`) --- @@ -104,8 +119,10 @@ Backend hooks allow modules to react to console lifecycle events. Each module ex ## Creating a Backend Module +1. **Create your module repository** with the following structure: + ```typescript -// src/lib/core/my-module/index.ts +// index.ts import type { Hooks } from "../../../types"; export default class implements Hooks { @@ -119,13 +136,15 @@ export default class implements Hooks { } ``` -Add to `console.config.json`: -```json -{ - "modules": ["core/my-module"] -} +2. **Publish to GitHub**: Push your module code to a GitHub repository (e.g., `github.com/yourusername/my-m3tering-module`) + +3. **Configure in `.env`**: +```bash +BACKEND_MODULES="core/arweave,core/prover,yourusername/my-m3tering-module" ``` +4. **Restart container**: The module will be automatically cloned from GitHub and loaded when the container starts + ## Hook Lifecycle Reference ### Initialization Phase @@ -180,8 +199,10 @@ UI Hooks allow modules to extend the web interface at `http://localhost:3000`. M ## Creating a UI Module +1. **Create your module repository** with the following structure: + ```typescript -// src/lib/core/my-module/ui.ts +// ui.ts (or index.ts) import type { UIHooks, UIAppIcon, UIAppWindow, UIAction } from "../../../types"; export default class implements UIHooks { @@ -222,15 +243,15 @@ export default class implements UIHooks { } ``` -Add to `console.config.json`: -```json -{ - "uiModules": { - "my-module": "core/my-module/ui" - } -} +2. **Publish to GitHub**: Push your module code to a GitHub repository (e.g., `github.com/yourusername/my-ui-module`) + +3. **Configure in `.env`**: +```bash +UI_MODULES="streamr:core/streamr/ui,my-module:yourusername/my-ui-module" ``` +4. **Restart container**: The module will be automatically cloned from GitHub and loaded when the container starts + ## UIHooks Interface | Method | Return Type | Description | @@ -311,7 +332,6 @@ Response: { success: boolean, message?: string, data?: any } | `core/prover` | Sends batched transactions to the prover node | | `core/streamr` | Publishes transactions to Streamr streams on a cron schedule | | `core/is_on` | Computes device on/off state based on balance | -| `core/prune_sync` | Cleans up old synchronized transactions | ## UI Modules diff --git a/console.config.json b/console.config.json deleted file mode 100644 index bbefcd2..0000000 --- a/console.config.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "modules": [ - "core/arweave", - "core/streamr", - "core/is_on", - "core/prune_sync" - ], - "uiModules": { - "streamr": "core/streamr/ui" - }, - "streamr": { - "streamId": [ - "0x567853282663b601bfdb9203819b1fbb3fe18926/m3tering/test" - ], - "cronSchedule": "0 * * * *" - }, - "prune_sync": { - "cronSchedule": "0 * * * *" - } -} diff --git a/console.example.config.json b/console.example.config.json deleted file mode 100644 index 628ab5f..0000000 --- a/console.example.config.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "modules": [ - "core/arweave", - "core/prover", - "core/streamr", - "core/is_on", - "core/prune_sync" - ], - "uiModules": { - "streamr": "core/streamr/ui" - }, - "streamr": { - "streamId": [ - "0x567853282663b601bfdb9203819b1fbb3fe18926/m3tering/test" - ], - "cronSchedule": "0 * * * *" - }, - "prune_sync": { - "cronSchedule": "0 * * * *" - } -} diff --git a/docker-compose.yaml b/docker-compose.yaml index c919398..893e1ba 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -2,8 +2,17 @@ services: console: image: ghcr.io/m3tering/console:main network_mode: host + restart: unless-stopped env_file: - .env ports: - "3000:3000" - restart: unless-stopped + volumes: + - ./.data:/opt/app/.data + - console_src_modules:/opt/app/src/lib + - console_dist_modules:/opt/app/dist/lib + +volumes: + console_db: + console_src_modules: + console_dist_modules: diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100644 index 0000000..d5c32ef --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env sh +set -e + +APP_DIR="/opt/app" +SRC_LIB="$APP_DIR/src/lib" +DIST_LIB="$APP_DIR/dist/lib" + +mkdir -p "$SRC_LIB" +mkdir -p "$DIST_LIB" + +if [ -n "$MODULES" ]; then + echo "Requested modules: $MODULES" + IFS=',' + + for repo in $MODULES; do + CLEAN_REPO=$(echo "$repo" | cut -d'#' -f1) + NAME=$(basename "$CLEAN_REPO") + VERSION=$(echo "$repo" | grep -o '#.*' | sed 's/#//') + + SRC_DEST="$SRC_LIB/$NAME" + DIST_DEST="$DIST_LIB/$NAME" + + # ----------------------------- + # 1️⃣ Download source if missing + # ----------------------------- + if [ -d "$SRC_DEST" ]; then + echo "✔ Module $NAME already downloaded" + else + echo "⬇ Downloading module: $NAME" + git clone --depth=1 "https://github.com/$CLEAN_REPO.git" "$SRC_DEST" + + if [ -n "$VERSION" ]; then + git -C "$SRC_DEST" fetch --tags + git -C "$SRC_DEST" checkout "$VERSION" + fi + fi + + # ----------------------------- + # 2️⃣ Install plugin dependencies (isolated) + # ----------------------------- + if [ -f "$SRC_DEST/package.json" ] && [ ! -d "$SRC_DEST/node_modules" ]; then + echo "📦 Installing dependencies for $NAME (isolated)" + cd "$SRC_DEST" + npm install --production + cd "$APP_DIR" + fi + + # ----------------------------- + # 3️⃣ Build or copy plugin + # ----------------------------- + if [ -f "$DIST_DEST/.built" ]; then + echo "🏁 Plugin $NAME already built" + continue + fi + + echo "🔧 Building plugin: $NAME" + mkdir -p "$DIST_DEST" + + # TypeScript project + if [ -f "$SRC_DEST/tsconfig.json" ]; then + npx tsc --project "$SRC_DEST/tsconfig.json" --outDir "$DIST_DEST" + + # Single TS file + elif [ -f "$SRC_DEST/index.ts" ]; then + npx tsc "$SRC_DEST/index.ts" --outDir "$DIST_DEST" + + # Plain JS project + else + cp -r "$SRC_DEST"/* "$DIST_DEST/" + fi + + # ----------------------------- + # 4️⃣ Copy plugin's node_modules to dist + # ----------------------------- + if [ -d "$SRC_DEST/node_modules" ]; then + echo "📦 Copying isolated dependencies for $NAME" + cp -r "$SRC_DEST/node_modules" "$DIST_DEST/" + fi + + # Mark as built to avoid rebuilding every restart + touch "$DIST_DEST/.built" + done +fi + +exec "$@" diff --git a/src/lib/core/prover/index.ts b/src/lib/core/prover/index.ts index e647812..2724a2e 100644 --- a/src/lib/core/prover/index.ts +++ b/src/lib/core/prover/index.ts @@ -1,7 +1,7 @@ import { buildBatchPayload } from "../../utils"; import type { BatchTransactionPayload, Hooks, TransactionRecord } from "../../../types"; -const PREFERRED_PROVER_NODE = process.env.PREFERRED_PROVER_NODE || "https://prover.m3ter.ing"; +const PREFERRED_PROVER_NODE = process.env.PREFERRED_PROVER_NODE; export default class implements Hooks { async onTransactionDistribution(_: any, __: any, pendingTransactions: TransactionRecord[]) { @@ -51,7 +51,7 @@ export default class implements Hooks { } async getProverURL(): Promise { - return PREFERRED_PROVER_NODE; + return PREFERRED_PROVER_NODE || null; } async sendPendingTransactionsToProver(proverURL: string, pendingTransactions: TransactionRecord[]) { diff --git a/src/lib/core/prune_sync/index.ts b/src/lib/core/prune_sync/index.ts deleted file mode 100644 index aed1354..0000000 --- a/src/lib/core/prune_sync/index.ts +++ /dev/null @@ -1,28 +0,0 @@ -import cron from "node-cron"; -import { Hooks } from "../../../types"; -import { getAllMeterRecords } from "../../../store/sqlite"; -import { loadConfigurations } from "../../utils"; -import { pruneAndSyncOnchain } from "../../sync"; - -export default class implements Hooks { - private config = loadConfigurations(); - - async onAfterInit() { - console.log("Registering prune_sync cron job..."); - - // Schedule a cron job to perform prune verified transactions and sync with onchain state - cron.schedule(this.config.prune_sync.cronSchedule, async () => { - const m3ters = getAllMeterRecords(); - for (const m3ter of m3ters) { - try { - pruneAndSyncOnchain(m3ter.publicKey); - } catch (error) { - console.error(`Error pruning and syncing meter ${m3ter.publicKey}:`, error); - } - } - }); - - console.log("prune_sync cron job registered."); - return; - } -} diff --git a/src/lib/core/streamr/index.ts b/src/lib/core/streamr/index.ts index a3f1375..f8525d6 100644 --- a/src/lib/core/streamr/index.ts +++ b/src/lib/core/streamr/index.ts @@ -1,7 +1,7 @@ import cron from "node-cron"; import { StreamrClient } from "@streamr/sdk"; import { getAllMeterRecords, getAllTransactionRecords } from "../../../store/sqlite"; -import { buildBatchPayload, loadConfigurations, retry } from "../../utils"; +import { buildBatchPayload, retry } from "../../utils"; import type { Hooks, TransactionRecord } from "../../../types"; import { pruneAndSyncOnchain } from "../../sync"; @@ -12,14 +12,15 @@ if (!ETHEREUM_PRIVATE_KEY) { } export default class implements Hooks { - private config = loadConfigurations(); + private streamIds: string[] = process.env.STREAMR_STREAM_ID ? process.env.STREAMR_STREAM_ID.split(",") : []; + private cronSchedule: string = process.env.STREAMR_CRONSCHEDULE || "0 * * * *"; async onAfterInit() { - console.log("Registering Streamr cron job..."); + console.log("Registering Streamr cron job... Schedule: ", this.cronSchedule, " Stream IDs: ", JSON.stringify(this.streamIds)); // Schedule a cron job to publish pending transactions cron.schedule( - this.config.streamr.cronSchedule, + this.cronSchedule, async () => { console.log("Streamr cron job started: Pruning meters and publishing pending transactions..."); const m3ters = getAllMeterRecords(); @@ -35,14 +36,16 @@ export default class implements Hooks { const pendingTransactions = await this.getPendingTransactions(); if (pendingTransactions.length > 0) { - for (const STREAMR_STREAM_ID of this.config.streamr.streamId) { - console.log(`Publishing to Streamr stream: ${STREAMR_STREAM_ID}`); - await retry( - () => this.publishPendingTransactionsToStreamr(STREAMR_STREAM_ID, pendingTransactions), - 3, - 2000, - ); - } + await Promise.all( + this.streamIds.map(async (STREAMR_STREAM_ID) => { + console.log(`Publishing to Streamr stream: ${STREAMR_STREAM_ID}`); + await retry( + () => this.publishPendingTransactionsToStreamr(STREAMR_STREAM_ID, pendingTransactions), + 3, + 2000, + ); + }), + ); } }, { name: "streamr-publish-pending-transactions", noOverlap: true }, @@ -65,9 +68,9 @@ export default class implements Hooks { try { console.log(`[streamr] Connecting to ${STREAMR_STREAM_ID}...`); - const stream = await retry(() => streamrClient.getStream(STREAMR_STREAM_ID!), 3, 10000); + const stream = await retry(() => streamrClient.getStream(STREAMR_STREAM_ID!), 3, 2000); - await new Promise((resolve) => setTimeout(resolve, 100000)); // wait for 100 seconds to ensure connection is established + await new Promise((resolve) => setTimeout(resolve, 2000)); // wait for 2 seconds to ensure connection is established console.log(`[streamr] Connected. Publishing ${pendingTransactions.length} transactions...`); const batchPayload = buildBatchPayload(pendingTransactions); diff --git a/src/lib/core/streamr/ui.ts b/src/lib/core/streamr/ui.ts index f947529..2f1a0bd 100644 --- a/src/lib/core/streamr/ui.ts +++ b/src/lib/core/streamr/ui.ts @@ -1,5 +1,5 @@ import { getAllTransactionRecords } from "../../../store/sqlite"; -import { buildBatchPayload, loadConfigurations, retry } from "../../utils"; +import { buildBatchPayload, retry } from "../../utils"; import type { UIHooks, UIAppIcon, UIAppWindow, UIAction, TransactionRecord } from "../../../types"; import { StreamrClient } from "@streamr/sdk"; @@ -11,7 +11,8 @@ const { ETHEREUM_PRIVATE_KEY } = process.env; * a panel showing stream configuration and a manual publish action */ export default class implements UIHooks { - private config = loadConfigurations(); + private streamIds: string[] = process.env.STREAMR_STREAM_ID ? process.env.STREAMR_STREAM_ID.split(",") : []; + private cronSchedule: string = process.env.STREAMR_CRONSCHEDULE || "Not configured"; private lastPublishTime: Date | null = null; private lastPublishStatus: "success" | "error" | null = null; @@ -26,8 +27,8 @@ export default class implements UIHooks { async getAppWindow(): Promise { const pendingCount = (await this.getPendingTransactions()).length; - const streamIds = this.config.streamr.streamId; - const cronSchedule = this.config.streamr.cronSchedule; + const streamIds = this.streamIds; + const cronSchedule = this.cronSchedule; return { id: "streamr", @@ -94,16 +95,18 @@ export default class implements UIHooks { } try { - for (const streamId of this.config.streamr.streamId) { - console.log(`[streamr-ui] Publishing to stream: ${streamId}`); - await retry(() => this.publishToStreamr(streamId, pendingTransactions), 3, 2000); - } + await Promise.all( + this.streamIds.map(async (streamId) => { + console.log(`[streamr-ui] Publishing to stream: ${streamId}`); + await retry(() => this.publishToStreamr(streamId, pendingTransactions), 3, 2000); + }), + ); this.lastPublishTime = new Date(); this.lastPublishStatus = "success"; return { - message: `Published ${pendingTransactions.length} transactions to ${this.config.streamr.streamId.length} stream(s)`, + message: `Published ${pendingTransactions.length} transactions to ${this.streamIds.length} stream(s)`, data: { count: pendingTransactions.length }, }; } catch (error: any) { @@ -119,8 +122,8 @@ export default class implements UIHooks { const pendingTransactions = await this.getPendingTransactions(); return { pendingCount: pendingTransactions.length, - streamIds: this.config.streamr.streamId, - cronSchedule: this.config.streamr.cronSchedule, + streamIds: this.streamIds, + cronSchedule: this.cronSchedule, lastPublishTime: this.lastPublishTime, lastPublishStatus: this.lastPublishStatus, }; @@ -142,9 +145,9 @@ export default class implements UIHooks { try { const stream = await retry(() => streamrClient.getStream(streamId), 3, 2000); const batchPayload = buildBatchPayload(pendingTransactions); - await new Promise((resolve) => setTimeout(resolve, 2000)); + await new Promise((resolve) => setTimeout(resolve, 2000)); // wait for 2 seconds to ensure connection is established await stream.publish(batchPayload); - await new Promise((resolve) => setTimeout(resolve, 2000)); + await new Promise((resolve) => setTimeout(resolve, 100000)); // wait for 100 seconds to ensure message is sent console.log(`[streamr-ui] Published ${pendingTransactions.length} transactions to stream ${streamId}`); } catch (error) { console.error(`[streamr-ui] Error publishing to Streamr:`, error); diff --git a/src/lib/utils.ts b/src/lib/utils.ts index 3835826..4707db8 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -1,4 +1,3 @@ -import fs from "fs"; import path from "path"; import { createPublicKey, verify } from "crypto"; import type { @@ -14,32 +13,33 @@ import type { const extensions: Hooks[] = []; const uiExtensions: Map = new Map(); -export const defaultConfigurations: AppConfig = { - modules: ["core/arweave", "core/prover", "core/streamr", "core/is_on", "core/prune_sync"], - uiModules: { - streamr: "core/streamr/ui", - }, - streamr: { - streamId: ["0x567853282663b601bfdb9203819b1fbb3fe18926/m3tering/test"], - cronSchedule: "0 * * * *", - }, - prune_sync: { - cronSchedule: "0 * * * *", - }, -}; -export function loadConfigurations(configPath: string = "console.config.json"): AppConfig { - try { - const config: AppConfig = JSON.parse(fs.readFileSync(configPath, "utf-8")); - return config; - } catch (error) { - console.warn(`Could not load configuration from ${configPath}, using default configurations.`); - return defaultConfigurations; - } +const { BACKEND_MODULES, UI_MODULES } = process.env; + +if (!BACKEND_MODULES) { + console.warn("[config] No BACKEND_MODULES specified"); } -export async function loadExtensionsFromConfig(configPath: string = "console.config.json"): Promise { - const config: AppConfig = loadConfigurations(configPath); +if (!UI_MODULES) { + console.warn("[config] No UI_MODULES specified"); +} + +export const defaultConfigurations: AppConfig = { + modules: (BACKEND_MODULES || "").split(",").filter(Boolean), // ["core/arweave", "core/prover", "core/streamr", "core/is_on"], + uiModules: Object.fromEntries( + (UI_MODULES || "") + .split(",") + .filter(Boolean) + .map((entry) => { + const [moduleId, modulePath] = entry.split(":"); + return [moduleId, modulePath]; + }) + .filter(([moduleId, modulePath]) => moduleId && modulePath), + ), +}; + +export async function loadExtensionsFromConfig(): Promise { + const config: AppConfig = defaultConfigurations; for (const modulePath of config.modules) { const resolved = path.resolve(__dirname, modulePath); @@ -67,21 +67,15 @@ export async function runHook(hook: K, ...args: Parameter return result; } -// ========================================== -// UI Extension System -// ========================================== - /** * Load UI extensions from configuration file * Looks for 'uiModules' key in config, which maps module IDs to their paths */ -export async function loadUIExtensionsFromConfig( - configPath: string = "console.config.json", -): Promise> { - const config = loadConfigurations(configPath) as AppConfig & { uiModules?: Record }; +export async function loadUIExtensionsFromConfig(): Promise> { + const config = defaultConfigurations as AppConfig & { uiModules?: Record }; if (!config.uiModules) { - console.log("[ui] No UI modules configured"); + console.warn("[ui] No UI modules configured"); return uiExtensions; } diff --git a/src/services/mqtt.ts b/src/services/mqtt.ts index b2c0a56..a771ebe 100644 --- a/src/services/mqtt.ts +++ b/src/services/mqtt.ts @@ -224,6 +224,7 @@ export async function handleMessage(blob: Buffer) { logger.info(`Updated meter nonce to: ${expectedNonce}`); const pendingTransactions = getAllTransactionRecords(); + console.log(`${pendingTransactions.length} Pending transactions: ${JSON.stringify(pendingTransactions)}`); await runHook("onTransactionDistribution", m3ter.tokenId, decoded, pendingTransactions); } diff --git a/src/store/sqlite.ts b/src/store/sqlite.ts index 13cb20f..61763ac 100644 --- a/src/store/sqlite.ts +++ b/src/store/sqlite.ts @@ -2,7 +2,6 @@ import fs from "fs"; import Database from "better-sqlite3"; import type { Database as DatabaseType, Statement as DatabaseStatementType } from "better-sqlite3"; import { MeterRecord, TransactionRecord } from "../types"; -import { get } from "http"; // meter queries let db: DatabaseType; @@ -24,7 +23,7 @@ let getTransactionByNonceQuery: DatabaseStatementType; * @param databaseName name of the database file */ export default function setupDatabase(databaseName = "m3tering.db") { - db = new Database(databaseName, {}); + db = new Database(`./.data/${databaseName}`, {}); initializeTransactionsTable(); initializeMetersTable(); @@ -36,7 +35,7 @@ export function deleteDatabase(databaseName = "m3tering.db") { db.exec(`DROP TABLE IF EXISTS meters`); db.exec(`DROP TABLE IF EXISTS transactions`); db.close(); - fs.unlinkSync(databaseName); + fs.unlinkSync(`./.data/${databaseName}`); } catch (err: any) { console.error("Failed to delete database:", err); } diff --git a/src/types.ts b/src/types.ts index 3004ac9..9e18412 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,16 +1,9 @@ import { MqttClient } from "mqtt/*"; -// Application configuration type (console.config.json) +// Application configuration type export type AppConfig = { modules: string[]; uiModules?: Record; - streamr: { - streamId: string[]; - cronSchedule: string; - }; - prune_sync: { - cronSchedule: string; - }; }; // Hooks type for lifecycle events