From 3510597d9dbf36f33d4c47a74fb6e69b931a8862 Mon Sep 17 00:00:00 2001 From: Emmo00 Date: Wed, 28 Jan 2026 23:21:22 +0100 Subject: [PATCH 01/14] refactor: remove legacy test workflow and add CI for testing and publishing Docker image --- .github/workflows/ci.yml | 151 +++++++++++++++++++++++++++++++++++++ .github/workflows/test.yml | 21 ------ docker-compose.yaml | 1 + 3 files changed, 152 insertions(+), 21 deletions(-) create mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..c6456be --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,151 @@ +name: M3tering Console Test and Publish Docker Image + +on: + workflow_dispatch: + push: + pull_request: + branches: + - main + +env: + REGISTRY: ghcr.io + +permissions: + contents: read + packages: write + attestations: write + id-token: write + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v2 + with: + node-version: "20" + + - name: Install dependencies + run: npm install + + - name: Run tests + run: npm test + + build-platform: + needs: [test] + strategy: + matrix: + include: + - platform: linux/amd64 + runner: ubuntu-latest + arch: amd64 + - platform: linux/arm64 + runner: ubuntu-24.04-arm + arch: arm64 + + runs-on: ${{ matrix.runner }} + + steps: + - name: Checkout repository + uses: actions/checkout@v5 + + - name: Normalize image names + run: | + echo "CONSOLE_IMAGE_NAME=${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV + + - name: Free disk space + uses: jlumbroso/free-disk-space@main + with: + tool-cache: true + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: true + swap-storage: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Console image - push by digest with repository name + - name: Build Console image (single platform) + id: build-console + uses: docker/build-push-action@v6 + with: + context: . + platforms: ${{ matrix.platform }} + push: ${{ github.event_name != 'pull_request' }} + outputs: type=image,name=${{ env.REGISTRY }}/${{ env.CONSOLE_IMAGE_NAME }},push-by-digest=true + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.CONSOLE_IMAGE_NAME }}:buildcache-${{ matrix.arch }} + cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.CONSOLE_IMAGE_NAME }}:buildcache-${{ matrix.arch }},mode=max + + - name: Save digests to files + if: github.event_name != 'pull_request' + run: | + echo ${{ steps.build-console.outputs.digest }} > digests-${{ matrix.arch }}-console-digest.txt + + - name: Upload digests artifact + if: github.event_name != 'pull_request' + uses: actions/upload-artifact@v4 + with: + name: digests-${{ matrix.arch }} + path: | + digests-${{ matrix.arch }}-console-digest.txt + + create-manifests: + needs: build-platform + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + + steps: + - name: Normalize image names + run: | + echo "CONSOLE_IMAGE_NAME=${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV + + - name: Download all digests + uses: actions/download-artifact@v4 + with: + pattern: digests-* + merge-multiple: true + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Extract metadata for Console image + id: meta-console + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.CONSOLE_IMAGE_NAME }} + + - name: Create multi-arch manifest for Console + run: | + AMD64_DIGEST=$(cat digests-amd64-console-digest.txt) + ARM64_DIGEST=$(cat digests-arm64-console-digest.txt) + echo AMD64_DIGEST=$AMD64_DIGEST + echo ARM64_DIGEST=$ARM64_DIGEST + TAGS=(${{ steps.meta-console.outputs.tags }}) + TAG_ARGS="" + for tag in "${TAGS[@]}"; do + TAG_ARGS="$TAG_ARGS --tag $tag" + done + docker buildx imagetools create $TAG_ARGS \ + ${{ env.REGISTRY }}/${{ env.CONSOLE_IMAGE_NAME }}@$AMD64_DIGEST \ + ${{ env.REGISTRY }}/${{ env.CONSOLE_IMAGE_NAME }}@$ARM64_DIGEST diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index de02cc3..0000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: M3tering Console Tests - -on: [workflow_dispatch, push, pull_request] - -jobs: - test: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Node.js - uses: actions/setup-node@v2 - with: - node-version: "20" - - - name: Install dependencies - run: npm install - - - name: Run tests - run: npm test diff --git a/docker-compose.yaml b/docker-compose.yaml index b57d3fa..6287647 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,5 +1,6 @@ services: console: + image: ghcr.io/m3tering/console:main build: . network_mode: host restart: unless-stopped From a8dc87580323fe8e90cd6a08f396cbec110327bd Mon Sep 17 00:00:00 2001 From: Emmo00 Date: Mon, 2 Feb 2026 12:32:22 +0100 Subject: [PATCH 02/14] refactor: update preferred prover node configuration to allow null fallback --- src/lib/core/prover/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/core/prover/index.ts b/src/lib/core/prover/index.ts index e647812..2724a2e 100644 --- a/src/lib/core/prover/index.ts +++ b/src/lib/core/prover/index.ts @@ -1,7 +1,7 @@ import { buildBatchPayload } from "../../utils"; import type { BatchTransactionPayload, Hooks, TransactionRecord } from "../../../types"; -const PREFERRED_PROVER_NODE = process.env.PREFERRED_PROVER_NODE || "https://prover.m3ter.ing"; +const PREFERRED_PROVER_NODE = process.env.PREFERRED_PROVER_NODE; export default class implements Hooks { async onTransactionDistribution(_: any, __: any, pendingTransactions: TransactionRecord[]) { @@ -51,7 +51,7 @@ export default class implements Hooks { } async getProverURL(): Promise { - return PREFERRED_PROVER_NODE; + return PREFERRED_PROVER_NODE || null; } async sendPendingTransactionsToProver(proverURL: string, pendingTransactions: TransactionRecord[]) { From 04dc70d31bcf34cfce6b148d1101d517d98b1025 Mon Sep 17 00:00:00 2001 From: Emmo00 Date: Tue, 3 Feb 2026 14:15:26 +0100 Subject: [PATCH 03/14] refactor: integrate meter pruning into Streamr cron job for improved transaction handling --- src/lib/core/prune_sync/index.ts | 28 ---------------------------- src/lib/core/streamr/index.ts | 17 ++++++++++++++--- 2 files changed, 14 insertions(+), 31 deletions(-) delete mode 100644 src/lib/core/prune_sync/index.ts diff --git a/src/lib/core/prune_sync/index.ts b/src/lib/core/prune_sync/index.ts deleted file mode 100644 index aed1354..0000000 --- a/src/lib/core/prune_sync/index.ts +++ /dev/null @@ -1,28 +0,0 @@ -import cron from "node-cron"; -import { Hooks } from "../../../types"; -import { getAllMeterRecords } from "../../../store/sqlite"; -import { loadConfigurations } from "../../utils"; -import { pruneAndSyncOnchain } from "../../sync"; - -export default class implements Hooks { - private config = loadConfigurations(); - - async onAfterInit() { - console.log("Registering prune_sync cron job..."); - - // Schedule a cron job to perform prune verified transactions and sync with onchain state - cron.schedule(this.config.prune_sync.cronSchedule, async () => { - const m3ters = getAllMeterRecords(); - for (const m3ter of m3ters) { - try { - pruneAndSyncOnchain(m3ter.publicKey); - } catch (error) { - console.error(`Error pruning and syncing meter ${m3ter.publicKey}:`, error); - } - } - }); - - console.log("prune_sync cron job registered."); - return; - } -} diff --git a/src/lib/core/streamr/index.ts b/src/lib/core/streamr/index.ts index 8c5ee35..3805e84 100644 --- a/src/lib/core/streamr/index.ts +++ b/src/lib/core/streamr/index.ts @@ -1,8 +1,9 @@ import cron from "node-cron"; import { StreamrClient } from "@streamr/sdk"; -import { getAllTransactionRecords } from "../../../store/sqlite"; +import { getAllMeterRecords, getAllTransactionRecords } from "../../../store/sqlite"; import { buildBatchPayload, loadConfigurations, retry } from "../../utils"; import type { Hooks, TransactionRecord } from "../../../types"; +import { pruneAndSyncOnchain } from "../../sync"; const { ETHEREUM_PRIVATE_KEY } = process.env; @@ -20,6 +21,16 @@ export default class implements Hooks { cron.schedule( this.config.streamr.cronSchedule, async () => { + console.log("Streamr cron job started: Pruning meters and publishing pending transactions..."); + const m3ters = getAllMeterRecords(); + for (const m3ter of m3ters) { + try { + pruneAndSyncOnchain(m3ter.publicKey); + } catch (error) { + console.error(`Error pruning and syncing meter ${m3ter.publicKey}:`, error); + } + } + console.log("Publishing pending transactions to Streamr..."); const pendingTransactions = await this.getPendingTransactions(); @@ -56,14 +67,14 @@ export default class implements Hooks { console.log(`[streamr] Connecting to ${STREAMR_STREAM_ID}...`); const stream = await retry(() => streamrClient.getStream(STREAMR_STREAM_ID!), 3, 2000); - await new Promise((resolve) => setTimeout(resolve, 2000)); + await new Promise((resolve) => setTimeout(resolve, 20000)); // wait for 20 seconds to ensure connection is established console.log(`[streamr] Connected. Publishing ${pendingTransactions.length} transactions...`); const batchPayload = buildBatchPayload(pendingTransactions); await stream.publish(batchPayload); console.log(`[streamr] Published ${pendingTransactions.length} transactions to stream ${STREAMR_STREAM_ID}`); - await new Promise((resolve) => setTimeout(resolve, 2000)); + await new Promise((resolve) => setTimeout(resolve, 20000)); // wait for 20 seconds to ensure message is sent } catch (error) { console.error(`[streamr] Error publishing to Streamr:`, error); throw error; From 000f8d76a2f1d359a5e73fce9f8e2fedec5b1b6c Mon Sep 17 00:00:00 2001 From: Emmo00 Date: Wed, 4 Feb 2026 13:18:28 +0100 Subject: [PATCH 04/14] refactor: modularize Streamr configuration and remove legacy console config --- .env.example | 3 +++ .gitignore | 3 --- Dockerfile | 1 - console.example.config.json | 21 --------------------- src/lib/core/streamr/index.ts | 27 +++++++++++++++------------ src/lib/core/streamr/ui.ts | 29 ++++++++++++++++------------- src/lib/utils.ts | 27 ++++----------------------- src/types.ts | 9 +-------- 8 files changed, 39 insertions(+), 81 deletions(-) delete mode 100644 console.example.config.json diff --git a/.env.example b/.env.example index 50f95b1..cd8f0a1 100644 --- a/.env.example +++ b/.env.example @@ -5,4 +5,7 @@ CONTRACT_LABEL="M3ters" CHIRPSTACK_HOST="localhost" MAINNET_RPC="https://sepolia.drpc.org" PREFERRED_PROVER_NODE="http://prover.m3ter.ing" +STREAMR_STREAM_ID="0x567853282663b601bfdb9203819b1fbb3fe18926/m3tering/test" ETHEREUM_PRIVATE_KEY="..." + +STREAMR_CRONSCHEDULE="0 * * * *" # Every hour diff --git a/.gitignore b/.gitignore index fa5348b..195e462 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,3 @@ -# console configuration file -console.config.json - #test scripts emulate.ts test-database.ts diff --git a/Dockerfile b/Dockerfile index 65f6043..1d84708 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,6 @@ COPY babel.config.js . COPY tsconfig.json . COPY .env . COPY src ./src -COPY console.config.jso[n] . # Build project RUN npm run build diff --git a/console.example.config.json b/console.example.config.json deleted file mode 100644 index 628ab5f..0000000 --- a/console.example.config.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "modules": [ - "core/arweave", - "core/prover", - "core/streamr", - "core/is_on", - "core/prune_sync" - ], - "uiModules": { - "streamr": "core/streamr/ui" - }, - "streamr": { - "streamId": [ - "0x567853282663b601bfdb9203819b1fbb3fe18926/m3tering/test" - ], - "cronSchedule": "0 * * * *" - }, - "prune_sync": { - "cronSchedule": "0 * * * *" - } -} diff --git a/src/lib/core/streamr/index.ts b/src/lib/core/streamr/index.ts index 3805e84..bbc168c 100644 --- a/src/lib/core/streamr/index.ts +++ b/src/lib/core/streamr/index.ts @@ -1,7 +1,7 @@ import cron from "node-cron"; import { StreamrClient } from "@streamr/sdk"; import { getAllMeterRecords, getAllTransactionRecords } from "../../../store/sqlite"; -import { buildBatchPayload, loadConfigurations, retry } from "../../utils"; +import { buildBatchPayload, retry } from "../../utils"; import type { Hooks, TransactionRecord } from "../../../types"; import { pruneAndSyncOnchain } from "../../sync"; @@ -12,14 +12,15 @@ if (!ETHEREUM_PRIVATE_KEY) { } export default class implements Hooks { - private config = loadConfigurations(); + private streamIds: string[] = process.env.STREAMR_STREAM_ID ? process.env.STREAMR_STREAM_ID.split(",") : []; + private cronSchedule: string = process.env.STREAMR_CRONSCHEDULE || "0 * * * *"; async onAfterInit() { console.log("Registering Streamr cron job..."); // Schedule a cron job to publish pending transactions cron.schedule( - this.config.streamr.cronSchedule, + this.cronSchedule, async () => { console.log("Streamr cron job started: Pruning meters and publishing pending transactions..."); const m3ters = getAllMeterRecords(); @@ -35,14 +36,16 @@ export default class implements Hooks { const pendingTransactions = await this.getPendingTransactions(); if (pendingTransactions.length > 0) { - for (const STREAMR_STREAM_ID of this.config.streamr.streamId) { - console.log(`Publishing to Streamr stream: ${STREAMR_STREAM_ID}`); - await retry( - () => this.publishPendingTransactionsToStreamr(STREAMR_STREAM_ID, pendingTransactions), - 3, - 2000, - ); - } + await Promise.all( + this.streamIds.map(async (STREAMR_STREAM_ID) => { + console.log(`Publishing to Streamr stream: ${STREAMR_STREAM_ID}`); + await retry( + () => this.publishPendingTransactionsToStreamr(STREAMR_STREAM_ID, pendingTransactions), + 3, + 2000, + ); + }), + ); } }, { name: "streamr-publish-pending-transactions", noOverlap: true }, @@ -67,7 +70,7 @@ export default class implements Hooks { console.log(`[streamr] Connecting to ${STREAMR_STREAM_ID}...`); const stream = await retry(() => streamrClient.getStream(STREAMR_STREAM_ID!), 3, 2000); - await new Promise((resolve) => setTimeout(resolve, 20000)); // wait for 20 seconds to ensure connection is established + await new Promise((resolve) => setTimeout(resolve, 2000)); // wait for 2 seconds to ensure connection is established console.log(`[streamr] Connected. Publishing ${pendingTransactions.length} transactions...`); const batchPayload = buildBatchPayload(pendingTransactions); diff --git a/src/lib/core/streamr/ui.ts b/src/lib/core/streamr/ui.ts index f947529..2c3d1b4 100644 --- a/src/lib/core/streamr/ui.ts +++ b/src/lib/core/streamr/ui.ts @@ -1,5 +1,5 @@ import { getAllTransactionRecords } from "../../../store/sqlite"; -import { buildBatchPayload, loadConfigurations, retry } from "../../utils"; +import { buildBatchPayload, retry } from "../../utils"; import type { UIHooks, UIAppIcon, UIAppWindow, UIAction, TransactionRecord } from "../../../types"; import { StreamrClient } from "@streamr/sdk"; @@ -11,7 +11,8 @@ const { ETHEREUM_PRIVATE_KEY } = process.env; * a panel showing stream configuration and a manual publish action */ export default class implements UIHooks { - private config = loadConfigurations(); + private streamIds: string[] = process.env.STREAMR_STREAM_ID ? process.env.STREAMR_STREAM_ID.split(",") : []; + private cronSchedule: string = process.env.STREAMR_CRONSCHEDULE || "Not configured"; private lastPublishTime: Date | null = null; private lastPublishStatus: "success" | "error" | null = null; @@ -26,8 +27,8 @@ export default class implements UIHooks { async getAppWindow(): Promise { const pendingCount = (await this.getPendingTransactions()).length; - const streamIds = this.config.streamr.streamId; - const cronSchedule = this.config.streamr.cronSchedule; + const streamIds = this.streamIds; + const cronSchedule = this.cronSchedule; return { id: "streamr", @@ -94,16 +95,18 @@ export default class implements UIHooks { } try { - for (const streamId of this.config.streamr.streamId) { - console.log(`[streamr-ui] Publishing to stream: ${streamId}`); - await retry(() => this.publishToStreamr(streamId, pendingTransactions), 3, 2000); - } + await Promise.all( + this.streamIds.map(async (streamId) => { + console.log(`[streamr-ui] Publishing to stream: ${streamId}`); + await retry(() => this.publishToStreamr(streamId, pendingTransactions), 3, 2000); + }), + ); this.lastPublishTime = new Date(); this.lastPublishStatus = "success"; return { - message: `Published ${pendingTransactions.length} transactions to ${this.config.streamr.streamId.length} stream(s)`, + message: `Published ${pendingTransactions.length} transactions to ${this.streamIds.length} stream(s)`, data: { count: pendingTransactions.length }, }; } catch (error: any) { @@ -119,8 +122,8 @@ export default class implements UIHooks { const pendingTransactions = await this.getPendingTransactions(); return { pendingCount: pendingTransactions.length, - streamIds: this.config.streamr.streamId, - cronSchedule: this.config.streamr.cronSchedule, + streamIds: this.streamIds, + cronSchedule: this.cronSchedule, lastPublishTime: this.lastPublishTime, lastPublishStatus: this.lastPublishStatus, }; @@ -142,9 +145,9 @@ export default class implements UIHooks { try { const stream = await retry(() => streamrClient.getStream(streamId), 3, 2000); const batchPayload = buildBatchPayload(pendingTransactions); - await new Promise((resolve) => setTimeout(resolve, 2000)); + await new Promise((resolve) => setTimeout(resolve, 2000)); // wait for 2 seconds to ensure connection is established await stream.publish(batchPayload); - await new Promise((resolve) => setTimeout(resolve, 2000)); + await new Promise((resolve) => setTimeout(resolve, 20000)); // wait for 20 seconds to ensure message is sent console.log(`[streamr-ui] Published ${pendingTransactions.length} transactions to stream ${streamId}`); } catch (error) { console.error(`[streamr-ui] Error publishing to Streamr:`, error); diff --git a/src/lib/utils.ts b/src/lib/utils.ts index 3835826..de22a3f 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -19,27 +19,10 @@ export const defaultConfigurations: AppConfig = { uiModules: { streamr: "core/streamr/ui", }, - streamr: { - streamId: ["0x567853282663b601bfdb9203819b1fbb3fe18926/m3tering/test"], - cronSchedule: "0 * * * *", - }, - prune_sync: { - cronSchedule: "0 * * * *", - }, }; -export function loadConfigurations(configPath: string = "console.config.json"): AppConfig { - try { - const config: AppConfig = JSON.parse(fs.readFileSync(configPath, "utf-8")); - return config; - } catch (error) { - console.warn(`Could not load configuration from ${configPath}, using default configurations.`); - return defaultConfigurations; - } -} - -export async function loadExtensionsFromConfig(configPath: string = "console.config.json"): Promise { - const config: AppConfig = loadConfigurations(configPath); +export async function loadExtensionsFromConfig(): Promise { + const config: AppConfig = defaultConfigurations; for (const modulePath of config.modules) { const resolved = path.resolve(__dirname, modulePath); @@ -75,10 +58,8 @@ export async function runHook(hook: K, ...args: Parameter * Load UI extensions from configuration file * Looks for 'uiModules' key in config, which maps module IDs to their paths */ -export async function loadUIExtensionsFromConfig( - configPath: string = "console.config.json", -): Promise> { - const config = loadConfigurations(configPath) as AppConfig & { uiModules?: Record }; +export async function loadUIExtensionsFromConfig(): Promise> { + const config = defaultConfigurations as AppConfig & { uiModules?: Record }; if (!config.uiModules) { console.log("[ui] No UI modules configured"); diff --git a/src/types.ts b/src/types.ts index 3004ac9..9e18412 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,16 +1,9 @@ import { MqttClient } from "mqtt/*"; -// Application configuration type (console.config.json) +// Application configuration type export type AppConfig = { modules: string[]; uiModules?: Record; - streamr: { - streamId: string[]; - cronSchedule: string; - }; - prune_sync: { - cronSchedule: string; - }; }; // Hooks type for lifecycle events From 358fee7a89fe20d903c5799e63a1d856dff0f4b4 Mon Sep 17 00:00:00 2001 From: Emmo00 Date: Wed, 4 Feb 2026 14:17:46 +0100 Subject: [PATCH 05/14] refactor: update database path handling in setupDatabase and deleteDatabase functions --- src/store/sqlite.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/store/sqlite.ts b/src/store/sqlite.ts index 13cb20f..2f1a78a 100644 --- a/src/store/sqlite.ts +++ b/src/store/sqlite.ts @@ -2,7 +2,6 @@ import fs from "fs"; import Database from "better-sqlite3"; import type { Database as DatabaseType, Statement as DatabaseStatementType } from "better-sqlite3"; import { MeterRecord, TransactionRecord } from "../types"; -import { get } from "http"; // meter queries let db: DatabaseType; @@ -24,7 +23,7 @@ let getTransactionByNonceQuery: DatabaseStatementType; * @param databaseName name of the database file */ export default function setupDatabase(databaseName = "m3tering.db") { - db = new Database(databaseName, {}); + db = new Database(`./data/${databaseName}`, {}); initializeTransactionsTable(); initializeMetersTable(); @@ -36,7 +35,7 @@ export function deleteDatabase(databaseName = "m3tering.db") { db.exec(`DROP TABLE IF EXISTS meters`); db.exec(`DROP TABLE IF EXISTS transactions`); db.close(); - fs.unlinkSync(databaseName); + fs.unlinkSync(`./data/${databaseName}`); } catch (err: any) { console.error("Failed to delete database:", err); } From 4260071d64ac9463bcceeb392518f01c6051dfb0 Mon Sep 17 00:00:00 2001 From: Emmo00 Date: Wed, 4 Feb 2026 14:55:55 +0100 Subject: [PATCH 06/14] refactor: enhance module configuration handling and add warnings for missing environment variables --- .env.example | 3 +++ src/lib/utils.ts | 33 +++++++++++++++++++++++---------- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/.env.example b/.env.example index cd8f0a1..c44e714 100644 --- a/.env.example +++ b/.env.example @@ -8,4 +8,7 @@ PREFERRED_PROVER_NODE="http://prover.m3ter.ing" STREAMR_STREAM_ID="0x567853282663b601bfdb9203819b1fbb3fe18926/m3tering/test" ETHEREUM_PRIVATE_KEY="..." +BACKEND_MODULES="core/arweave,core/is_on,core/prover,core/streamr" +UI_MODULES="streamr:core/streamr/ui" + STREAMR_CRONSCHEDULE="0 * * * *" # Every hour diff --git a/src/lib/utils.ts b/src/lib/utils.ts index de22a3f..4707db8 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -1,4 +1,3 @@ -import fs from "fs"; import path from "path"; import { createPublicKey, verify } from "crypto"; import type { @@ -14,11 +13,29 @@ import type { const extensions: Hooks[] = []; const uiExtensions: Map = new Map(); + +const { BACKEND_MODULES, UI_MODULES } = process.env; + +if (!BACKEND_MODULES) { + console.warn("[config] No BACKEND_MODULES specified"); +} + +if (!UI_MODULES) { + console.warn("[config] No UI_MODULES specified"); +} + export const defaultConfigurations: AppConfig = { - modules: ["core/arweave", "core/prover", "core/streamr", "core/is_on", "core/prune_sync"], - uiModules: { - streamr: "core/streamr/ui", - }, + modules: (BACKEND_MODULES || "").split(",").filter(Boolean), // ["core/arweave", "core/prover", "core/streamr", "core/is_on"], + uiModules: Object.fromEntries( + (UI_MODULES || "") + .split(",") + .filter(Boolean) + .map((entry) => { + const [moduleId, modulePath] = entry.split(":"); + return [moduleId, modulePath]; + }) + .filter(([moduleId, modulePath]) => moduleId && modulePath), + ), }; export async function loadExtensionsFromConfig(): Promise { @@ -50,10 +67,6 @@ export async function runHook(hook: K, ...args: Parameter return result; } -// ========================================== -// UI Extension System -// ========================================== - /** * Load UI extensions from configuration file * Looks for 'uiModules' key in config, which maps module IDs to their paths @@ -62,7 +75,7 @@ export async function loadUIExtensionsFromConfig(): Promise const config = defaultConfigurations as AppConfig & { uiModules?: Record }; if (!config.uiModules) { - console.log("[ui] No UI modules configured"); + console.warn("[ui] No UI modules configured"); return uiExtensions; } From 56ff3844094f49efe5b442c16cb5f21911a79d4b Mon Sep 17 00:00:00 2001 From: Emmo00 Date: Wed, 4 Feb 2026 14:56:21 +0100 Subject: [PATCH 07/14] refactor: update Dockerfile and entrypoint script for improved module handling --- Dockerfile | 7 +++- docker-compose.yaml | 10 +++++- entrypoint.sh | 85 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+), 2 deletions(-) create mode 100644 entrypoint.sh diff --git a/Dockerfile b/Dockerfile index 1d84708..9d222f8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ FROM node:20-alpine # Create working directory WORKDIR /opt/app -RUN apk add --no-cache cmake make g++ python3 openssl-dev py3-setuptools +RUN apk add --no-cache cmake make g++ python3 openssl-dev py3-setuptools git # Copy and install dependencies COPY package*.json ./ @@ -22,5 +22,10 @@ RUN npm run build # Expose application port EXPOSE 3000 +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] + # Start app CMD [ "npm", "start" ] diff --git a/docker-compose.yaml b/docker-compose.yaml index 6287647..dcba00c 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,8 +1,16 @@ services: console: image: ghcr.io/m3tering/console:main - build: . network_mode: host restart: unless-stopped ports: - "3000:3000" + volumes: + - console_db:/data + - console_src_modules:/opt/app/src/lib + - console_dist_modules:/opt/app/dist/lib + +volumes: + console_db: + console_src_modules: + console_dist_modules: diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100644 index 0000000..d5c32ef --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env sh +set -e + +APP_DIR="/opt/app" +SRC_LIB="$APP_DIR/src/lib" +DIST_LIB="$APP_DIR/dist/lib" + +mkdir -p "$SRC_LIB" +mkdir -p "$DIST_LIB" + +if [ -n "$MODULES" ]; then + echo "Requested modules: $MODULES" + IFS=',' + + for repo in $MODULES; do + CLEAN_REPO=$(echo "$repo" | cut -d'#' -f1) + NAME=$(basename "$CLEAN_REPO") + VERSION=$(echo "$repo" | grep -o '#.*' | sed 's/#//') + + SRC_DEST="$SRC_LIB/$NAME" + DIST_DEST="$DIST_LIB/$NAME" + + # ----------------------------- + # 1️⃣ Download source if missing + # ----------------------------- + if [ -d "$SRC_DEST" ]; then + echo "✔ Module $NAME already downloaded" + else + echo "⬇ Downloading module: $NAME" + git clone --depth=1 "https://github.com/$CLEAN_REPO.git" "$SRC_DEST" + + if [ -n "$VERSION" ]; then + git -C "$SRC_DEST" fetch --tags + git -C "$SRC_DEST" checkout "$VERSION" + fi + fi + + # ----------------------------- + # 2️⃣ Install plugin dependencies (isolated) + # ----------------------------- + if [ -f "$SRC_DEST/package.json" ] && [ ! -d "$SRC_DEST/node_modules" ]; then + echo "📦 Installing dependencies for $NAME (isolated)" + cd "$SRC_DEST" + npm install --production + cd "$APP_DIR" + fi + + # ----------------------------- + # 3️⃣ Build or copy plugin + # ----------------------------- + if [ -f "$DIST_DEST/.built" ]; then + echo "🏁 Plugin $NAME already built" + continue + fi + + echo "🔧 Building plugin: $NAME" + mkdir -p "$DIST_DEST" + + # TypeScript project + if [ -f "$SRC_DEST/tsconfig.json" ]; then + npx tsc --project "$SRC_DEST/tsconfig.json" --outDir "$DIST_DEST" + + # Single TS file + elif [ -f "$SRC_DEST/index.ts" ]; then + npx tsc "$SRC_DEST/index.ts" --outDir "$DIST_DEST" + + # Plain JS project + else + cp -r "$SRC_DEST"/* "$DIST_DEST/" + fi + + # ----------------------------- + # 4️⃣ Copy plugin's node_modules to dist + # ----------------------------- + if [ -d "$SRC_DEST/node_modules" ]; then + echo "📦 Copying isolated dependencies for $NAME" + cp -r "$SRC_DEST/node_modules" "$DIST_DEST/" + fi + + # Mark as built to avoid rebuilding every restart + touch "$DIST_DEST/.built" + done +fi + +exec "$@" From 75f99b42ec115d162bbd748915b58b2cc8cd3745 Mon Sep 17 00:00:00 2001 From: Emmo00 Date: Wed, 4 Feb 2026 15:33:13 +0100 Subject: [PATCH 08/14] refactor: reorganize environment variables in README and .env.example for clarity --- .env.example | 29 ++++++++---- .vscode/settings.json | 7 +-- README.md | 104 +++++++++++++++++++++++++----------------- 3 files changed, 85 insertions(+), 55 deletions(-) diff --git a/.env.example b/.env.example index c44e714..6e4ea80 100644 --- a/.env.example +++ b/.env.example @@ -1,14 +1,23 @@ -PORT="3000" -API_TOKEN="..." -APPLICATION_ID="..." -CONTRACT_LABEL="M3ters" -CHIRPSTACK_HOST="localhost" -MAINNET_RPC="https://sepolia.drpc.org" -PREFERRED_PROVER_NODE="http://prover.m3ter.ing" -STREAMR_STREAM_ID="0x567853282663b601bfdb9203819b1fbb3fe18926/m3tering/test" -ETHEREUM_PRIVATE_KEY="..." +# Server Configuration +PORT=3000 +# Module Configuration BACKEND_MODULES="core/arweave,core/is_on,core/prover,core/streamr" UI_MODULES="streamr:core/streamr/ui" -STREAMR_CRONSCHEDULE="0 * * * *" # Every hour +# ChirpStack Configuration +API_TOKEN=... +APPLICATION_ID=... +CHIRPSTACK_HOST=localhost + +# Contract & Network Configuration +CONTRACT_LABEL=M3ters +MAINNET_RPC=https://sepolia.drpc.org +ETHEREUM_PRIVATE_KEY="..." + +# Streamr Configuration +STREAMR_STREAM_ID="0x567853282663b601bfdb9203819b1fbb3fe18926/m3tering/test" +STREAMR_CRONSCHEDULE="0 * * * *" # Every hour + +# Optional: Prover Node (disabled publishing to prover if not set) +# PREFERRED_PROVER_NODE="http://prover.m3ter.ing" \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index 7d1e594..a6fb97c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,9 +2,10 @@ "cSpell.words": [ "ardrive", "arweave", - "m3ters", - "Emmo00", "ccip", - "Mauchly", + "CRONSCHEDULE", + "Emmo00", + "m3ters", + "Mauchly" ] } \ No newline at end of file diff --git a/README.md b/README.md index a04dea3..bbbe6a3 100644 --- a/README.md +++ b/README.md @@ -22,14 +22,29 @@ A modular, extensible service console for providers on the M3tering protocol. Fe Create `.env` file: ``` + # Server Configuration PORT=3000 + + # Module Configuration + BACKEND_MODULES="core/arweave,core/is_on,core/prover,core/streamr" + UI_MODULES="streamr:core/streamr/ui" + + # ChirpStack Configuration API_TOKEN=... APPLICATION_ID=... - CONTRACT_LABEL=M3ters CHIRPSTACK_HOST=localhost + + # Contract & Network Configuration + CONTRACT_LABEL=M3ters MAINNET_RPC=https://sepolia.drpc.org - PREFERRED_PROVER_NODE=http://34.244.149.153 ETHEREUM_PRIVATE_KEY="..." + + # Streamr Configuration + STREAMR_STREAM_ID="0x567853282663b601bfdb9203819b1fbb3fe18926/m3tering/test" + STREAMR_CRONSCHEDULE="0 * * * *" # Every hour + + # Optional: Prover Node (defaults to automatic selection) + # PREFERRED_PROVER_NODE="http://prover.m3ter.ing" ``` 3. **Docker Build and Run** @@ -65,36 +80,36 @@ The M3tering Console provides two complementary extension systems: 1. **Backend Hooks** - Hook into the console lifecycle (MQTT, database, message processing) 2. **UI Hooks** - Add custom icons, panels, and actions to the web interface -Both systems use a config-driven approach where modules are loaded dynamically from paths specified in `console.config.json`. +Both systems use an environment-driven approach where modules are loaded dynamically from paths specified in your `.env` file. ## Configuration -```json -{ - "modules": [ - "core/arweave", - "core/prover", - "core/streamr", - "core/is_on", - "core/prune_sync" - ], - "uiModules": { - "streamr": "core/streamr/ui" - }, - "streamr": { - "streamId": [ - "0x567853282663b601bfdb9203819b1fbb3fe18926/m3tering/test" - ], - "cronSchedule": "0 * * * *" - }, - "prune_sync": { - "cronSchedule": "0 * * * *" - } -} +Modules are configured via environment variables. Modules are automatically pulled from GitHub repositories when the container starts up. + +```bash +# Backend Modules (comma-separated GitHub repositories) +BACKEND_MODULES="core/arweave,core/is_on,core/prover,core/streamr,username/my-custom-module" + +# UI Modules (colon-separated format: moduleId:github_repo) +UI_MODULES="streamr:core/streamr/ui,my-module:username/my-custom-module" + +# Module-specific configuration +STREAMR_STREAM_ID="0x567853282663b601bfdb9203819b1fbb3fe18926/m3tering/test" +STREAMR_CRONSCHEDULE="0 * * * *" # Every hour ``` -- **`modules`**: Array of paths to backend hook modules (relative to `src/lib/`) -- **`uiModules`**: Object mapping module IDs to UI module paths (relative to `src/lib/`) +- **`BACKEND_MODULES`**: Comma-separated list of GitHub repositories in the format `/` or built-in paths like `core/arweave` +- **`UI_MODULES`**: Comma-separated list of `moduleId:/` pairs +- **Module-specific variables**: Each module can have its own configuration variables (e.g., `STREAMR_STREAM_ID`) + +### Publishing Custom Modules + +To use your own extensions: + +1. Publish your module code to a GitHub repository +2. Reference it in your `.env` file using the format `/` +3. The extension code is automatically cloned from GitHub when the Docker container starts up +4. For specific versions, append `#` or `#` (e.g., `username/my-module#v1.0.0`) --- @@ -104,8 +119,10 @@ Backend hooks allow modules to react to console lifecycle events. Each module ex ## Creating a Backend Module +1. **Create your module repository** with the following structure: + ```typescript -// src/lib/core/my-module/index.ts +// index.ts import type { Hooks } from "../../../types"; export default class implements Hooks { @@ -119,13 +136,15 @@ export default class implements Hooks { } ``` -Add to `console.config.json`: -```json -{ - "modules": ["core/my-module"] -} +2. **Publish to GitHub**: Push your module code to a GitHub repository (e.g., `github.com/yourusername/my-m3tering-module`) + +3. **Configure in `.env`**: +```bash +BACKEND_MODULES="core/arweave,core/prover,yourusername/my-m3tering-module" ``` +4. **Restart container**: The module will be automatically cloned from GitHub and loaded when the container starts + ## Hook Lifecycle Reference ### Initialization Phase @@ -180,8 +199,10 @@ UI Hooks allow modules to extend the web interface at `http://localhost:3000`. M ## Creating a UI Module +1. **Create your module repository** with the following structure: + ```typescript -// src/lib/core/my-module/ui.ts +// ui.ts (or index.ts) import type { UIHooks, UIAppIcon, UIAppWindow, UIAction } from "../../../types"; export default class implements UIHooks { @@ -222,15 +243,15 @@ export default class implements UIHooks { } ``` -Add to `console.config.json`: -```json -{ - "uiModules": { - "my-module": "core/my-module/ui" - } -} +2. **Publish to GitHub**: Push your module code to a GitHub repository (e.g., `github.com/yourusername/my-ui-module`) + +3. **Configure in `.env`**: +```bash +UI_MODULES="streamr:core/streamr/ui,my-module:yourusername/my-ui-module" ``` +4. **Restart container**: The module will be automatically cloned from GitHub and loaded when the container starts + ## UIHooks Interface | Method | Return Type | Description | @@ -311,7 +332,6 @@ Response: { success: boolean, message?: string, data?: any } | `core/prover` | Sends batched transactions to the prover node | | `core/streamr` | Publishes transactions to Streamr streams on a cron schedule | | `core/is_on` | Computes device on/off state based on balance | -| `core/prune_sync` | Cleans up old synchronized transactions | ## UI Modules From edd1509157523c2de3f029f22b59217bee9a7000 Mon Sep 17 00:00:00 2001 From: Emmo00 Date: Wed, 4 Feb 2026 15:41:47 +0100 Subject: [PATCH 09/14] refactor: adjust retry delay for Streamr connection to improve stability --- console.config.json | 20 -------------------- src/lib/core/streamr/index.ts | 2 +- 2 files changed, 1 insertion(+), 21 deletions(-) delete mode 100644 console.config.json diff --git a/console.config.json b/console.config.json deleted file mode 100644 index bbefcd2..0000000 --- a/console.config.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "modules": [ - "core/arweave", - "core/streamr", - "core/is_on", - "core/prune_sync" - ], - "uiModules": { - "streamr": "core/streamr/ui" - }, - "streamr": { - "streamId": [ - "0x567853282663b601bfdb9203819b1fbb3fe18926/m3tering/test" - ], - "cronSchedule": "0 * * * *" - }, - "prune_sync": { - "cronSchedule": "0 * * * *" - } -} diff --git a/src/lib/core/streamr/index.ts b/src/lib/core/streamr/index.ts index c774fc5..225ed28 100644 --- a/src/lib/core/streamr/index.ts +++ b/src/lib/core/streamr/index.ts @@ -68,7 +68,7 @@ export default class implements Hooks { try { console.log(`[streamr] Connecting to ${STREAMR_STREAM_ID}...`); - const stream = await retry(() => streamrClient.getStream(STREAMR_STREAM_ID!), 3, 10000); + const stream = await retry(() => streamrClient.getStream(STREAMR_STREAM_ID!), 3, 2000); await new Promise((resolve) => setTimeout(resolve, 2000)); // wait for 2 seconds to ensure connection is established From 5238af2047a0fab745223d34bf148a27954b0e91 Mon Sep 17 00:00:00 2001 From: Emmo00 Date: Wed, 4 Feb 2026 15:53:22 +0100 Subject: [PATCH 10/14] refactor: log pending transactions count in handleMessage function --- src/services/mqtt.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/services/mqtt.ts b/src/services/mqtt.ts index b2c0a56..a771ebe 100644 --- a/src/services/mqtt.ts +++ b/src/services/mqtt.ts @@ -224,6 +224,7 @@ export async function handleMessage(blob: Buffer) { logger.info(`Updated meter nonce to: ${expectedNonce}`); const pendingTransactions = getAllTransactionRecords(); + console.log(`${pendingTransactions.length} Pending transactions: ${JSON.stringify(pendingTransactions)}`); await runHook("onTransactionDistribution", m3ter.tokenId, decoded, pendingTransactions); } From 73477e6b77cb10f82d6ef9768d358f3213d92a1a Mon Sep 17 00:00:00 2001 From: Emmo00 Date: Wed, 4 Feb 2026 16:10:34 +0100 Subject: [PATCH 11/14] refactor: add .private file for sensitive data management --- data/.private | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 data/.private diff --git a/data/.private b/data/.private new file mode 100644 index 0000000..e69de29 From cb557fb96d5cb8f2ebeca86ba41638d422ed8079 Mon Sep 17 00:00:00 2001 From: Emmo00 Date: Wed, 4 Feb 2026 16:22:36 +0100 Subject: [PATCH 12/14] refactor: update database path and manage sensitive data with .private file --- {data => .data}/.private | 0 docker-compose.yaml | 2 +- src/store/sqlite.ts | 4 ++-- 3 files changed, 3 insertions(+), 3 deletions(-) rename {data => .data}/.private (100%) diff --git a/data/.private b/.data/.private similarity index 100% rename from data/.private rename to .data/.private diff --git a/docker-compose.yaml b/docker-compose.yaml index dcba00c..5c0f8db 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -6,7 +6,7 @@ services: ports: - "3000:3000" volumes: - - console_db:/data + - ./.data:/opt/app/.data - console_src_modules:/opt/app/src/lib - console_dist_modules:/opt/app/dist/lib diff --git a/src/store/sqlite.ts b/src/store/sqlite.ts index 2f1a78a..61763ac 100644 --- a/src/store/sqlite.ts +++ b/src/store/sqlite.ts @@ -23,7 +23,7 @@ let getTransactionByNonceQuery: DatabaseStatementType; * @param databaseName name of the database file */ export default function setupDatabase(databaseName = "m3tering.db") { - db = new Database(`./data/${databaseName}`, {}); + db = new Database(`./.data/${databaseName}`, {}); initializeTransactionsTable(); initializeMetersTable(); @@ -35,7 +35,7 @@ export function deleteDatabase(databaseName = "m3tering.db") { db.exec(`DROP TABLE IF EXISTS meters`); db.exec(`DROP TABLE IF EXISTS transactions`); db.close(); - fs.unlinkSync(`./data/${databaseName}`); + fs.unlinkSync(`./.data/${databaseName}`); } catch (err: any) { console.error("Failed to delete database:", err); } From 68397c6a05c0d3d22acfcf0219a14c66d6e4a797 Mon Sep 17 00:00:00 2001 From: Emmo00 Date: Wed, 4 Feb 2026 16:54:32 +0100 Subject: [PATCH 13/14] refactor: add env_file configuration to docker-compose for environment variables --- docker-compose.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yaml b/docker-compose.yaml index 5c0f8db..893e1ba 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -3,6 +3,8 @@ services: image: ghcr.io/m3tering/console:main network_mode: host restart: unless-stopped + env_file: + - .env ports: - "3000:3000" volumes: From d2eb3d2448500fc41f8365e2112c385232503345 Mon Sep 17 00:00:00 2001 From: Emmo00 Date: Wed, 4 Feb 2026 17:45:07 +0100 Subject: [PATCH 14/14] refactor: enhance logging for Streamr cron job and extend wait time for message publishing --- src/lib/core/streamr/index.ts | 2 +- src/lib/core/streamr/ui.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/core/streamr/index.ts b/src/lib/core/streamr/index.ts index 225ed28..f8525d6 100644 --- a/src/lib/core/streamr/index.ts +++ b/src/lib/core/streamr/index.ts @@ -16,7 +16,7 @@ export default class implements Hooks { private cronSchedule: string = process.env.STREAMR_CRONSCHEDULE || "0 * * * *"; async onAfterInit() { - console.log("Registering Streamr cron job..."); + console.log("Registering Streamr cron job... Schedule: ", this.cronSchedule, " Stream IDs: ", JSON.stringify(this.streamIds)); // Schedule a cron job to publish pending transactions cron.schedule( diff --git a/src/lib/core/streamr/ui.ts b/src/lib/core/streamr/ui.ts index 2c3d1b4..2f1a0bd 100644 --- a/src/lib/core/streamr/ui.ts +++ b/src/lib/core/streamr/ui.ts @@ -147,7 +147,7 @@ export default class implements UIHooks { const batchPayload = buildBatchPayload(pendingTransactions); await new Promise((resolve) => setTimeout(resolve, 2000)); // wait for 2 seconds to ensure connection is established await stream.publish(batchPayload); - await new Promise((resolve) => setTimeout(resolve, 20000)); // wait for 20 seconds to ensure message is sent + await new Promise((resolve) => setTimeout(resolve, 100000)); // wait for 100 seconds to ensure message is sent console.log(`[streamr-ui] Published ${pendingTransactions.length} transactions to stream ${streamId}`); } catch (error) { console.error(`[streamr-ui] Error publishing to Streamr:`, error);