From 54840c2b656df17b7a50e4ef773186515fdb6b24 Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Mon, 8 Dec 2025 13:34:29 +0100 Subject: [PATCH 01/25] Add a view to export datapackage as zip files --- app/projects/urls.py | 5 +++++ app/projects/views.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/app/projects/urls.py b/app/projects/urls.py index 16fb3b13b..d21bd956a 100644 --- a/app/projects/urls.py +++ b/app/projects/urls.py @@ -143,6 +143,11 @@ name="scenario_duplicate", ), path("scenario/export/", scenario_export, name="scenario_export"), + path( + "scenario/export/datapackage/", + scenario_export_as_datapackage, + name="scenario_export_as_datapackage", + ), path("scenario/upload/", scenario_upload, name="scenario_upload"), # path('scenario/upload/', LoadScenarioFromFileView.as_view(), name='scenario_upload'), # Timeseries Model diff --git a/app/projects/views.py b/app/projects/views.py index a97fb8d91..b017c9b5a 100644 --- a/app/projects/views.py +++ b/app/projects/views.py @@ -1,4 +1,8 @@ # from bootstrap_modal_forms.generic import BSModalCreateView +import tempfile +from pathlib import Path +import zipfile + from django.contrib.auth.decorators import login_required import datetime from django.http import ( @@ -1360,6 +1364,35 @@ def scenario_export(request, proj_id): return response +@login_required +@require_http_methods(["GET"]) +def scenario_export_as_datapackage(request, scen_id): + + scenario = get_object_or_404(Scenario, id=int(scen_id)) + + with tempfile.TemporaryDirectory() as temp_dir: + destination_path = Path(temp_dir) + # write the content of the scenario into a temp directory + scenario.to_datapackage(destination_path) + + zip_buffer = io.BytesIO() + with zipfile.ZipFile(zip_buffer, "w") as zip_file: + for file_path in destination_path.rglob("*"): # Recursively walk all files + if file_path.is_file(): + # Relative path inside ZIP + arcname = file_path.relative_to(destination_path) + with open(file_path, "rb") as f: + zip_file.writestr(str(arcname), f.read()) + + zip_buffer.seek(0) + response = HttpResponse(zip_buffer.getvalue(), content_type="application/zip") + response["Content-Disposition"] = ( + f'attachment; filename="datapackage_scenario_{scen_id}.zip"' + ) + + return response + + @login_required @require_http_methods(["POST"]) def scenario_delete(request, scen_id): From f00fd3bca7a2788536563bc91b0b5cb1d49cd8a8 Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Mon, 8 Dec 2025 13:35:38 +0100 Subject: [PATCH 02/25] Add option to download as datapackage from progession bar This need to be placed somewhere else and is currently only for testing purposes --- app/templates/scenario/scenario_progression.html | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/app/templates/scenario/scenario_progression.html b/app/templates/scenario/scenario_progression.html index 0da181fd8..5978043fd 100644 --- a/app/templates/scenario/scenario_progression.html +++ b/app/templates/scenario/scenario_progression.html @@ -36,10 +36,16 @@

- -
+ + {% if scen_id %} + + {% endif %} + + From 55282156c1510d606f590ac6e7fcb3737deb596a Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Mon, 8 Dec 2025 13:47:07 +0100 Subject: [PATCH 03/25] Move the to_datapackage of a scenario to the Model definition --- .../management/commands/datapackage.py | 93 +------------------ app/projects/models/base_models.py | 93 +++++++++++++++++++ 2 files changed, 96 insertions(+), 90 deletions(-) diff --git a/app/projects/management/commands/datapackage.py b/app/projects/management/commands/datapackage.py index 0910e2bba..c89fb2645 100644 --- a/app/projects/management/commands/datapackage.py +++ b/app/projects/management/commands/datapackage.py @@ -1,16 +1,6 @@ from django.core.management.base import BaseCommand, CommandError -from projects.models import ( - Asset, - AssetType, - TopologyNode, - Scenario, - Timeseries, - ConnectionLink, - Bus, -) -import pandas as pd +from projects.models import Scenario from pathlib import Path -import numpy as np import shutil @@ -20,93 +10,16 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument("scen_id", nargs="+", type=int) - parser.add_argument( - "--overwrite", action="store_true", help="Overwrite the datapackage" - ) - def handle(self, *args, **options): - overwrite = options["overwrite"] for scen_id in options["scen_id"]: try: scenario = Scenario.objects.get(pk=scen_id) except Scenario.DoesNotExist: raise CommandError('Scenario "%s" does not exist' % scen_id) - destination_path = Path(__file__).resolve().parents[4] - # Create a folder with a datapackage structure scenario_folder = destination_path / f"scenario_{scen_id}" - create_folder = True - if scenario_folder.exists(): - if not overwrite: - create_folder = False - else: - shutil.rmtree(scenario_folder) - - elements_folder = scenario_folder / "data" / "elements" - sequences_folder = scenario_folder / "data" / "sequences" - - if create_folder: - # create subfolders - (scenario_folder / "scripts").mkdir(parents=True) - elements_folder.mkdir(parents=True) - sequences_folder.mkdir(parents=True) - - # List all components of the scenario (except the busses) - qs_assets = Asset.objects.filter(scenario=scenario) - # List all distinct components' assettypes (or facade name) - facade_names = qs_assets.distinct().values_list( - "asset_type__asset_type", flat=True - ) - - bus_resource_records = [] - profile_resource_records = {} - for facade_name in facade_names: - resource_records = [] - for i, asset in enumerate( - qs_assets.filter(asset_type__asset_type=facade_name) - ): - resource_rec, bus_resource_rec, profile_resource_rec = ( - asset.to_datapackage() - ) - resource_records.append(resource_rec) - # those constitute the busses and sequences used by this asset - bus_resource_records.extend(bus_resource_rec) - profile_resource_records.update(profile_resource_rec) - - if resource_records: - out_path = elements_folder / f"{facade_name}.csv" - Path(out_path).parent.mkdir(parents=True, exist_ok=True) - df = pd.DataFrame(resource_records) - df.to_csv(out_path, index=False) - - # Save all unique busses to a elements resource - if bus_resource_records: - out_path = elements_folder / f"bus.csv" - Path(out_path).parent.mkdir(parents=True, exist_ok=True) - df = pd.DataFrame(bus_resource_records) - df.drop_duplicates("name").to_csv(out_path, index=False) - - # Save all profiles to a sequences resource - if profile_resource_records: - out_path = sequences_folder / f"profiles.csv" - Path(out_path).parent.mkdir(parents=True, exist_ok=True) - # add timestamps to the profiles - profile_resource_records["timeindex"] = scenario.get_timestamps() - try: - df = pd.DataFrame(profile_resource_records) - except ValueError as e: - # If not all profiles have the same length we pad the shorter profiles with np.nan - max_len = max(len(v) for v in profile_resource_records.values()) - profile_resource_records = { - k: v + [np.nan] * (max_len - len(v)) - for k, v in profile_resource_records.items() - } - df = pd.DataFrame(profile_resource_records) - print( - f"Some profiles have more timesteps that other profiles in scenario {scenario.name}({scen_id}) --> the shorter profiles will be expanded with NaN values" - ) - # TODO check if there are column duplicates - df.set_index("timeindex").to_csv(out_path, index=True) + shutil.rmtree(scenario_folder) + scenario.to_datapackage(destination_path) diff --git a/app/projects/models/base_models.py b/app/projects/models/base_models.py index 91c553405..fee0dae91 100644 --- a/app/projects/models/base_models.py +++ b/app/projects/models/base_models.py @@ -1,7 +1,12 @@ import datetime import json +import logging import uuid from datetime import timedelta +import pandas as pd +from pathlib import Path +import numpy as np + import oemof.thermal.compression_heatpumps_and_chillers as cmpr_hp_chiller from django.conf import settings @@ -106,6 +111,17 @@ def export(self, bind_scenario_data=True): dm["scenario_set_data"] = scenario_data return dm + def to_datapackage(self): + """""" + dp = model_to_dict(self.economic_data, exclude=["id", "currency"]) + dp["name"] = self.name + dp["type"] = "project" + dp["discount_factor"] = dp.pop("discount") + dp["lifetime"] = dp.pop("duration") + dp["shortage_cost"] = 999 + dp["excess_costs"] = 99 + return dp + def add_viewer_if_not_exist(self, email=None, share_rights=""): user = None success = False @@ -297,6 +313,83 @@ def export(self, bind_project_data=False): dm["busses"] = busses return dm + def to_datapackage(self, destination_path): + + # Create a folder with a datapackage structure + scenario_folder = destination_path / f"scenario_{self.id}" + + elements_folder = scenario_folder / "data" / "elements" + sequences_folder = scenario_folder / "data" / "sequences" + + # create subfolders + (scenario_folder / "scripts").mkdir(parents=True) + elements_folder.mkdir(parents=True) + sequences_folder.mkdir(parents=True) + + # Save the project specifics + proj = self.project + out_path = elements_folder / f"project.csv" + Path(out_path).parent.mkdir(parents=True, exist_ok=True) + df = pd.DataFrame([proj.to_datapackage()]) + df.drop_duplicates("name").to_csv(out_path, index=False) + + # List all components of the scenario (except the busses) + qs_assets = Asset.objects.filter(scenario=self) + # List all distinct components' assettypes (or facade name) + facade_names = qs_assets.distinct().values_list( + "asset_type__asset_type", flat=True + ) + + bus_resource_records = [] + profile_resource_records = {} + for facade_name in facade_names: + resource_records = [] + for i, asset in enumerate( + qs_assets.filter(asset_type__asset_type=facade_name) + ): + resource_rec, bus_resource_rec, profile_resource_rec = ( + asset.to_datapackage() + ) + resource_records.append(resource_rec) + # those constitute the busses and sequences used by this asset + bus_resource_records.extend(bus_resource_rec) + profile_resource_records.update(profile_resource_rec) + + if resource_records: + out_path = elements_folder / f"{facade_name}.csv" + Path(out_path).parent.mkdir(parents=True, exist_ok=True) + df = pd.DataFrame(resource_records) + df.to_csv(out_path, index=False) + + # Save all unique busses to a elements resource + if bus_resource_records: + out_path = elements_folder / f"bus.csv" + Path(out_path).parent.mkdir(parents=True, exist_ok=True) + df = pd.DataFrame(bus_resource_records) + df.drop_duplicates("name").to_csv(out_path, index=False) + + # Save all profiles to a sequences resource + if profile_resource_records: + out_path = sequences_folder / f"profiles.csv" + Path(out_path).parent.mkdir(parents=True, exist_ok=True) + # add timestamps to the profiles + profile_resource_records["timeindex"] = self.get_timestamps() + try: + df = pd.DataFrame(profile_resource_records) + except ValueError as e: + # If not all profiles have the same length we pad the shorter profiles with np.nan + max_len = max(len(v) for v in profile_resource_records.values()) + profile_resource_records = { + k: v + [np.nan] * (max_len - len(v)) + for k, v in profile_resource_records.items() + } + df = pd.DataFrame(profile_resource_records) + logging.warning( + f"Some profiles have more timesteps that other profiles in scenario {self.name}({self.id}) --> the shorter profiles will be expanded with NaN values" + ) + # TODO check if there are column duplicates + df.set_index("timeindex").to_csv(out_path, index=True) + def get_default_timeseries(): return list([]) From 2ae4acf71c830d89c37817c1b93e50fc7e09ab08 Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Mon, 8 Dec 2025 13:55:27 +0100 Subject: [PATCH 04/25] Prevent export of scenario not belonging to user --- app/projects/views.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/projects/views.py b/app/projects/views.py index b017c9b5a..3850ca853 100644 --- a/app/projects/views.py +++ b/app/projects/views.py @@ -1370,6 +1370,9 @@ def scenario_export_as_datapackage(request, scen_id): scenario = get_object_or_404(Scenario, id=int(scen_id)) + if scenario.project.user != request.user: + raise PermissionDenied + with tempfile.TemporaryDirectory() as temp_dir: destination_path = Path(temp_dir) # write the content of the scenario into a temp directory From 9dcf1bc26a69a09345150d744a51955dcae7ce2b Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Tue, 9 Dec 2025 09:28:24 +0100 Subject: [PATCH 05/25] Save project.csv in data folder --- app/projects/models/base_models.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/projects/models/base_models.py b/app/projects/models/base_models.py index fee0dae91..883970daa 100644 --- a/app/projects/models/base_models.py +++ b/app/projects/models/base_models.py @@ -318,8 +318,9 @@ def to_datapackage(self, destination_path): # Create a folder with a datapackage structure scenario_folder = destination_path / f"scenario_{self.id}" - elements_folder = scenario_folder / "data" / "elements" - sequences_folder = scenario_folder / "data" / "sequences" + data_folder = scenario_folder / "data" + elements_folder = data_folder / "elements" + sequences_folder = data_folder / "sequences" # create subfolders (scenario_folder / "scripts").mkdir(parents=True) @@ -328,7 +329,7 @@ def to_datapackage(self, destination_path): # Save the project specifics proj = self.project - out_path = elements_folder / f"project.csv" + out_path = data_folder / f"project.csv" Path(out_path).parent.mkdir(parents=True, exist_ok=True) df = pd.DataFrame([proj.to_datapackage()]) df.drop_duplicates("name").to_csv(out_path, index=False) From f671c2fca7bb18244764993cfa7a66412642a358 Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Tue, 9 Dec 2025 15:04:17 +0100 Subject: [PATCH 06/25] Add the project name to the asset csv In order to compute the annuities --- app/projects/models/base_models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/projects/models/base_models.py b/app/projects/models/base_models.py index 883970daa..8f35d1a0c 100644 --- a/app/projects/models/base_models.py +++ b/app/projects/models/base_models.py @@ -756,7 +756,9 @@ def input_timeseries_values(self): return answer def to_datapackage(self): + """Return the asset's attributes in a datapackage form""" dp = {"type": self.asset_type.asset_type} + dp["project_data"] = self.scenario.project.name # to collect the timeseries used by the asset profile_resource_rec = {} for field in self.asset_type.visible_fields: From d05ea26b5d417e5a890476120df4adc8306bf860 Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Tue, 9 Dec 2025 15:05:48 +0100 Subject: [PATCH 07/25] Generate a datapackage.json file The foreign keys are not yet correctly built/linked --- .../management/commands/datapackage.py | 19 +++++++++++++++++++ app/requirements/base.txt | 6 +++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/app/projects/management/commands/datapackage.py b/app/projects/management/commands/datapackage.py index c89fb2645..2ef004f03 100644 --- a/app/projects/management/commands/datapackage.py +++ b/app/projects/management/commands/datapackage.py @@ -2,6 +2,8 @@ from projects.models import Scenario from pathlib import Path import shutil +from oemof.tabular.datapackage import building +import datapackage as dp class Command(BaseCommand): @@ -23,3 +25,20 @@ def handle(self, *args, **options): if scenario_folder.exists(): shutil.rmtree(scenario_folder) scenario.to_datapackage(destination_path) + + dp_json = scenario_folder / "datapackage.json" + + if dp_json.exists(): + print("Only inferring metadata") + p = dp.Package(dp_json) + building.infer_package_foreign_keys(p) + p.descriptor["resources"].sort(key=lambda x: (x["path"], x["name"])) + p.commit() + p.save(dp_json) + + else: + print("Creating datapackage.json") + building.infer_metadata_from_data( + package_name=f"scenario_{scen_id}", + path=scenario_folder, + ) diff --git a/app/requirements/base.txt b/app/requirements/base.txt index 01a1acc43..45154a085 100644 --- a/app/requirements/base.txt +++ b/app/requirements/base.txt @@ -2,6 +2,7 @@ Django==4.2.4 django-bootstrap-modal-forms==2.0.0 django-compressor==4.1 django-crispy-forms==1.9.2 +django-environ django-extensions==3.0.9 django-jsonview==2.0.0 django-q==1.3.4 @@ -12,12 +13,11 @@ httpx==0.23.0 jsonschema==4.4.0 libsass==0.21.0 numpy>=1.22.4 -oemof-solph==0.4.4 +oemof-tabular==0.0.6.dev0 oemof-thermal==0.0.5 openpyxl==3.0.10 plotly==5.6.0 psycopg2-binary==2.9.3 requests==2.24.0 -XlsxWriter==1.3.9 -django-environ whitenoise==6.9.0 +XlsxWriter==1.3.9 From 9ea5c4835e2153f9eec0c4407ad1e9d630fd5422 Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Mon, 15 Dec 2025 14:22:03 +0100 Subject: [PATCH 08/25] Change oemof-tabular version --- app/requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/requirements/base.txt b/app/requirements/base.txt index 45154a085..ee7c53299 100644 --- a/app/requirements/base.txt +++ b/app/requirements/base.txt @@ -8,12 +8,12 @@ django-jsonview==2.0.0 django-q==1.3.4 django-sass-processor==1.2.2 exchangelib==4.4.0 +git+https://github.com/oemof/oemof-tabular@dev gunicorn==20.0.4 httpx==0.23.0 jsonschema==4.4.0 libsass==0.21.0 numpy>=1.22.4 -oemof-tabular==0.0.6.dev0 oemof-thermal==0.0.5 openpyxl==3.0.10 plotly==5.6.0 From a3bd5ea41e6a2d5870f0c8ef6e0365e66dc3e357 Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Mon, 15 Dec 2025 14:41:31 +0100 Subject: [PATCH 09/25] Install git in docker container --- app/compose/production/app_postgres/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/compose/production/app_postgres/Dockerfile b/app/compose/production/app_postgres/Dockerfile index 0f96f7409..ae0a1dd99 100644 --- a/app/compose/production/app_postgres/Dockerfile +++ b/app/compose/production/app_postgres/Dockerfile @@ -10,16 +10,16 @@ RUN mkdir ${CONFIG_ROOT} COPY app/requirements/base.txt ${CONFIG_ROOT}/base.txt COPY app/requirements/production.txt ${CONFIG_ROOT}/production.txt +# Install gettext utilities for translations +RUN apt-get update && \ + apt-get install -y --no-install-recommends gettext-base gettext git && \ + rm -rf /var/lib/apt/lists/* RUN pip install --upgrade pip \ && pip install --no-cache-dir -r ${CONFIG_ROOT}/production.txt WORKDIR ${APP_ROOT} -# Install gettext utilities for translations -RUN apt-get update && \ - apt-get install -y --no-install-recommends gettext-base gettext && \ - rm -rf /var/lib/apt/lists/* ADD app/ ${APP_ROOT} From df65e2ed764c55b72217bec72e036c6c4b440d5c Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Mon, 15 Dec 2025 14:48:14 +0100 Subject: [PATCH 10/25] Update oemof-thermal --- app/requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/requirements/base.txt b/app/requirements/base.txt index ee7c53299..9084596e0 100644 --- a/app/requirements/base.txt +++ b/app/requirements/base.txt @@ -14,7 +14,7 @@ httpx==0.23.0 jsonschema==4.4.0 libsass==0.21.0 numpy>=1.22.4 -oemof-thermal==0.0.5 +oemof-thermal==0.0.8 openpyxl==3.0.10 plotly==5.6.0 psycopg2-binary==2.9.3 From 7878b2bb0cc2a6cc1430287a727a1baa6fce4afc Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Mon, 15 Dec 2025 15:41:32 +0100 Subject: [PATCH 11/25] Add datapackage.json file --- app/projects/models/base_models.py | 2 ++ app/projects/views.py | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/app/projects/models/base_models.py b/app/projects/models/base_models.py index 8f35d1a0c..6c6d51058 100644 --- a/app/projects/models/base_models.py +++ b/app/projects/models/base_models.py @@ -391,6 +391,8 @@ def to_datapackage(self, destination_path): # TODO check if there are column duplicates df.set_index("timeindex").to_csv(out_path, index=True) + return scenario_folder + def get_default_timeseries(): return list([]) diff --git a/app/projects/views.py b/app/projects/views.py index 3850ca853..c0a825431 100644 --- a/app/projects/views.py +++ b/app/projects/views.py @@ -2,6 +2,9 @@ import tempfile from pathlib import Path import zipfile +import datapackage as dp +from oemof.tabular.datapackage import building + from django.contrib.auth.decorators import login_required import datetime @@ -1376,7 +1379,19 @@ def scenario_export_as_datapackage(request, scen_id): with tempfile.TemporaryDirectory() as temp_dir: destination_path = Path(temp_dir) # write the content of the scenario into a temp directory - scenario.to_datapackage(destination_path) + scenario_folder = scenario.to_datapackage(destination_path) + + building.infer_metadata_from_data( + package_name=f"scenario_{scen_id}", + path=scenario_folder, + ) + dp_json = scenario_folder / "datapackage.json" + + p = dp.Package(str(dp_json)) + building.infer_package_foreign_keys(p) + p.descriptor["resources"].sort(key=lambda x: (x["path"], x["name"])) + p.commit() + p.save(dp_json) zip_buffer = io.BytesIO() with zipfile.ZipFile(zip_buffer, "w") as zip_file: From 3f401d3d9594acea96b2229d8dc2550f772f84e6 Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Mon, 15 Dec 2025 23:14:24 +0100 Subject: [PATCH 12/25] Look for foreign keys to resource 'project' Move the infer_metadata call to the to_datapackage method of the Scenario model class --- app/projects/models/base_models.py | 8 ++++++++ app/projects/views.py | 14 -------------- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/app/projects/models/base_models.py b/app/projects/models/base_models.py index 6c6d51058..478f8a6c9 100644 --- a/app/projects/models/base_models.py +++ b/app/projects/models/base_models.py @@ -6,6 +6,8 @@ import pandas as pd from pathlib import Path import numpy as np +import datapackage as dp +from oemof.tabular.datapackage import building import oemof.thermal.compression_heatpumps_and_chillers as cmpr_hp_chiller @@ -391,6 +393,12 @@ def to_datapackage(self, destination_path): # TODO check if there are column duplicates df.set_index("timeindex").to_csv(out_path, index=True) + # creating datapackage.json metadata file at the root of the datapackage + building.infer_metadata_from_data( + package_name=f"scenario_{self.id}", + path=scenario_folder, + fk_targets=["project"], + ) return scenario_folder diff --git a/app/projects/views.py b/app/projects/views.py index c0a825431..23f90a104 100644 --- a/app/projects/views.py +++ b/app/projects/views.py @@ -2,8 +2,6 @@ import tempfile from pathlib import Path import zipfile -import datapackage as dp -from oemof.tabular.datapackage import building from django.contrib.auth.decorators import login_required @@ -1381,18 +1379,6 @@ def scenario_export_as_datapackage(request, scen_id): # write the content of the scenario into a temp directory scenario_folder = scenario.to_datapackage(destination_path) - building.infer_metadata_from_data( - package_name=f"scenario_{scen_id}", - path=scenario_folder, - ) - dp_json = scenario_folder / "datapackage.json" - - p = dp.Package(str(dp_json)) - building.infer_package_foreign_keys(p) - p.descriptor["resources"].sort(key=lambda x: (x["path"], x["name"])) - p.commit() - p.save(dp_json) - zip_buffer = io.BytesIO() with zipfile.ZipFile(zip_buffer, "w") as zip_file: for file_path in destination_path.rglob("*"): # Recursively walk all files From 30d595e23e73fc88a728f5d398ca28f1b7205f30 Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Mon, 15 Dec 2025 23:34:45 +0100 Subject: [PATCH 13/25] Do not associate demand with project_data --- app/projects/models/base_models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/projects/models/base_models.py b/app/projects/models/base_models.py index 478f8a6c9..5a3ddce87 100644 --- a/app/projects/models/base_models.py +++ b/app/projects/models/base_models.py @@ -768,7 +768,8 @@ def input_timeseries_values(self): def to_datapackage(self): """Return the asset's attributes in a datapackage form""" dp = {"type": self.asset_type.asset_type} - dp["project_data"] = self.scenario.project.name + if self.asset_type.asset_type not in ("demand"): + dp["project_data"] = self.scenario.project.name # to collect the timeseries used by the asset profile_resource_rec = {} for field in self.asset_type.visible_fields: From 26741caa6b930df983a88fb07c1fad84746bbff8 Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Mon, 15 Dec 2025 23:35:13 +0100 Subject: [PATCH 14/25] Update to_datapackage method of Bus Model class --- app/projects/models/base_models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/projects/models/base_models.py b/app/projects/models/base_models.py index 5a3ddce87..770d904e4 100644 --- a/app/projects/models/base_models.py +++ b/app/projects/models/base_models.py @@ -936,7 +936,8 @@ class Bus(TopologyNode): def to_datapackage(self): dm = model_to_dict(self, fields=["type", "name"]) - dm["facade"] = "bus" + dm["carrier"] = dm["type"] + dm["type"] = "CarrierBus" dm["balanced"] = "True" dm["excess"] = "False" dm["excess_costs"] = "0.0" From f527a378629d948b5927de14da224f7610805b22 Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Mon, 15 Dec 2025 23:37:22 +0100 Subject: [PATCH 15/25] Fix typo --- app/projects/models/base_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/projects/models/base_models.py b/app/projects/models/base_models.py index 770d904e4..91a2bc661 100644 --- a/app/projects/models/base_models.py +++ b/app/projects/models/base_models.py @@ -121,7 +121,7 @@ def to_datapackage(self): dp["discount_factor"] = dp.pop("discount") dp["lifetime"] = dp.pop("duration") dp["shortage_cost"] = 999 - dp["excess_costs"] = 99 + dp["excess_cost"] = 99 return dp def add_viewer_if_not_exist(self, email=None, share_rights=""): From b8ee258560f1c2b5159bf756484313297517a162 Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Mon, 15 Dec 2025 23:55:40 +0100 Subject: [PATCH 16/25] Do not provide project_data for dso either --- app/projects/models/base_models.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/projects/models/base_models.py b/app/projects/models/base_models.py index 91a2bc661..5361d9d87 100644 --- a/app/projects/models/base_models.py +++ b/app/projects/models/base_models.py @@ -768,7 +768,10 @@ def input_timeseries_values(self): def to_datapackage(self): """Return the asset's attributes in a datapackage form""" dp = {"type": self.asset_type.asset_type} - if self.asset_type.asset_type not in ("demand"): + if ( + "demand" not in self.asset_type.asset_type + and "dso" not in self.asset_type.asset_type + ): dp["project_data"] = self.scenario.project.name # to collect the timeseries used by the asset profile_resource_rec = {} From bdeccaafd7aaba4382d8f95a9b5a96ce8dbd20ed Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Mon, 15 Dec 2025 23:56:08 +0100 Subject: [PATCH 17/25] WIP need to fix bus column label --- app/projects/models/base_models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/projects/models/base_models.py b/app/projects/models/base_models.py index 5361d9d87..ae82a0c48 100644 --- a/app/projects/models/base_models.py +++ b/app/projects/models/base_models.py @@ -807,6 +807,7 @@ def to_datapackage(self): qs_bus = self.connectionlink_set.filter( flow_direction=direction, asset_connection_port=port ) + # TODO use bus names for the resource column label according to a convention with the facade attributes if qs_bus.exists(): connection = qs_bus.get() dp[field] = connection.bus.name From ba8137a92031d4e0ce3efa245e5209891a0c7cfe Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Tue, 16 Dec 2025 00:01:19 +0100 Subject: [PATCH 18/25] Adapt local export to datapackage via command --- app/projects/management/commands/datapackage.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/app/projects/management/commands/datapackage.py b/app/projects/management/commands/datapackage.py index 2ef004f03..b222adee6 100644 --- a/app/projects/management/commands/datapackage.py +++ b/app/projects/management/commands/datapackage.py @@ -24,21 +24,17 @@ def handle(self, *args, **options): scenario_folder = destination_path / f"scenario_{scen_id}" if scenario_folder.exists(): shutil.rmtree(scenario_folder) - scenario.to_datapackage(destination_path) dp_json = scenario_folder / "datapackage.json" if dp_json.exists(): print("Only inferring metadata") - p = dp.Package(dp_json) - building.infer_package_foreign_keys(p) + p = dp.Package(str(dp_json)) + building.infer_package_foreign_keys(p, fk_targets=["project"]) p.descriptor["resources"].sort(key=lambda x: (x["path"], x["name"])) p.commit() p.save(dp_json) else: print("Creating datapackage.json") - building.infer_metadata_from_data( - package_name=f"scenario_{scen_id}", - path=scenario_folder, - ) + scenario.to_datapackage(destination_path) From b3d917ee41a0c7b4308e1ba5e917931f97c789cd Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Thu, 22 Jan 2026 23:42:12 +0100 Subject: [PATCH 19/25] Fix initial assignement in forms --- app/projects/forms.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/projects/forms.py b/app/projects/forms.py index c9ce93dc6..10c129bb7 100644 --- a/app/projects/forms.py +++ b/app/projects/forms.py @@ -834,7 +834,7 @@ def __init__(self, *args, **kwargs): """ for field in self.fields: if field == "renewable_asset" and self.asset_type_name in RENEWABLE_ASSETS: - self.fields[field].initial = True + self.initial[field] = True self.fields[field].widget.attrs.update({f"df-{field}": ""}) if field == "input_timeseries": self.fields[field].required = self.is_input_timeseries_empty() @@ -1130,15 +1130,15 @@ def __init__(self, *args, **kwargs): asset_type_name = kwargs.pop("asset_type", None) super(StorageForm, self).__init__(*args, asset_type="capacity", **kwargs) self.fields["dispatchable"].widget = forms.HiddenInput() - self.fields["dispatchable"].initial = True + self.initial["dispatchable"] = True if asset_type_name != "hess": self.fields["fixed_thermal_losses_relative"].widget = forms.HiddenInput() - self.fields["fixed_thermal_losses_relative"].initial = 0 + self.initial["fixed_thermal_losses_relative"] = 0 self.fields["fixed_thermal_losses_absolute"].widget = forms.HiddenInput() - self.fields["fixed_thermal_losses_absolute"].initial = 0 + self.initial["fixed_thermal_losses_absolute"] = 0 self.fields["thermal_loss_rate"].widget = forms.HiddenInput() - self.fields["thermal_loss_rate"].initial = 0 + self.initial["thermal_loss_rate"] = 0 else: field_name = "fixed_thermal_losses_relative" help_text = self.fields[field_name].help_text From 2c1ee53baf01e9fad8bb13cccd665dbe37ace6e1 Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Mon, 2 Feb 2026 22:48:39 +0100 Subject: [PATCH 20/25] Update bus port names --- app/static/assettypes_list.csv | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/app/static/assettypes_list.csv b/app/static/assettypes_list.csv index 05d6c700c..806798127 100644 --- a/app/static/assettypes_list.csv +++ b/app/static/assettypes_list.csv @@ -1,9 +1,9 @@ asset_type,asset_category,energy_vector,mvs_type,asset_fields,unit,ports -dso,energy_provider,Electricity,source,"[name,energy_price,feedin_tariff,peak_demand_pricing,peak_demand_pricing_period,renewable_share,feedin_cap]",,"{'output_1': ['to_bus','Electricity']}" -gas_dso,energy_provider,Gas,source,"[name,energy_price,feedin_tariff,peak_demand_pricing,peak_demand_pricing_period,renewable_share,feedin_cap]",,"{'output_1': ['to_bus','Gas']}" -h2_dso,energy_provider,H2,source,"[name,energy_price,feedin_tariff,peak_demand_pricing,peak_demand_pricing_period,renewable_share,feedin_cap]",,"{'output_1': ['to_bus','H2']}" -heat_dso,energy_provider,Heat,source,"[name,energy_price,feedin_tariff,peak_demand_pricing,peak_demand_pricing_period,renewable_share,feedin_cap]",,"{'output_1': ['to_bus','Heat']}" -demand,energy_consumption,Electricity,sink,"[name,input_timeseries]",kWh,"{'input_1':['from_bus','Electricity']}" +dso,energy_provider,Electricity,source,"[name,energy_price,feedin_tariff,peak_demand_pricing,peak_demand_pricing_period,renewable_share,feedin_cap]",,"{'output_1': ['bus_electricity','Electricity']}" +gas_dso,energy_provider,Gas,source,"[name,energy_price,feedin_tariff,peak_demand_pricing,peak_demand_pricing_period,renewable_share,feedin_cap]",,"{'output_1': ['bus_gas','Gas']}" +h2_dso,energy_provider,H2,source,"[name,energy_price,feedin_tariff,peak_demand_pricing,peak_demand_pricing_period,renewable_share,feedin_cap]",,"{'output_1': ['bus_h2','H2']}" +heat_dso,energy_provider,Heat,source,"[name,energy_price,feedin_tariff,peak_demand_pricing,peak_demand_pricing_period,renewable_share,feedin_cap]",,"{'output_1': ['bus_heat','Heat']}" +demand,energy_consumption,Electricity,sink,"[name,input_timeseries]",kWh,"{'input_1':['bus_in_electricity','Electricity']}" gas_demand,energy_consumption,Gas,sink,"[name,input_timeseries]",,"{'input_1':['from_bus','Gas']}" h2_demand,energy_consumption,H2,sink,"[name,input_timeseries]",,"{'input_1':['from_bus','H2']}" heat_demand,energy_consumption,Heat,sink,"[name,input_timeseries]",,"{'input_1':['from_bus','Heat']}" @@ -17,15 +17,15 @@ fuel_cell,energy_conversion,Electricity,transformer,"[name,age_installed,install gas_boiler,energy_conversion,Gas,transformer,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,maximum_capacity,efficiency]",kW,"{'input_1':['from_bus','Gas'],'output_1': ['to_bus','Heat']}" electrolyzer,energy_conversion,H2,transformer,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,maximum_capacity,efficiency,efficiency_multiple]",kVA,"{'input_1':['el_bus','Electricity'],'output_1': ['heat_bus','Heat'], 'output_2': ['h2_bus','H2']}" heat_pump,energy_conversion,Heat,transformer,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_var,opex_fix,lifetime,optimize_cap,maximum_capacity,efficiency]",kVA,"{'input_1':['electricity_bus','Electricity'], 'input_2': ['heat_bus','Heat'],'output_1': ['to_bus','Heat']}" -pv_plant,energy_production,Electricity,source,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,maximum_capacity,renewable_asset,input_timeseries]",kWp,"{'output_1': ['to_bus','Electricity']}" -wind_plant,energy_production,Electricity,source,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,maximum_capacity,renewable_asset,input_timeseries]",kW,"{'output_1': ['to_bus','Electricity']}" +pv_plant,energy_production,Electricity,source,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,maximum_capacity,renewable_asset,input_timeseries]",kWp,"{'output_1': ['bus_out_electricity','Electricity']}" +wind_plant,energy_production,Electricity,source,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,maximum_capacity,renewable_asset,input_timeseries]",kW,"{'output_1': ['bus_out_electricity','Electricity']}" biogas_plant,energy_production,Gas,source,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,maximum_capacity,renewable_asset,input_timeseries]",,"{'output_1': ['to_bus','Gas']}" geothermal_conversion,energy_production,Electricity,source,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,maximum_capacity,renewable_asset,input_timeseries]",kWh,"{'output_1': ['to_bus','Heat']}" solar_thermal_plant,energy_production,Heat,source,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,maximum_capacity,renewable_asset,input_timeseries]",m³,"{'output_1': ['to_bus','Heat']}" charging_power,energy_storage,Electricity,storage,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,crate,efficiency,dispatchable]",kW,"{'input_1':['charge','Electricity'],'output_1': ['discharge','Electricity']}" discharging_power,energy_storage,Electricity,storage,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,crate,efficiency,dispatchable]",kW,"{'input_1':['charge','Electricity'],'output_1': ['discharge','Electricity']}" capacity,energy_storage,Electricity,storage,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,maximum_capacity,efficiency,soc_max,soc_min,crate,dispatchable,thermal_loss_rate,fixed_thermal_losses_relative,fixed_thermal_losses_absolute]",kWh,"{'input_1':['charge','Electricity'],'output_1': ['discharge','Electricity']}" -bess,energy_storage,Electricity,storage,[name],kW (el),"{'input_1':['charge','Electricity'],'output_1': ['discharge','Electricity']}" +bess,energy_storage,Electricity,storage,[name],kW (el),"{'input_1':['bus_in_electricity','Electricity'],'output_1': ['bus_out_electricity','Electricity']}" gess,energy_storage,Gas,storage,[name],l,"{'input_1':['charge','Gas'],'output_1': ['discharge','Gas']}" h2ess,energy_storage,H2,storage,[name],kgH2,"{'input_1':['charge','H2'],'output_1': ['discharge','H2']}" hess,energy_storage,Heat,storage,[name],kW (therm),"{'input_1':['charge','Heat'],'output_1': ['discharge','Heat']}" From 8a7a2fddf1fd3ec0ec951d3387357c23ff98d892 Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Mon, 2 Feb 2026 23:13:49 +0100 Subject: [PATCH 21/25] Adapt datapackage export for storage --- app/projects/models/base_models.py | 84 +++++++++++++++++++++++------- 1 file changed, 65 insertions(+), 19 deletions(-) diff --git a/app/projects/models/base_models.py b/app/projects/models/base_models.py index ae82a0c48..27e8ebfd5 100644 --- a/app/projects/models/base_models.py +++ b/app/projects/models/base_models.py @@ -318,7 +318,7 @@ def export(self, bind_project_data=False): def to_datapackage(self, destination_path): # Create a folder with a datapackage structure - scenario_folder = destination_path / f"scenario_{self.id}" + scenario_folder = destination_path / f"scenario_{self.name}".replace(" ", "_") data_folder = scenario_folder / "data" elements_folder = data_folder / "elements" @@ -338,9 +338,11 @@ def to_datapackage(self, destination_path): # List all components of the scenario (except the busses) qs_assets = Asset.objects.filter(scenario=self) - # List all distinct components' assettypes (or facade name) - facade_names = qs_assets.distinct().values_list( - "asset_type__asset_type", flat=True + # List all distinct components' assettypes (or facade name) which are not children + facade_names = ( + qs_assets.filter(parent_asset__isnull=True) + .distinct() + .values_list("asset_type__asset_type", flat=True) ) bus_resource_records = [] @@ -395,7 +397,7 @@ def to_datapackage(self, destination_path): # creating datapackage.json metadata file at the root of the datapackage building.infer_metadata_from_data( - package_name=f"scenario_{self.id}", + package_name=f"scenario_{self.name}".replace(" ", "_"), path=scenario_folder, fk_targets=["project"], ) @@ -775,21 +777,66 @@ def to_datapackage(self): dp["project_data"] = self.scenario.project.name # to collect the timeseries used by the asset profile_resource_rec = {} - for field in self.asset_type.visible_fields: - value = getattr(self, field) - # if the field is a candidate for a scalar/list - if isinstance(value, str) and field != "name": - value = json.loads(value) - if isinstance(value, list): - col = f"{self.name}__{field}" - profile_resource_rec[col] = value + + # Storage assets are the only one to have children (namely `capacity`, `charge` and `discharge` + qs_children = Asset.objects.filter(parent_asset__id=self.id) + if qs_children.exists(): + # Only keep the values from the capacity children asset of the storage + capacity = qs_children.get(asset_type__asset_type="capacity") + for attribute in [ + "capex_fix", + "capex_var", + "opex_fix", + "opex_var", + "lifetime", + "crate", + "efficiency", + "soc_max", + "soc_min", + "maximum_capacity", + "optimize_cap", + "installed_capacity", + "age_installed", + "thermal_loss_rate", # only for hess + "fixed_thermal_losses_relative", # only for hess + "fixed_thermal_losses_absolute", # only for hess + ]: + setattr(self, attribute, getattr(capacity, attribute)) + + if self.asset_type.asset_type != "hess": + attributes = [ + f + for f in AssetType.objects.get(asset_type="capacity").visible_fields + if f + not in ( + "thermal_loss_rate", + "fixed_thermal_losses_relative", + "fixed_thermal_losses_absolute", + ) + ] + else: + attributes = AssetType.objects.get(asset_type="capacity").visible_fields + else: + attributes = self.asset_type.visible_fields + + for field in attributes: + if ( + field != "dispatchable" + ): # TODO remove this when `dispatchable` not a visible field anymore + value = getattr(self, field) + # if the field is a candidate for a scalar/list + if isinstance(value, str) and field != "name": + value = json.loads(value) + if isinstance(value, list): + col = f"{self.name}__{field}" + profile_resource_rec[col] = value + value = col + elif isinstance(value, Timeseries): + col = value.name + profile_resource_rec[col] = value.values value = col - elif isinstance(value, Timeseries): - col = value.name - profile_resource_rec[col] = value.values - value = col - dp[field] = value + dp[field] = value # to collect the bus(ses) used by the asset bus_resource_rec = [] @@ -807,7 +854,6 @@ def to_datapackage(self): qs_bus = self.connectionlink_set.filter( flow_direction=direction, asset_connection_port=port ) - # TODO use bus names for the resource column label according to a convention with the facade attributes if qs_bus.exists(): connection = qs_bus.get() dp[field] = connection.bus.name From c247c9d9ecbc5517f7a07b6dc08ebcdbdd0de1cb Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Tue, 3 Feb 2026 23:48:32 +0100 Subject: [PATCH 22/25] Map efficiency and efficiency multiple for chp_fixed_ratio --- app/projects/models/base_models.py | 6 ++++++ app/static/assettypes_list.csv | 22 +++++++++++----------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/app/projects/models/base_models.py b/app/projects/models/base_models.py index 27e8ebfd5..db12e7b84 100644 --- a/app/projects/models/base_models.py +++ b/app/projects/models/base_models.py @@ -836,6 +836,12 @@ def to_datapackage(self): profile_resource_rec[col] = value.values value = col + if self.asset_type.asset_type == "chp_fixed_ratio": + if field == "efficiency": + field = "conversion_factor_to_electricity" + elif field == "efficiency_multiple": + field = "conversion_factor_to_heat" + dp[field] = value # to collect the bus(ses) used by the asset diff --git a/app/static/assettypes_list.csv b/app/static/assettypes_list.csv index 806798127..9b63b0734 100644 --- a/app/static/assettypes_list.csv +++ b/app/static/assettypes_list.csv @@ -1,12 +1,12 @@ asset_type,asset_category,energy_vector,mvs_type,asset_fields,unit,ports dso,energy_provider,Electricity,source,"[name,energy_price,feedin_tariff,peak_demand_pricing,peak_demand_pricing_period,renewable_share,feedin_cap]",,"{'output_1': ['bus_electricity','Electricity']}" -gas_dso,energy_provider,Gas,source,"[name,energy_price,feedin_tariff,peak_demand_pricing,peak_demand_pricing_period,renewable_share,feedin_cap]",,"{'output_1': ['bus_gas','Gas']}" +gas_dso,energy_provider,Gas,source,"[name,energy_price,feedin_tariff,peak_demand_pricing,peak_demand_pricing_period,renewable_share,feedin_cap]",,"{'output_1': ['bus_fuel','Gas']}" h2_dso,energy_provider,H2,source,"[name,energy_price,feedin_tariff,peak_demand_pricing,peak_demand_pricing_period,renewable_share,feedin_cap]",,"{'output_1': ['bus_h2','H2']}" heat_dso,energy_provider,Heat,source,"[name,energy_price,feedin_tariff,peak_demand_pricing,peak_demand_pricing_period,renewable_share,feedin_cap]",,"{'output_1': ['bus_heat','Heat']}" demand,energy_consumption,Electricity,sink,"[name,input_timeseries]",kWh,"{'input_1':['bus_in_electricity','Electricity']}" -gas_demand,energy_consumption,Gas,sink,"[name,input_timeseries]",,"{'input_1':['from_bus','Gas']}" -h2_demand,energy_consumption,H2,sink,"[name,input_timeseries]",,"{'input_1':['from_bus','H2']}" -heat_demand,energy_consumption,Heat,sink,"[name,input_timeseries]",,"{'input_1':['from_bus','Heat']}" +gas_demand,energy_consumption,Gas,sink,"[name,input_timeseries]",,"{'input_1':['bus_in_fuel','Gas']}" +h2_demand,energy_consumption,H2,sink,"[name,input_timeseries]",,"{'input_1':['bus_in_h2','H2']}" +heat_demand,energy_consumption,Heat,sink,"[name,input_timeseries]",,"{'input_1':['bus_in_heat','Heat']}" transformer_station_in,energy_conversion,Electricity,transformer,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,maximum_capacity,efficiency]",kVA,"{'input_1':['from_bus','Electricity'],'output_1': ['to_bus','Electricity']}" transformer_station_out,energy_conversion,Electricity,transformer,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,maximum_capacity,efficiency]",kVA,"{'input_1':['from_bus','Electricity'],'output_1': ['to_bus','Electricity']}" storage_charge_controller_in,energy_conversion,Electricity,transformer,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,efficiency]",kVA,"{'input_1':['from_bus','Electricity'],'output_1': ['to_bus','Electricity']}" @@ -14,20 +14,20 @@ storage_charge_controller_out,energy_conversion,Electricity,transformer,"[name,a solar_inverter,energy_conversion,Electricity,transformer,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,efficiency]",kVA,"{'input_1':['from_bus','Electricity'],'output_1': ['to_bus','Electricity']}" diesel_generator,energy_conversion,Electricity,transformer,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,maximum_capacity,efficiency]",kVA,"{'input_1':['from_bus','Gas'],'output_1': ['to_bus','Electricity']}" fuel_cell,energy_conversion,Electricity,transformer,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,efficiency]",kVA,"{'input_1':['from_bus','H2'],'output_1': ['to_bus','Electricity']}" -gas_boiler,energy_conversion,Gas,transformer,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,maximum_capacity,efficiency]",kW,"{'input_1':['from_bus','Gas'],'output_1': ['to_bus','Heat']}" +gas_boiler,energy_conversion,Gas,transformer,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,maximum_capacity,efficiency]",kW,"{'input_1':['bus_in_fuel','Gas'],'output_1': ['bus_out_heat','Heat']}" electrolyzer,energy_conversion,H2,transformer,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,maximum_capacity,efficiency,efficiency_multiple]",kVA,"{'input_1':['el_bus','Electricity'],'output_1': ['heat_bus','Heat'], 'output_2': ['h2_bus','H2']}" heat_pump,energy_conversion,Heat,transformer,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_var,opex_fix,lifetime,optimize_cap,maximum_capacity,efficiency]",kVA,"{'input_1':['electricity_bus','Electricity'], 'input_2': ['heat_bus','Heat'],'output_1': ['to_bus','Heat']}" pv_plant,energy_production,Electricity,source,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,maximum_capacity,renewable_asset,input_timeseries]",kWp,"{'output_1': ['bus_out_electricity','Electricity']}" wind_plant,energy_production,Electricity,source,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,maximum_capacity,renewable_asset,input_timeseries]",kW,"{'output_1': ['bus_out_electricity','Electricity']}" biogas_plant,energy_production,Gas,source,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,maximum_capacity,renewable_asset,input_timeseries]",,"{'output_1': ['to_bus','Gas']}" geothermal_conversion,energy_production,Electricity,source,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,maximum_capacity,renewable_asset,input_timeseries]",kWh,"{'output_1': ['to_bus','Heat']}" -solar_thermal_plant,energy_production,Heat,source,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,maximum_capacity,renewable_asset,input_timeseries]",m³,"{'output_1': ['to_bus','Heat']}" +solar_thermal_plant,energy_production,Heat,source,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,maximum_capacity,renewable_asset,input_timeseries]",m³,"{'output_1': ['bus_out_heat','Heat']}" charging_power,energy_storage,Electricity,storage,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,crate,efficiency,dispatchable]",kW,"{'input_1':['charge','Electricity'],'output_1': ['discharge','Electricity']}" discharging_power,energy_storage,Electricity,storage,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,crate,efficiency,dispatchable]",kW,"{'input_1':['charge','Electricity'],'output_1': ['discharge','Electricity']}" capacity,energy_storage,Electricity,storage,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,maximum_capacity,efficiency,soc_max,soc_min,crate,dispatchable,thermal_loss_rate,fixed_thermal_losses_relative,fixed_thermal_losses_absolute]",kWh,"{'input_1':['charge','Electricity'],'output_1': ['discharge','Electricity']}" bess,energy_storage,Electricity,storage,[name],kW (el),"{'input_1':['bus_in_electricity','Electricity'],'output_1': ['bus_out_electricity','Electricity']}" -gess,energy_storage,Gas,storage,[name],l,"{'input_1':['charge','Gas'],'output_1': ['discharge','Gas']}" -h2ess,energy_storage,H2,storage,[name],kgH2,"{'input_1':['charge','H2'],'output_1': ['discharge','H2']}" -hess,energy_storage,Heat,storage,[name],kW (therm),"{'input_1':['charge','Heat'],'output_1': ['discharge','Heat']}" -chp,energy_conversion,Electricity,extractionTurbineCHP,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_var,opex_fix,lifetime,optimize_cap,maximum_capacity,efficiency_multiple,efficiency,thermal_loss_rate]",kW,"{'input_1':['fuel','Gas'],'output_1': ['heat_bus','Heat'], 'output_2': ['electricity_bus','Electricity']}" -chp_fixed_ratio,energy_conversion,Electricity,transformer,"[name,age_installed,installed_capacity,capex_var,opex_fix,lifetime,optimize_cap,maximum_capacity,efficiency_multiple,efficiency]",kW,"{'input_1':['fuel','Gas'],'output_1': ['heat_bus','Heat'], 'output_2': ['electricity_bus','Electricity']}" +gess,energy_storage,Gas,storage,[name],l,"{'input_1':['bus_in_fuel','Gas'],'output_1': ['bus_out_fuel','Gas']}" +h2ess,energy_storage,H2,storage,[name],kgH2,"{'input_1':['bus_in_h2','H2'],'output_1': ['bus_out_h2','H2']}" +hess,energy_storage,Heat,storage,[name],kW (therm),"{'input_1':['bus_in_heat','Heat'],'output_1': ['bus_out_heat','Heat']}" +chp,energy_conversion,Electricity,extractionTurbineCHP,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_var,opex_fix,lifetime,optimize_cap,maximum_capacity,efficiency_multiple,efficiency,thermal_loss_rate]",kW,"{'input_1':['bus_in_fuel','Gas'],'output_1': ['bus_out_heat','Heat'], 'output_2': ['bus_out_electricity','Electricity']}" +chp_fixed_ratio,energy_conversion,Electricity,transformer,"[name,age_installed,installed_capacity,capex_var,opex_fix,lifetime,optimize_cap,maximum_capacity,efficiency_multiple,efficiency]",kW,"{'input_1':['bus_in_fuel','Gas'],'output_1': ['bus_out_heat','Heat'], 'output_2': ['bus_out_electricity','Electricity']}" From cb8a296224527a95c1ee821a5979e0a18294d2aa Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Tue, 3 Feb 2026 23:51:44 +0100 Subject: [PATCH 23/25] Change oemof-tabular for oemof-datapackage dependency --- app/projects/management/commands/datapackage.py | 2 +- app/projects/models/base_models.py | 3 +-- app/requirements/base.txt | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/app/projects/management/commands/datapackage.py b/app/projects/management/commands/datapackage.py index b222adee6..3e4ff370d 100644 --- a/app/projects/management/commands/datapackage.py +++ b/app/projects/management/commands/datapackage.py @@ -2,7 +2,7 @@ from projects.models import Scenario from pathlib import Path import shutil -from oemof.tabular.datapackage import building +from oemof.datapackage.datapackage import building import datapackage as dp diff --git a/app/projects/models/base_models.py b/app/projects/models/base_models.py index db12e7b84..381f869c8 100644 --- a/app/projects/models/base_models.py +++ b/app/projects/models/base_models.py @@ -6,8 +6,7 @@ import pandas as pd from pathlib import Path import numpy as np -import datapackage as dp -from oemof.tabular.datapackage import building +from oemof.datapackage.datapackage import building import oemof.thermal.compression_heatpumps_and_chillers as cmpr_hp_chiller diff --git a/app/requirements/base.txt b/app/requirements/base.txt index 9084596e0..3b78bb46a 100644 --- a/app/requirements/base.txt +++ b/app/requirements/base.txt @@ -8,12 +8,12 @@ django-jsonview==2.0.0 django-q==1.3.4 django-sass-processor==1.2.2 exchangelib==4.4.0 -git+https://github.com/oemof/oemof-tabular@dev gunicorn==20.0.4 httpx==0.23.0 jsonschema==4.4.0 libsass==0.21.0 numpy>=1.22.4 +oemof-datapackage oemof-thermal==0.0.8 openpyxl==3.0.10 plotly==5.6.0 From 49b5c9a991e24c31ef5881b69b8419991622ac9c Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Tue, 3 Feb 2026 23:52:21 +0100 Subject: [PATCH 24/25] Add outpath option for datapackage export command --- app/projects/management/commands/datapackage.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/app/projects/management/commands/datapackage.py b/app/projects/management/commands/datapackage.py index 3e4ff370d..b9ab85495 100644 --- a/app/projects/management/commands/datapackage.py +++ b/app/projects/management/commands/datapackage.py @@ -11,6 +11,7 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument("scen_id", nargs="+", type=int) + parser.add_argument("-o", "--outfile", type=str, nargs="?", const="") def handle(self, *args, **options): @@ -19,9 +20,15 @@ def handle(self, *args, **options): scenario = Scenario.objects.get(pk=scen_id) except Scenario.DoesNotExist: raise CommandError('Scenario "%s" does not exist' % scen_id) - destination_path = Path(__file__).resolve().parents[4] + destination_path = options["outfile"] + if destination_path == "": + destination_path = Path(__file__).resolve().parents[4] + else: + destination_path = Path(destination_path) - scenario_folder = destination_path / f"scenario_{scen_id}" + scenario_folder = destination_path / f"scenario_{scenario.name}".replace( + " ", "_" + ) if scenario_folder.exists(): shutil.rmtree(scenario_folder) From 34ada2160723777fdc30ce1846f9de969f92c60c Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Mon, 9 Feb 2026 23:56:24 +0100 Subject: [PATCH 25/25] Enable chp and electrolyzer to be exported to datapackage --- app/projects/models/base_models.py | 13 +++++++++++++ app/static/assettypes_list.csv | 22 +++++++++++----------- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/app/projects/models/base_models.py b/app/projects/models/base_models.py index 381f869c8..3a4c626e5 100644 --- a/app/projects/models/base_models.py +++ b/app/projects/models/base_models.py @@ -840,7 +840,20 @@ def to_datapackage(self): field = "conversion_factor_to_electricity" elif field == "efficiency_multiple": field = "conversion_factor_to_heat" + elif self.asset_type.asset_type == "chp": + if field == "thermal_loss_rate": + field = "beta" + elif field == "efficiency": + field = "conversion_factor_to_electricity" + elif field == "efficiency_multiple": + field = "conversion_factor_to_heat" + elif self.asset_type.asset_type == "heat_pump": + if field == "efficiency": + field = "cop" + elif self.asset_type.asset_type == "electrolyzer": + if field == "efficiency_multiple": + field = "efficiency_heat" dp[field] = value # to collect the bus(ses) used by the asset diff --git a/app/static/assettypes_list.csv b/app/static/assettypes_list.csv index 9b63b0734..71fc4c15e 100644 --- a/app/static/assettypes_list.csv +++ b/app/static/assettypes_list.csv @@ -7,20 +7,20 @@ demand,energy_consumption,Electricity,sink,"[name,input_timeseries]",kWh,"{'inpu gas_demand,energy_consumption,Gas,sink,"[name,input_timeseries]",,"{'input_1':['bus_in_fuel','Gas']}" h2_demand,energy_consumption,H2,sink,"[name,input_timeseries]",,"{'input_1':['bus_in_h2','H2']}" heat_demand,energy_consumption,Heat,sink,"[name,input_timeseries]",,"{'input_1':['bus_in_heat','Heat']}" -transformer_station_in,energy_conversion,Electricity,transformer,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,maximum_capacity,efficiency]",kVA,"{'input_1':['from_bus','Electricity'],'output_1': ['to_bus','Electricity']}" -transformer_station_out,energy_conversion,Electricity,transformer,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,maximum_capacity,efficiency]",kVA,"{'input_1':['from_bus','Electricity'],'output_1': ['to_bus','Electricity']}" -storage_charge_controller_in,energy_conversion,Electricity,transformer,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,efficiency]",kVA,"{'input_1':['from_bus','Electricity'],'output_1': ['to_bus','Electricity']}" -storage_charge_controller_out,energy_conversion,Electricity,transformer,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,efficiency]",kVA,"{'input_1':['from_bus','Electricity'],'output_1': ['to_bus','Electricity']}" -solar_inverter,energy_conversion,Electricity,transformer,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,efficiency]",kVA,"{'input_1':['from_bus','Electricity'],'output_1': ['to_bus','Electricity']}" -diesel_generator,energy_conversion,Electricity,transformer,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,maximum_capacity,efficiency]",kVA,"{'input_1':['from_bus','Gas'],'output_1': ['to_bus','Electricity']}" -fuel_cell,energy_conversion,Electricity,transformer,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,efficiency]",kVA,"{'input_1':['from_bus','H2'],'output_1': ['to_bus','Electricity']}" +transformer_station_in,energy_conversion,Electricity,transformer,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,maximum_capacity,efficiency]",kVA,"{'input_1':['bus_in_electricity','Electricity'],'output_1': ['bus_out_electricity','Electricity']}" +transformer_station_out,energy_conversion,Electricity,transformer,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,maximum_capacity,efficiency]",kVA,"{'input_1':['bus_in_electricity','Electricity'],'output_1': ['bus_out_electricity','Electricity']}" +storage_charge_controller_in,energy_conversion,Electricity,transformer,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,efficiency]",kVA,"{'input_1':['bus_in_electricity','Electricity'],'output_1': ['bus_out_electricity','Electricity']}" +storage_charge_controller_out,energy_conversion,Electricity,transformer,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,efficiency]",kVA,"{'input_1':['bus_in_electricity','Electricity'],'output_1': ['bus_out_electricity','Electricity']}" +solar_inverter,energy_conversion,Electricity,transformer,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,efficiency]",kVA,"{'input_1':['bus_in_electricity','Electricity'],'output_1': ['bus_out_electricity','Electricity']}" +diesel_generator,energy_conversion,Electricity,transformer,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,maximum_capacity,efficiency]",kVA,"{'input_1':['bus_in_fuel','Gas'],'output_1': ['bus_out_electricity','Electricity']}" +fuel_cell,energy_conversion,Electricity,transformer,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,efficiency]",kVA,"{'input_1':['bus_in_h2','H2'],'output_1': ['bus_out_electricity','Electricity']}" gas_boiler,energy_conversion,Gas,transformer,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,maximum_capacity,efficiency]",kW,"{'input_1':['bus_in_fuel','Gas'],'output_1': ['bus_out_heat','Heat']}" -electrolyzer,energy_conversion,H2,transformer,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,maximum_capacity,efficiency,efficiency_multiple]",kVA,"{'input_1':['el_bus','Electricity'],'output_1': ['heat_bus','Heat'], 'output_2': ['h2_bus','H2']}" -heat_pump,energy_conversion,Heat,transformer,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_var,opex_fix,lifetime,optimize_cap,maximum_capacity,efficiency]",kVA,"{'input_1':['electricity_bus','Electricity'], 'input_2': ['heat_bus','Heat'],'output_1': ['to_bus','Heat']}" +electrolyzer,energy_conversion,H2,transformer,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,maximum_capacity,efficiency,efficiency_multiple]",kVA,"{'input_1':['bus_in_electricity','Electricity'],'output_1': ['bus_out_heat','Heat'], 'output_2': ['bus_out_h2','H2']}" +heat_pump,energy_conversion,Heat,transformer,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_var,opex_fix,lifetime,optimize_cap,maximum_capacity,efficiency]",kVA,"{'input_1':['bus_in_electricity','Electricity'], 'input_2': ['bus_in_heat','Heat'],'output_1': ['bus_out_heat','Heat']}" pv_plant,energy_production,Electricity,source,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,maximum_capacity,renewable_asset,input_timeseries]",kWp,"{'output_1': ['bus_out_electricity','Electricity']}" wind_plant,energy_production,Electricity,source,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,maximum_capacity,renewable_asset,input_timeseries]",kW,"{'output_1': ['bus_out_electricity','Electricity']}" -biogas_plant,energy_production,Gas,source,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,maximum_capacity,renewable_asset,input_timeseries]",,"{'output_1': ['to_bus','Gas']}" -geothermal_conversion,energy_production,Electricity,source,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,maximum_capacity,renewable_asset,input_timeseries]",kWh,"{'output_1': ['to_bus','Heat']}" +biogas_plant,energy_production,Gas,source,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,maximum_capacity,renewable_asset,input_timeseries]",,"{'output_1': ['bus_out_fuel','Gas']}" +geothermal_conversion,energy_production,Electricity,source,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,maximum_capacity,renewable_asset,input_timeseries]",kWh,"{'output_1': ['bus_out_heat','Heat']}" solar_thermal_plant,energy_production,Heat,source,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,optimize_cap,maximum_capacity,renewable_asset,input_timeseries]",m³,"{'output_1': ['bus_out_heat','Heat']}" charging_power,energy_storage,Electricity,storage,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,crate,efficiency,dispatchable]",kW,"{'input_1':['charge','Electricity'],'output_1': ['discharge','Electricity']}" discharging_power,energy_storage,Electricity,storage,"[name,age_installed,installed_capacity,capex_fix,capex_var,opex_fix,opex_var,lifetime,crate,efficiency,dispatchable]",kW,"{'input_1':['charge','Electricity'],'output_1': ['discharge','Electricity']}"