From 03b59bd010602c495f62e4adcae1cce3b197bdc2 Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Thu, 24 Feb 2022 13:51:57 +0100 Subject: [PATCH 01/17] Add a column to flag sensitivity analysis parameter --- app/static/MVS_parameters_list.csv | 108 ++++++++++++++--------------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/app/static/MVS_parameters_list.csv b/app/static/MVS_parameters_list.csv index ccec70d8..9f8a2c28 100644 --- a/app/static/MVS_parameters_list.csv +++ b/app/static/MVS_parameters_list.csv @@ -1,54 +1,54 @@ -verbose,:Default:,:Definition_Short:,:Definition_Long:,:Example:,:Restrictions:,:Type:,:Unit:,label,ref,category,see_also,,,, -None,5,The number of years the asset has already been in operation,,10,Natural number,numeric,Year,age_installed,age_ins-label,conversion,production,,,, -None,1,The rate at which the storage can be charged or discharged relative to the nominal capacity of the storage, A C-rate of 1 implies that the battery can be fully charged or discharged completely in a single timestep. A C-rate of 0.5 implies that the battery needs at least 2 timesteps to be fully charged or discharged.,"*storage capacity*: NaN, *input power*: 1, *output power*: 1","Real number between 0 and 1. Only the columns 'input power' and 'output power' require a value, in column 'storage capacity' c_rate should be set to NaN.",numeric,Factor,c-rate,crate-label,storage_csv,,,,, -None,None,The name of the country where the project is being deployed,,Norway,None,str,None,country,country-label,project_data,,,,, -None,None,The currency of the country where the project is implemented,,EUR,None,str,None,currency,currency-label,economic_data,,,,, -None,0,Fixed project costs,This could be planning and development costs which do not depend on the (optimized) capacities of the assets.,10000,Positive real number,numeric,€,development_costs,developmentcosts-label,conversion,storage_csv,production,fixcost,, -Discount factor,0.05,"The factor which accounts for the depreciation in the value of money in the future, compared to the current value of the same money. The common method is to calculate the weighted average cost of capital (WACC) and use it as the discount rate.",,0.06,Between 0 and 1,numeric,Factor,discount_factor,discountfactor-label,economic_data,,,,, -None,0.10,Costs associated with a flow through/from the asset,e.g. in EUR/kWh,0.64,"In 'storage_xx.csv' only the columns 'input power' and 'output power' require a value, in column 'storage capacity' dispatch_price should be set to NaN.",numeric,€/kWh,dispatch_price,dispatchprice-label,conversion,production,storage_csv,fixcost,, -None,False,"Stands for Demand Side Management. Currently, not implemented.",,False,Acceptable values are either True or False.,boolean,None,dsm,dsm-label,hidden,,,,, -None,0.8,Ratio of energy output/energy input,"The battery efficiency is the ratio of the energy taken out from the battery, to the energy put into the battery. It means that it is not possible to retrieve as much energy as provided to the battery due to the discharge losses. The efficiency of the 'input power' and 'ouput power' columns should be set equal to the charge and discharge efficiencies respectively, while the 'storage capacity' efficiency should be equal to the storage's efficiency/ability to hold charge over time (`= 1 - self-discharge/decay`), which is usually in the range of 0.95 to 1 for electrical storages.",0.95,Between 0 and 1,numeric,Factor,efficiency,efficiency-label,conversion,storage_csv,,,, -None,0.20,Emissions per unit dispatch of an asset,,14.4,Positive real number,numeric,kgCO2eq/asset unit,emission_factor,emissionfactor-label,providers,production,,,, -None,0.30,Price of the energy carrier sourced from the utility grid.,,0.1,None,numeric,€/energy carrier unit,energy_price,energyprice-label,providers,,,,, -None,Electricity,"Energy vector/commodity. Convention: For an energy conversion asset define energyVector of the output. For a sink define based on inflow. For a source define based on output flow. For a storage, define based on stored energy carrier.",,Electricity,"One of “Electricity”, “Gas”, “Bio-Gas”, “Diesel”, “Heat”, “H2”",str,None,energyVector,energyvector-label,busses,consumption,production,storage,providers,conversion -None,365,The number of days for which the simulation is to be run,,365,Natural number,numeric,Day,evaluated_period,evaluatedperiod-label,simulation_settings,,,,, -None,0.10,Price received for feeding electricity into the grid,,0.7,Real number between 0 and 1,numeric,€/kWh,feedin_tariff,feedintariff-label,providers,,,,, -None,None,Name of a csv file containing the input generation or demand timeseries.,,demand_harbor.csv,This file must be placed in a folder named “time_series” inside your input folder.,str,None,file_name,filename-label,consumption,production,storage,,, -None,0,Thermal losses of storage independent of state of charge and independent of nominal storage capacity between two consecutive timesteps.,,0.0003,Between 0 and 1,numeric,factor,fixed_thermal_losses_absolute,fixed_thermal_losses_absolute-label,storage_csv,,,,, -None,0,Thermal losses of storage independent of state of charge between two consecutive timesteps relative to nominal storage capacity.,,0.0016,Between 0 and 1,numeric,factor,fixed_thermal_losses_relative,fixed_thermal_losses_relative-label,storage_csv,,,,, -None,None,The label of the bus/component from which the energyVector is arriving into the asset.,,Electricity,None,str,None,inflow_direction,inflowdirection-label,consumption,conversion,providers,storage,, -None,0,The already existing installed capacity in-place,"If the project lasts longer than its remaining lifetime, its replacement costs will be taken into account.",50,Each component in the “energy production” category must have a value.,numeric,kWp,installedCap,installedcap-label,conversion,production,storage_csv,,, -None,None,Name of the asset for display purposes,,pv_plant_01,"Input the names in a computer friendly format, preferably with underscores instead of spaces, and avoiding special characters",str,None,label,labl-label,fixcost,,,,, -Location's latitude,None,Latitude coordinate of the project's geographical location,,45.641603,Should follow geographical convention,numeric,None,latitude,latitude-label,project_data,,,,, -None,20,Number of operational years of the asset until it has to be replaced,,30,Natural number,numeric,Year,lifetime,lifetime-label,conversion,production,storage_csv,fixcost,, -Location's longitude,None,Longitude coordinate of the project's geographical location,,10.95787,Should follow geographical convention,numeric,None,longitude,longitude-label,project_data,,,,, -None,None,The maximum amount of total emissions which are allowed in the optimized energy system,,100000,Acceptable values are either a positive real number or None,numeric,kgCO2eq/a,maximum_emissions,maxemissions-label,constraints,,,,, -None,None,The maximum total capacity of an asset that can be installed at the project site,"This includes the installed and the also the maximum additional capacity possible. An example would be that a roof can only carry 50 kWp PV (maximumCap), whereas the installed capacity is already 10 kWp. The optimization would only be allowed to add 40 kWp PV at maximum.",1050,Acceptable values are either a positive real number or None,numeric,kWp,maximumCap,maxcap-label,production,,,,, -None,None,The minimum degree of autonomy that needs to be met by the optimization,,0.3,Between 0 and 1,numeric,factor,minimal_degree_of_autonomy,minda-label,constraints,,,,, -None,None,The minimum share of energy supplied by renewable generation that needs to be met by the optimization,Insert the value 0 to deactivate this constraint.,0.7,Between 0 and 1,numeric,factor,minimal_renewable_factor,minrenshare-label,constraints,,,,, -None,False,Specifies whether optimization needs to result into a net zero energy system (True) or not (False),,True,Acceptable values are either True or False.,boolean,None,net_zero_energy,nzeconstraint-label,constraints,,,,, -None,False,Choose if capacity optimization should be performed for this asset.,,True,Permissible values are either True or False,boolean,None,optimizeCap,optimizecap-label,conversion,production,providers,storage,, -None,None,The label of bus/component towards which the energyVector is leaving from the asset.,,PV plant (mono),None,str,None,outflow_direction,outflowdirec-label,consumption,conversion,providers,storage,, -None,False,"Enable the generation of a file with the linear equation system describing the simulation, ie., with the objective function and all the constraints. This lp file enables the user look at the underlying equations of the optimization.",,False,Acceptable values are either True or False,boolean,None,output_lp_file,outputlpfile-label,simulation_settings,,,,, -None,None,Price to be paid additionally for energy consumption based on the peak demand of a given period,,60,None,numeric,€/kW,peak_demand_pricing,peakdemand-label,providers,peakdemandperiod-label,,,, -None,1,Number of reference periods in one year for the peak demand pricing,,2,"Only one of the following are acceptable values: 1 (yearly), 2, 3 ,4, 6, 12 (monthly)",numeric,"times per year (1,2,3,4,6,12)",peak_demand_pricing_period,peakdemandperiod-label,providers,peakdemand-label,,,, -Project duration,20,The number of years the project is intended to be operational,The project duration also sets the installation time of the assets used in the simulation. After the project ends these assets are 'sold' and the refund is charged against the initial investment costs.,30,Natural number,numeric,Years,project_duration,projectduration-label,economic_data,,,,, -None,None,Assign a project ID as per your preference.,,1,Cannot be the same as an already existing project,str,None,project_id,projectid-label,project_data,,,,, -None,None,Assign a project name as per your preference.,,Borg Havn,None,str,None,project_name,projectname-label,project_data,,,,, -None,0.44,The share of renewables in the generation mix of the energy supplied by the DSO utility,,0.1,Real number between 0 and 1,numeric,Factor,renewable_share,renshare-label,providers,,,,, -None,None,Choose if this asset should be tagged as renewable.,,True,Acceptable values are either True or False,boolean,None,renewableAsset,renewableasset-label,production,,,,, -None,None,Brief description of the scenario being simulated,,This scenario simulates a sector-coupled energy system,None,str,None,scenario_description,scenariodescription-label,project_data,,,,, -None,None,Assign a scenario id as per your preference.,,1,Cannot be the same as an already existing scenario within the project,str,None,scenario_id,scenarioid-label,project_data,,,,, -None,None,Assign a scenario name as per your preference.,,Warehouse 14,None,str,None,scenario_name,scenarioname-label,project_data,,,,, -None,None,The level of charge (as a factor of the actual capacity) in the storage in the zeroth time-step.,,":code:`storage capacity`: None, :code:`input power`: NaN, :code:`output power`: NaN","Acceptable values are either None or the factor. Only the column :code:`storage capacity` requires a value, in column :code:`input power` and :code:`output power` :code:`soc_initial` should be set to NaN. The :code:`soc_initial` has to be within the [0,1] interval.",numeric,None or factor,soc_initial,socin-label,storage_csv,,,,, -None,1,"The maximum permissible level of charge in the battery (generally, it is when the battery is filled to its nominal capacity), represented by the value 1.0. Users can also specify a certain value as a factor of the actual capacity.",,":code:`storage capacity`: 1, :code:`input power`: NaN, :code:`output power`: NaN","Only the column :code:`storage capacity` requires a value, in column :code:`input power` and :code:`output power` :code:`soc_max` should be set to NaN. The :code:`soc_max` has to be in the [0,1] interval.",numeric,Factor,soc_max,socmax-label,storage_csv,,,,, -None,0,The minimum permissible level of charge in the battery as a factor of the nominal capacity of the battery.,,":code:`storage capacity`:0.2, :code:`input power`: NaN, :code:`output power`: NaN","Only the column :code:`storage capacity` requires a value, in column :code:`input power` and :code:`output power` :code:`soc_min` should be set to NaN. The soc_min has to be in the [0,1] interval.",numeric,Factor,soc_min,socmin-label,storage_csv,,,,, -None,4000,"Actual CAPEX of an asset, i.e., specific investment costs",,4000,None,numeric,€/unit,specific_costs,specificcosts-label,conversion,production,storage_csv,fixcost,, -None,120,"Actual OPEX of an asset, i.e., specific operational and maintenance costs.",,120,None,numeric,€/unit/year,specific_costs_om,specificomcosts-label,conversion,production,storage_csv,fixcost,, -None,None,The date and time on which the simulation starts at the first step.,,2018-01-01 00:00:00,Acceptable format is YYYY-MM-DD HH:MM:SS,str,None,start_date,startdate-label,simulation_settings,,,,, -None,None,Name of a csv file containing the properties of a storage component,,storage_01.csv,Follows the convention of 'storage_xx.csv' where 'xx' is a number. This file must be placed in a folder named “csv_elements” inside your input folder.,str,None,storage_filename,storagefilename-label,storage,,,,, -Tax factor,0.07,Tax factor.,,0,Between 0 and 1,numeric,Factor,tax,tax-label,economic_data,,,,, -None,60,Length of the time-steps.,,60,Can only be 60 at the moment,numeric,Minutes,timestep,timestep-label,simulation_settings,,,,, -None,None,The type of the component.,,demand,*demand*,str,None,type_asset,typeasset-label,hidden,,,,, -None,None,"Input the type of OEMOF component. For example, a PV plant would be a source, a solar inverter would be a transformer, etc. The `type_oemof` will later on be determined through the EPA.",,sink,*sink* or *source* or one of the other component classes of OEMOF.,str,None,type_oemof,typeoemof-label,consumption,conversion,production,providers,storage, -None,None,Unit associated with the capacity of the component,,"Storage could have units like kW or kWh, transformer station could have kVA, and so on.",Appropriate scientific unit,str,NA,unit,unit-label,consumption,conversion,production,providers,storage_csv, +verbose,:Default:,:Definition_Short:,:Definition_Long:,:Example:,:Restrictions:,:Type:,:Unit:,sensitivity_analysis,label,ref,category,see_also +None,5,The number of years the asset has already been in operation,,10,Natural number,numeric,Year,1,age_installed,age_ins-label,conversion:production, +None,1,The rate at which the storage can be charged or discharged relative to the nominal capacity of the storage, A C-rate of 1 implies that the battery can be fully charged or discharged completely in a single timestep. A C-rate of 0.5 implies that the battery needs at least 2 timesteps to be fully charged or discharged.,"*storage capacity*: NaN, *input power*: 1, *output power*: 1","Real number between 0 and 1. Only the columns 'input power' and 'output power' require a value, in column 'storage capacity' c_rate should be set to NaN.",numeric,Factor,1,c-rate,crate-label,storage_csv, +None,None,The name of the country where the project is being deployed,,Norway,None,str,None,0,country,country-label,project_data, +None,None,The currency of the country where the project is implemented,,EUR,None,str,None,0,currency,currency-label,economic_data, +None,0,Fixed project costs,This could be planning and development costs which do not depend on the (optimized) capacities of the assets.,10000,Positive real number,numeric,€,1,development_costs,developmentcosts-label,conversion;storage_csv;production;fixcost, +Discount factor,0.05,"The factor which accounts for the depreciation in the value of money in the future, compared to the current value of the same money. The common method is to calculate the weighted average cost of capital (WACC) and use it as the discount rate.",,0.06,Between 0 and 1,numeric,Factor,0,discount_factor,discountfactor-label,economic_data, +None,0.1,Costs associated with a flow through/from the asset,e.g. in EUR/kWh,0.64,"In 'storage_xx.csv' only the columns 'input power' and 'output power' require a value, in column 'storage capacity' dispatch_price should be set to NaN.",numeric,€/kWh,1,dispatch_price,dispatchprice-label,conversion;storage_csv;production;fixcost, +None,False,"Stands for Demand Side Management. Currently, not implemented.",,False,Acceptable values are either True or False.,boolean,None,0,dsm,dsm-label,hidden, +None,0.8,Ratio of energy output/energy input,"The battery efficiency is the ratio of the energy taken out from the battery, to the energy put into the battery. It means that it is not possible to retrieve as much energy as provided to the battery due to the discharge losses. The efficiency of the 'input power' and 'ouput power' columns should be set equal to the charge and discharge efficiencies respectively, while the 'storage capacity' efficiency should be equal to the storage's efficiency/ability to hold charge over time (`= 1 - self-discharge/decay`), which is usually in the range of 0.95 to 1 for electrical storages.",0.95,Between 0 and 1,numeric,Factor,1,efficiency,efficiency-label,conversion;storage_csv, +None,0.2,Emissions per unit dispatch of an asset,,14.4,Positive real number,numeric,kgCO2eq/asset unit,1,emission_factor,emissionfactor-label,providers;production, +None,0.3,Price of the energy carrier sourced from the utility grid.,,0.1,None,numeric,€/energy carrier unit,1,energy_price,energyprice-label,providers, +None,Electricity,"Energy vector/commodity. Convention: For an energy conversion asset define energyVector of the output. For a sink define based on inflow. For a source define based on output flow. For a storage, define based on stored energy carrier.",,Electricity,"One of “Electricity”, “Gas”, “Bio-Gas”, “Diesel”, “Heat”, “H2”",str,None,0,energyVector,energyvector-label,busses;consumption;production;storage;providers;conversion, +None,365,The number of days for which the simulation is to be run,,365,Natural number,numeric,Day,1,evaluated_period,evaluatedperiod-label,simulation_settings, +None,0.1,Price received for feeding electricity into the grid,,0.7,Real number between 0 and 1,numeric,€/kWh,1,feedin_tariff,feedintariff-label,providers, +None,None,Name of a csv file containing the input generation or demand timeseries.,,demand_harbor.csv,This file must be placed in a folder named “time_series” inside your input folder.,str,None,0,file_name,filename-label,consumption;production;storage_csv, +None,0,Thermal losses of storage independent of state of charge and independent of nominal storage capacity between two consecutive timesteps.,,0.0003,Between 0 and 1,numeric,factor,1,fixed_thermal_losses_absolute,fixed_thermal_losses_absolute-label,storage_csv, +None,0,Thermal losses of storage independent of state of charge between two consecutive timesteps relative to nominal storage capacity.,,0.0016,Between 0 and 1,numeric,factor,1,fixed_thermal_losses_relative,fixed_thermal_losses_relative-label,storage_csv, +None,None,The label of the bus/component from which the energyVector is arriving into the asset.,,Electricity,None,str,None,0,inflow_direction,inflowdirection-label,consumption;conversion;providers;storage, +None,0,The already existing installed capacity in-place,"If the project lasts longer than its remaining lifetime, its replacement costs will be taken into account.",50,Each component in the “energy production” category must have a value.,numeric,kWp,1,installedCap,installedcap-label,conversion;production;storage_csv, +None,None,Name of the asset for display purposes,,pv_plant_01,"Input the names in a computer friendly format, preferably with underscores instead of spaces, and avoiding special characters",str,None,0,label,labl-label,fixcost, +Location's latitude,None,Latitude coordinate of the project's geographical location,,45.641603,Should follow geographical convention,numeric,None,0,latitude,latitude-label,project_data, +None,20,Number of operational years of the asset until it has to be replaced,,30,Natural number,numeric,Year,1,lifetime,lifetime-label,conversion;storage_csv;production;fixcost, +Location's longitude,None,Longitude coordinate of the project's geographical location,,10.95787,Should follow geographical convention,numeric,None,0,longitude,longitude-label,project_data, +None,None,The maximum amount of total emissions which are allowed in the optimized energy system,,100000,Acceptable values are either a positive real number or None,numeric,kgCO2eq/a,1,maximum_emissions,maxemissions-label,constraints, +None,None,The maximum total capacity of an asset that can be installed at the project site,"This includes the installed and the also the maximum additional capacity possible. An example would be that a roof can only carry 50 kWp PV (maximumCap), whereas the installed capacity is already 10 kWp. The optimization would only be allowed to add 40 kWp PV at maximum.",1050,Acceptable values are either a positive real number or None,numeric,kWp,1,maximumCap,maxcap-label,production, +None,None,The minimum degree of autonomy that needs to be met by the optimization,,0.3,Between 0 and 1,numeric,factor,1,minimal_degree_of_autonomy,minda-label,constraints, +None,None,The minimum share of energy supplied by renewable generation that needs to be met by the optimization,Insert the value 0 to deactivate this constraint.,0.7,Between 0 and 1,numeric,factor,1,minimal_renewable_factor,minrenshare-label,constraints, +None,False,Specifies whether optimization needs to result into a net zero energy system (True) or not (False),,True,Acceptable values are either True or False.,boolean,None,0,net_zero_energy,nzeconstraint-label,constraints, +None,False,Choose if capacity optimization should be performed for this asset.,,True,Permissible values are either True or False,boolean,None,0,optimizeCap,optimizecap-label,conversion;storage;providers;production, +None,None,The label of bus/component towards which the energyVector is leaving from the asset.,,PV plant (mono),None,str,None,0,outflow_direction,outflowdirec-label,consumption;conversion;storage;providers, +None,False,"Enable the generation of a file with the linear equation system describing the simulation, ie., with the objective function and all the constraints. This lp file enables the user look at the underlying equations of the optimization.",,False,Acceptable values are either True or False,boolean,None,0,output_lp_file,outputlpfile-label,simulation_settings, +None,None,Price to be paid additionally for energy consumption based on the peak demand of a given period,,60,None,numeric,€/kW,1,peak_demand_pricing,peakdemand-label,providers,peakdemandperiod-label +None,1,Number of reference periods in one year for the peak demand pricing,,2,"Only one of the following are acceptable values: 1 (yearly), 2, 3 ,4, 6, 12 (monthly)",numeric,"times per year (1,2,3,4,6,12)",1,peak_demand_pricing_period,peakdemandperiod-label,providers,peakdemand-label +Project duration,20,The number of years the project is intended to be operational,The project duration also sets the installation time of the assets used in the simulation. After the project ends these assets are 'sold' and the refund is charged against the initial investment costs.,30,Natural number,numeric,Years,0,project_duration,projectduration-label,economic_data, +None,None,Assign a project ID as per your preference.,,1,Cannot be the same as an already existing project,str,None,0,project_id,projectid-label,project_data, +None,None,Assign a project name as per your preference.,,Borg Havn,None,str,None,0,project_name,projectname-label,project_data, +None,0.44,The share of renewables in the generation mix of the energy supplied by the DSO utility,,0.1,Real number between 0 and 1,numeric,Factor,1,renewable_share,renshare-label,providers, +None,None,Choose if this asset should be tagged as renewable.,,True,Acceptable values are either True or False,boolean,None,0,renewableAsset,renewableasset-label,production, +None,None,Brief description of the scenario being simulated,,This scenario simulates a sector-coupled energy system,None,str,None,0,scenario_description,scenariodescription-label,project_data, +None,None,Assign a scenario id as per your preference.,,1,Cannot be the same as an already existing scenario within the project,str,None,0,scenario_id,scenarioid-label,project_data, +None,None,Assign a scenario name as per your preference.,,Warehouse 14,None,str,None,0,scenario_name,scenarioname-label,project_data, +None,None,The level of charge (as a factor of the actual capacity) in the storage in the zeroth time-step.,,":code:`storage capacity`: None, :code:`input power`: NaN, :code:`output power`: NaN","Acceptable values are either None or the factor. Only the column :code:`storage capacity` requires a value, in column :code:`input power` and :code:`output power` :code:`soc_initial` should be set to NaN. The :code:`soc_initial` has to be within the [0,1] interval.",numeric,None or factor,1,soc_initial,socin-label,storage_csv, +None,1,"The maximum permissible level of charge in the battery (generally, it is when the battery is filled to its nominal capacity), represented by the value 1.0. Users can also specify a certain value as a factor of the actual capacity.",,":code:`storage capacity`: 1, :code:`input power`: NaN, :code:`output power`: NaN","Only the column :code:`storage capacity` requires a value, in column :code:`input power` and :code:`output power` :code:`soc_max` should be set to NaN. The :code:`soc_max` has to be in the [0,1] interval.",numeric,Factor,1,soc_max,socmax-label,storage_csv, +None,0,The minimum permissible level of charge in the battery as a factor of the nominal capacity of the battery.,,":code:`storage capacity`:0.2, :code:`input power`: NaN, :code:`output power`: NaN","Only the column :code:`storage capacity` requires a value, in column :code:`input power` and :code:`output power` :code:`soc_min` should be set to NaN. The soc_min has to be in the [0,1] interval.",numeric,Factor,1,soc_min,socmin-label,storage_csv, +None,4000,"Actual CAPEX of an asset, i.e., specific investment costs",,4000,None,numeric,€/unit,1,specific_costs,specificcosts-label,conversion;storage_csv;production;fixcost, +None,120,"Actual OPEX of an asset, i.e., specific operational and maintenance costs.",,120,None,numeric,€/unit/year,1,specific_costs_om,specificomcosts-label,conversion;storage_csv;production;fixcost, +None,None,The date and time on which the simulation starts at the first step.,,2018-01-01 00:00:00,Acceptable format is YYYY-MM-DD HH:MM:SS,str,None,0,start_date,startdate-label,simulation_settings, +None,None,Name of a csv file containing the properties of a storage component,,storage_01.csv,Follows the convention of 'storage_xx.csv' where 'xx' is a number. This file must be placed in a folder named “csv_elements” inside your input folder.,str,None,0,storage_filename,storagefilename-label,storage, +Tax factor,0.07,Tax factor.,,0,Between 0 and 1,numeric,Factor,0,tax,tax-label,economic_data, +None,60,Length of the time-steps.,,60,Can only be 60 at the moment,numeric,Minutes,1,timestep,timestep-label,simulation_settings, +None,None,The type of the component.,,demand,*demand*,str,None,0,type_asset,typeasset-label,hidden, +None,None,"Input the type of OEMOF component. For example, a PV plant would be a source, a solar inverter would be a transformer, etc. The `type_oemof` will later on be determined through the EPA.",,sink,*sink* or *source* or one of the other component classes of OEMOF.,str,None,0,type_oemof,typeoemof-label,conversion;storage;production;fixcost;consumption;providers, +None,None,Unit associated with the capacity of the component,,"Storage could have units like kW or kWh, transformer station could have kVA, and so on.",Appropriate scientific unit,str,NA,0,unit,unit-label,conversion;storage_csv;production;fixcost;consumption;providers, From daca2f35321ccfa872bd6fde261606358dfb8b7b Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Thu, 24 Feb 2022 22:52:30 +0100 Subject: [PATCH 02/17] Add properties to list asset visible fields in models --- app/projects/models.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/app/projects/models.py b/app/projects/models.py index 3945e9b6..730ac1c9 100644 --- a/app/projects/models.py +++ b/app/projects/models.py @@ -1,6 +1,7 @@ import uuid import json from django.core.validators import MinValueValidator, MaxValueValidator +from django.utils.timezone import now from django.conf import settings from django.db import models from django.core.validators import MaxValueValidator, MinValueValidator @@ -8,7 +9,7 @@ from django.forms.models import model_to_dict from django.utils.translation import gettext_lazy as _ from django.core.exceptions import ValidationError -from .constants import ( +from projects.constants import ( ASSET_CATEGORY, ASSET_TYPE, COUNTRY, @@ -17,6 +18,7 @@ FLOW_DIRECTION, MVS_TYPE, SIMULATION_STATUS, + PENDING, TRUE_FALSE_CHOICES, BOOL_CHOICES, USER_RATING, @@ -174,6 +176,10 @@ def export(self): dm = model_to_dict(self, exclude=["id"]) return dm + @property + def visible_fields(self): + return self.asset_fields.replace("[", "").replace("]", "").split(",") + class TopologyNode(models.Model): name = models.CharField(max_length=60, null=False, blank=False) @@ -279,6 +285,28 @@ def save(self, *args, **kwargs): def fields(self): return [f.name for f in self._meta.fields + self._meta.many_to_many] + @property + def visible_fields(self): + return self.asset_type.visible_fields + + def has_parameter(self, param_name): + return param_name in self.visible_fields + + def parameter_path(self, param_name): + # TODO for storage + if self.has_parameter(param_name): + # TODO if (unit, value) formatting, add "value" at the end + if self.asset_type.asset_category == "energy_provider": + asset_category = "energy_providers" + else: + asset_category = self.asset_type.asset_category + if param_name == "optimize_cap": + param_name = "optimize_capacity" + answer = (asset_category, self.name, param_name) + else: + answer = None + return answer + @property def timestamps(self): return self.scenario.get_timestamps() From d6f67adb5b2acae21c3ed4fdad54f7d3c8281760 Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Thu, 24 Feb 2022 22:53:31 +0100 Subject: [PATCH 03/17] Split Simulation model into 2 classes Some of the attribute will be reused for the sensitivity analysis --- app/projects/models.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/app/projects/models.py b/app/projects/models.py index 730ac1c9..282c4d2c 100644 --- a/app/projects/models.py +++ b/app/projects/models.py @@ -421,15 +421,24 @@ class ScenarioFile(models.Model): file = models.FileField(upload_to="tempFiles/", null=True, blank=True) -class Simulation(models.Model): +class AbstractSimulation(models.Model): + start_date = models.DateTimeField(auto_now_add=True, null=False) end_date = models.DateTimeField(null=True) elapsed_seconds = models.FloatField(null=True) mvs_token = models.CharField(max_length=200, null=True) - status = models.CharField(max_length=20, choices=SIMULATION_STATUS, null=False) + status = models.CharField( + max_length=20, choices=SIMULATION_STATUS, null=False, default=PENDING + ) + results = models.TextField(null=True, max_length=30e6) + errors = models.TextField(null=True) + + class Meta: + abstract = True + + +class Simulation(AbstractSimulation): scenario = models.OneToOneField(Scenario, on_delete=models.CASCADE, null=False) user_rating = models.PositiveSmallIntegerField( null=True, choices=USER_RATING, default=None ) - results = models.TextField(null=True, max_length=30e6) - errors = models.TextField(null=True) From 824fd622d1e08f9b203ce6401f8975f2f3e14ddb Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Thu, 24 Feb 2022 22:56:17 +0100 Subject: [PATCH 04/17] Move projects.models into folder structure There would be circular import with projects.dtos otherwise, or import at the end of the projects.models.py file which is not elegant --- app/projects/models/__init__.py | 1 + .../{models.py => models/base_models.py} | 9 ------ app/projects/models/simulation_models.py | 28 +++++++++++++++++++ 3 files changed, 29 insertions(+), 9 deletions(-) create mode 100644 app/projects/models/__init__.py rename app/projects/{models.py => models/base_models.py} (94%) create mode 100644 app/projects/models/simulation_models.py diff --git a/app/projects/models/__init__.py b/app/projects/models/__init__.py new file mode 100644 index 00000000..9f4c9d4d --- /dev/null +++ b/app/projects/models/__init__.py @@ -0,0 +1 @@ +from .base_models import * \ No newline at end of file diff --git a/app/projects/models.py b/app/projects/models/base_models.py similarity index 94% rename from app/projects/models.py rename to app/projects/models/base_models.py index 282c4d2c..ff17a1ce 100644 --- a/app/projects/models.py +++ b/app/projects/models/base_models.py @@ -1,7 +1,5 @@ import uuid import json -from django.core.validators import MinValueValidator, MaxValueValidator -from django.utils.timezone import now from django.conf import settings from django.db import models from django.core.validators import MaxValueValidator, MinValueValidator @@ -435,10 +433,3 @@ class AbstractSimulation(models.Model): class Meta: abstract = True - - -class Simulation(AbstractSimulation): - scenario = models.OneToOneField(Scenario, on_delete=models.CASCADE, null=False) - user_rating = models.PositiveSmallIntegerField( - null=True, choices=USER_RATING, default=None - ) diff --git a/app/projects/models/simulation_models.py b/app/projects/models/simulation_models.py new file mode 100644 index 00000000..d1ff15ff --- /dev/null +++ b/app/projects/models/simulation_models.py @@ -0,0 +1,28 @@ +from .base_models import AbstractSimulation, Scenario + + +import json +import jsonschema +from django.db import models +from django.core.validators import MaxValueValidator, MinValueValidator + +from django.utils.translation import gettext_lazy as _ + +from projects.constants import USER_RATING +from projects.helpers import ( + sensitivity_analysis_payload, + SA_OUPUT_NAMES_SCHEMA, + sa_output_values_schema_generator, + SA_MVS_TOKEN_SCHEMA, + format_scenario_for_mvs, +) + +from dashboard.helpers import nested_dict_crawler + + +class Simulation(AbstractSimulation): + scenario = models.OneToOneField(Scenario, on_delete=models.CASCADE, null=False) + user_rating = models.PositiveSmallIntegerField( + null=True, choices=USER_RATING, default=None + ) + From 6e3af568ac6df731e3953faeead8c1775f67320a Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Thu, 24 Feb 2022 23:03:38 +0100 Subject: [PATCH 05/17] Create SensitivityAnalysis model --- app/projects/models/__init__.py | 3 +- app/projects/models/simulation_models.py | 111 +++++++++++++++++++++++ 2 files changed, 113 insertions(+), 1 deletion(-) diff --git a/app/projects/models/__init__.py b/app/projects/models/__init__.py index 9f4c9d4d..97d33510 100644 --- a/app/projects/models/__init__.py +++ b/app/projects/models/__init__.py @@ -1 +1,2 @@ -from .base_models import * \ No newline at end of file +from .base_models import * +from .simulation_models import Simulation, SensitivityAnalysis diff --git a/app/projects/models/simulation_models.py b/app/projects/models/simulation_models.py index d1ff15ff..c8011b55 100644 --- a/app/projects/models/simulation_models.py +++ b/app/projects/models/simulation_models.py @@ -3,6 +3,8 @@ import json import jsonschema +import numpy as np +import logging from django.db import models from django.core.validators import MaxValueValidator, MinValueValidator @@ -26,3 +28,112 @@ class Simulation(AbstractSimulation): null=True, choices=USER_RATING, default=None ) + +class SensitivityAnalysis(AbstractSimulation): + name = models.CharField(max_length=50) + # attribute linked to output_parameter_names + output_parameters_names = models.TextField() + # attributes linked to variable_parameter_name + variable_name = models.CharField(max_length=100) + # attributes to compute the variable_parameter_range + variable_min = models.FloatField() + variable_max = models.FloatField() + variable_step = models.FloatField(validators=[MinValueValidator(0.0)]) + # attributes linked to variable_parameter_ref_val + variable_reference = models.FloatField( + help_text=_( + "The value of the variable parameter for the reference scenario of the sensitivity analysis, if is it different than the value of the current scenario user need to decide whether to take current scenario as reference with its actual value for this parameter, duplicate the scenario and use the duplicata with chosen reference value or keep current scenario but change parameter value to chosen one" + ) + ) # label=_("Variable parameter of the reference scenario"), + scenario = models.ForeignKey(Scenario, on_delete=models.CASCADE, null=True) + # attribute linked to the output values of the sensitivity analysis + output_parameters_values = models.TextField() + nested_dict_pathes = None + + # TODO look at jsonschema to create a custon TextField --> https://dev.to/saadullahaleem/adding-validation-support-for-json-in-django-models-5fbm + + def save(self, *args, **kwargs): + # if self.output_parameters_names is not None: + # self.output_parameters_names = json.dumps(self.output_parameters_names) + super().save(*args, **kwargs) + if self.scenario is not None: + self.nested_dict_pathes = nested_dict_crawler( + format_scenario_for_mvs(self.scenario) + ) + + def set_reference_scenario(self, scenario): + self.scenario = scenario + self.save() + + def collect_simulations_tokens(self, server_response): + try: + jsonschema.validate(server_response, SA_MVS_TOKEN_SCHEMA) + except jsonschema.exceptions.ValidationError: + answer = {} + + @property + def variable_range(self): + return np.arange( + self.variable_min, self.variable_max, self.variable_step + ).tolist() + + @property + def output_names(self): + try: + answer = json.loads(self.output_parameters_names) + try: + jsonschema.validate(answer, SA_OUPUT_NAMES_SCHEMA) + except jsonschema.exceptions.ValidationError: + answer = [] + except json.decoder.JSONDecodeError: + answer = [] + + return answer + + @property + def output_values(self): + try: + answer = json.loads(self.output_parameters_values) + try: + jsonschema.validate( + answer, sa_output_values_schema_generator(self.output_names) + ) + except jsonschema.exceptions.ValidationError: + answer = {} + except json.decoder.JSONDecodeError: + answer = {} + return answer + + @property + def payload(self): + return sensitivity_analysis_payload( + variable_parameter_name=self.variable_name_path, + variable_parameter_range=self.variable_range, + variable_parameter_ref_val=self.variable_reference, + output_parameter_names=self.output_names, + ) + + @property + def variable_name_path(self): + """Provided with a (nested) dict, find the path to the variable_name""" + if self.nested_dict_pathes is None: + variable_name_path = self.variable_name + else: + if "." in self.variable_name: + asset_name, variable_name = self.variable_name.split(".") + else: + variable_name = self.variable_name + asset_name = None + variable_name_path = self.nested_dict_pathes.get(variable_name, None) + if variable_name_path is None: + if asset_name is not None: + asset = self.scenario.asset_set.get(name=asset_name) + variable_name_path = asset.parameter_path(variable_name) + if variable_name_path is None: + logging.error( + f"The variable '{self.variable_name}' cannot be found in the scenario json structure" + ) + + if isinstance(variable_name_path, list): + variable_name_path = variable_name_path[0] + return variable_name_path From e1a20c2c789aef4015bd9e26ec692363baeed047 Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Thu, 24 Feb 2022 23:06:09 +0100 Subject: [PATCH 06/17] Rename get_topology_json to format_scenario_for_mvs And move it from projects.scenario_topology_helpers to projects.helpers --- app/projects/dtos.py | 11 ++- app/projects/helpers.py | 100 ++++++++++++++++++++++ app/projects/scenario_topology_helpers.py | 36 -------- app/projects/views.py | 7 +- 4 files changed, 113 insertions(+), 41 deletions(-) create mode 100644 app/projects/helpers.py diff --git a/app/projects/dtos.py b/app/projects/dtos.py index 40b7e5fb..b485fdf6 100644 --- a/app/projects/dtos.py +++ b/app/projects/dtos.py @@ -4,7 +4,16 @@ from numpy.core import long from datetime import date, datetime, time -from projects.models import * +from projects.models import ( + ConnectionLink, + Scenario, + Project, + EconomicData, + Asset, + Bus, + Constraint, + ValueType, +) class ProjectDataDto: diff --git a/app/projects/helpers.py b/app/projects/helpers.py new file mode 100644 index 00000000..2c4c7d13 --- /dev/null +++ b/app/projects/helpers.py @@ -0,0 +1,100 @@ +import json +from projects.dtos import convert_to_dto + + +# Helper method to clean dict data from empty values +def remove_empty_elements(d): + def empty(x): + return x is None or x == {} or x == [] + + if not isinstance(d, (dict, list)): + return d + elif isinstance(d, list): + return [v for v in (remove_empty_elements(v) for v in d) if not empty(v)] + else: + return { + k: v + for k, v in ((k, remove_empty_elements(v)) for k, v in d.items()) + if not empty(v) + } + + +# Helper to convert Scenario data to MVS importable json +def format_scenario_for_mvs(scenario_to_convert): + mvs_request_dto = convert_to_dto(scenario_to_convert) + dumped_data = json.loads( + json.dumps(mvs_request_dto.__dict__, default=lambda o: o.__dict__) + ) + + # format the constraints in MVS format directly, thus avoiding the need to maintain MVS-EPA + # parser in multi-vector-simulator package + constraint_dict = {} + for constraint in dumped_data["constraints"]: + constraint_dict[constraint["label"]] = constraint["value"] + dumped_data["constraints"] = constraint_dict + + # Remove None values + return remove_empty_elements(dumped_data) + + +def sensitivity_analysis_payload( + variable_parameter_name="", + variable_parameter_range="", + variable_parameter_ref_val="", + output_parameter_names=None, +): + """format the parameters required to request a sensitivity analysis in a specific JSON""" + if output_parameter_names is None: + output_parameter_names = [] + return { + "sensitivity_analysis_settings": { + "variable_parameter_name": variable_parameter_name, + "variable_parameter_range": variable_parameter_range, + "variable_parameter_ref_val": variable_parameter_ref_val, + "output_parameter_names": output_parameter_names, + } + } + + +SA_MVS_TOKEN_SCHEMA = { + "type": "object", + "required": ["ref_sim_id", "sensitivity_analysis_ids"], + "properties": { + "ref_sim_id": {"type": "string"}, + "sensitivity_analysis_ids": {"type": "array", "items": {"type": "string"}}, + }, + "additionalProperties": False, +} + + +# Used to proof the json objects stored as text in the db +SA_OUPUT_NAMES_SCHEMA = {"type": "array", "items": {"type": "string"}} + + +def sa_output_values_schema_generator(output_names): + return { + "type": "object", + "required": output_names, + "properties": { + output_name: { + "type": "object", + "required": ["value", "path"], + "properties": { + "value": { + "oneOf": [ + {"type": "null"}, + {"type": "array", "items": {"type": "number"}}, + ] + }, + "path": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}}, + ] + }, + }, + } + for output_name in output_names + }, + "additionalProperties": False, + } diff --git a/app/projects/scenario_topology_helpers.py b/app/projects/scenario_topology_helpers.py index 9fd1238c..f8e509fe 100644 --- a/app/projects/scenario_topology_helpers.py +++ b/app/projects/scenario_topology_helpers.py @@ -14,7 +14,6 @@ from projects.forms import AssetCreateForm, BusForm, StorageForm # region sent db nodes to js -from projects.dtos import convert_to_dto from crispy_forms.templatetags import crispy_forms_filters from django.http import JsonResponse import logging @@ -502,38 +501,3 @@ def create_ESS_objects(all_ess_assets_node_list, scen_id): if asset.name == "capacity": # check if there is a connection link to a bus pass - - -# Helper method to clean dict data from empty values -def remove_empty_elements(d): - def empty(x): - return x is None or x == {} or x == [] - - if not isinstance(d, (dict, list)): - return d - elif isinstance(d, list): - return [v for v in (remove_empty_elements(v) for v in d) if not empty(v)] - else: - return { - k: v - for k, v in ((k, remove_empty_elements(v)) for k, v in d.items()) - if not empty(v) - } - - -# Helper to convert Scenario data to MVS importable json -def get_topology_json(scenario_to_convert): - mvs_request_dto = convert_to_dto(scenario_to_convert) - dumped_data = json.loads( - json.dumps(mvs_request_dto.__dict__, default=lambda o: o.__dict__) - ) - - # format the constraints in MVS format directly, thus avoiding the need to maintain MVS-EPA - # parser in multi-vector-simulator package - constraint_dict = {} - for constraint in dumped_data["constraints"]: - constraint_dict[constraint["label"]] = constraint["value"] - dumped_data["constraints"] = constraint_dict - - # Remove None values - return remove_empty_elements(dumped_data) diff --git a/app/projects/views.py b/app/projects/views.py index badf5348..7ada672e 100644 --- a/app/projects/views.py +++ b/app/projects/views.py @@ -33,9 +33,9 @@ update_deleted_objects_from_database, duplicate_scenario_objects, duplicate_scenario_connections, - get_topology_json, load_scenario_from_dict, ) +from projects.helpers import format_scenario_for_mvs from .constants import DONE, ERROR, MODIFIED from .services import ( create_or_delete_simulation_scheduler, @@ -1042,8 +1042,7 @@ def view_mvs_data_input(request, scen_id=0): raise PermissionDenied try: - data_clean = get_topology_json(scenario) - print(data_clean) + data_clean = format_scenario_for_mvs(scenario) except Exception as e: logger.error( @@ -1072,7 +1071,7 @@ def request_mvs_simulation(request, scen_id=0): # Load scenario scenario = Scenario.objects.get(pk=scen_id) try: - data_clean = get_topology_json(scenario) + data_clean = format_scenario_for_mvs(scenario) # err = 1/0 except Exception as e: error_msg = f"Scenario Serialization ERROR! User: {scenario.project.user.username}. Scenario Id: {scenario.id}. Thrown Exception: {e}." From 1093b936f45631625edda5d03562e70bed1e9a64 Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Thu, 24 Feb 2022 23:06:41 +0100 Subject: [PATCH 07/17] Add endpoints for sensitivity analysis --- app/projects/urls.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/app/projects/urls.py b/app/projects/urls.py index 37b5b35b..1bb90b3a 100644 --- a/app/projects/urls.py +++ b/app/projects/urls.py @@ -138,6 +138,27 @@ update_simulation_results, name="update_simulation_results", ), + # Sensitivity analysis + path( + "scenario//sensitivity-analysis/create", + sensitivity_analysis_create, + name="sensitivity_analysis_create", + ), + path( + "scenario//sensitivity-analysis/", + sensitivity_analysis_create, + name="sensitivity_analysis_review", + ), + path( + "scenario//sensitivity-analysis/run", + sensitivity_analysis_create, + name="sensitivity_analysis_run", + ), + path( + "scenario//sensitivity-analysis/error", + sensitivity_analysis_create, + name="sensitivity_analysis_error", + ), # User Feedback path("user_feedback", user_feedback, name="user_feedback"), ] From d42d28a2e4186e5e13a70a162a027e2c370da9b4 Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Thu, 24 Feb 2022 23:08:16 +0100 Subject: [PATCH 08/17] Create a new request to MVS server for sensitivity analysis --- app/epa/settings.py | 1 + app/projects/requests.py | 29 ++++++++++++++++++++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/app/epa/settings.py b/app/epa/settings.py index 68b98868..ac901d91 100644 --- a/app/epa/settings.py +++ b/app/epa/settings.py @@ -184,6 +184,7 @@ MVS_POST_URL = f"{MVS_API_HOST}/sendjson/openplan" MVS_GET_URL = f"{MVS_API_HOST}/check/" MVS_LP_FILE_URL = f"{MVS_API_HOST}/get_lp_file/" +MVS_SA_POST_URL = f"{MVS_API_HOST}/sendjson/openplan/sensitivity-analysis" # Allow iframes to show in page X_FRAME_OPTIONS = "SAMEORIGIN" diff --git a/app/projects/requests.py b/app/projects/requests.py index 7da0b2ef..e5a2ab01 100644 --- a/app/projects/requests.py +++ b/app/projects/requests.py @@ -3,7 +3,7 @@ import json # from requests.exceptions import HTTPError -from epa.settings import PROXY_CONFIG, MVS_POST_URL, MVS_GET_URL +from epa.settings import PROXY_CONFIG, MVS_POST_URL, MVS_GET_URL, MVS_SA_POST_URL from dashboard.models import AssetsResults, KPICostsMatrixResults, KPIScalarResults from projects.constants import DONE, PENDING, ERROR import logging @@ -149,3 +149,30 @@ def parse_mvs_results(simulation, response_results): assets_list=json.dumps(data_subdict), simulation=simulation ) return response_results + + +def mvs_sensitivity_analysis_request(data: dict): + + headers = {"content-type": "application/json"} + payload = json.dumps(data) + + try: + response = requests.post( + MVS_SA_POST_URL, + data=payload, + headers=headers, + proxies=PROXY_CONFIG, + verify=False, + ) + + # If the response was successful, no Exception will be raised + response.raise_for_status() + except requests.HTTPError as http_err: + logger.error(f"HTTP error occurred: {http_err}") + return None + except Exception as err: + logger.error(f"Other error occurred: {err}") + return None + else: + logger.info("The simulation was sent successfully to MVS API.") + return json.loads(response.text) From ae46666a1b420e5a72e0ae3a42c7a64694cbf785 Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Thu, 24 Feb 2022 23:14:47 +0100 Subject: [PATCH 09/17] Add Sensitivity Analysis form Modify the PARAMETERS dict loaded from the MVS_parameters_list.csv file --- app/dashboard/helpers.py | 10 ++++++- app/projects/forms.py | 58 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/app/dashboard/helpers.py b/app/dashboard/helpers.py index 5ea7eeb3..c70a3b2c 100644 --- a/app/dashboard/helpers.py +++ b/app/dashboard/helpers.py @@ -25,6 +25,7 @@ EMPTY_SUBCAT = "none" KPI_PARAMETERS = {} +KPI_PARAMETERS_ASSETS = {} if os.path.exists(staticfiles_storage.path("MVS_kpis_list.csv")) is True: with open(staticfiles_storage.path("MVS_kpis_list.csv")) as csvfile: @@ -65,17 +66,24 @@ for i, row in enumerate(csvreader): if i == 0: hdr = [el.replace(" ", "_").replace(":", "").lower() for el in row] - print(hdr) label_idx = hdr.index("label") cat_idx = hdr.index("category") + scope_idx = hdr.index("scope") else: label = row[label_idx] category = row[cat_idx] + scope = row[scope_idx] + if category != "files": KPI_PARAMETERS[label] = { k: _(v) if k == "verbose" or k == "definition" else v for k, v in zip(hdr, row) } + if "asset" in scope: + KPI_PARAMETERS_ASSETS[label] = { + k: _(v) if k == "verbose" or k == "definition" else v + for k, v in zip(hdr, row) + } #### FUNCTIONS #### diff --git a/app/projects/forms.py b/app/projects/forms.py index 2155fd0f..5b4bd665 100644 --- a/app/projects/forms.py +++ b/app/projects/forms.py @@ -23,6 +23,8 @@ from projects.models import * from projects.constants import MAP_EPA_MVS, RENEWABLE_ASSETS +from dashboard.helpers import KPI_PARAMETERS_ASSETS + from django.utils.translation import ugettext_lazy as _ from django.conf import settings as django_settings @@ -36,7 +38,11 @@ label_idx = hdr.index("label") else: label = row[label_idx] - PARAMETERS[label] = {k: v for k, v in zip(hdr, row)} + PARAMETERS[label] = {} + for k, v in zip(hdr, row): + if k == "sensitivity_analysis": + v = bool(int(v)) + PARAMETERS[label][k] = v def gettext_variables(some_string, lang="de"): @@ -71,7 +77,7 @@ def set_parameter_info(param_name, field, parameters=PARAMETERS): verbose = None default_value = None if param_name in PARAMETERS: - help_text = PARAMETERS[param_name][":Definition:"] + help_text = PARAMETERS[param_name][":Definition_Short:"] unit = PARAMETERS[param_name][":Unit:"] verbose = PARAMETERS[param_name]["verbose"] default_value = PARAMETERS[param_name][":Default:"] @@ -481,6 +487,54 @@ class Meta: exclude = ["scenario", "value"] +class SensitivityAnalysisForm(ModelForm): + output_parameters_names = forms.MultipleChoiceField( + choices=[ + (v, _(KPI_PARAMETERS_ASSETS[v]["verbose"])) for v in KPI_PARAMETERS_ASSETS + ] + ) + + class Meta: + model = SensitivityAnalysis + fields = [ + "variable_name", + "variable_min", + "variable_max", + "variable_step", + "variable_reference", + "output_parameters_names", + ] + + def __init__(self, *args, **kwargs): + scen_id = kwargs.pop("scen_id", None) + super().__init__(*args, **kwargs) + + forbidden_parameters_for_sa = ("name", "input_timeseries") + + if scen_id is not None: + scenario = Scenario.objects.get(id=scen_id) + asset_parameters = [] + for asset in scenario.asset_set.all(): + asset_parameters += [ + (f"{asset.name}.{p}", _(p) + f" ({asset.name})") + for p in asset.visible_fields + if p not in forbidden_parameters_for_sa + ] + self.fields["variable_name"] = forms.ChoiceField(choices=asset_parameters) + # self.fields["output_parameters_names"] = forms.MultipleChoiceField(choices = [(v, _(KPI_PARAMETERS_ASSETS[v]["verbose"])) for v in KPI_PARAMETERS_ASSETS]) + # TODO restrict possible parameters here + self.fields["output_parameters_names"].choices = [ + (v, _(KPI_PARAMETERS_ASSETS[v]["verbose"])) + for v in KPI_PARAMETERS_ASSETS + ] + + def clean_output_parameters_names(self): + """method which gets called upon form validation""" + data = self.cleaned_data["output_parameters_names"] + data_js = json.dumps(data) + return data_js + + class BusForm(OpenPlanModelForm): def __init__(self, *args, **kwargs): bus_type_name = kwargs.pop("asset_type", None) # always = bus From e31f3e5c5047d295529cd375d449f43b6ee5afbe Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Thu, 24 Feb 2022 23:16:03 +0100 Subject: [PATCH 10/17] Add a new html template for sensitivity analysis view Also add a link to the create view from scenario_review view --- app/templates/scenario/scenario_step4.html | 5 +- .../scenario/sensitivity_analysis.html | 63 +++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 app/templates/scenario/sensitivity_analysis.html diff --git a/app/templates/scenario/scenario_step4.html b/app/templates/scenario/scenario_step4.html index 0779aea1..99748ecb 100644 --- a/app/templates/scenario/scenario_step4.html +++ b/app/templates/scenario/scenario_step4.html @@ -53,10 +53,13 @@ {% endif %} +
+ + {% translate "Perform a sensitivity analysis" %} +
{% else %}
{% translate "You can now click on 'Run simulation' button" %}
{% endif %} - diff --git a/app/templates/scenario/sensitivity_analysis.html b/app/templates/scenario/sensitivity_analysis.html new file mode 100644 index 00000000..47fc3d63 --- /dev/null +++ b/app/templates/scenario/sensitivity_analysis.html @@ -0,0 +1,63 @@ +{% extends 'scenario/scenario_progression.html' %} +{% load static %} +{% load crispy_forms_tags %} +{% load custom_filters %} +{% load i18n %} + +{% block head_block %} + +{% endblock head_block %} + + +{% block progression_content %} + + + +
+ +
+
+ + {% if simulation_status %} + + {% else %} +
{% translate "There is no simulation_status" %}
+ {% endif %} +
+
+ {% csrf_token %} + {{ sa_form|crispy }} + +
+
+
+
+ + +
+ + + +{% endblock progression_content %} + + +{% block end_body_scripts %} + + +{% endblock end_body_scripts %} + +{% block footer %} +
+ + +
+{% endblock footer %} \ No newline at end of file From 2eab46dfb74577bd24b2012be7f4dfe7f1319bd0 Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Thu, 24 Feb 2022 23:22:17 +0100 Subject: [PATCH 11/17] Add view to create a sensitivity analysis and request it --- app/projects/views.py | 100 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 99 insertions(+), 1 deletion(-) diff --git a/app/projects/views.py b/app/projects/views.py index 7ada672e..b6d7b776 100644 --- a/app/projects/views.py +++ b/app/projects/views.py @@ -22,6 +22,7 @@ mvs_simulation_request, mvs_simulation_check_status, get_mvs_simulation_results, + mvs_sensitivity_analysis_request, ) from .models import * from .scenario_topology_helpers import ( @@ -328,7 +329,6 @@ def project_duplicate(request, proj_id): # duplicate the project project.pk = None - print(project.economic_data.pk) economic_data = project.economic_data economic_data.pk = None economic_data.save() @@ -491,6 +491,7 @@ def scenario_create_parameters(request, proj_id, scen_id=None, step_id=1, max_st "form": form, "proj_id": proj_id, "proj_name": project.name, + "scenario": scenario, "scen_id": scen_id, "step_id": step_id, "step_list": STEP_LIST, @@ -892,6 +893,103 @@ def scenario_delete(request, scen_id): # endregion Scenario +@login_required +@require_http_methods(["GET", "POST"]) +def sensitivity_analysis_create(request, scen_id, sa_id=None, step_id=5): + excuses_design_under_development(request) + scenario = get_object_or_404(Scenario, id=scen_id) + if scenario.project.user != request.user: + raise PermissionDenied + + if request.method == "GET": + if sa_id is not None: + sa_item = get_object_or_404(SensitivityAnalysis, id=sa_id) + sa_form = SensitivityAnalysisForm(scen_id=scen_id, instance=sa_item) + else: + sa_form = SensitivityAnalysisForm( + scen_id=scen_id, + ) + + answer = render( + request, + "scenario/sensitivity_analysis.html", + { + "proj_id": scenario.project.id, + "proj_name": scenario.project.name, + "scenario": scenario, + "scen_id": scen_id, + "step_id": step_id, + "step_list": STEP_LIST + [_("Sensitivity analysis")], + "max_step": 5, + "sa_form": sa_form, + }, + ) + + if request.method == "POST": + qs = request.POST + sa_form = SensitivityAnalysisForm(qs) + + if sa_form.is_valid(): + sa_item = sa_form.save(commit=False) + # TODO if the reference value is not the same as in the current scenario, duplicate the scenario and bind the duplicate to sa_item + # TODO check if the scenario is already bound to a SA + sa_item.set_reference_scenario(scenario) + try: + data_clean = format_scenario_for_mvs(scenario) + except Exception as e: + error_msg = f"Scenario Serialization ERROR! User: {scenario.project.user.username}. Scenario Id: {scenario.id}. Thrown Exception: {e}." + logger.error(error_msg) + messages.error(request, error_msg) + answer = JsonResponse( + {"error": f"Scenario Serialization ERROR! Thrown Exception: {e}."}, + status=500, + content_type="application/json", + ) + + sa_item.save() + + # Add the information about the sensitivity analysis to the json + data_clean.update(sa_item.payload) + # Make simulation request to MVS + results = mvs_sensitivity_analysis_request(data_clean) + + if results is None: + error_msg = "Could not communicate with the simulation server." + logger.error(error_msg) + messages.error(request, error_msg) + # TODO redirect to prefilled feedback form / bug form + answer = JsonResponse( + {"status": "error", "error": error_msg}, + status=407, + content_type="application/json", + ) + else: + sa_item.mvs_token = results["id"] if results["id"] else None + + if "status" in results.keys() and ( + results["status"] == DONE or results["status"] == ERROR + ): + sa_item.status = results["status"] + sa_item.results = results["results"] + # Simulation.objects.filter(scenario_id=scen_id).delete() + # TODO the reference scenario should have its simulation replaced by this one if successful, this can be done via the mvs_token of the simulation + + sa_item.end_date = datetime.now() + else: # PENDING + sa_item.status = results["status"] + # create a task which will update simulation status + # TODO check it does the right thing with sensitivity analysis + # create_or_delete_simulation_scheduler(mvs_token=sa_item.mvs_token) + + sa_item.elapsed_seconds = (datetime.now() - sa_item.start_date).seconds + + answer = HttpResponseRedirect( + reverse("sensitivity_analysis_review", args=[scen_id, sa_item.id]) + ) + + return answer + + # region Asset From 1494feb97b8a1ae9620cc975574c09773db66bb7 Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Thu, 24 Feb 2022 23:35:32 +0100 Subject: [PATCH 12/17] Improve page after requesting sensitivity to MVS server --- app/epa/settings.py | 1 + app/projects/views.py | 7 +++++-- app/templates/scenario/sensitivity_analysis.html | 14 ++++++++++++-- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/app/epa/settings.py b/app/epa/settings.py index ac901d91..a25a9ce0 100644 --- a/app/epa/settings.py +++ b/app/epa/settings.py @@ -185,6 +185,7 @@ MVS_GET_URL = f"{MVS_API_HOST}/check/" MVS_LP_FILE_URL = f"{MVS_API_HOST}/get_lp_file/" MVS_SA_POST_URL = f"{MVS_API_HOST}/sendjson/openplan/sensitivity-analysis" +MVS_SA_GET_URL = f"{MVS_API_HOST}/check-sensitivity-analysis/" # Allow iframes to show in page X_FRAME_OPTIONS = "SAMEORIGIN" diff --git a/app/projects/views.py b/app/projects/views.py index b6d7b776..94829d48 100644 --- a/app/projects/views.py +++ b/app/projects/views.py @@ -16,7 +16,7 @@ from datetime import datetime from users.models import CustomUser from django.db.models import Q -from epa.settings import MVS_GET_URL, MVS_LP_FILE_URL +from epa.settings import MVS_GET_URL, MVS_LP_FILE_URL, MVS_SA_GET_URL from .forms import * from .requests import ( mvs_simulation_request, @@ -906,6 +906,7 @@ def sensitivity_analysis_create(request, scen_id, sa_id=None, step_id=5): sa_item = get_object_or_404(SensitivityAnalysis, id=sa_id) sa_form = SensitivityAnalysisForm(scen_id=scen_id, instance=sa_item) else: + sa_item = None sa_form = SensitivityAnalysisForm( scen_id=scen_id, ) @@ -921,7 +922,9 @@ def sensitivity_analysis_create(request, scen_id, sa_id=None, step_id=5): "step_id": step_id, "step_list": STEP_LIST + [_("Sensitivity analysis")], "max_step": 5, + "MVS_SA_GET_URL": MVS_SA_GET_URL, "sa_form": sa_form, + "sa_item": sa_item, }, ) @@ -982,7 +985,7 @@ def sensitivity_analysis_create(request, scen_id, sa_id=None, step_id=5): # create_or_delete_simulation_scheduler(mvs_token=sa_item.mvs_token) sa_item.elapsed_seconds = (datetime.now() - sa_item.start_date).seconds - + sa_item.save() answer = HttpResponseRedirect( reverse("sensitivity_analysis_review", args=[scen_id, sa_item.id]) ) diff --git a/app/templates/scenario/sensitivity_analysis.html b/app/templates/scenario/sensitivity_analysis.html index 47fc3d63..471b1f92 100644 --- a/app/templates/scenario/sensitivity_analysis.html +++ b/app/templates/scenario/sensitivity_analysis.html @@ -22,11 +22,19 @@
- {% if simulation_status %} + {% if sa_item %} +
+ + {% if sa_item.status != "MODIFIED" %} + The simulation {{ sa_item.mvs_token }} has been started, its status is {{ sa_item.status }} + {% else %} + The parameters of the scenario linked to the simulation {{ mvs_token }} have been changed, you can rerun the simulation to update the results, please note that existing results will be erased only if the simulation does not have errors + {% endif %} +
+ {% else %}
{% translate "There is no simulation_status" %}
- {% endif %}
{% csrf_token %} @@ -34,6 +42,8 @@
+ {% endif %} +
From e0fd7a8f1a291299e306d2928f8d522c17a12cca Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Fri, 25 Feb 2022 22:27:33 +0100 Subject: [PATCH 13/17] Add js module to check simulation status periodically --- app/static/js/simulation_requests.js | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 app/static/js/simulation_requests.js diff --git a/app/static/js/simulation_requests.js b/app/static/js/simulation_requests.js new file mode 100644 index 00000000..a8406c74 --- /dev/null +++ b/app/static/js/simulation_requests.js @@ -0,0 +1,20 @@ + +const myInterval = setInterval(check_if_simulation_is_done, 3000); + +function check_if_simulation_is_done(url=checkSimulationUrl){ + + $.ajax({ + type: "GET", + url: url, + success: function (resp) { + console.log(resp); + if(resp.areResultReady == true){ + clearInterval(myInterval); + location.reload(); + } + }, + error: function (XMLHttpRequest, textStatus, errorThrown) { + clearInterval(myInterval); + } + }); +}; From 568d16969f34b567a4b411f644a73e401ca4038e Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Fri, 25 Feb 2022 22:30:59 +0100 Subject: [PATCH 14/17] Add endpoint to check simulations on mvs server For normal simulations or sensitivity analysis --- app/projects/urls.py | 21 ++++++++------------- app/projects/views.py | 39 ++++++++++++++++++++------------------- 2 files changed, 28 insertions(+), 32 deletions(-) diff --git a/app/projects/urls.py b/app/projects/urls.py index 1bb90b3a..433a86c5 100644 --- a/app/projects/urls.py +++ b/app/projects/urls.py @@ -123,20 +123,10 @@ name="update_simulation_rating", ), # path('topology/simulation_status/', check_simulation_status, name='check_simulation_status'), - re_path( - r"^topology/simulation_status/(?P\d+)?$", - check_simulation_status, - name="check_simulation_status_regex", - ), - path( - "topology/simulation_status/", - check_simulation_status, - name="check_simulation_status", - ), path( - "project//scenario/", - update_simulation_results, - name="update_simulation_results", + "simulation/fetch-results/", + fetch_simulation_results, + name="fetch_simulation_results", ), # Sensitivity analysis path( @@ -159,6 +149,11 @@ sensitivity_analysis_create, name="sensitivity_analysis_error", ), + path( + "sensitivity-analysis/fetch-results/", + fetch_sensitivity_analysis_results, + name="fetch_sensitivity_analysis_results", + ), # User Feedback path("user_feedback", user_feedback, name="user_feedback"), ] diff --git a/app/projects/views.py b/app/projects/views.py index 94829d48..da937f82 100644 --- a/app/projects/views.py +++ b/app/projects/views.py @@ -20,9 +20,9 @@ from .forms import * from .requests import ( mvs_simulation_request, - mvs_simulation_check_status, - get_mvs_simulation_results, + fetch_mvs_simulation_results, mvs_sensitivity_analysis_request, + fetch_mvs_sa_results, ) from .models import * from .scenario_topology_helpers import ( @@ -1255,27 +1255,28 @@ def update_simulation_rating(request): @json_view @login_required -@require_http_methods(["GET", "POST"]) -def check_simulation_status(request, scen_id): - scenario = get_object_or_404(Scenario, pk=scen_id) - if scenario.simulation: - return JsonResponse( - mvs_simulation_check_status(scenario.simulation.mvs_token), - status=200, - content_type="application/json", - ) +@require_http_methods(["GET"]) +def fetch_simulation_results(request, sim_id): + simulation = get_object_or_404(Simulation, id=sim_id) + are_result_ready = fetch_mvs_simulation_results(simulation) + return JsonResponse( + dict(areResultReady=are_result_ready), + status=200, + content_type="application/json", + ) +@json_view @login_required @require_http_methods(["GET"]) -def update_simulation_results(request, proj_id, scen_id): - scenario = get_object_or_404(Scenario, pk=scen_id) - - simulation = scenario.simulation - - get_mvs_simulation_results(simulation) - - return HttpResponseRedirect(reverse("scenario_review", args=[proj_id, scen_id])) +def fetch_sensitivity_analysis_results(request, sa_id): + sa_item = get_object_or_404(SensitivityAnalysis, id=sa_id) + are_result_ready = fetch_mvs_sa_results(sa_item) + return JsonResponse( + dict(areResultReady=are_result_ready), + status=200, + content_type="application/json", + ) # endregion MVS JSON Related From 0138a02114fdb4cf60a6282c9ef65083b968a6ab Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Fri, 25 Feb 2022 22:33:37 +0100 Subject: [PATCH 15/17] Update html template to use the js periodic check of results --- app/projects/views.py | 6 +++++- app/templates/scenario/scenario_step4.html | 12 ++++++------ app/templates/scenario/sensitivity_analysis.html | 14 ++++++++++---- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/app/projects/views.py b/app/projects/views.py index da937f82..fd3e3e2f 100644 --- a/app/projects/views.py +++ b/app/projects/views.py @@ -749,6 +749,7 @@ def scenario_review(request, proj_id, scen_id, step_id=4, max_step=5): simulation = qs.first() context.update( { + "sim_id": simulation.id, "simulation_status": simulation.status, "secondsElapsed": simulation.elapsed_seconds, "rating": simulation.user_rating, @@ -905,8 +906,10 @@ def sensitivity_analysis_create(request, scen_id, sa_id=None, step_id=5): if sa_id is not None: sa_item = get_object_or_404(SensitivityAnalysis, id=sa_id) sa_form = SensitivityAnalysisForm(scen_id=scen_id, instance=sa_item) + sa_status = sa_item.status else: sa_item = None + sa_status = None sa_form = SensitivityAnalysisForm( scen_id=scen_id, ) @@ -924,7 +927,8 @@ def sensitivity_analysis_create(request, scen_id, sa_id=None, step_id=5): "max_step": 5, "MVS_SA_GET_URL": MVS_SA_GET_URL, "sa_form": sa_form, - "sa_item": sa_item, + "sa_status": sa_status, + "sa_id": sa_id, }, ) diff --git a/app/templates/scenario/scenario_step4.html b/app/templates/scenario/scenario_step4.html index 99748ecb..6acef653 100644 --- a/app/templates/scenario/scenario_step4.html +++ b/app/templates/scenario/scenario_step4.html @@ -39,7 +39,6 @@
{% if simulation_status == "DONE" %} - {% translate "Check results dashboard" %} {% elif simulation_status == "ERROR" %}

{{ simulation_error_msg }}

@@ -47,10 +46,7 @@ {% translate "Link to send bug report automatically" %} {% elif simulation_status == "PENDING" %} -
- {% csrf_token %} - -
+ The simulation {{ mvs_token }} has been started, its status is {{ simulation_status }} {% endif %}
@@ -72,9 +68,13 @@ {% block end_body_scripts %} +{% if simulation_status == "PENDING" %} + +{% endif %} {% endblock end_body_scripts %} diff --git a/app/templates/scenario/sensitivity_analysis.html b/app/templates/scenario/sensitivity_analysis.html index 471b1f92..bc6e57dc 100644 --- a/app/templates/scenario/sensitivity_analysis.html +++ b/app/templates/scenario/sensitivity_analysis.html @@ -22,11 +22,11 @@
- {% if sa_item %} + {% if sa_status %}
- {% if sa_item.status != "MODIFIED" %} - The simulation {{ sa_item.mvs_token }} has been started, its status is {{ sa_item.status }} + {% if sa_status != "MODIFIED" %} + The simulation {{ sa_item.mvs_token }} has been started, its status is {{ sa_status }} {% else %} The parameters of the scenario linked to the simulation {{ mvs_token }} have been changed, you can rerun the simulation to update the results, please note that existing results will be erased only if the simulation does not have errors {% endif %} @@ -56,9 +56,15 @@ {% block end_body_scripts %} - + +{% endif %} {% endblock end_body_scripts %} From c08a5a2dadfeea0b2920fc4a55bc5a7c40edfa78 Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Fri, 25 Feb 2022 22:36:25 +0100 Subject: [PATCH 16/17] Modify helper function to fetch results on mvs server --- app/projects/requests.py | 59 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 3 deletions(-) diff --git a/app/projects/requests.py b/app/projects/requests.py index e5a2ab01..3a3e4558 100644 --- a/app/projects/requests.py +++ b/app/projects/requests.py @@ -3,7 +3,13 @@ import json # from requests.exceptions import HTTPError -from epa.settings import PROXY_CONFIG, MVS_POST_URL, MVS_GET_URL, MVS_SA_POST_URL +from epa.settings import ( + PROXY_CONFIG, + MVS_POST_URL, + MVS_GET_URL, + MVS_SA_POST_URL, + MVS_SA_GET_URL, +) from dashboard.models import AssetsResults, KPICostsMatrixResults, KPIScalarResults from projects.constants import DONE, PENDING, ERROR import logging @@ -53,7 +59,24 @@ def mvs_simulation_check_status(token): return json.loads(response.text) -def fetch_mvs_simulation_status(simulation): +def mvs_sa_check_status(token): + try: + response = requests.get( + MVS_SA_GET_URL + token, proxies=PROXY_CONFIG, verify=False + ) + response.raise_for_status() + except requests.HTTPError as http_err: + logger.error(f"HTTP error occurred: {http_err}") + return None + except Exception as err: + logger.error(f"Other error occurred: {err}") + return None + else: + logger.info("Success!") + return json.loads(response.text) + + +def fetch_mvs_simulation_results(simulation): if simulation.status == PENDING: response = mvs_simulation_check_status(token=simulation.mvs_token) try: @@ -79,6 +102,36 @@ def fetch_mvs_simulation_status(simulation): ) simulation.save() + return simulation.status != PENDING + + +def fetch_mvs_sa_results(simulation): + if simulation.status == PENDING: + response = mvs_sa_check_status(token=simulation.mvs_token) + try: + simulation.status = response["status"] + simulation.errors = ( + json.dumps(response["results"][ERROR]) + if simulation.status == ERROR + else None + ) + # TODO here fetch the results of the reference scenario in a separate Simulation instance + # and parse the + simulation.results = ( + response["results"] if simulation.status == DONE else None + ) + logger.info(f"The simulation {simulation.id} is finished") + except: + simulation.status = ERROR + simulation.results = None + + simulation.elapsed_seconds = (datetime.now() - simulation.start_date).seconds + simulation.end_date = ( + datetime.now() if response["status"] in [ERROR, DONE] else None + ) + simulation.save() + return simulation.status != PENDING + def get_mvs_simulation_results(simulation): # TODO do not repeat if the simulation is not on the server anymore, or if the results are already loaded @@ -99,7 +152,7 @@ def get_mvs_simulation_results(simulation): simulation.save() else: - fetch_mvs_simulation_status(simulation) + fetch_mvs_simulation_results(simulation) def parse_mvs_results(simulation, response_results): From d28d9194e9bfde6e09a182c0bb57145cfbfc6cb4 Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Mon, 28 Feb 2022 13:40:20 +0100 Subject: [PATCH 17/17] Parse the sensitivity analysis results to the database Make sure of the format of the json to avoid possible html injection --- app/projects/helpers.py | 27 ++++++++-- app/projects/models/simulation_models.py | 69 ++++++++++++++++++------ app/projects/requests.py | 24 ++------- app/projects/views.py | 4 +- 4 files changed, 83 insertions(+), 41 deletions(-) diff --git a/app/projects/helpers.py b/app/projects/helpers.py index 2c4c7d13..a513267a 100644 --- a/app/projects/helpers.py +++ b/app/projects/helpers.py @@ -56,10 +56,26 @@ def sensitivity_analysis_payload( } -SA_MVS_TOKEN_SCHEMA = { +SA_RESPONSE_SCHEMA = { "type": "object", - "required": ["ref_sim_id", "sensitivity_analysis_ids"], + "required": ["server_info", "mvs_version", "id", "status", "results"], "properties": { + "server_info": {"type": "string"}, + "mvs_version": {"type": "string"}, + "id": {"type": "string"}, + "status": {"type": "string"}, + "results": { + "type": "object", + "required": ["reference_simulation_id", "sensitivity_analysis_steps"], + "properties": { + "reference_simulation_id": {"type": "string"}, + "sensitivity_analysis_steps": { + "type": "array", + "items": {"type": "object"}, + }, + }, + "additionalProperties": False, + }, "ref_sim_id": {"type": "string"}, "sensitivity_analysis_ids": {"type": "array", "items": {"type": "string"}}, }, @@ -83,7 +99,12 @@ def sa_output_values_schema_generator(output_names): "value": { "oneOf": [ {"type": "null"}, - {"type": "array", "items": {"type": "number"}}, + { + "type": "array", + "items": { + "anyOf": [{"type": "number"}, {"type": "null"}] + }, + }, ] }, "path": { diff --git a/app/projects/models/simulation_models.py b/app/projects/models/simulation_models.py index c8011b55..22e96468 100644 --- a/app/projects/models/simulation_models.py +++ b/app/projects/models/simulation_models.py @@ -5,17 +5,20 @@ import jsonschema import numpy as np import logging + +logger = logging.getLogger(__name__) from django.db import models from django.core.validators import MaxValueValidator, MinValueValidator from django.utils.translation import gettext_lazy as _ +from datetime import datetime -from projects.constants import USER_RATING +from projects.constants import USER_RATING, DONE, PENDING, ERROR from projects.helpers import ( sensitivity_analysis_payload, SA_OUPUT_NAMES_SCHEMA, sa_output_values_schema_generator, - SA_MVS_TOKEN_SCHEMA, + SA_RESPONSE_SCHEMA, format_scenario_for_mvs, ) @@ -65,12 +68,6 @@ def set_reference_scenario(self, scenario): self.scenario = scenario self.save() - def collect_simulations_tokens(self, server_response): - try: - jsonschema.validate(server_response, SA_MVS_TOKEN_SCHEMA) - except jsonschema.exceptions.ValidationError: - answer = {} - @property def variable_range(self): return np.arange( @@ -93,17 +90,59 @@ def output_names(self): @property def output_values(self): try: - answer = json.loads(self.output_parameters_values) - try: - jsonschema.validate( - answer, sa_output_values_schema_generator(self.output_names) - ) - except jsonschema.exceptions.ValidationError: - answer = {} + out_values = json.loads(self.output_parameters_values) + answer = {} + for in_value, sa_step in zip(self.variable_range, out_values): + try: + jsonschema.validate( + sa_step, sa_output_values_schema_generator(self.output_names) + ) + answer[in_value] = sa_step + except jsonschema.exceptions.ValidationError: + answer[in_value] = None except json.decoder.JSONDecodeError: answer = {} return answer + def parse_server_response(self, sa_results): + try: + # make sure the response is formatted as expected + jsonschema.validate(sa_results, SA_RESPONSE_SCHEMA) + self.status = sa_results["status"] + self.errors = ( + json.dumps(sa_results["results"][ERROR]) + if self.status == ERROR + else None + ) + if self.status == DONE: + sa_steps = sa_results["results"]["sensitivity_analysis_steps"] + sa_steps_processed = [] + # make sure that each step is formatted as expected + for step_idx, sa_step in enumerate(sa_steps): + try: + jsonschema.validate( + sa_step, + sa_output_values_schema_generator(self.output_names), + ) + sa_steps_processed.append(sa_step) + except jsonschema.exceptions.ValidationError as e: + logger.error( + f"Could not parse the results of the sensitivity analysis {self.id} for step {step_idx}" + ) + sa_steps_processed.append(None) + self.output_parameters_values = json.dumps(sa_steps_processed) + + except jsonschema.exceptions.ValidationError as e: + self.status = ERROR + self.output_parameters_values = "" + self.errors = str(e) + + self.elapsed_seconds = (datetime.now() - self.start_date).seconds + self.end_date = ( + datetime.now() if sa_results["status"] in [ERROR, DONE] else None + ) + self.save() + @property def payload(self): return sensitivity_analysis_payload( diff --git a/app/projects/requests.py b/app/projects/requests.py index 3a3e4558..1fe67164 100644 --- a/app/projects/requests.py +++ b/app/projects/requests.py @@ -108,28 +108,12 @@ def fetch_mvs_simulation_results(simulation): def fetch_mvs_sa_results(simulation): if simulation.status == PENDING: response = mvs_sa_check_status(token=simulation.mvs_token) - try: - simulation.status = response["status"] - simulation.errors = ( - json.dumps(response["results"][ERROR]) - if simulation.status == ERROR - else None - ) - # TODO here fetch the results of the reference scenario in a separate Simulation instance - # and parse the - simulation.results = ( - response["results"] if simulation.status == DONE else None - ) + + simulation.parse_server_response(response) + + if simulation.status == DONE: logger.info(f"The simulation {simulation.id} is finished") - except: - simulation.status = ERROR - simulation.results = None - simulation.elapsed_seconds = (datetime.now() - simulation.start_date).seconds - simulation.end_date = ( - datetime.now() if response["status"] in [ERROR, DONE] else None - ) - simulation.save() return simulation.status != PENDING diff --git a/app/projects/views.py b/app/projects/views.py index fd3e3e2f..e1911074 100644 --- a/app/projects/views.py +++ b/app/projects/views.py @@ -910,9 +910,7 @@ def sensitivity_analysis_create(request, scen_id, sa_id=None, step_id=5): else: sa_item = None sa_status = None - sa_form = SensitivityAnalysisForm( - scen_id=scen_id, - ) + sa_form = SensitivityAnalysisForm(scen_id=scen_id) answer = render( request,