Merge branch 'dev' into sp/#35-ext-sim-termination-msg

# Conflicts:
#	CHANGELOG.md

.github/dependabot.yml

@@ -12,11 +12,22 @@ updates:
   - johanneshiry
   - t-ober
   - sensarmad
+  - sebastian-peter
+  - danielfeismann
   ignore:
     - dependency-name: org.spockframework:spock-core
       versions:
-        - 2.1-groovy-3.0-SNAPSHOT
         - 2.1-groovy-2.5-SNAPSHOT
+        - 2.1-groovy-3.0-SNAPSHOT
+        - 2.2-groovy-4.0-SNAPSHOT
+        - 2.2-groovy-2.5-SNAPSHOT
+        - 2.2-groovy-3.0-SNAPSHOT
+        - 2.2-M1-groovy-2.5
+        - 2.2-M1-groovy-3.0
+        - 2.2-M1-groovy-4.0
+        - 2.2-M2-groovy-2.5
+        - 2.2-M2-groovy-3.0
+        - 2.2-M2-groovy-4.0
     - dependency-name: org.scalatest:scalatest_2.13
       versions:
         - 3.3.0-SNAP+

CHANGELOG.md

@@ -5,14 +5,21 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+
+### Added
+- Implement SQL source for primary data [#34](https://github.com/ie3-institute/simona/issues/34), [#101](https://github.com/ie3-institute/simona/issues/101)
+
 ### Changed
 - Re-organizing test resources into their respective packages [#105](https://github.com/ie3-institute/simona/issues/105)
 - BREAKING: Using snapshot version of PSDM
 - Simplified PrimaryServiceProxy due to changes in PSDM [#120](https://github.com/ie3-institute/simona/issues/120)
+- Improved handling of weights and their sum in determination of weather data [#173](https://github.com/ie3-institute/simona/issues/173)
 - Improving code readability in EvcsAgent by moving FreeLotsRequest to separate methods [#19](https://github.com/ie3-institute/simona/issues/19)
 - Sending termination message to external simulation on expected and unexpected shutdowns of SIMONA [#35](https://github.com/ie3-institute/simona/issues/35)
 
 ### Fixed
 - Location of `vn_simona` test grid (was partially in Berlin and Dortmund)
+- Let `ParticipantAgent` die after failed registration with secondary services (prevents stuck simulation)
+- Fix default resolution of weather source wrapper [#78](https://github.com/ie3-institute/simona/issues/78)
 
 [Unreleased]: https://github.com/ie3-institute/simona/compare/a14a093239f58fca9b2b974712686b33e5e5f939...HEAD

build.gradle

@@ -7,10 +7,10 @@ plugins {
   id 'signing'
   id 'maven-publish' // publish to a maven repo (local or mvn central, has to be defined)
   id 'pmd' // code check, working on source code
-  id 'com.diffplug.spotless' version '6.2.2'// code format
+  id 'com.diffplug.spotless' version '6.4.2'// code format
   id 'com.github.onslip.gradle-one-jar' version '1.0.6' // pack a self contained jar
   id "com.github.ben-manes.versions" version '0.42.0'
-  id "de.undercouch.download" version "5.0.1" // downloads plugin
+  id "de.undercouch.download" version "5.0.4" // downloads plugin
   id "kr.motd.sphinx" version "2.10.1" // documentation generation
   id "com.github.johnrengelman.shadow" version "7.1.2" // fat jar
   id "org.sonarqube" version "3.3" // sonarqube
@@ -26,10 +26,12 @@ ext {
 
   scalaVersion = '2.13'
   scalaBinaryVersion = '2.13.8'
-  akkaVersion = '2.6.18'
+  akkaVersion = '2.6.19'
   tscfgVersion = '0.9.997'
   scapegoatVersion = '1.4.12'
 
+  testContainerVersion = '0.40.5'
+
   scriptsLocation = 'gradle' + File.separator + 'scripts' + File.separator // location of script plugins
 }
 
@@ -96,17 +98,21 @@ dependencies {
 
   /* logging */
   implementation "com.typesafe.scala-logging:scala-logging_${scalaVersion}:3.9.4" // akka scala logging
-  implementation "ch.qos.logback:logback-classic:1.2.10"
+  implementation "ch.qos.logback:logback-classic:1.2.11"
 
   /* testing */
-  testImplementation 'org.spockframework:spock-core:2.1-M2-groovy-3.0'
+  testImplementation 'org.spockframework:spock-core:2.1-groovy-3.0'
   testImplementation 'org.scalatestplus:mockito-3-4_2.13:3.2.10.0'
-  implementation 'org.mockito:mockito-core:4.3.1' // mocking framework
+  implementation 'org.mockito:mockito-core:4.4.0' // mocking framework
   testImplementation "org.scalatest:scalatest_${scalaVersion}:3.2.11"
   testRuntimeClasspath 'com.vladsch.flexmark:flexmark-all:0.64.0'
   testImplementation group: 'org.pegdown', name: 'pegdown', version: '1.6.0'
   testImplementation "com.typesafe.akka:akka-testkit_${scalaVersion}:${akkaVersion}" // akka testkit
 
+  // testcontainers
+  testImplementation "com.dimafeng:testcontainers-scala-scalatest_${scalaVersion}:${testContainerVersion}"
+  testImplementation "com.dimafeng:testcontainers-scala-postgresql_${scalaVersion}:${testContainerVersion}"
+
   /* --- Scala libs --- */
   /* CORE Scala */
   implementation "org.scala-lang:scala-library:${scalaBinaryVersion}"
@@ -136,13 +142,13 @@ dependencies {
   scalaCompilerPlugin "com.sksamuel.scapegoat:scalac-scapegoat-plugin_${scalaBinaryVersion}:${scapegoatVersion}"
 
   implementation 'org.apache.commons:commons-math3:3.6.1' // apache commons math3
-  implementation 'org.apache.poi:poi-ooxml:5.2.0' // used for FilenameUtils
+  implementation 'org.apache.poi:poi-ooxml:5.2.2' // used for FilenameUtils
   implementation 'javax.measure:unit-api:2.1.3'
-  implementation 'tech.units:indriya:2.1.2' // quantities
+  implementation 'tech.units:indriya:2.1.3' // quantities
   implementation 'org.apache.commons:commons-csv:1.9.0'
   implementation 'org.scalanlp:breeze_2.13:1.3' // scientific calculations (http://www.scalanlp.org/)
   implementation 'de.lmu.ifi.dbs.elki:elki:0.7.5' // Statistics (for random load model)
-  implementation 'com.google.guava:guava:31.0.1-jre' // Building threads
+  implementation 'com.google.guava:guava:31.1-jre' // Building threads
   implementation 'org.jgrapht:jgrapht-core:1.5.1'
 }
 

docs/readthedocs/usersguide.rst

@@ -17,11 +17,11 @@ To run and customize the project you need a Java Development Kit (JDK) installat
 Installation
 ============
 
-You can find and download the source code of the latest stable SIMONA version `here <https://git.ie3.e-technik.tu-dortmund.de/SIMONACrew/SIMONA>`_. Go ahead and clone the repository using git:
+You can find and download the source code of the latest stable SIMONA version `here <https://github.com/ie3-institute/simona>`_. Go ahead and clone the repository using git:
 
 .. code-block:: none
 
-   $ git clone https://git.ie3.e-technik.tu-dortmund.de/SIMONACrew/SIMONA.git
+   $ git clone https://github.com/ie3-institute/simona.git
 
 
 Running a Standalone Simulation
@@ -171,7 +171,7 @@ SIMONA is capable of running an external sub-simulation by integration within th
 The information flow between SIMONA and the external simulation is partitioned into a control stream (see ``edu.ie3.simona.api.ExtSimAdapter``) and a number of optional data streams.
 Currently, only a data stream transporting electric vehicle movement information is implemented (see ``edu.ie3.simona.service.ev.ExtEvDataService``).
 
-An external simulation has to depend on `SimonaAPI <https://git.ie3.e-technik.tu-dortmund.de/SIMONACrew/SimonaAPI>`_ and make use of some of its interfaces (see below).
+An external simulation has to depend on `SimonaAPI <https://github.com/ie3-institute/simonaAPI>`_ and make use of some of its interfaces (see below).
 In order to run an external simulation, several requirements have to be fulfilled and a bunch of preparation steps have to be followed.
 
 .. note::
@@ -201,3 +201,25 @@ These steps have to be performed each time updates to the external simulation ne
 - Copy the resulting *jar* (usually placed inside <external project>/build/libs) to ``./input/ext_sim/``.
 
 Now, when a simulation with SIMONA is started (see `above <#running-a-standalone-simulation>`_), the external simulation is triggered at each tick that it requested.
+
+Troubleshooting
+===============
+
+My power flow calculation isn't converging - why is that?
+---------------------------------------------------------
+
+When your power flow is not converging it means that the load situation in the grid during the time of the power flow calculation is not physically feasible.
+
+This can have basically one of the following two reasons:
+
+#.
+    There is more load in the grid than it can physically handle.
+
+#.
+    There is more generation in the grid than it can physically handle.
+
+One of the main reasons is a misconfiguration of the grid and its assets.
+Assess the power of the load and generation units and check if the values make sense.
+Keep in mind the metric prefixes that are assumed for the models, which are listed in the `PSDM docs <https://powersystemdatamodel.readthedocs.io/en/latest/index.html>`_.
+If everything seems to be configured correctly it could also be the case that the grid itself is incorrectly configured.
+Do a similar sanity check for the grids assets.

docs/uml/main/ParticipantModelling.puml

@@ -41,9 +41,9 @@ package edu.ie3.edu.ie3.simona {
             }
             DateTime --|> SecondaryData
 
-            Class Weather{
-              + diffRad: Quantity[Irradiation]
-              + dirRad: Quantity[Irradiation]
+            Class WeatherData{
+              + diffIrr: Quantity[Irradiation]
+              + dirIrr: Quantity[Irradiation]
               + temp: Quantity[Temperature]
               + windVel: Quantity[Speed]
             }

gradle/scripts/tscfg.gradle

@@ -15,7 +15,7 @@ task genConfigClass {
       args = [
         "build/tscfg-${tscfgVersion}.jar",
         "--spec",
-        "src/main/resources/config/simona-config-template.conf",
+        "src/main/resources/config/config-template.conf",
         "--scala",
         "--durations",
         "--pn",

input/samples/vn_146_lv_small/fullGrid/transformer_2_w_type_input.csv

@@ -1,6 +1,6 @@
 "uuid","b_m","d_phi","d_v","g_m","id","r_sc","s_rated","tap_max","tap_min","tap_neutr","tap_side","v_rated_a","v_rated_b","x_sc"
 14b1798a-6903-49d6-8578-ad2a7d399341,0.0,0.0,1.5,0.0,HS-MS_1,45.375,20000.0,10,-10,0,false,110.0,20.0,102.759
-97735722-05cc-4ca8-8a8d-c08ac3ded19a,1.27,0.0,1.5,555.5,HöS-HS_1,5.415,200000.0,5,-5,0,false,380.0,110.0,108.165
+97735722-05cc-4ca8-8a8d-c08ac3ded19a,-1.27,0.0,1.5,555.5,HöS-HS_1,5.415,200000.0,5,-5,0,false,380.0,110.0,108.165
 f88989c7-9812-4b3e-9bc0-3df29f1e5ae1,0.0,0.0,0.5,0.0,MS-NS_1,10.078,630.0,10,-10,0,false,20.0,0.4,23.312
 cf7b1102-8dbd-4da2-a469-90800b3394b6,0.0,0.0,1.5,0.0,HS-MS_1,45.375,20000.0,10,-10,0,false,110.0,20.0,102.759
 1214c366-826e-4aeb-88f5-af8f40acaa04,0.0,0.0,1.5,0.0,HS-MS_1,45.375,20000.0,10,-10,0,false,110.0,20.0,102.759

input/samples/vn_simona/fullGrid/pv_input.csv

@@ -1,4 +1,4 @@
-"uuid","albedo","azimuth","cos_phi_rated","eta_conv","height","id","k_g","k_t","market_reaction","node","operates_from","operates_until","operator","q_characteristics","s_rated"
+"uuid","albedo","azimuth","cos_phi_rated","eta_conv","elevation_angle","id","k_g","k_t","market_reaction","node","operates_from","operates_until","operator","q_characteristics","s_rated"
 5b38af42-1ee4-4a41-b666-ea141187df37,0.20000000298023224,-11.463644027709961,0.8999999761581421,96.0,33.62879943847656,NS_NET146_F2_(3)_PV,0.8999999761581421,1.0,false,0170837a-1876-45f9-a613-666f9991964d,,,,cosPhiFixed:{(0.00,0.90)},10.0
 e447506e-3d43-4bce-8aab-a7ca8b7fbc45,0.20000000298023224,3.8914573192596436,0.8999999761581421,98.0,42.77021408081055,NS_NET146_F4_(9)_PV,0.8999999761581421,1.0,false,9b889b73-c108-4b38-b6eb-3377841e0c83,,,,cosPhiFixed:{(0.00,0.90)},10.0
 6cac0624-6336-4418-bcf0-990abcdb824b,0.20000000298023224,-8.097375869750977,0.8999999761581421,98.0,44.90728759765625,NS_NET146_F4_(16)_PV,0.8999999761581421,1.0,false,9f7599de-c488-46c5-b053-1279a511f7b9,,,,cosPhiFixed:{(0.00,0.90)},30.0

input/samples/vn_simona/fullGrid/transformer_2_w_type_input.csv

@@ -1,6 +1,6 @@
 "uuid","b_m","d_phi","d_v","g_m","id","r_sc","s_rated","tap_max","tap_min","tap_neutr","tap_side","v_rated_a","v_rated_b","x_sc"
 14b1798a-6903-49d6-8578-ad2a7d399341,0.0,0.0,1.5,0.0,HS-MS_1,45.375,20000.0,10,-10,0,false,110.0,20.0,102.759
-97735722-05cc-4ca8-8a8d-c08ac3ded19a,1.27,0.0,1.5,555.5,HöS-HS_1,5.415,200000.0,5,-5,0,false,380.0,110.0,108.165
+97735722-05cc-4ca8-8a8d-c08ac3ded19a,-1.27,0.0,1.5,555.5,HöS-HS_1,5.415,200000.0,5,-5,0,false,380.0,110.0,108.165
 f88989c7-9812-4b3e-9bc0-3df29f1e5ae1,0.0,0.0,0.5,0.0,MS-NS_1,10.078,630.0,10,-10,0,false,20.0,0.4,23.312
 cf7b1102-8dbd-4da2-a469-90800b3394b6,0.0,0.0,1.5,0.0,HS-MS_1,45.375,20000.0,10,-10,0,false,110.0,20.0,102.759
 1214c366-826e-4aeb-88f5-af8f40acaa04,0.0,0.0,1.5,0.0,HS-MS_1,45.375,20000.0,10,-10,0,false,110.0,20.0,102.759

src/main/resources/config/config-template.conf

@@ -101,9 +101,7 @@ simona.input.primary = {
       jdbcUrl:  string
       userName: string
       password: string
-      weatherTableName: string
       schemaName: string | "public"
-      timeColumnName: string
       timePattern: string | "yyyy-MM-dd'T'HH:mm:ss[.S[S][S]]'Z'" # default pattern from PSDM:TimeBasedSimpleValueFactory
     }
     #@optional
@@ -150,9 +148,8 @@ simona.input.weather.datasource = {
       jdbcUrl:  string
       userName: string
       password: string
-      weatherTableName: string
+      tableName: string
       schemaName: string | "public"
-      timeColumnName: string
     }
     #@optional
     couchbaseParams = {

src/main/scala/edu/ie3/simona/agent/participant/ParticipantAgentFundamentals.scala

@@ -6,7 +6,7 @@
 
 package edu.ie3.simona.agent.participant
 
-import akka.actor.{ActorRef, FSM}
+import akka.actor.{ActorRef, FSM, PoisonPill}
 import akka.event.LoggingAdapter
 import akka.util
 import akka.util.Timeout
@@ -452,6 +452,7 @@ protected trait ParticipantAgentFundamentals[
           )
         }
       case RegistrationResponseMessage.RegistrationFailedMessage =>
+        self ! PoisonPill
         throw new ActorNotRegisteredException(
           s"Registration of actor $actorName for ${sender()} failed."
         )

src/main/scala/edu/ie3/simona/agent/participant/pv/PVAgentFundamentals.scala

@@ -230,8 +230,8 @@ protected trait PVAgentFundamentals
                 PVRelevantData(
                   dateTime,
                   tickInterval,
-                  weatherData.diffRad,
-                  weatherData.dirRad
+                  weatherData.diffIrr,
+                  weatherData.dirIrr
                 )
 
               val power = pvModel.calculatePower(

src/main/scala/edu/ie3/simona/config/SimonaConfig.scala

@@ -1,5 +1,5 @@
 /*
- * © 2021. TU Dortmund University,
+ * © 2022. TU Dortmund University,
  * Institute of Energy Systems, Energy Efficiency and Energy Economics,
  * Research group Distribution grid planning and operation
  */
@@ -908,10 +908,8 @@ object SimonaConfig {
             jdbcUrl: java.lang.String,
             password: java.lang.String,
             schemaName: java.lang.String,
-            timeColumnName: java.lang.String,
             timePattern: java.lang.String,
-            userName: java.lang.String,
-            weatherTableName: java.lang.String
+            userName: java.lang.String
         )
         object SqlParams {
           def apply(
@@ -925,14 +923,10 @@ object SimonaConfig {
               schemaName =
                 if (c.hasPathOrNull("schemaName")) c.getString("schemaName")
                 else "public",
-              timeColumnName =
-                $_reqStr(parentPath, c, "timeColumnName", $tsCfgValidator),
               timePattern =
                 if (c.hasPathOrNull("timePattern")) c.getString("timePattern")
                 else "yyyy-MM-dd'T'HH:mm:ss[.S[S][S]]'Z'",
-              userName = $_reqStr(parentPath, c, "userName", $tsCfgValidator),
-              weatherTableName =
-                $_reqStr(parentPath, c, "weatherTableName", $tsCfgValidator)
+              userName = $_reqStr(parentPath, c, "userName", $tsCfgValidator)
             )
           }
           private def $_reqStr(
@@ -1277,9 +1271,8 @@ object SimonaConfig {
               jdbcUrl: java.lang.String,
               password: java.lang.String,
               schemaName: java.lang.String,
-              timeColumnName: java.lang.String,
-              userName: java.lang.String,
-              weatherTableName: java.lang.String
+              tableName: java.lang.String,
+              userName: java.lang.String
           )
           object SqlParams {
             def apply(
@@ -1293,11 +1286,9 @@ object SimonaConfig {
                 schemaName =
                   if (c.hasPathOrNull("schemaName")) c.getString("schemaName")
                   else "public",
-                timeColumnName =
-                  $_reqStr(parentPath, c, "timeColumnName", $tsCfgValidator),
-                userName = $_reqStr(parentPath, c, "userName", $tsCfgValidator),
-                weatherTableName =
-                  $_reqStr(parentPath, c, "weatherTableName", $tsCfgValidator)
+                tableName =
+                  $_reqStr(parentPath, c, "tableName", $tsCfgValidator),
+                userName = $_reqStr(parentPath, c, "userName", $tsCfgValidator)
               )
             }
             private def $_reqStr(