Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add CN-MN exchange and parser #6283

Open
wants to merge 7 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions config/exchanges/CN_MN.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
lonlat:
- 107.4573
- 42.4151
parsers:
exchange: MN.fetch_exchange
rotation: 0
42 changes: 42 additions & 0 deletions parsers/MN.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,12 @@

from electricitymap.contrib.config import ZoneKey
from electricitymap.contrib.lib.models.event_lists import (
ExchangeList,
ProductionBreakdownList,
TotalConsumptionList,
)
from electricitymap.contrib.lib.models.events import ProductionMix
from electricitymap.contrib.parsers.RU import fetch_exchange as ru_fetch_exchange
from parsers.lib.exceptions import ParserException

NDC_GENERATION = "https://disnews.energy.mn/convertt.php"
Expand Down Expand Up @@ -143,6 +145,46 @@ def fetch_consumption(
return consumption_list.to_list()


def fetch_exchange(
zone_key1: ZoneKey,
zone_key2: ZoneKey,
session: Session = Session(),
target_datetime: datetime | None = None,
logger: Logger = getLogger(__name__),
) -> list[dict]:
if target_datetime:
raise NotImplementedError("This parser is not yet able to parse past dates.")

sorted_zone_keys = ZoneKey("->".join(sorted([zone_key1, zone_key2])))

if sorted_zone_keys != "CN->MN":
raise NotImplementedError(
"This parser is only able to fetch CN->MN exchange data."
)

query_data = query(session, logger, ZoneKey("MN"))

russia_data = ru_fetch_exchange("MN", "RU-2", session, logger=logger)

exchange_list = ExchangeList(logger)

for data in russia_data:
if (
data["datetime"] == query_data["time"]
and data["sortedZoneKeys"] == "MN->RU-2"
and "netFlow" in data.keys()
):
exchange_list.append(
datetime=query_data["time"],
zoneKey=sorted_zone_keys,
# We calculate the flow with the total imports and the MN->RU-2 exchange, as Mongolia has only two connections
netFlow=query_data["importMW"] + data["netFlow"],
source="https://ndc.energy.mn/",
VIKTORVAV99 marked this conversation as resolved.
Show resolved Hide resolved
Copy link
Member

@VIKTORVAV99 VIKTORVAV99 Jan 8, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The current implementation seems to be inversed.
Please double check this yourself.

It would also be good if we could add a test or two for this so we can validate the expected behavior.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

seems to be inversed

I think it isn't.
Let's see:
CN->MN + RU->MN = Total Imports
CN->MN = Total Imports + MN->RU

I'll work on those tests. It would be optimal to be able to test the parser without requiring the sources to have matching data.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the easiest way would probably be to add snapshot tests using mock data. Like I did in #6285.

That way it don't rely on an online source, changing data or creating manual tests are are both logic intensive and hard to maintain long term.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was thinking of doing something similar to what I did for the REE parser but I will take a look at these snapshot tests.

)

return exchange_list.to_list()


if __name__ == "__main__":
print("fetch_production() ->")
print(fetch_production(ZoneKey("MN")))
Expand Down
1 change: 1 addition & 0 deletions parsers/test/mocks/MN/GetData.json
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"$id":"1","Flows":[{"$id":"2","Id":34,"Name":null,"PositiveDirection":1,"IsVisible":true,"NumValue":2264.29638671875,"ArrowClass":"arrow-backward","Value":"2264,3 МВт","VisibilityClass":""},{"$id":"3","Id":35,"Name":null,"PositiveDirection":1,"IsVisible":true,"NumValue":-8.4750795364379883,"ArrowClass":"arrow-forward","Value":"8,48 МВт","VisibilityClass":""},{"$id":"4","Id":36,"Name":null,"PositiveDirection":0,"IsVisible":true,"NumValue":378.20269775390625,"ArrowClass":"arrow-forward","Value":"378,2 МВт","VisibilityClass":""},{"$id":"5","Id":37,"Name":null,"PositiveDirection":0,"IsVisible":true,"NumValue":-33.196315765380859,"ArrowClass":"arrow-backward","Value":"33,2 МВт","VisibilityClass":""},{"$id":"6","Id":96,"Name":null,"PositiveDirection":0,"IsVisible":true,"NumValue":569.170654296875,"ArrowClass":"arrow-forward","Value":"569,17 МВт","VisibilityClass":""},{"$id":"7","Id":187,"Name":null,"PositiveDirection":0,"IsVisible":false,"NumValue":0.0,"ArrowClass":"arrow-empty","Value":"0 МВт","VisibilityClass":"flow-invisible"},{"$id":"8","Id":276,"Name":null,"PositiveDirection":1,"IsVisible":true,"NumValue":-3.2802648544311523,"ArrowClass":"arrow-forward","Value":"3,28 МВт","VisibilityClass":""},{"$id":"9","Id":321,"Name":null,"PositiveDirection":1,"IsVisible":true,"NumValue":-280.58309936523438,"ArrowClass":"arrow-forward","Value":"280,58 МВт","VisibilityClass":""},{"$id":"10","Id":329,"Name":null,"PositiveDirection":1,"IsVisible":true,"NumValue":112.51697444915771,"ArrowClass":"arrow-backward","Value":"112,52 МВт","VisibilityClass":""},{"$id":"11","Id":598,"Name":null,"PositiveDirection":1,"IsVisible":true,"NumValue":18.350364685058594,"ArrowClass":"arrow-backward","Value":"18,35 МВт","VisibilityClass":""},{"$id":"12","Id":752,"Name":null,"PositiveDirection":0,"IsVisible":true,"NumValue":0.0,"ArrowClass":"arrow-empty","Value":"0 МВт","VisibilityClass":"flow-empty"},{"$id":"13","Id":764,"Name":null,"PositiveDirection":1,"IsVisible":true,"NumValue":-107.35099029541016,"ArrowClass":"arrow-forward","Value":"107,35 МВт","VisibilityClass":""},{"$id":"14","Id":785,"Name":null,"PositiveDirection":0,"IsVisible":true,"NumValue":742.91796875,"ArrowClass":"arrow-forward","Value":"742,92 МВт","VisibilityClass":""},{"$id":"15","Id":1611,"Name":null,"PositiveDirection":1,"IsVisible":true,"NumValue":641.18646240234375,"ArrowClass":"arrow-backward","Value":"641,19 МВт","VisibilityClass":""},{"$id":"16","Id":880,"Name":null,"PositiveDirection":1,"IsVisible":true,"NumValue":0.0,"ArrowClass":"arrow-empty","Value":"0 МВт","VisibilityClass":"flow-empty"},{"$id":"17","Id":139,"Name":null,"PositiveDirection":0,"IsVisible":true,"NumValue":-145.84413146972656,"ArrowClass":"arrow-backward","Value":"145,84 МВт","VisibilityClass":""},{"$id":"18","Id":212,"Name":null,"PositiveDirection":0,"IsVisible":true,"NumValue":-23.509000778198242,"ArrowClass":"arrow-backward","Value":"23,51 МВт","VisibilityClass":""},{"$id":"19","Id":5688,"Name":null,"PositiveDirection":0,"IsVisible":false,"NumValue":103.27338409423828,"ArrowClass":"arrow-forward","Value":"103,27 МВт","VisibilityClass":"flow-invisible"},{"$id":"20","Id":344,"Name":null,"PositiveDirection":0,"IsVisible":true,"NumValue":875.9818115234375,"ArrowClass":"arrow-forward","Value":"875,98 МВт","VisibilityClass":""},{"$id":"21","Id":2394,"Name":null,"PositiveDirection":1,"IsVisible":true,"NumValue":-1600.6649169921875,"ArrowClass":"arrow-forward","Value":"1600,66 МВт","VisibilityClass":""}],"Nodes":[{"$id":"22","Id":530000,"UsePlan":34809.19140625,"UseFact":34741.234375,"GenPlan":32536.01171875,"GenFact":32591.28515625,"AvgPrice":1589.83289},{"$id":"23","Id":550000,"UsePlan":14529.388671875,"UseFact":14543.515625,"GenPlan":14028.3115234375,"GenFact":14535.3212890625,"AvgPrice":1882.88989},{"$id":"24","Id":600000,"UsePlan":14716.34375,"UseFact":14529.84765625,"GenPlan":15506.5703125,"GenFact":15423.591796875,"AvgPrice":1494.66736},{"$id":"25","Id":610000,"UsePlan":29660.05078125,"UseFact":29811.658203125,"GenPlan":29011.0,"GenFact":28723.923828125,"AvgPrice":1345.82288},{"$id":"26","Id":630000,"UsePlan":32734.31640625,"UseFact":32959.5546875,"GenPlan":33946.71875,"GenFact":34167.22265625,"AvgPrice":1365.97742},{"$id":"27","Id":840000,"UsePlan":13460.2626953125,"UseFact":13515.1337890625,"GenPlan":15824.591796875,"GenFact":15872.9345703125,"AvgPrice":1292.67468},{"$id":"28","Id":540000,"UsePlan":6414.75341796875,"UseFact":6355.0107421875,"GenPlan":6594.83203125,"GenFact":6531.5498046875,"AvgPrice":null}]}
1 change: 1 addition & 0 deletions parsers/test/mocks/MN/convertt.php
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"date":"2024-02-17 06:00:00","syssum":"1051.48","sumnar":24.88,"sums":65.4,"energyimport":"1.9","t":"-1"}
25 changes: 25 additions & 0 deletions parsers/test/snapshots/snap_test_MN.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
# snapshottest: v1 - https://goo.gl/zC4yUc

from snapshottest import Snapshot

snapshots = Snapshot()

snapshots["test_exchange 1"] = [
{
"datetime": "2024-02-17T06:00:00+08:00",
"netFlow": 5.180265,
"sortedZoneKeys": "CN->MN",
"source": "https://ndc.energy.mn/",
"sourceType": "measured",
}
]

snapshots["test_production 1"] = [
{
"datetime": "2024-02-17T06:00:00+08:00",
"production": {"solar": 24.88, "unknown": 959.3, "wind": 65.4},
"source": "https://ndc.energy.mn/",
"sourceType": "measured",
"zoneKey": "MN",
}
]
72 changes: 72 additions & 0 deletions parsers/test/test_MN.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
from requests import Session
from requests_mock import GET, Adapter

from electricitymap.contrib.lib.types import ZoneKey
from parsers.MN import NDC_GENERATION, fetch_exchange, fetch_production
from parsers.RU import BASE_EXCHANGE_URL


def test_production(snapshot):
session = Session()
adapter = Adapter()
session.mount("https://", adapter)
mock_file = open("parsers/test/mocks/MN/convertt.php", "rb")
adapter.register_uri(
GET,
NDC_GENERATION,
content=mock_file.read(),
)

production = fetch_production(
zone_key=ZoneKey("MN"),
session=session,
)

snapshot.assert_match(
[
{
"datetime": element["datetime"].isoformat(),
"production": element["production"],
"source": element["source"],
"zoneKey": element["zoneKey"],
"sourceType": element["sourceType"].value,
}
for element in production
]
)


def test_exchange(snapshot):
session = Session()
adapter = Adapter()
session.mount("https://", adapter)
mock_file = open("parsers/test/mocks/MN/convertt.php", "rb")
adapter.register_uri(
GET,
NDC_GENERATION,
content=mock_file.read(),
)
mock_file2 = open("parsers/test/mocks/MN/GetData.json", "rb")
adapter.register_uri(
GET,
BASE_EXCHANGE_URL,
content=mock_file2.read(),
)
exchange = fetch_exchange(
zone_key1=ZoneKey("CN"),
zone_key2=ZoneKey("MN"),
session=session,
)

snapshot.assert_match(
[
{
"datetime": element["datetime"].isoformat(),
"source": element["source"],
"netFlow": element["netFlow"],
"sortedZoneKeys": element["sortedZoneKeys"],
"sourceType": element["sourceType"].value,
}
for element in exchange
]
)