Updated PUN tariff to use the new GME website

tariff/pun.go

@@ -1,11 +1,14 @@
 package tariff
 
 import (
+	"archive/zip"
+	"bytes"
 	"encoding/xml"
 	"fmt"
 	"io"
+	"net/http"
 	"net/http/cookiejar"
-	"net/url"
+
 	"slices"
 	"strconv"
 	"strings"
@@ -16,7 +19,6 @@ import (
 	"github.com/evcc-io/evcc/api"
 	"github.com/evcc-io/evcc/util"
 	"github.com/evcc-io/evcc/util/request"
-	"golang.org/x/net/html"
 )
 
 type Pun struct {
@@ -79,7 +81,7 @@ func (t *Pun) run(done chan error) {
 		var today api.Rates
 		if err := backoff.Retry(func() error {
 			var err error
-
+			t.log.TRACE.Println(`Current day PUN hourly tariffs import started`)
 			today, err = t.getData(time.Now())
 
 			return err
@@ -91,20 +93,23 @@ func (t *Pun) run(done chan error) {
 		}
 
 		res, err := backoff.RetryWithData(func() (api.Rates, error) {
+			t.log.TRACE.Println(`Next day PUN hourly tariffs import started`)
 			res, err := t.getData(time.Now().AddDate(0, 0, 1))
 			return res, backoffPermanentError(err)
 		}, bo())
 		if err != nil {
 			once.Do(func() { done <- err })
-			t.log.ERROR.Println(err)
 			continue
 		}
 
 		// merge today and tomorrow data
 		data := append(today, res...)
 
 		mergeRates(t.data, data)
-		once.Do(func() { close(done) })
+		once.Do(func() {
+			close(done)
+			t.log.DEBUG.Println(`PUN hourly tariffs import completed`)
+		})
 	}
 }
 
@@ -127,44 +132,71 @@ func (t *Pun) getData(day time.Time) (api.Rates, error) {
 	client := request.NewClient(t.log)
 	client.Jar, _ = cookiejar.New(nil)
 
-	// Erster Request
-	uri := "https://storico.mercatoelettrico.org/It/WebServerDataStore/MGP_Prezzi/" + day.Format("20060102") + "MGPPrezzi.xml"
-	resp, err := client.Get(uri)
+	// Request the ZIP file
+	uri := "https://gme.mercatoelettrico.org/DesktopModules/GmeDownload/API/ExcelDownload/downloadzipfile?DataInizio=" + day.Format("20060102") + "&DataFine=" + day.Format("20060102") + "&Date=" + day.Format("20060102") + "&Mercato=MGP&Settore=Prezzi&FiltroDate=InizioFine"
+
+	req, err := http.NewRequest("GET", uri, nil)
 	if err != nil {
 		return nil, err
 	}
+
+	req.Header = http.Header{
+		"moduleid":           []string{"12103"},
+		"Referer":            []string{"https://gme.mercatoelettrico.org/en-us/Home/Results/Electricity/MGP/Download?valore=Prezzi"},
+		"sec-ch-ua-mobile":   []string{"?0"},
+		"sec-ch-ua-platform": []string{"Windows"},
+		"sec-fetch-dest":     []string{"empty"},
+		"sec-fetch-mode":     []string{"cors"},
+		"sec-fetch-site":     []string{"same-origin"},
+		"sec-gpc":            []string{"1"},
+		"tabid":              []string{"1749"},
+		"userid":             []string{"-1"},
+	}
+
+	t.log.TRACE.Println(`PUN Zip file requested`)
+
+	resp, err := client.Do(req)
+	if err != nil || resp.StatusCode != http.StatusOK {
+		t.log.TRACE.Println(`PUN Zip file request failed:`, err)
+		return nil, err
+	}
+	t.log.TRACE.Println(`PUN Zip file request status:`, resp.Status)
 	defer resp.Body.Close()
 
-	formData, err := parseFormFields(resp.Body)
+	// Process the ZIP file
+	body, err := io.ReadAll(resp.Body)
 	if err != nil {
-		return nil, fmt.Errorf("form fields: %w", err)
+		return nil, err
 	}
+	t.log.TRACE.Println(`Decompressing Zip file`)
 
-	redirectURL := resp.Request.URL.String()
+	zipReader, err := zip.NewReader(bytes.NewReader(body), int64(len(body)))
+	if err != nil {
+		return nil, err
+	}
+	t.log.TRACE.Println(`Zip file decompressed, now looking for the XML file`)
 
-	// Hinzufügen der spezifizierten Parameter
-	formData.Set("ctl00$ContentPlaceHolder1$CBAccetto1", "on")
-	formData.Set("ctl00$ContentPlaceHolder1$CBAccetto2", "on")
-	formData.Set("ctl00$ContentPlaceHolder1$Button1", "Accetto")
+	if len(zipReader.File) != 1 {
+		return nil, fmt.Errorf("unexpected number of files in the ZIP archive")
+	}
 
-	// Formular senden
-	resp, err = client.PostForm(redirectURL, formData)
+	zipFile := zipReader.File[0]
+	f, err := zipFile.Open()
 	if err != nil {
-		fmt.Println("Error submitting form:", err)
 		return nil, err
 	}
-	defer resp.Body.Close()
+	t.log.TRACE.Println(`Opening XML file`)
+	defer f.Close()
 
-	// Erneuter Request auf die ursprüngliche URL
-	resp, err = client.Get(uri)
+	xmlFile, err := io.ReadAll(f)
 	if err != nil {
 		return nil, err
 	}
-	defer resp.Body.Close()
+	t.log.TRACE.Println(`Reading XML file`)
 
-	// Verarbeitung der erhaltenen Daten
+	// Processing the received data
 	var dataSet NewDataSet
-	if err := xml.NewDecoder(resp.Body).Decode(&dataSet); err != nil {
+	if err := xml.NewDecoder(bytes.NewReader(xmlFile)).Decode(&dataSet); err != nil {
 		return nil, err
 	}
 
@@ -181,6 +213,12 @@ func (t *Pun) getData(day time.Time) (api.Rates, error) {
 			return nil, fmt.Errorf("parse hour: %w", err)
 		}
 
+		// Adjust hour to handle edge case where p.Ora is "00"
+		if hour == 0 {
+			hour = 24
+			date = date.AddDate(0, 0, -1)
+		}
+
 		location, err := time.LoadLocation("Europe/Rome")
 		if err != nil {
 			return nil, fmt.Errorf("load location: %w", err)
@@ -198,39 +236,9 @@ func (t *Pun) getData(day time.Time) (api.Rates, error) {
 			Price: t.totalPrice(price/1e3, ts),
 		}
 		data = append(data, ar)
+		t.log.TRACE.Println(`PUN data:`, ar.Start, ar.End, ar.Price)
 	}
 
 	data.Sort()
 	return data, nil
 }
-
-func parseFormFields(body io.Reader) (url.Values, error) {
-	data := url.Values{}
-	doc, err := html.Parse(body)
-	if err != nil {
-		return data, err
-	}
-	var f func(*html.Node)
-	f = func(n *html.Node) {
-		if n.Type == html.ElementNode && n.Data == "input" {
-			var inputType, inputName, inputValue string
-			for _, a := range n.Attr {
-				if a.Key == "type" {
-					inputType = a.Val
-				} else if a.Key == "name" {
-					inputName = a.Val
-				} else if a.Key == "value" {
-					inputValue = a.Val
-				}
-			}
-			if inputType == "hidden" && inputName != "" {
-				data.Set(inputName, inputValue)
-			}
-		}
-		for c := n.FirstChild; c != nil; c = c.NextSibling {
-			f(c)
-		}
-	}
-	f(doc)
-	return data, nil
-}