From c96d3726d8af2a74197782619f734484d2d159d8 Mon Sep 17 00:00:00 2001 From: Florian <1technophile@users.noreply.github.com> Date: Tue, 26 Sep 2023 11:26:37 -0500 Subject: [PATCH 1/2] [SYS] Configuration portal for Ethernet board Enable Wifi Manager as a configuration portal for Ethernet boards, also leverage the Wifi Access point as a fallback connectivity method Add link to the WebUI from HA and reduce key length --- docs/upload/portal.md | 24 ++++-- main/User_config.h | 5 +- main/ZmqttDiscovery.ino | 24 ++++-- main/main.ino | 171 ++++++++++++++++++++++------------------ 4 files changed, 128 insertions(+), 96 deletions(-) diff --git a/docs/upload/portal.md b/docs/upload/portal.md index e316ed6bd0..ffbea0a2ee 100644 --- a/docs/upload/portal.md +++ b/docs/upload/portal.md @@ -1,15 +1,19 @@ -# WiFi and MQTT configuration +# Wifi and MQTT configuration -## WiFi and MQTT configuration portal for WiFi boards +## Configuration portal Once loaded into your ESP, and if you don't use the manual configuration, you have to set your network parameters with WiFi Manager portal. -From your smartphone, search for  OpenMQTTGateway or OMG WiFi network and connect to it with the `WifiManager_password` you have defined (the default password is **"your_password"**), or if you are using a device sold by Theengs or the macro `-DWM_PWD_FROM_MAC=true`, the password will be the last eight digits of the device MAC Address with upper case. +From your smartphone, search for  OpenMQTTGateway or OMG_ WiFi network and connect to it with the `WifiManager_password` you have defined (the default password is **"your_password"**), or if you are using a device sold by Theengs or the macro `-DWM_PWD_FROM_MAC=true`, the password will be the last eight digits of the device MAC Address with upper case. Example, the password would be `CCDDEEFF` for a MAC Address `AABBCCDDEEFF`. -For Theengs devices, the MAC Address can be found on the device sticker, and for all the devices, it is printed on the serial monitor logs. +For the Theengs Plug, the MAC Address can be found on the device sticker, and for all the devices, it is printed on the serial monitor logs. Once connected to the WiFi, a web page should appear. On Android, you may also have a popup asking you if you want to connect to it without an internet connection. Answer yes always/all the time. If the web page doesn't appear, click on the WiFi Access Point and "Manage router". +::: tip +For boards with an ethernet port you can also access to the Wifi Manager portal through the LAN board IP address. +::: + ![WiFi manager menu](../img/OpenMQTTGateway_Wifi_Manager_menu.png) * Click on Configure WiFi @@ -17,7 +21,12 @@ Once connected to the WiFi, a web page should appear. On Android, you may also h ![WiFi manager parameters](../img/OpenMQTTGateway_Wifi_Manager_enter_parameters.png) * Select your WiFi -* Set your WiFi password +* Set your WiFi password + +::: note +If the board is connected by ethernet, the Wifi and password can be empty. If you fill them the board will use this wifi as a fallback connectivity method. +::: + * Set your MQTT Server IP * Set your MQTT Server Port (default: 1883) * Set the MQTT secure connection box to select whether or not the connection should be secure @@ -37,12 +46,11 @@ The ESP restart and connect to your network. Note that your credentials are save Once done the gateway should connect to your network and your broker, you should see it into the broker in the form of the following messages: ``` home/OpenMQTTGateway/LWT Online -home/OpenMQTTGateway/version ``` -Note that the web portal appears only on first boot, if you want to configure again the setting you can do a long press on TRIGGER_GPIO or [erase the settings](../use/gateway.md#erase-the-esp-settings). +Note that the web portal appears only on first boot, if you want to configure again the setting you can do a long press on TRIGGER_GPIO, [erase the settings](../use/gateway.md#erase-the-esp-settings) or Reset the configuration from the WebUI. -## MQTT and network configuration for Ethernet and WiFi boards +## Build time configuration You can configure your MQTT server credentials and network configuration before building the application. It can be done either in Arduino or in Platformio IDE thought the User_config.h file. Note that with Platformio IDE you can also set your credentials into the platformio.ini file or an *_env.ini file, here is an example with the Olimex ESP32 gateway: diff --git a/main/User_config.h b/main/User_config.h index a81a566246..a0d607014f 100644 --- a/main/User_config.h +++ b/main/User_config.h @@ -140,9 +140,8 @@ const byte mac[] = {0xDE, 0xED, 0xBA, 0xFE, 0x54, 0x95}; //W5100 ethernet shield /*-------------DEFINE YOUR ADVANCED NETWORK PARAMETERS BELOW----------------*/ //#define MDNS_SD //uncomment if you want to use mDNS for discovering automatically your IP server, please note that mDNS with ESP32 can cause the BLE to not work -#define maxConnectionRetry 10 //maximum MQTT connection attempts before going to wifimanager setup if never connected once -#define maxConnectionRetryWifi 5 //maximum Wifi connection attempts with existing credential at start (used to bypass ESP32 issue on wifi connect) -#define maxRetryWatchDog 11 //maximum Wifi or MQTT re-connection attempts before restarting +#define maxConnectionRetryNetwork 5 //maximum Wifi connection attempts with existing credential at start (used to bypass ESP32 issue on wifi connect) +#define maxRetryWatchDog 11 //maximum Wifi or MQTT re-connection attempts before restarting //set minimum quality of signal so it ignores AP's under that quality #define MinimumWifiSignalQuality 8 diff --git a/main/ZmqttDiscovery.ino b/main/ZmqttDiscovery.ino index 92ecb937cb..8b0cddf349 100644 --- a/main/ZmqttDiscovery.ino +++ b/main/ZmqttDiscovery.ino @@ -346,7 +346,7 @@ void createDiscovery(const char* sensor_type, StaticJsonDocument jsonDeviceBuffer; JsonObject device = jsonDeviceBuffer.to(); - JsonArray identifiers = device.createNestedArray("identifiers"); + JsonArray identifiers = device.createNestedArray("ids"); if (gateway_entity) { //device representing the board @@ -354,17 +354,25 @@ void createDiscovery(const char* sensor_type, # ifndef GATEWAY_MODEL String model = ""; serializeJson(modules, model); - device["model"] = model; + device["mdl"] = model; # else - device["model"] = GATEWAY_MODEL; + device["mdl"] = GATEWAY_MODEL; # endif - device["manufacturer"] = GATEWAY_MANUFACTURER; - device["sw_version"] = OMG_VERSION; + device["mf"] = GATEWAY_MANUFACTURER; + if (ethConnected) { +# ifdef ESP32_ETHERNET + device["cu"] = String("http://") + String(ETH.localIP().toString()) + String("/"); //configuration_url +# endif + } else { + device["cu"] = String("http://") + String(WiFi.localIP().toString()) + String("/"); //configuration_url + } + + device["sw"] = OMG_VERSION; identifiers.add(String(getMacAddress())); } else { //The Connections if (device_id[0]) { - JsonArray connections = device.createNestedArray("connections"); + JsonArray connections = device.createNestedArray("cns"); JsonArray connection_mac = connections.createNestedArray(); connection_mac.add("mac"); connection_mac.add(device_id); @@ -374,11 +382,11 @@ void createDiscovery(const char* sensor_type, } if (device_manufacturer[0]) { - device["manufacturer"] = device_manufacturer; + device["mf"] = device_manufacturer; } if (device_model[0]) { - device["model"] = device_model; + device["mdl"] = device_model; } // generate unique device name by adding the second half of the device_id only if device_name and device_id are different diff --git a/main/main.ino b/main/main.ino index b76fc787c5..b345436b0d 100644 --- a/main/main.ino +++ b/main/main.ino @@ -78,6 +78,7 @@ bool ready_to_sleep = false; StaticJsonDocument modulesBuffer; JsonArray modules = modulesBuffer.to(); +bool ethConnected = false; #ifndef ZgatewayGFSunInverter // Arduino IDE compiles, it automatically creates all the header declarations for all the functions you have in your *.ino file. @@ -264,7 +265,6 @@ bool ProcessLock = false; // Process lock when we want to use a critical functio # ifdef ESP32_ETHERNET # include void WiFiEvent(WiFiEvent_t event); -static bool esp32EthConnected = false; # endif # include @@ -952,18 +952,27 @@ void setup() { modules.add(ZactuatorONOFF); # endif -# ifdef ESP32_ETHERNET - setup_ethernet_esp32(); -# else // WIFI ESP -# if defined(ESPWifiManualSetup) +# if defined(ESPWifiManualSetup) setup_wifi(); -# else - // In failSafeMode we don't want to setup wifi manager as it has already been done before - if (!failSafeMode) setup_wifimanager(false); +# else + if (loadConfigFromFlash()) { + Log.notice(F("Config loaded from flash" CR)); +# ifdef ESP32_ETHERNET + setup_ethernet_esp32(); +# endif + if (!failSafeMode && !ethConnected) setup_wifimanager(false); + } else { +# ifdef ESP32_ETHERNET + setup_ethernet_esp32(); # endif + Log.notice(F("No config in flash, launching wifi manager" CR)); + // In failSafeMode we don't want to setup wifi manager as it has already been done before + if (!failSafeMode) setup_wifimanager(false); + } + +# endif Log.trace(F("OpenMQTTGateway mac: %s" CR), WiFi.macAddress().c_str()); Log.trace(F("OpenMQTTGateway ip: %s" CR), WiFi.localIP().toString().c_str()); -# endif setOTA(); #else // In case of arduino platform @@ -1189,9 +1198,9 @@ void setup() { bool wifi_reconnect_bypass() { uint8_t wifi_autoreconnect_cnt = 0; # ifdef ESP32 - while (WiFi.status() != WL_CONNECTED && wifi_autoreconnect_cnt < maxConnectionRetryWifi) { + while (WiFi.status() != WL_CONNECTED && wifi_autoreconnect_cnt < maxConnectionRetryNetwork) { # else - while (WiFi.waitForConnectResult() != WL_CONNECTED && wifi_autoreconnect_cnt < maxConnectionRetryWifi) { + while (WiFi.waitForConnectResult() != WL_CONNECTED && wifi_autoreconnect_cnt < maxConnectionRetryNetwork) { # endif Log.notice(F("Attempting Wifi connection with saved AP: %d" CR), wifi_autoreconnect_cnt); @@ -1202,7 +1211,7 @@ bool wifi_reconnect_bypass() { delay(1000); wifi_autoreconnect_cnt++; } - if (wifi_autoreconnect_cnt < maxConnectionRetryWifi) { + if (wifi_autoreconnect_cnt < maxConnectionRetryNetwork) { return true; } else { return false; @@ -1316,6 +1325,10 @@ void setupTLS(bool self_signed, uint8_t index) { # endif } } +#else +bool wifi_reconnect_bypass() { + return true; +} #endif /* @@ -1372,7 +1385,7 @@ void setup_wifi() { failure_number_ntwk++; # if defined(ESP32) && defined(ZgatewayBT) if (lowpowermode) { - if (failure_number_ntwk > maxConnectionRetryWifi) { + if (failure_number_ntwk > maxConnectionRetryNetwork) { lowPowerESP32(); } } else { @@ -1494,28 +1507,9 @@ void saveMqttConfig() { configFile.close(); } -void setup_wifimanager(bool reset_settings) { - delay(10); - WiFi.mode(WIFI_STA); - - if (reset_settings) - eraseAndRestart(); - - String s = WiFi.macAddress(); -# ifdef USE_MAC_AS_GATEWAY_NAME - sprintf(gateway_name, "%.2s%.2s%.2s%.2s%.2s%.2s", - s.c_str(), s.c_str() + 3, s.c_str() + 6, s.c_str() + 9, s.c_str() + 12, s.c_str() + 15); - snprintf(WifiManager_ssid, MAC_NAME_MAX_LEN, "%s_%.2s%.2s", Gateway_Short_Name, s.c_str(), s.c_str() + 3); - strcpy(ota_hostname, WifiManager_ssid); - Log.notice(F("OTA Hostname: %s.local" CR), ota_hostname); -# endif -# ifdef WM_PWD_FROM_MAC // From ESP Mac Address, last 8 digits as the password - sprintf(WifiManager_password, "%.2s%.2s%.2s%.2s", - s.c_str() + 6, s.c_str() + 9, s.c_str() + 12, s.c_str() + 15); -# endif - - //read configuration from FS json +bool loadConfigFromFlash() { Log.trace(F("mounting FS..." CR)); + bool result = false; if (SPIFFS.begin()) { Log.trace(F("mounted file system" CR)); @@ -1561,12 +1555,35 @@ void setup_wifimanager(bool reset_settings) { strcpy(ota_pass, json["ota_pass"]); if (json.containsKey("ota_server_cert")) ota_server_cert = json["ota_server_cert"].as(); + result = true; } else { Log.warning(F("failed to load json config" CR)); } configFile.close(); } } + return result; +} + +void setup_wifimanager(bool reset_settings) { + delay(10); + WiFi.mode(WIFI_STA); + + if (reset_settings) + eraseAndRestart(); + + String s = WiFi.macAddress(); +# ifdef USE_MAC_AS_GATEWAY_NAME + sprintf(gateway_name, "%.2s%.2s%.2s%.2s%.2s%.2s", + s.c_str(), s.c_str() + 3, s.c_str() + 6, s.c_str() + 9, s.c_str() + 12, s.c_str() + 15); + snprintf(WifiManager_ssid, MAC_NAME_MAX_LEN, "%s_%.2s%.2s", Gateway_Short_Name, s.c_str(), s.c_str() + 3); + strcpy(ota_hostname, WifiManager_ssid); + Log.notice(F("OTA Hostname: %s.local" CR), ota_hostname); +# endif +# ifdef WM_PWD_FROM_MAC // From ESP Mac Address, last 8 digits as the password + sprintf(WifiManager_password, "%.2s%.2s%.2s%.2s", + s.c_str() + 6, s.c_str() + 9, s.c_str() + 12, s.c_str() + 15); +# endif wifiManager.setDebugOutput(WM_DEBUG_LEVEL); @@ -1623,6 +1640,10 @@ void setup_wifimanager(bool reset_settings) { //set minimum quality of signal so it ignores AP's under that quality wifiManager.setMinimumSignalQuality(MinimumWifiSignalQuality); +# ifdef ESP32_ETHERNET + wifiManager.setBreakAfterConfig(true); // If ethernet is used, we don't want to block the connection by keeping the portal up +# endif + if (!wifi_reconnect_bypass()) // if we didn't connect with saved credential we start Wifimanager web portal { # ifdef ESP32 @@ -1643,21 +1664,23 @@ void setup_wifimanager(bool reset_settings) { //if it does not connect it starts an access point with the specified name //and goes into a blocking loop awaiting configuration if (!wifiManager.autoConnect(WifiManager_ssid, WifiManager_password)) { - Log.warning(F("failed to connect and hit timeout" CR)); - delay(3000); + if (!ethConnected) { // If we are using ethernet, we don't want to restart the ESP + Log.warning(F("failed to connect and hit timeout" CR)); + delay(3000); # ifdef ESP32 - /* Workaround for bug in arduino core that causes the AP to become unsecure on reboot */ - esp_wifi_set_mode(WIFI_MODE_AP); - esp_wifi_start(); - wifi_config_t conf; - esp_wifi_get_config(WIFI_IF_AP, &conf); - conf.ap.ssid_hidden = 1; - esp_wifi_set_config(WIFI_IF_AP, &conf); + /* Workaround for bug in arduino core that causes the AP to become unsecure on reboot */ + esp_wifi_set_mode(WIFI_MODE_AP); + esp_wifi_start(); + wifi_config_t conf; + esp_wifi_get_config(WIFI_IF_AP, &conf); + conf.ap.ssid_hidden = 1; + esp_wifi_set_config(WIFI_IF_AP, &conf); # endif - //restart and try again - ESPRestart(3); + //restart and try again + ESPRestart(3); + } } InfoIndicatorOFF(); ErrorIndicatorOFF(); @@ -1721,9 +1744,10 @@ void setup_ethernet_esp32() { ethBeginSuccess = ETH.begin(); # endif Log.trace(F("Connecting to Ethernet" CR)); - while (!esp32EthConnected) { + while (!ethConnected && failure_number_ntwk <= maxConnectionRetryNetwork) { delay(500); Log.trace(F("." CR)); + failure_number_ntwk++; } } @@ -1740,15 +1764,15 @@ void WiFiEvent(WiFiEvent_t event) { Log.trace(F("OpenMQTTGateway MAC: %s" CR), ETH.macAddress().c_str()); Log.trace(F("OpenMQTTGateway IP: %s" CR), ETH.localIP().toString().c_str()); Log.trace(F("OpenMQTTGateway link speed: %d Mbps" CR), ETH.linkSpeed()); - esp32EthConnected = true; + ethConnected = true; break; case ARDUINO_EVENT_ETH_DISCONNECTED: Log.error(F("Ethernet Disconnected" CR)); - esp32EthConnected = false; + ethConnected = false; break; case ARDUINO_EVENT_ETH_STOP: Log.error(F("Ethernet Stopped" CR)); - esp32EthConnected = false; + ethConnected = false; break; default: break; @@ -1824,11 +1848,10 @@ void loop() { } #if defined(ESP8266) || defined(ESP32) -# ifdef ESP32_ETHERNET - if (esp32EthConnected) { -# else - if (WiFi.status() == WL_CONNECTED) { -# endif + if (ethConnected || WiFi.status() == WL_CONNECTED) { + if (ethConnected && WiFi.status() == WL_CONNECTED) { + WiFi.disconnect(); // we disconnect the wifi as we are connected to ethernet + } ArduinoOTA.handle(); #else if ((Ethernet.hardwareStatus() != EthernetW5100 && Ethernet.linkStatus() == LinkON) || (Ethernet.hardwareStatus() == EthernetW5100)) { //we are able to detect disconnection only on w5200 and w5500 @@ -2008,7 +2031,7 @@ void loop() { esp_task_wdt_reset(); #endif connected = false; - Log.warning(F("Network disconnected:" CR)); + Log.warning(F("Network disconnected" CR)); ErrorIndicatorON(); delay(2000); // add a delay to avoid ESP32 crash and reset #ifdef ESP32 @@ -2016,14 +2039,7 @@ void loop() { #endif ErrorIndicatorOFF(); delay(2000); -#if defined(ESP8266) || defined(ESP32) && !defined(ESP32_ETHERNET) -# ifdef ESP32 // If used with ESP8266 this method prevent the reconnection - WiFi.reconnect(); -# endif - Log.warning(F("wifi" CR)); -#else - Log.warning(F("ethernet" CR)); -#endif + wifi_reconnect_bypass(); } // Function that doesn't need an active connection #if defined(ZboardM5STICKC) || defined(ZboardM5STICKCP) || defined(ZboardM5STACK) || defined(ZboardM5TOUGH) @@ -2169,22 +2185,23 @@ String stateMeasures() { # endif SYSdata["freestack"] = uxTaskGetStackHighWaterMark(NULL); # endif + + SYSdata["ethernet"] = ethConnected; + if (ethConnected) { # ifdef ESP32_ETHERNET - SYSdata["mac"] = (char*)ETH.macAddress().c_str(); - SYSdata["ip"] = ip2CharArray(ETH.localIP()); - ETH.fullDuplex() ? SYSdata["fd"] = (bool)"true" : SYSdata["fd"] = (bool)"false"; - SYSdata["linkspeed"] = (int)ETH.linkSpeed(); -# else - long rssi = WiFi.RSSI(); - SYSdata["rssi"] = rssi; - String SSID = WiFi.SSID(); - SYSdata["SSID"] = SSID; - String BSSID = WiFi.BSSIDstr(); - SYSdata["BSSID"] = BSSID; - SYSdata["ip"] = ip2CharArray(WiFi.localIP()); - String mac = WiFi.macAddress(); - SYSdata["mac"] = (char*)mac.c_str(); + SYSdata["mac"] = (char*)ETH.macAddress().c_str(); + SYSdata["ip"] = ip2CharArray(ETH.localIP()); + ETH.fullDuplex() ? SYSdata["fd"] = (bool)"true" : SYSdata["fd"] = (bool)"false"; + SYSdata["linkspeed"] = (int)ETH.linkSpeed(); # endif + } else { + SYSdata["rssi"] = (long)WiFi.RSSI(); + SYSdata["SSID"] = (char*)WiFi.SSID().c_str(); + SYSdata["BSSID"] = (char*)WiFi.BSSIDstr().c_str(); + SYSdata["ip"] = ip2CharArray(WiFi.localIP()); + SYSdata["mac"] = (char*)WiFi.macAddress().c_str(); + } + # endif # ifdef ZgatewayBT # ifdef ESP32 From 2eb1c391263487423a7186bf9514b332ace5f500 Mon Sep 17 00:00:00 2001 From: Florian <1technophile@users.noreply.github.com> Date: Thu, 28 Sep 2023 14:54:17 -0500 Subject: [PATCH 2/2] Remove unnecessary environment Now that one environment cans support wifi and ethernet, merge the 2 --- .github/workflows/build.yml | 1 - environments.ini | 31 +------------------------------ platformio.ini | 1 - 3 files changed, 1 insertion(+), 32 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8fc4fcb087..912ad07ca3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -77,7 +77,6 @@ jobs: - "airm2m_core_esp32c3" - "lolin_c3_mini" - "thingpulse-espgateway" - - "thingpulse-espgateway-ethernet" runs-on: ubuntu-latest name: Build with PlatformIO steps: diff --git a/environments.ini b/environments.ini index dcb8db0b49..7f5d08576c 100644 --- a/environments.ini +++ b/environments.ini @@ -1623,34 +1623,6 @@ build_flags = '-DZgatewayBT="BT"' '-DLED_SEND_RECEIVE=2' '-DLED_SEND_RECEIVE_ON=0' - '-DpubBLEServiceUUID=true' - '-DGateway_Name="OMG_THINGPULSE_BLE"' - '-DANEOPIX_IND_DATA_GPIO=32' - '-DANEOPIX_IND_NUM_LEDS=4' - '-DANEOPIX_INFO_LED=0' - '-DANEOPIX_SEND_RECEIVE_LED=1' - '-DANEOPIX_ERROR_LED=2' - '-DANEOPIX_BRIGHTNESS=255' - '-DRGB_INDICATORS=true' -; '-DsimplePublishing=true' -custom_description = BLE Gateway using Wifi -custom_hardware = ThingPulse gateway single ESP32 - -[env:thingpulse-espgateway-ethernet] -platform = ${com.esp32_platform} -board = esp32dev -board_build.partitions = min_spiffs.csv -lib_deps = - ${com-esp32.lib_deps} - ${libraries.ble} - ${libraries.adafruit_neopixel} - ${libraries.decoder} -build_flags = - ${com-esp32.build_flags} - '-DZgatewayBT="BT"' - '-DLED_SEND_RECEIVE=2' - '-DLED_SEND_RECEIVE_ON=0' - '-DpubBLEServiceUUID=true' '-DGateway_Name="OMG_THINGPULSE_ETH_BLE"' '-DESP32_ETHERNET=true' '-DETH_PHY_TYPE=ETH_PHY_LAN8720' @@ -1667,6 +1639,5 @@ build_flags = '-DANEOPIX_BRIGHTNESS=255' '-DRGB_INDICATORS=true' ; '-DsimplePublishing=true' -custom_description = BLE Gateway using ethernet, requires PIO configuration +custom_description = BLE Gateway using ethernet or wifi with external antenna custom_hardware = ThingPulse gateway single ESP32 - diff --git a/platformio.ini b/platformio.ini index a634df017f..a1cdd34dc0 100644 --- a/platformio.ini +++ b/platformio.ini @@ -100,7 +100,6 @@ extra_configs = ;default_envs = esp32c3-dev-c2-ble ;default_envs = lolin_c3_mini ;default_envs = thingpulse-espgateway -;default_envs = thingpulse-espgateway-ethernet ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ; ENVIRONMENTS PARAMETERS ;