When SerializeJson(doc, file) to a file the last charachters are missing in the file

[chess-levin]

Describe the issue
I'm trying to write a struct with config data to a json file. But the last few characters (always the closing curly brackets) are always missing when I read the file again. My work around is to serialize the data to a json String buffer and then writing this buffer to the file. Why does serializeJson(doc, file) not work correctly?

The complete process looks like this:

1. While setup of ESP a json file with default_config data is read from SPIFFS.
2. The config data is deserialized an stored in config data struct.
3. The user can change config data via web interface.
4. The browser sends data of changed config as json to ESP8266Server.
5. Changed config data is deserialized and written into existing config data struct.
6. Config data struct is serialized and written to a user_config.json file.

Troubleshooter report

ArduinoJson Troubleshooter's report

1. The issue happens at run time
2. The issue concerns serialization
3. Output contains garbage
4. serializeJson() produces garbage
5. Program doesn’t call deserializeJson()
6. Program calls serializeJson(const JsonDocument&, ...)
7. Program uses DynamicJsonDocument
8. Program doesn’t use String

Environment
Here is the environment that I'm using':

• Visual Studio Code & PlatformIO
• wemos d1 mini
• bblanchon/ArduinoJson@^6.19.1
  PLATFORM: Espressif 8266 (3.2.0) > Espressif ESP8266 ESP-12E
  HARDWARE: ESP8266 80MHz, 80KB RAM, 4MB Flash
  PACKAGES:

• framework-arduinoespressif8266 3.30002.0 (3.0.2)
• tool-esptool 1.413.0 (4.13)
• tool-esptoolpy 1.30000.201119 (3.0.0)
• tool-mklittlefs 1.203.210628 (2.3)
• tool-mkspiffs 1.200.0 (2.0)
• toolchain-xtensa 2.100300.210717 (10.3.0)
  LDF: Library Dependency Finder -> https://bit.ly/configure-pio-ldf
  LDF Modes: Finder ~ chain, Compatibility ~ soft

Reproduction
This is the json structure

{
  "wifi": {
    "hostname": "ESP8266",
    "mdns_host": "esp8266",
    "ssid": "XYZABCAND",
    "key": "123456789012345678901234"
  },
  "ntp": {
    "server": "de.pool.ntp.org",
    "timezone": "CET-1CEST,M3.5.0,M10.5.0/3"
  },
  "assigments": {
    "LED1": 16,
    "LED2": 15,
    "LED3": 14,
    "LED4": 2,
    "LED5": 0,
    "PIR1": 12,
    "PIR2": 13
  },
  "pwm_level": {
    "off": 0,
    "low": 25,
    "high": 100
  },
  "lux": {
    "threshold": 5,
    "readInterval": 10
  },
  "dim": {
    "motionTimeout": 60,
    "pwmStepUp": 8,
    "pwmStepDown": 2,
    "stepInterval": 15
  },
  "schedule": {
    "SO": {
      "on": "08:00",
      "off": "23:30"
    },
    "MO": {
      "on": "06:30",
      "off": "23:15"
    },
    "DI": {
      "on": "06:30",
      "off": "23:15"
    },
    "MI": {
      "on": "06:30",
      "off": "23:15"
    },
    "DO": {
      "on": "06:30",
      "off": "23:15"
    },
    "FR": {
      "on": "06:30",
      "off": "23:30"
    },
    "SA": {
      "on": "07:30",
      "off": "23:30"
    }
  }
}

This is the config struct that holds the config data im memory:

struct WifiCfg {
  char hostname[32];
  char mdns_host[32];
  char ssid[64];
  char key[64];
};

struct NtpCfg {
  char server[32];
  char timezone[32];
};

struct DailySwitching {
  char on[6];
  char off[6];
};

struct GpioAssignments {
  uint8_t ledGpioMapping[5];
  uint8_t pirGpioMapping[2];
};


struct Config {
  struct GpioAssignments assignments;
  struct WifiCfg wifiCfg;
  struct NtpCfg ntpCfg;
  struct DailySwitching schedule[7]; 
  int lux_threshold;
  int lux_readInterval;
  int pwm_off;
  int pwm_low;
  int pwm_high;
  int dim_motionTimeout;
  int dim_pwmStepUp;
  int dim_pwmStepDown;
  int dim_stepInterval;
};

extern struct Config globalCfg;

and this my function to serialize the struct into a DynamicJsonDocument:

void serializeConfig(Config &cfg, bool noCredentials, JsonDocument &doc) {
  PRINTSLN(">> serializeConfig >>");

  // wifi
  doc["wifi"]["hostname"] = cfg.wifiCfg.hostname;
  doc["wifi"]["mdns_host"] = cfg.wifiCfg.mdns_host;
  doc["wifi"]["ssid"] = cfg.wifiCfg.ssid;
  doc["wifi"]["key"] = (noCredentials)? KEY_PLACEHOLDER : cfg.wifiCfg.key;

  //ntp
  doc["ntp"]["server"] = cfg.ntpCfg.server;
  doc["ntp"]["timezone"] = cfg.ntpCfg.timezone;

  // gpio assignments
  doc["assigments"]["LED1"]=cfg.assignments.ledGpioMapping[0];
  doc["assigments"]["LED2"]=cfg.assignments.ledGpioMapping[1];
  doc["assigments"]["LED3"]=cfg.assignments.ledGpioMapping[2];
  doc["assigments"]["LED4"]=cfg.assignments.ledGpioMapping[3];
  doc["assigments"]["LED5"]=cfg.assignments.ledGpioMapping[4];
  doc["assigments"]["PIR1"]=cfg.assignments.pirGpioMapping[0];
  doc["assigments"]["PIR2"]=cfg.assignments.pirGpioMapping[1];

  // pwmLevel
  doc["pwm_level"]["off"] = cfg.pwm_off;
  doc["pwm_level"]["low"] = cfg.pwm_low;
  doc["pwm_level"]["high"] = cfg.pwm_high;

  // lux
  doc["lux"]["threshold"] = cfg.lux_threshold;
  doc["lux"]["readInterval"] = cfg.lux_readInterval;

  // dim
  doc["dim"]["motionTimeout"] = cfg.dim_motionTimeout;
  doc["dim"]["pwmStepUp"] = cfg.dim_pwmStepUp;
  doc["dim"]["pwmStepDown"] = cfg.dim_pwmStepDown;
  doc["dim"]["stepInterval"] = cfg.dim_stepInterval;

  // schedule
  for (int i = 0; i < 7; i++)
  {
    doc["schedule"][weekdays[i]]["on"] = cfg.schedule[i].on; 
    doc["schedule"][weekdays[i]]["off"] = cfg.schedule[i].off;
  }

  serializeJsonPretty(doc, Serial);
  PRINTSLN("\n<< serializeConfig() <<");
}

Program output

Expected output:

{"wifi":{"hostname":"ESP8266","mdns_host":"esp8266","ssid":"XYZABCAND","key":"123456789012345678901234"},"ntp":{"server":"de.pool.ntp.org","timezone":"CET-1CEST,M3.5.0,M10.5.0/3"},"assigments":{"LED1":2,"LED2":16,"LED3":0,"LED4":14,"LED5":15,"PIR1":12,"PIR2":13},"pwm_level":{"off":0,"low":25,"high":100},"lux":{"threshold":5,"readInterval":10},"dim":{"motionTimeout":60,"pwmStepUp":8,"pwmStepDown":2,"stepInterval":15},"schedule":{"SO":{"on":"08:00","off":"23:30"},"MO":{"on":"06:30","off":"23:15"},"DI":{"on":"06:30","off":"23:15"},"MI":{"on":"06:30","off":"23:15"},"DO":{"on":"06:30","off":"23:15"},"FR":{"on":"06:30","off":"23:30"},"SA":{"on":"07:30","off":"23:30"}}}

Actual output:

{"wifi":{"hostname":"ESP8266","mdns_host":"esp8266","ssid":"XYZABCAND","key":"123456789012345678901234"},"ntp":{"server":"de.pool.ntp.org","timezone":"CET-1CEST,M3.5.0,M10.5.0/3"},"assigments":{"LED1":2,"LED2":16,"LED3":0,"LED4":14,"LED5":15,"PIR1":12,"PIR2":13},"pwm_level":{"off":0,"low":25,"high":100},"lux":{"threshold":5,"readInterval":10},"dim":{"motionTimeout":60,"pwmStepUp":8,"pwmStepDown":2,"stepInterval":15},"schedule":{"SO":{"on":"08:00","off":"23:30"},"MO":{"on":"06:30","off":"23:15"},"DI":{"on":"06:30","off":"23:15"},"MI":{"on":"06:30","off":"23:15"},"DO":{"on":"06:30","off":"23:15"},"FR":{"on":"06:30","off":"23:30"},"SA":{"on":"07:30","off":"23:30"���

[bblanchon]

Hi @chess-levin,

First, make sure that you call file.close() after serializeJson(doc, file).

This issue may be a duplicate of #1695, which seems to be caused by esp8266/Arduino#8372.
Please downgrade the ESP8266 Core to 2.7.4 and try again.

Best regards,
Benoit

[chess-levin]

Hello @bblanchon, thank you for your Feedback.
Here is my save function that produces the defect file. And yes, I'm using file.close().

bool saveConfig(Config &cfg, const char* fn) {
  PRINTLN("Saving config to ", fn);

  DynamicJsonDocument doc(configJsonDocSize);   // StaticJsonDocument leads to crash
  printConfigStruct(cfg);
  serializeConfig(cfg, false, doc);

  if (SPIFFS.remove(fn)) {
    PRINTLN("Deleted exisiting file ", fn);
  }

  File file = SPIFFS.open(fn, FILE_WRITE);
  if (!file) {
    PRINTLN("Failed to create file ", fn);
    return false;
  }
  
  if (serializeJson(doc, file) == 0) {
    PRINTLN("Failed to write file ", fn);
    return false;
  } 

  file.close();

  PRINTLN("Config successfully written to ", fn);
  
  printDir2Console();

  return true;  
}

If it is a problem with long Strings as mentioned here #1695 why is my workaround successful? I do serialize the whole JSON struct to a String buf and write it to a SPIFFS file. See workarond code below:

bool saveConfig(Config &cfg, const char* fn) {
  PRINTLN("Saving config to ", fn);

  DynamicJsonDocument doc(configJsonDocSize);   // StaticJsonDocument leads to crash
  printConfigStruct(cfg);
  serializeConfig(cfg, false, doc);

  String buf;
  serializeJson(doc, buf);

  if (SPIFFS.remove(fn)) {
    PRINTLN("Deleted exisiting file ", fn);
  }

  File file = SPIFFS.open(fn, FILE_WRITE);
  if (!file) {
    PRINTLN("Failed to create file ", fn);
    return false;
  }
  
  if (!file.print(buf)) {
  //if (serializeJson(doc, file) == 0) {
    PRINTLN("Failed to write file ", fn);
    return false;
  } 

  file.close();

  PRINTLN("Config successfully written to ", fn);
  
  printDir2Console();

  return true;  
}

How do I downgrade the ESP Core Lib? Can't find the dependency in my project platform.ini lib_deps section. I have no idea...

[bblanchon]

To downgrade ESP8266 core to version 2.7.4, set the following value in your platform.ini file:

platform = espressif8266@2.6.2

PlatformIO should download the requested version automatically.

Note that the "platform" version is 2.6.2, and not 2.7.4, as you may expect.
You can see which version of the "core" is embedded in each version of the "platform" by looking at the release notes.