Large time consumption in Julia deserialization (to read time series data?)

[joaquimg]

I set HiGHS gap limit to a very low tolerance of 0.9 and ran the Sienna performance test.
The julia deserialization function is taking 50%+ of the total time.
Hence, the main bottleneck by far.

I just ran the following:

# Extracted from:
# https://github.com/NREL-Sienna/PowerSimulations.jl/blob/main/test/performance/performance_test.jl

if isempty(ARGS)
    push!(ARGS, "Sienna01")
end

precompile_time = @timed using PowerSimulations

using PowerSimulations
const PSI = PowerSimulations
using PowerSystems
const PSY = PowerSystems
using Logging
using PowerSystemCaseBuilder
using PowerNetworkMatrices
using HydroPowerSimulations
using HiGHS
using Dates


using Profile
using PProf

@info pkgdir(PowerSimulations)

open("precompile_time.txt", "a") do io
    write(io, "| $(ARGS[1]) | $(precompile_time.time) |\n")
end


sys_rts_da = build_system(PSISystems, "modified_RTS_GMLC_DA_sys")
sys_rts_rt = build_system(PSISystems, "modified_RTS_GMLC_RT_sys")
sys_rts_realization = build_system(PSISystems, "modified_RTS_GMLC_realization_sys")

for i in 2:2
    template_uc = ProblemTemplate(
        NetworkModel(
            PTDFPowerModel;
            use_slacks = true,
            PTDF_matrix = PTDF(sys_rts_da),
            duals = [CopperPlateBalanceConstraint],
        ),
    )

    set_device_model!(template_uc, ThermalMultiStart, ThermalCompactUnitCommitment)
    set_device_model!(template_uc, ThermalStandard, ThermalCompactUnitCommitment)
    set_device_model!(template_uc, RenewableDispatch, RenewableFullDispatch)
    set_device_model!(template_uc, PowerLoad, StaticPowerLoad)
    set_device_model!(template_uc, DeviceModel(Line, StaticBranch))
    set_device_model!(template_uc, Transformer2W, StaticBranchUnbounded)
    set_device_model!(template_uc, TapTransformer, StaticBranchUnbounded)
    set_device_model!(template_uc, HydroDispatch, FixedOutput)
    set_device_model!(template_uc, HydroEnergyReservoir, HydroDispatchRunOfRiver)
    set_service_model!(
        template_uc,
        ServiceModel(VariableReserve{ReserveUp}, RangeReserve),
    )
    set_service_model!(
        template_uc,
        ServiceModel(VariableReserve{ReserveDown}, RangeReserve),
    )

    template_ed = deepcopy(template_uc)
    set_device_model!(template_ed, ThermalMultiStart, ThermalBasicDispatch)
    set_device_model!(template_ed, ThermalStandard, ThermalBasicDispatch)
    set_device_model!(template_ed, HydroDispatch, HydroDispatchRunOfRiver)
    set_device_model!(template_ed, HydroEnergyReservoir, HydroDispatchRunOfRiver)

    template_em = deepcopy(template_ed)
    set_device_model!(template_ed, Line, StaticBranchUnbounded)
    empty!(template_em.services)

    models = SimulationModels(;
        decision_models = [
            DecisionModel(
                template_uc,
                sys_rts_da;
                name = "UC",
                optimizer = optimizer_with_attributes(HiGHS.Optimizer,
                    "mip_rel_gap" => 0.9),
                    # "mip_rel_gap" => 0.01),
                system_to_file = false,
                initialize_model = true,
                optimizer_solve_log_print = true,
                direct_mode_optimizer = true,
                check_numerical_bounds = false,
            ),
            DecisionModel(
                template_ed,
                sys_rts_rt;
                name = "ED",
                optimizer = optimizer_with_attributes(HiGHS.Optimizer,
                    "mip_rel_gap" => 0.9),
                    # "mip_rel_gap" => 0.01),
                system_to_file = false,
                initialize_model = true,
                check_numerical_bounds = false,
                #export_pwl_vars = true,
                optimizer_solve_log_print = true,
            ),
        ],
        emulation_model = EmulationModel(
            template_em,
            sys_rts_realization;
            name = "PF",
            optimizer = optimizer_with_attributes(HiGHS.Optimizer),
            optimizer_solve_log_print = true,
        ),
    )

    sequence = SimulationSequence(;
        models = models,
        feedforwards = Dict(
            "ED" => [
                SemiContinuousFeedforward(;
                    component_type = ThermalStandard,
                    source = OnVariable,
                    affected_values = [ActivePowerVariable],
                ),
            ],
            "PF" => [
                SemiContinuousFeedforward(;
                    component_type = ThermalStandard,
                    source = OnVariable,
                    affected_values = [ActivePowerVariable],
                ),
            ],
        ),
        ini_cond_chronology = InterProblemChronology(),
    )

    sim = Simulation(;
        name = "compact_sim",
        steps = 1,
        # steps = 3,
        models = models,
        sequence = sequence,
        initial_time = DateTime("2020-01-01T00:00:00"),
        simulation_folder = mktempdir(; cleanup = true),
    )

    build_out, time_build, _, _ =
        @timed build!(sim; console_level = Logging.Error, serialize = false)

    if build_out == PSI.SimulationBuildStatus.BUILT
        name = i > 1 ? "Postcompile" : "Precompile"
        open("build_time.txt", "a") do io
            write(io, "| $(ARGS[1])-Build Time $name | $(time_build) |\n")
        end
    else
        open("build_time.txt", "a") do io
            write(io, "| $(ARGS[1])- Build Time $name | FAILED TO TEST |\n")
        end
    end

    Profile.clear()
@profile solve_out, time_solve, _, _ = @timed execute!(sim; enable_progress_bar = false)

    # if solve_out == PSI.RunStatus.SUCCESSFULLY_FINALIZED
    #     name = i > 1 ? "Postcompile" : "Precompile"
    #     open("solve_time.txt", "a") do io
    #         write(io, "| $(ARGS[1])-Solve Time $name | $(time_solve) |\n")
    #     end
    # else
    #     open("solve_time.txt", "a") do io
    #         write(io, "| $(ARGS[1])- Solve Time $name | FAILED TO TEST |\n")
    #     end
    # end
end


pprof()

and got the flame graph:

In particular:

My versions:

(sienna_param) pkg> st
Status `C:\Users\joaquimgarcia\Desktop\BT\sketch\sienna_param\Project.toml`
  [31a5f54b] Debugger v0.7.10
  [87dc4568] HiGHS v1.10.2
  [fc1677e0] HydroPowerSimulations v0.9.3
  [4076af6c] JuMP v1.23.3
  [98e50ef6] JuliaFormatter v1.0.62
  [b8f27783] MathOptInterface v1.33.0
  [e4faabce] PProf v3.1.3
  [0ce4ce61] ParametricOptInterface v0.8.2
  [bed98974] PowerNetworkMatrices v0.11.1
  [e690365d] PowerSimulations v0.28.3 at Main branch today
  [f00506e0] PowerSystemCaseBuilder v1.3.7
  [bcd98974] PowerSystems v4.4.0
  [295af30f] Revise v3.6.2

At Julia 1.10.5

[daniel-thom]

The linked PowerSimulations issue partially solves this problem. There are additional issues in InfrastructureSystems and I'll post more soon.

[jd-lara]

we should run again the code in PSI main and this to see what the results are

[jd-lara]

@daniel-thom This is a massive improvement

zooming into the update parameters, we have this