Skip to content

Commit 1d32682

Browse files
psafontpsafont
authored
Date: Accept all valid timezones from client, allow sub-second precision (#5950)
The Date module has a very convoluted way to deal with timezones because of its historic clients. While we can't remove all the issues, we can remove most of them. - Dates (timestamps, really) without timezones do not contain a frame of reference, and hence placing them in the UTC timeline can't be done accurately in the general case. This means that comparing timestamps gives incorrect results. - XMLRPC enforces dates to be encoded in ISO8601format. This means timestamps can lack a timezone. - Xapi uses xmlrpc's dates, adding this unfortunate limitation. - While Date allows these timestamps, it assumes that these are in the UTC timezone. On top of that, it refuses to process any timestamps that are in any timezone other than UTC (`Z`) - The Date module tries really hard to hide the timezone of timestamps that lack it when printing them. This means that timezoneless timestamps can persist in the database, for no good reason, as they are treated as UTC timestamps. - Host.localtime is the only call that returns timezoneless timestamps, all the other calls correctly return timestamps in the UTC timezone. Because the call on purpose does not want to return the current time in UTC, changing this might break clients not ready to process any timezone, mainly SDK-built ones #5802 - Dates are stored as a tuple of date, hour, minutes, seconds. This has very limited precision, which might be unexpected. This PR does the following mitigation / fixes: - Document all calls with Datetimes as parameters that the timestamps will be interpreted as UTC ones if they miss the timezone. - Remove the limitation to process any valid timezone - Timezoneless timstamps are immediately processed as UTC timestamps, as refusing these timestamps can break clients. Issues that the PR does not fix - Host.localtime produces timestamps without timezones, this is needed as adding non-zero timestamps breaks the SDK clients. - The server does not reject timezoneless timestamp, this might break migrations, RPUs, or normal clients, so I've held back on this change. - Printed timestamp do _not_ retain sub-second precision Drafting as tests are ongoing
2 parents 85f9271 + 96195f7 commit 1d32682

File tree

95 files changed

+526
-384
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

95 files changed

+526
-384
lines changed

‎doc/content/xapi/storage/sxm.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -450,8 +450,8 @@ but we've still got a bit of thinking to do: we sort the VDIs to copy based on a
450450
let compare_fun v1 v2 =
451451
let r = Int64.compare v1.size v2.size in
452452
if r = 0 then
453-
let t1 = Date.to_float (Db.VDI.get_snapshot_time ~__context ~self:v1.vdi) in
454-
let t2 = Date.to_float (Db.VDI.get_snapshot_time ~__context ~self:v2.vdi) in
453+
let t1 = Date.to_unix_time (Db.VDI.get_snapshot_time ~__context ~self:v1.vdi) in
454+
let t2 = Date.to_unix_time (Db.VDI.get_snapshot_time ~__context ~self:v2.vdi) in
455455
compare t1 t2
456456
else r in
457457
let all_vdis = all_vdis |> List.sort compare_fun in

‎ocaml/alerts/expiry_alert.ml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,15 +50,15 @@ let all_messages rpc session_id =
5050

5151
let message_body msg expiry =
5252
Printf.sprintf "<body><message>%s</message><date>%s</date></body>" msg
53-
(Date.to_string expiry)
53+
(Date.to_rfc3339 expiry)
5454

5555
let expired_message obj = Printf.sprintf "%s has expired." obj
5656

5757
let expiring_message obj = Printf.sprintf "%s is expiring soon." obj
5858

5959
let maybe_generate_alert now obj_description alert_conditions expiry =
6060
let remaining_days =
61-
days_until_expiry (Date.to_float now) (Date.to_float expiry)
61+
days_until_expiry (Date.to_unix_time now) (Date.to_unix_time expiry)
6262
in
6363
alert_conditions
6464
|> List.sort (fun (a, _) (b, _) -> compare a b)

‎ocaml/idl/datamodel.ml

Lines changed: 42 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -217,9 +217,8 @@ module Session = struct
217217
session instance has is_local_superuser set, then the value of \
218218
this field is undefined."
219219
; field ~in_product_since:rel_george ~qualifier:DynamicRO
220-
~default_value:(Some (VDateTime (Date.of_float 0.)))
221-
~ty:DateTime "validation_time"
222-
"time when session was last validated"
220+
~default_value:(Some (VDateTime Date.epoch)) ~ty:DateTime
221+
"validation_time" "time when session was last validated"
223222
; field ~in_product_since:rel_george ~qualifier:DynamicRO
224223
~default_value:(Some (VString "")) ~ty:String "auth_user_sid"
225224
"the subject identifier of the user that was externally \
@@ -3895,9 +3894,11 @@ module VDI = struct
38953894
; {
38963895
param_type= DateTime
38973896
; param_name= "snapshot_time"
3898-
; param_doc= "Storage-specific config"
3897+
; param_doc=
3898+
"Storage-specific config. When the timezone is missing, UTC is \
3899+
assumed"
38993900
; param_release= tampa_release
3900-
; param_default= Some (VDateTime Date.never)
3901+
; param_default= Some (VDateTime Date.epoch)
39013902
}
39023903
; {
39033904
param_type= Ref _vdi
@@ -4089,7 +4090,11 @@ module VDI = struct
40894090
~params:
40904091
[
40914092
(Ref _vdi, "self", "The VDI to modify")
4092-
; (DateTime, "value", "The snapshot time of this VDI.")
4093+
; ( DateTime
4094+
, "value"
4095+
, "The snapshot time of this VDI. When the timezone is missing, UTC \
4096+
is assumed"
4097+
)
40934098
]
40944099
~flags:[`Session] ~doc:"Sets the snapshot time of this VDI."
40954100
~hide_from_docs:true ~allowed_roles:_R_LOCAL_ROOT_ONLY ()
@@ -4468,7 +4473,7 @@ module VDI = struct
44684473
~ty:(Set (Ref _vdi)) ~doc_tags:[Snapshots] "snapshots"
44694474
"List pointing to all the VDIs snapshots."
44704475
; field ~in_product_since:rel_orlando
4471-
~default_value:(Some (VDateTime Date.never)) ~qualifier:DynamicRO
4476+
~default_value:(Some (VDateTime Date.epoch)) ~qualifier:DynamicRO
44724477
~ty:DateTime ~doc_tags:[Snapshots] "snapshot_time"
44734478
"Date/time when this snapshot was created."
44744479
; field ~writer_roles:_R_VM_OP ~in_product_since:rel_orlando
@@ -4752,7 +4757,7 @@ module VBD_metrics = struct
47524757
uid _vbd_metrics
47534758
; namespace ~name:"io" ~contents:iobandwidth ()
47544759
; field ~qualifier:DynamicRO ~ty:DateTime
4755-
~default_value:(Some (VDateTime Date.never))
4760+
~default_value:(Some (VDateTime Date.epoch))
47564761
~lifecycle:
47574762
[
47584763
(Published, rel_rio, "")
@@ -5511,7 +5516,11 @@ module VMPP = struct
55115516
~params:
55125517
[
55135518
(Ref _vmpp, "self", "The protection policy")
5514-
; (DateTime, "value", "the value to set")
5519+
; ( DateTime
5520+
, "value"
5521+
, "When was the last backup was done. When the timezone is missing, \
5522+
UTC is assumed"
5523+
)
55155524
]
55165525
()
55175526

@@ -5521,7 +5530,11 @@ module VMPP = struct
55215530
~params:
55225531
[
55235532
(Ref _vmpp, "self", "The protection policy")
5524-
; (DateTime, "value", "the value to set")
5533+
; ( DateTime
5534+
, "value"
5535+
, "When was the last archive was done. When the timezone is missing, \
5536+
UTC is assumed"
5537+
)
55255538
]
55265539
()
55275540

@@ -5669,7 +5682,7 @@ module VMPP = struct
56695682
"true if this protection policy's backup is running"
56705683
; field ~lifecycle:removed ~qualifier:DynamicRO ~ty:DateTime
56715684
"backup_last_run_time" "time of the last backup"
5672-
~default_value:(Some (VDateTime (Date.of_float 0.)))
5685+
~default_value:(Some (VDateTime Date.epoch))
56735686
; field ~lifecycle:removed ~qualifier:StaticRO ~ty:archive_target_type
56745687
"archive_target_type" "type of the archive target config"
56755688
~default_value:(Some (VEnum "none"))
@@ -5693,7 +5706,7 @@ module VMPP = struct
56935706
"true if this protection policy's archive is running"
56945707
; field ~lifecycle:removed ~qualifier:DynamicRO ~ty:DateTime
56955708
"archive_last_run_time" "time of the last archive"
5696-
~default_value:(Some (VDateTime (Date.of_float 0.)))
5709+
~default_value:(Some (VDateTime Date.epoch))
56975710
; field ~lifecycle:removed ~qualifier:DynamicRO ~ty:(Set (Ref _vm))
56985711
"VMs" "all VMs attached to this protection policy"
56995712
; field ~lifecycle:removed ~qualifier:StaticRO ~ty:Bool
@@ -5782,7 +5795,11 @@ module VMSS = struct
57825795
~params:
57835796
[
57845797
(Ref _vmss, "self", "The snapshot schedule")
5785-
; (DateTime, "value", "the value to set")
5798+
; ( DateTime
5799+
, "value"
5800+
, "When was the schedule was last run. When a timezone is missing, \
5801+
UTC is assumed"
5802+
)
57865803
]
57875804
()
57885805

@@ -5856,7 +5873,7 @@ module VMSS = struct
58565873
~default_value:(Some (VMap []))
58575874
; field ~qualifier:DynamicRO ~ty:DateTime "last_run_time"
58585875
"time of the last snapshot"
5859-
~default_value:(Some (VDateTime (Date.of_float 0.)))
5876+
~default_value:(Some (VDateTime Date.epoch))
58605877
; field ~qualifier:DynamicRO ~ty:(Set (Ref _vm)) "VMs"
58615878
"all VMs attached to this snapshot schedule"
58625879
]
@@ -6311,15 +6328,24 @@ module Message = struct
63116328
[
63126329
(cls, "cls", "The class of object")
63136330
; (String, "obj_uuid", "The uuid of the object")
6314-
; (DateTime, "since", "The cutoff time")
6331+
; ( DateTime
6332+
, "since"
6333+
, "The cutoff time. When the timezone is missing, UTC is assumed"
6334+
)
63156335
]
63166336
~flags:[`Session]
63176337
~result:(Map (Ref _message, Record _message), "The relevant messages")
63186338
~allowed_roles:_R_READ_ONLY ()
63196339

63206340
let get_since =
63216341
call ~name:"get_since" ~in_product_since:rel_orlando
6322-
~params:[(DateTime, "since", "The cutoff time")]
6342+
~params:
6343+
[
6344+
( DateTime
6345+
, "since"
6346+
, "The cutoff time. When the timezone is missing, UTC is assumed"
6347+
)
6348+
]
63236349
~flags:[`Session]
63246350
~result:(Map (Ref _message, Record _message), "The relevant messages")
63256351
~allowed_roles:_R_READ_ONLY ()

‎ocaml/idl/datamodel_certificate.ml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,10 +59,10 @@ let t =
5959
~default_value:(Some (VRef null_ref))
6060
"The host where the certificate is installed"
6161
; field ~qualifier:StaticRO ~lifecycle ~ty:DateTime "not_before"
62-
~default_value:(Some (VDateTime Date.never))
62+
~default_value:(Some (VDateTime Date.epoch))
6363
"Date after which the certificate is valid"
6464
; field ~qualifier:StaticRO ~lifecycle ~ty:DateTime "not_after"
65-
~default_value:(Some (VDateTime Date.never))
65+
~default_value:(Some (VDateTime Date.epoch))
6666
"Date before which the certificate is valid"
6767
; field ~qualifier:StaticRO
6868
~lifecycle:

‎ocaml/idl/datamodel_host.ml

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -935,7 +935,9 @@ let create_params =
935935
; {
936936
param_type= DateTime
937937
; param_name= "last_software_update"
938-
; param_doc= "Date and time when the last software update was applied."
938+
; param_doc=
939+
"Date and time when the last software update was applied. When the \
940+
timezone is missing, UTC is assumed"
939941
; param_release= dundee_release
940942
; param_default= Some (VDateTime Date.epoch)
941943
}
@@ -2188,8 +2190,7 @@ let t =
21882190
"tls_verification_enabled" ~default_value:(Some (VBool false))
21892191
"True if this host has TLS verifcation enabled"
21902192
; field ~qualifier:DynamicRO ~lifecycle:[] ~ty:DateTime
2191-
"last_software_update"
2192-
~default_value:(Some (VDateTime (Date.of_float 0.0)))
2193+
"last_software_update" ~default_value:(Some (VDateTime Date.epoch))
21932194
"Date and time when the last software update was applied"
21942195
; field ~qualifier:DynamicRO ~lifecycle:[] ~ty:Bool
21952196
~default_value:(Some (VBool false)) "https_only"

‎ocaml/idl/datamodel_types.ml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,12 +26,12 @@
2626
2727
*)
2828
module Date = struct
29-
open Xapi_stdext_date
29+
module Date = Xapi_stdext_date.Date
3030
include Date
3131

32-
let iso8601_of_rpc rpc = Date.of_string (Rpc.string_of_rpc rpc)
32+
let t_of_rpc rpc = Date.of_iso8601 (Rpc.string_of_rpc rpc)
3333

34-
let rpc_of_iso8601 date = Rpc.rpc_of_string (Date.to_string date)
34+
let rpc_of_t date = Rpc.rpc_of_string (Date.to_rfc3339 date)
3535
end
3636

3737
(* useful constants for product vsn tracking *)
@@ -418,7 +418,7 @@ type api_value =
418418
| VInt of int64
419419
| VFloat of float
420420
| VBool of bool
421-
| VDateTime of Date.iso8601
421+
| VDateTime of Date.t
422422
| VEnum of string
423423
| VMap of (api_value * api_value) list
424424
| VSet of api_value list

‎ocaml/idl/datamodel_types.mli

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
module Date : sig
22
include module type of Xapi_stdext_date.Date
33

4-
val iso8601_of_rpc : Rpc.t -> Xapi_stdext_date.Date.iso8601
4+
val t_of_rpc : Rpc.t -> Xapi_stdext_date.Date.t
55

6-
val rpc_of_iso8601 : Xapi_stdext_date.Date.iso8601 -> Rpc.t
6+
val rpc_of_t : Xapi_stdext_date.Date.t -> Rpc.t
77
end
88

99
val oss_since_303 : string option
@@ -115,7 +115,7 @@ type api_value =
115115
| VInt of int64
116116
| VFloat of float
117117
| VBool of bool
118-
| VDateTime of Date.iso8601
118+
| VDateTime of Date.t
119119
| VEnum of string
120120
| VMap of (api_value * api_value) list
121121
| VSet of api_value list

‎ocaml/idl/datamodel_values.ml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ let rec to_rpc v =
4040
| VBool b ->
4141
Rpc.Bool b
4242
| VDateTime d ->
43-
Rpc.String (Date.to_string d)
43+
Rpc.String (Date.to_rfc3339 d)
4444
| VEnum e ->
4545
Rpc.String e
4646
| VMap vvl ->
@@ -94,7 +94,7 @@ let to_db v =
9494
| VBool false ->
9595
String "false"
9696
| VDateTime d ->
97-
String (Date.to_string d)
97+
String (Date.to_rfc3339 d)
9898
| VEnum e ->
9999
String e
100100
| VMap vvl ->
@@ -117,7 +117,7 @@ let gen_empty_db_val t =
117117
| Bool ->
118118
Value.String "false"
119119
| DateTime ->
120-
Value.String (Date.to_string Date.never)
120+
Value.String Date.(to_rfc3339 epoch)
121121
| Enum (_, (enum_value, _) :: _) ->
122122
Value.String enum_value
123123
| Enum (_, []) ->

‎ocaml/idl/datamodel_vm.ml

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -339,7 +339,11 @@ let update_snapshot_metadata =
339339
[
340340
(Ref _vm, "vm", "The VM to update")
341341
; (Ref _vm, "snapshot_of", "")
342-
; (DateTime, "snapshot_time", "")
342+
; ( DateTime
343+
, "snapshot_time"
344+
, "The timestamp the snapshot was taken. When a timezone is missing, \
345+
UTC is assumed"
346+
)
343347
; (String, "transportable_snapshot_id", "")
344348
]
345349
~allowed_roles:_R_POOL_OP ()
@@ -2112,7 +2116,7 @@ let t =
21122116
"List pointing to all the VM snapshots."
21132117
; field ~writer_roles:_R_VM_POWER_ADMIN ~qualifier:DynamicRO
21142118
~in_product_since:rel_orlando
2115-
~default_value:(Some (VDateTime Date.never)) ~ty:DateTime
2119+
~default_value:(Some (VDateTime Date.epoch)) ~ty:DateTime
21162120
"snapshot_time" "Date/time when this snapshot was created."
21172121
; field ~writer_roles:_R_VM_POWER_ADMIN ~qualifier:DynamicRO
21182122
~in_product_since:rel_orlando ~default_value:(Some (VString ""))

‎ocaml/idl/dune

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,11 +45,12 @@
4545
(action (run %{x} -closed -markdown))
4646
)
4747

48-
(test
49-
(name schematest)
48+
(tests
49+
(names schematest test_datetimes)
5050
(modes exe)
51-
(modules schematest)
51+
(modules schematest test_datetimes)
5252
(libraries
53+
astring
5354
rpclib.core
5455
rpclib.json
5556
xapi_datamodel

0 commit comments

Comments
 (0)