From e69c6055b77bb1bbc450335aa1e302b0857f6606 Mon Sep 17 00:00:00 2001 From: dongdongcai Date: Thu, 10 May 2018 10:18:12 -0400 Subject: [PATCH 01/66] Add CosmosDb adapter --- Equinox.sln | 2 + src/Equinox.Cosmos/Cosmos.fs | 680 +++++++++++++++++++++++ src/Equinox.Cosmos/Equinox.Cosmos.fsproj | 36 ++ src/Equinox.EventStore/Infrastructure.fs | 15 + 4 files changed, 733 insertions(+) create mode 100644 src/Equinox.Cosmos/Cosmos.fs create mode 100644 src/Equinox.Cosmos/Equinox.Cosmos.fsproj diff --git a/Equinox.sln b/Equinox.sln index fa7f0e393..d5c5b92ba 100644 --- a/Equinox.sln +++ b/Equinox.sln @@ -46,6 +46,8 @@ Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Equinox.Codec", "src\Equino EndProject Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Equinox.Tool", "tools\Equinox.Tool\Equinox.Tool.fsproj", "{C8992C1C-6DC5-42CD-A3D7-1C5663433FED}" EndProject +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Equinox.Cosmos", "src\Equinox.Cosmos\Equinox.Cosmos.fsproj", "{54EA6187-9F9F-4D67-B602-163D011E43E6}" +EndProject Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "TodoBackend", "samples\TodoBackend\TodoBackend.fsproj", "{EC2EC658-3D85-44F3-AD2F-52AFCAFF8871}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Samples", "Samples", "{8F3EB30C-8BA3-4CC0-8361-0EA47C19ABB9}" diff --git a/src/Equinox.Cosmos/Cosmos.fs b/src/Equinox.Cosmos/Cosmos.fs new file mode 100644 index 000000000..4a376dc53 --- /dev/null +++ b/src/Equinox.Cosmos/Cosmos.fs @@ -0,0 +1,680 @@ +namespace Equinox.Cosmos + +open Equinox +open Equinox.Store +open FSharp.Control +open Microsoft.Azure.Documents +open Newtonsoft.Json +open Serilog +open System +open TypeShape + +type SN = int64 + +[] +module SN = + + /// The first sequence number. + let [] zero : SN = 0L + + /// The last sequence number + let [] last : SN = -1L + + /// Computes the next sequence number. + let inline next (sn : SN) : SN = sn + 1L + + /// Computes the previous sequence number + let inline prev (sn: SN): SN = sn - 1L + + /// Compares two sequence numbers. + let inline compare (sn1 : SN) (sn2 : SN) : int = Operators.compare sn1 sn2 + +type StreamId = string + +[] +module ArraySegmentExtensions = + + open System.Text + + type Encoding with + member x.GetString(data:ArraySegment) = x.GetString(data.Array, data.Offset, data.Count) + +type ByteArrayConverter() = + inherit JsonConverter() + + override this.ReadJson(reader, _, _, serializer) = + let s = serializer.Deserialize(reader, typeof) :?> string + if s = null + then + let arr: byte[] = [||] + // Why Array.Empty :> obj doesn't work here.... + arr :> obj + else + System.Text.Encoding.UTF8.GetBytes(s) :> obj + + override this.CanConvert(objectType) = + typeof.Equals(objectType) + + override this.WriteJson(writer, value, serializer) = + let array = value :?> byte[] + if Array.length array = 0 + then + serializer.Serialize(writer, null) + else + serializer.Serialize(writer, System.Text.Encoding.UTF8.GetString(array)) + +[] +/// Event data. +type EventData = { + eventType : string + data : byte[] + metadata : byte[] option } + with + + static member create (eventType: string, data: byte[], ?metadata: byte[]) = + { + eventType = eventType + data = data + metadata = + match metadata with + | None -> None + | Some md -> md |> Some } + +/// Operations on event data. +[] +[] +module EventData = + + let eventType (ed:EventData) = ed.eventType + let data (ed:EventData) = ed.data + let metadata (ed:EventData) = ed.metadata + +[] +type EquinoxEvent = { + id : string + s : StreamId + k : string + ts : DateTimeOffset + sn : SN + et : string + + [)>] + d : byte[] + + [)>] + md : byte[] } + +type Connection = IDocumentClient * Uri + +[] +type Direction = Forward | Backward with + override this.ToString() = match this with Forward -> "Forward" | Backward -> "Backward" + +module Log = + [] + type Measurement = { stream: string; interval: StopwatchInterval; bytes: int; count: int; ru: int } + [] + type Event = + | WriteSuccess of Measurement + | WriteConflict of Measurement + | Slice of Direction * Measurement + | Batch of Direction * slices: int * Measurement + let prop name value (log : ILogger) = log.ForContext(name, value) + open Serilog.Events + /// Attach a property to the log context to hold the metrics + // Sidestep Log.ForContext converting to a string; see https://github.com/serilog/serilog/issues/1124 + let event (value : Event) (log : ILogger) = + let enrich (e : LogEvent) = e.AddPropertyIfAbsent(LogEventProperty("eqxEvt", ScalarValue(value))) + log.ForContext({ new Serilog.Core.ILogEventEnricher with member __.Enrich(evt,_) = enrich evt }) + let withLoggedRetries<'t> retryPolicy (contextLabel : string) (f : ILogger -> Async<'t>) log: Async<'t> = + match retryPolicy with + | None -> f log + | Some retryPolicy -> + let withLoggingContextWrapping count = + let log = if count = 1 then log else log |> prop contextLabel count + f log + retryPolicy withLoggingContextWrapping + let (|BlobLen|) = function null -> 0 | (x : byte[]) -> x.Length + +[] +type EqxSyncResult = Written of SN * float | Conflict of float + +module private Write = + + let private eventDataToEquinoxEvent (streamId:StreamId) (sequenceNumber:SN) (ed: EventData) = + { + EquinoxEvent.et = ed.eventType + EquinoxEvent.id = (sprintf "%s-e-%d" streamId sequenceNumber) + EquinoxEvent.s = streamId + EquinoxEvent.k = streamId + EquinoxEvent.d = ed.data + EquinoxEvent.md = + match ed.metadata with + | Some x -> x + | None -> [||] + EquinoxEvent.sn = sequenceNumber + EquinoxEvent.ts = DateTimeOffset.UtcNow + } + + let [] private multiDocInsert = "AtomicMultiDocInsert" + + let inline private sprocUri (sprocName : string) (collectionUri : Uri) = + (collectionUri.ToString()) + "/sprocs/" + sprocName // TODO: do this elegantly + + /// Appends the single EventData using the sdk CreateDocumentAsync + let private appendSingleEvent ((client, collectionUri) : Connection) streamId sequenceNumber eventData : Async = + async { + let sequenceNumber = (SN.next sequenceNumber) + + let equinoxEvent = + eventData + |> eventDataToEquinoxEvent streamId sequenceNumber + + let! res = + client.CreateDocumentAsync (collectionUri, equinoxEvent) + |> Async.AwaitTaskCorrect + + return (sequenceNumber, res.RequestCharge) + } + + /// Appends the given EventData batch using the atomic stored procedure + // This requires store procuedure in CosmosDB, is there other ways to do this? + let private appendEventBatch ((client, collectionUri) : Connection) streamId sequenceNumber eventsData : Async = + async { + let sequenceNumber = (SN.next sequenceNumber) + let res, sn = + eventsData + |> Seq.mapFold (fun sn ed -> (eventDataToEquinoxEvent streamId sn ed) |> JsonConvert.SerializeObject, SN.next sn) sequenceNumber + + let requestOptions = + Client.RequestOptions(PartitionKey = PartitionKey(streamId)) + + let! res = + client.ExecuteStoredProcedureAsync(collectionUri |> sprocUri multiDocInsert, requestOptions, res:> obj) + |> Async.AwaitTaskCorrect + + return (sn - 1L), res.RequestCharge + } + + let private append connection streamId sequenceNumber (eventsData: EventData seq) = + match Seq.length eventsData with + | l when l = 0 -> invalidArg "eventsData" "must be non-empty" + | l when l = 1 -> + eventsData + |> Seq.head + |> appendSingleEvent connection streamId sequenceNumber + | _ -> appendEventBatch connection streamId sequenceNumber eventsData + + let private writeEventsAsync (log : ILogger) (conn : Connection) (streamName : string) (version : int64) (events : EventData[]) + : Async = async { + try + let! wr = append conn streamName version events + return EqxSyncResult.Written wr + with ex -> + // change this for store procudure + match ex with + | :? DocumentClientException as dce -> + // Improve this? + if dce.Message.Contains "already" + then + log.Information(ex, "Ges TrySync WrongExpectedVersionException") + return EqxSyncResult.Conflict dce.RequestCharge + else + return raise dce + | e -> return raise e } + + let eventDataBytes events = + let eventDataLen (x : EventData) = + let data = x.data + let metaData = + match x.metadata with + | None -> [||] + | Some x -> x + match data, metaData with Log.BlobLen bytes, Log.BlobLen metaBytes -> bytes + metaBytes + events |> Array.sumBy eventDataLen + + let private writeEventsLogged (conn : Connection) (streamName : string) (version : int64) (events : EventData[]) (log : ILogger) + : Async = async { + let bytes, count = eventDataBytes events, events.Length + let log = log |> Log.prop "bytes" bytes + let writeLog = log |> Log.prop "stream" streamName |> Log.prop "expectedVersion" version |> Log.prop "count" count + let! t, result = writeEventsAsync writeLog conn streamName version events |> Stopwatch.Time + let reqMetric : Log.Measurement = { stream = streamName; interval = t; bytes = bytes; count = count; ru = 0} + let resultLog, evt, (ru: float) = + match result, reqMetric with + | EqxSyncResult.Conflict ru, m -> log, Log.WriteConflict { m with ru = Convert.ToInt32(ru) }, ru + | EqxSyncResult.Written (x, ru), m -> + log |> Log.prop "nextExpectedVersion" x |> Log.prop "ru" ru, + Log.WriteSuccess { m with ru = Convert.ToInt32(ru) }, + ru + // TODO drop expectedVersion when consumption no longer requires that literal; ditto stream when literal formatting no longer required + (resultLog |> Log.event evt).Information("Eqx{action:l} stream={stream} count={count} expectedVersion={expectedVersion} conflict={conflict}, RequestCharge={ru}", + "Write", streamName, events.Length, version, (match evt with Log.WriteConflict _ -> true | _ -> false), ru) + return result } + + let writeEvents (log : ILogger) retryPolicy (conn : Connection) (streamName : string) (version : int64) (events : EventData[]) + : Async = + let call = writeEventsLogged conn streamName version events + Log.withLoggedRetries retryPolicy "writeAttempt" call log + +module private Read = + open Microsoft.Azure.Documents.Linq + open System.Linq + + let private getQuery ((client, collectionUri): Connection) streamId (direction: Direction) batchSize sequenceNumber = + + let sequenceNumber = + match direction, sequenceNumber with + | Direction.Backward, SN.last -> Int64.MaxValue + | _ -> sequenceNumber + + let feedOptions = new Client.FeedOptions() + feedOptions.PartitionKey <- PartitionKey(streamId) + feedOptions.MaxItemCount <- Nullable(batchSize) + let sql = + match direction with + | Direction.Backward -> + let query = """ + SELECT * FROM c + WHERE c.s = @streamId + AND c.sn <= @sequenceNumber + ORDER BY c.sn DESC""" + SqlQuerySpec query + | Direction.Forward -> + let query = """ + SELECT * FROM c + WHERE c.s = @streamId + AND c.sn >= @sequenceNumber + ORDER BY c.sn ASC """ + SqlQuerySpec query + sql.Parameters <- SqlParameterCollection + [| + SqlParameter("@streamId", streamId) + SqlParameter("@sequenceNumber", sequenceNumber) + |] + client.CreateDocumentQuery(collectionUri, sql, feedOptions).AsDocumentQuery() + + let (|EquinoxEventLen|) (x : EquinoxEvent) = match x.d, x.md with Log.BlobLen bytes, Log.BlobLen metaBytes -> bytes + metaBytes + + let private lastSequenceNumber (xs:EquinoxEvent seq) : SN = + match xs |> Seq.tryLast with + | None -> SN.last + | Some last -> last.sn + + let private queryExecution (query: IDocumentQuery<'T>) = + async { + let! res = query.ExecuteNextAsync<'T>() |> Async.AwaitTaskCorrect + return res.ToArray(), res.RequestCharge } + + let private loggedQueryExecution streamName direction batchSize startPos (query: IDocumentQuery) (log: ILogger) + : Async = async { + let! t, (slice, ru) = queryExecution query |> Stopwatch.Time + let bytes, count = slice |> Array.sumBy (|EquinoxEventLen|), slice.Length + let reqMetric : Log.Measurement ={ stream = streamName; interval = t; bytes = bytes; count = count; ru = Convert.ToInt32(ru) } + let evt = Log.Slice (direction, reqMetric) + (log |> Log.prop "startPos" startPos |> Log.prop "bytes" bytes |> Log.prop "ru" ru |> Log.event evt).Information( + // TODO drop sliceLength, totalPayloadSize when consumption no longer requires that literal; ditto stream when literal formatting no longer required + "Eqx{action:l} stream={stream} count={count} version={version} sliceLength={sliceLength} totalPayloadSize={totalPayloadSize} RequestCharge={ru}", + "Read", streamName, count, (lastSequenceNumber slice), batchSize, bytes, ru) + return slice, ru } + + let private readBatches (log : ILogger) (readSlice: IDocumentQuery -> ILogger -> Async) (maxPermittedBatchReads: int option) (query: IDocumentQuery) + : AsyncSeq = + let rec loop batchCount : AsyncSeq = asyncSeq { + match maxPermittedBatchReads with + | Some mpbr when batchCount >= mpbr -> log.Information "batch Limit exceeded"; invalidOp "batch Limit exceeded" + | _ -> () + + let batchLog = log |> Log.prop "batchIndex" batchCount + let! slice = readSlice query batchLog + yield slice + if query.HasMoreResults then + yield! loop (batchCount + 1)} + //| x -> raise <| System.ArgumentOutOfRangeException("SliceReadStatus", x, "Unknown result value") } + loop 0 + + let equinoxEventBytes events = events |> Array.sumBy (|EquinoxEventLen|) + + let logBatchRead direction streamName t events batchSize version (ru: float) (log : ILogger) = + let bytes, count = equinoxEventBytes events, events.Length + let reqMetric : Log.Measurement = { stream = streamName; interval = t; bytes = bytes; count = count; ru = Convert.ToInt32(ru) } + let batches = (events.Length - 1)/batchSize + 1 + let action = match direction with Direction.Forward -> "LoadF" | Direction.Backward -> "LoadB" + let evt = Log.Event.Batch (direction, batches, reqMetric) + (log |> Log.prop "bytes" bytes |> Log.event evt).Information( + "Eqx{action:l} stream={stream} count={count}/{batches} version={version} RequestCharge={ru}", + action, streamName, count, batches, version, ru) + + let loadForwardsFrom (log : ILogger) retryPolicy conn batchSize maxPermittedBatchReads streamName startPosition + : Async = async { + let mutable ru = 0.0 + let mergeBatches (batches: AsyncSeq) = async { + let! (events : EquinoxEvent[]) = + batches + |> AsyncSeq.map (fun (events, r) -> ru <- ru + r; events) + |> AsyncSeq.concatSeq + |> AsyncSeq.toArrayAsync + return events, ru } + let query = getQuery conn streamName Direction.Forward batchSize startPosition + let call q = loggedQueryExecution streamName Direction.Forward batchSize startPosition q + let retryingLoggingReadSlice q = Log.withLoggedRetries retryPolicy "readAttempt" (call q) + let direction = Direction.Forward + let log = log |> Log.prop "batchSize" batchSize |> Log.prop "direction" direction |> Log.prop "stream" streamName + let batches : AsyncSeq = readBatches log retryingLoggingReadSlice maxPermittedBatchReads query + let! t, (events, ru) = mergeBatches batches |> Stopwatch.Time + (query :> IDisposable).Dispose() + let version = lastSequenceNumber events + log |> logBatchRead direction streamName t events batchSize version ru + return version, events } + + let partitionPayloadFrom firstUsedEventNumber : EquinoxEvent[] -> int * int = + let acc (tu,tr) ((EquinoxEventLen bytes) as y) = if y.sn < firstUsedEventNumber then tu, tr + bytes else tu + bytes, tr + Array.fold acc (0,0) + let loadBackwardsUntilCompactionOrStart (log : ILogger) retryPolicy conn batchSize maxPermittedBatchReads streamName isCompactionEvent + : Async = async { + let mergeFromCompactionPointOrStartFromBackwardsStream (log : ILogger) (batchesBackward : AsyncSeq) + : Async = async { + let lastBatch = ref None + let mutable ru = 0.0 + let! tempBackward = + batchesBackward + |> AsyncSeq.map (fun (events, r) -> lastBatch := Some events; ru <- ru + r; events) + |> AsyncSeq.concatSeq + |> AsyncSeq.takeWhileInclusive (fun x -> + if not (isCompactionEvent x) then true // continue the search + else + match !lastBatch with + | None -> log.Information("EqxStop stream={stream} at={eventNumber}", streamName, x.sn) + | Some batch -> + let used, residual = batch |> partitionPayloadFrom x.sn + log.Information("EqxStop stream={stream} at={eventNumber} used={used} residual={residual}", streamName, x.sn, used, residual) + false) + |> AsyncSeq.toArrayAsync + let eventsForward = Array.Reverse(tempBackward); tempBackward // sic - relatively cheap, in-place reverse of something we own + return eventsForward, ru } + let query = getQuery conn streamName Direction.Backward batchSize SN.last + let call q = loggedQueryExecution streamName Direction.Backward batchSize SN.last q + let retryingLoggingReadSlice q = Log.withLoggedRetries retryPolicy "readAttempt" (call q) + let log = log |> Log.prop "batchSize" batchSize |> Log.prop "stream" streamName + let direction = Direction.Backward + let readlog = log |> Log.prop "direction" direction + let batchesBackward : AsyncSeq = readBatches readlog retryingLoggingReadSlice maxPermittedBatchReads query + let! t, (events, ru) = mergeFromCompactionPointOrStartFromBackwardsStream log batchesBackward |> Stopwatch.Time + (query :> IDisposable).Dispose() + let version = lastSequenceNumber events + log |> logBatchRead direction streamName t events batchSize version ru + return version, events } + +module UnionEncoderAdapters = + let private encodedEventOfResolvedEvent (x : EquinoxEvent) : UnionEncoder.EncodedUnion = + { CaseName = x.et; Payload = x.d } + let private eventDataOfEncodedEvent (x : UnionEncoder.EncodedUnion) = + EventData.create(x.CaseName, x.Payload, [||]) + let encodeEvents (codec : UnionEncoder.IUnionEncoder<'event, byte[]>) (xs : 'event seq) : EventData[] = + xs |> Seq.map (codec.Encode >> eventDataOfEncodedEvent) |> Seq.toArray + let decodeKnownEvents (codec : UnionEncoder.IUnionEncoder<'event, byte[]>) (xs : EquinoxEvent[]) : 'event seq = + xs |> Seq.map encodedEventOfResolvedEvent |> Seq.choose codec.TryDecode + +type Token = { streamVersion: int64; compactionEventNumber: int64 option } + +[] +module Token = + let private create compactionEventNumber batchCapacityLimit streamVersion : Storage.StreamToken = + { value = box { streamVersion = streamVersion; compactionEventNumber = compactionEventNumber }; batchCapacityLimit = batchCapacityLimit } + /// No batching / compaction; we only need to retain the StreamVersion + let ofNonCompacting streamVersion : Storage.StreamToken = + create None None streamVersion + // headroom before compaction is necessary given the stated knowledge of the last (if known) `compactionEventNumberOption` + let private batchCapacityLimit compactedEventNumberOption unstoredEventsPending (batchSize : int) (streamVersion : int64) : int = + match compactedEventNumberOption with + | Some (compactionEventNumber : int64) -> (batchSize - unstoredEventsPending) - int (streamVersion - compactionEventNumber + 1L) |> max 0 + | None -> (batchSize - unstoredEventsPending) - (int streamVersion + 1) - 1 |> max 0 + let (*private*) ofCompactionEventNumber compactedEventNumberOption unstoredEventsPending batchSize streamVersion : Storage.StreamToken = + let batchCapacityLimit = batchCapacityLimit compactedEventNumberOption unstoredEventsPending batchSize streamVersion + create compactedEventNumberOption (Some batchCapacityLimit) streamVersion + /// Assume we have not seen any compaction events; use the batchSize and version to infer headroom + let ofUncompactedVersion batchSize streamVersion : Storage.StreamToken = + ofCompactionEventNumber None 0 batchSize streamVersion + /// Use previousToken plus the data we are adding and the position we are adding it to infer a headroom + let ofPreviousTokenAndEventsLength (previousToken : Storage.StreamToken) eventsLength batchSize streamVersion : Storage.StreamToken = + let compactedEventNumber = (unbox previousToken.value).compactionEventNumber + ofCompactionEventNumber compactedEventNumber eventsLength batchSize streamVersion + /// Use an event just read from the stream to infer headroom + let ofCompactionResolvedEventAndVersion (compactionEvent: EquinoxEvent) batchSize streamVersion : Storage.StreamToken = + ofCompactionEventNumber (Some compactionEvent.sn) 0 batchSize streamVersion + /// Use an event we are about to write to the stream to infer headroom + let ofPreviousStreamVersionAndCompactionEventDataIndex prevStreamVersion compactionEventDataIndex eventsLength batchSize streamVersion' : Storage.StreamToken = + ofCompactionEventNumber (Some (prevStreamVersion + 1L + int64 compactionEventDataIndex)) eventsLength batchSize streamVersion' + let private unpackGesStreamVersion (x : Storage.StreamToken) = let x : Token = unbox x.value in x.streamVersion + let supersedes current x = + let currentVersion, newVersion = unpackGesStreamVersion current, unpackGesStreamVersion x + newVersion > currentVersion + +type EqxConnection(connection, ?readRetryPolicy, ?writeRetryPolicy) = + member __.Connection = connection + member __.ReadRetryPolicy = readRetryPolicy + member __.WriteRetryPolicy = writeRetryPolicy + +type EqxBatchingPolicy(getMaxBatchSize : unit -> int, ?batchCountLimit) = + new (maxBatchSize) = EqxBatchingPolicy(fun () -> maxBatchSize) + member __.BatchSize = getMaxBatchSize() + member __.MaxBatches = batchCountLimit + +[] +type GatewaySyncResult = Written of Storage.StreamToken | Conflict + +type EqxGateway(conn : EqxConnection, batching : EqxBatchingPolicy) = + let isResolvedEventEventType predicate (x:EquinoxEvent) = predicate x.et + let tryIsResolvedEventEventType predicateOption = predicateOption |> Option.map isResolvedEventEventType + member __.LoadBatched streamName log isCompactionEventType: Async = async { + let! version, events = Read.loadForwardsFrom log conn.ReadRetryPolicy conn.Connection batching.BatchSize batching.MaxBatches streamName 0L + match tryIsResolvedEventEventType isCompactionEventType with + | None -> return Token.ofNonCompacting version, events + | Some isCompactionEvent -> + match events |> Array.tryFindBack isCompactionEvent with + | None -> return Token.ofUncompactedVersion batching.BatchSize version, events + | Some resolvedEvent -> return Token.ofCompactionResolvedEventAndVersion resolvedEvent batching.BatchSize version, events } + member __.LoadBackwardsStoppingAtCompactionEvent streamName log isCompactionEventType: Async = async { + let isCompactionEvent = isResolvedEventEventType isCompactionEventType + let! version, events = + Read.loadBackwardsUntilCompactionOrStart log conn.ReadRetryPolicy conn.Connection batching.BatchSize batching.MaxBatches streamName isCompactionEvent + match Array.tryHead events |> Option.filter isCompactionEvent with + | None -> return Token.ofUncompactedVersion batching.BatchSize version, events + | Some resolvedEvent -> return Token.ofCompactionResolvedEventAndVersion resolvedEvent batching.BatchSize version, events } + member __.LoadFromToken streamName log (token : Storage.StreamToken) isCompactionEventType + : Async = async { + let streamPosition = (unbox token.value).streamVersion + let! version, events = Read.loadForwardsFrom log conn.ReadRetryPolicy conn.Connection batching.BatchSize batching.MaxBatches streamName streamPosition + match tryIsResolvedEventEventType isCompactionEventType with + | None -> return Token.ofNonCompacting version, events + | Some isCompactionEvent -> + match events |> Array.tryFindBack isCompactionEvent with + | None -> return Token.ofPreviousTokenAndEventsLength token events.Length batching.BatchSize version, events + | Some resolvedEvent -> return Token.ofCompactionResolvedEventAndVersion resolvedEvent batching.BatchSize version, events } + member __.TrySync streamName log (token : Storage.StreamToken) (encodedEvents: EventData array) isCompactionEventType : Async = async { + let streamVersion = (unbox token.value).streamVersion + let! wr = Write.writeEvents log conn.WriteRetryPolicy conn.Connection streamName streamVersion encodedEvents + match wr with + | EqxSyncResult.Conflict _ -> return GatewaySyncResult.Conflict + | EqxSyncResult.Written (wr, _) -> + + let version' = wr + let token = + match isCompactionEventType with + | None -> Token.ofNonCompacting version' + | Some isCompactionEvent -> + let isEventDataEventType predicate (x:EventData) = predicate x.eventType + match encodedEvents |> Array.tryFindIndexBack (isEventDataEventType isCompactionEvent) with + | None -> Token.ofPreviousTokenAndEventsLength token encodedEvents.Length batching.BatchSize version' + | Some compactionEventIndex -> + Token.ofPreviousStreamVersionAndCompactionEventDataIndex streamVersion compactionEventIndex encodedEvents.Length batching.BatchSize version' + return GatewaySyncResult.Written token } + +type EqxCategory<'event, 'state>(gateway : EqxGateway, codec : UnionEncoder.IUnionEncoder<'event, byte[]>, ?compactionStrategy) = + let loadAlgorithm load streamName initial log = + let batched = load initial (gateway.LoadBatched streamName log None) + let compacted predicate = load initial (gateway.LoadBackwardsStoppingAtCompactionEvent streamName log predicate) + match compactionStrategy with + | Some predicate -> compacted predicate + | None -> batched + let load (fold: 'state -> 'event seq -> 'state) initial f = async { + let! token, events = f + return token, fold initial (UnionEncoderAdapters.decodeKnownEvents codec events) } + member __.Load (fold: 'state -> 'event seq -> 'state) (initial: 'state) (streamName : string) (log : ILogger) : Async = + loadAlgorithm (load fold) streamName initial log + member __.LoadFromToken (fold: 'state -> 'event seq -> 'state) (state: 'state) (streamName : string) token (log : ILogger) : Async = + (load fold) state (gateway.LoadFromToken streamName log token compactionStrategy) + member __.TrySync (fold: 'state -> 'event seq -> 'state) streamName (log : ILogger) (token, state) (events : 'event list) : Async> = async { + let encodedEvents : EventData[] = UnionEncoderAdapters.encodeEvents codec (Seq.ofList events) + let! syncRes = gateway.TrySync streamName log token encodedEvents compactionStrategy + match syncRes with + | GatewaySyncResult.Conflict -> return Storage.SyncResult.Conflict (load fold state (gateway.LoadFromToken streamName log token compactionStrategy)) + | GatewaySyncResult.Written token' -> return Storage.SyncResult.Written (token', fold state (Seq.ofList events)) } + +module Caching = + open System.Runtime.Caching + [] + type CacheEntry<'state>(initialToken : Storage.StreamToken, initialState :'state) = + let mutable currentToken, currentState = initialToken, initialState + member __.UpdateIfNewer (other : CacheEntry<'state>) = + lock __ <| fun () -> + let otherToken, otherState = other.Value + if otherToken |> Token.supersedes currentToken then + currentToken <- otherToken + currentState <- otherState + member __.Value : Storage.StreamToken * 'state = + lock __ <| fun () -> + currentToken, currentState + + type Cache(name, sizeMb : int) = + let cache = + let config = System.Collections.Specialized.NameValueCollection(1) + config.Add("cacheMemoryLimitMegabytes", string sizeMb); + new MemoryCache(name, config) + member __.UpdateIfNewer (policy : CacheItemPolicy) (key : string) entry = + match cache.AddOrGetExisting(key, box entry, policy) with + | null -> () + | :? CacheEntry<'state> as existingEntry -> existingEntry.UpdateIfNewer entry + | x -> failwithf "UpdateIfNewer Incompatible cache entry %A" x + member __.TryGet (key : string) = + match cache.Get key with + | null -> None + | :? CacheEntry<'state> as existingEntry -> Some existingEntry.Value + | x -> failwithf "TryGet Incompatible cache entry %A" x + + /// Forwards all state changes in all streams of an ICategory to a `tee` function + type CategoryTee<'event, 'state>(inner: ICategory<'event, 'state>, tee : string -> Storage.StreamToken * 'state -> unit) = + let intercept streamName tokenAndState = + tee streamName tokenAndState + tokenAndState + let interceptAsync load streamName = async { + let! tokenAndState = load + return intercept streamName tokenAndState } + interface ICategory<'event, 'state> with + member __.Load (streamName : string) (log : ILogger) : Async = + interceptAsync (inner.Load streamName log) streamName + member __.TrySync streamName (log : ILogger) (token, state) (events : 'event list) : Async> = async { + let! syncRes = inner.TrySync streamName log (token, state) events + match syncRes with + | Storage.SyncResult.Conflict resync -> return Storage.SyncResult.Conflict (interceptAsync resync streamName) + | Storage.SyncResult.Written (token', state') -> return Storage.SyncResult.Written (token', state') } + + let applyCacheUpdatesWithSlidingExpiration + (cache: Cache) + (prefix: string) + (slidingExpiration : TimeSpan) + (category: ICategory<'event, 'state>) + : ICategory<'event, 'state> = + let policy = new CacheItemPolicy(SlidingExpiration = slidingExpiration) + let addOrUpdateSlidingExpirationCacheEntry streamName = CacheEntry >> cache.UpdateIfNewer policy (prefix + streamName) + CategoryTee<'event,'state>(category, addOrUpdateSlidingExpirationCacheEntry) :> _ + +type EqxFolder<'event, 'state>(category : EqxCategory<'event, 'state>, fold: 'state -> 'event seq -> 'state, initial: 'state, ?readCache) = + let loadAlgorithm streamName initial log = + let batched = category.Load fold initial streamName log + let cached token state = category.LoadFromToken fold state streamName token log + match readCache with + | None -> batched + | Some (cache : Caching.Cache, prefix : string) -> + match cache.TryGet(prefix + streamName) with + | None -> batched + | Some (token, state) -> cached token state + interface ICategory<'event, 'state> with + member __.Load (streamName : string) (log : ILogger) : Async = + loadAlgorithm streamName initial log + member __.TrySync streamName (log : ILogger) (token, state) (events : 'event list) : Async> = async { + let! syncRes = category.TrySync fold streamName log (token, state) events + match syncRes with + | Storage.SyncResult.Conflict resync -> return Storage.SyncResult.Conflict resync + | Storage.SyncResult.Written (token',state') -> return Storage.SyncResult.Written (token',state') } + +[] +type CompactionStrategy = + | EventType of string + | Predicate of (string -> bool) + +[] +type CachingStrategy = + | SlidingWindow of Caching.Cache * window: TimeSpan + /// Prefix is used to distinguish multiple folds per stream + | SlidingWindowPrefixed of Caching.Cache * window: TimeSpan * prefix: string + +type EqxStreamBuilder<'event, 'state>(gateway, codec, fold, initial, ?compaction, ?caching) = + member __.Create streamName : Equinox.IStream<'event, 'state> = + let compactionPredicateOption = + match compaction with + | None -> None + | Some (CompactionStrategy.Predicate predicate) -> Some predicate + | Some (CompactionStrategy.EventType eventType) -> Some (fun x -> x = eventType) + let eqxCategory = EqxCategory<'event, 'state>(gateway, codec, ?compactionStrategy = compactionPredicateOption) + + let readCacheOption = + match caching with + | None -> None + | Some (CachingStrategy.SlidingWindow(cache, _)) -> Some(cache, null) + | Some (CachingStrategy.SlidingWindowPrefixed(cache, _, prefix)) -> Some(cache, prefix) + let folder = EqxFolder<'event, 'state>(eqxCategory, fold, initial, ?readCache = readCacheOption) + + let category : ICategory<_,_> = + match caching with + | None -> folder :> _ + | Some (CachingStrategy.SlidingWindow(cache, window)) -> + Caching.applyCacheUpdatesWithSlidingExpiration cache null window folder + | Some (CachingStrategy.SlidingWindowPrefixed(cache, window, prefix)) -> + Caching.applyCacheUpdatesWithSlidingExpiration cache prefix window folder + + Equinox.Stream.create category streamName + +[] +type Discovery = + | UriAndKey of Uri * string * string * string + //| ConnecionString of string * string * string -> support this later + +type EquinoxConnection = IDocumentClient * Uri + +type EqxConnector + ( requestTimeout: TimeSpan, maxRetryAttemptsOnThrottledRequests: int, maxRetryWaitTimeInSeconds: int, + ?maxConnectionLimit) = + let connPolicy = + let cp = Client.ConnectionPolicy.Default + cp.ConnectionMode <- Client.ConnectionMode.Direct + cp.ConnectionProtocol <- Client.Protocol.Tcp + cp.RetryOptions <-Client.RetryOptions(MaxRetryAttemptsOnThrottledRequests = maxRetryAttemptsOnThrottledRequests, MaxRetryWaitTimeInSeconds = maxRetryWaitTimeInSeconds) + cp.RequestTimeout <- requestTimeout + match maxConnectionLimit with | Some x -> cp.MaxConnectionLimit <- x | None -> cp.MaxConnectionLimit <- 1000 + cp + + /// Yields an IEventStoreConfiguration configured and Connect()ed to a node (or the cluster) per the requested `discovery` strategy + member __.Connect (discovery : Discovery) : Async = async { + let client (uri: Uri) (key: string) (dbId: string) (collectionName: string) = + let collectionUri = Client.UriFactory.CreateDocumentCollectionUri(dbId, collectionName) + let client = new Client.DocumentClient(uri, key, connPolicy, Nullable(ConsistencyLevel.Session)) + client, collectionUri + + let client, uri = + match discovery with + | Discovery.UriAndKey (uri, key, dbName, collName) -> client uri key dbName collName + + do! (client.OpenAsync() |> Async.AwaitTaskCorrect) + + return (client :> IDocumentClient, uri) } \ No newline at end of file diff --git a/src/Equinox.Cosmos/Equinox.Cosmos.fsproj b/src/Equinox.Cosmos/Equinox.Cosmos.fsproj new file mode 100644 index 000000000..4a8448ddf --- /dev/null +++ b/src/Equinox.Cosmos/Equinox.Cosmos.fsproj @@ -0,0 +1,36 @@ + + + + net461 + + 5 + false + true + true + $(DefineConstants);NET461 + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/Equinox.EventStore/Infrastructure.fs b/src/Equinox.EventStore/Infrastructure.fs index 22ce54431..bc00a8e32 100644 --- a/src/Equinox.EventStore/Infrastructure.fs +++ b/src/Equinox.EventStore/Infrastructure.fs @@ -15,6 +15,21 @@ module Seq = Some res else None + let arrMapFold f acc (array: _[]) = + match array.Length with + | 0 -> [| |], acc + | len -> + let f = OptimizedClosures.FSharpFunc<_,_,_>.Adapt(f) + let mutable acc = acc + let res = Array.zeroCreate len + for i = 0 to array.Length-1 do + let h',s' = f.Invoke(acc,array.[i]) + res.[i] <- h' + acc <- s' + res, acc + let mapFold<'T,'State,'Result> (mapping: 'State -> 'T -> 'Result * 'State) state source = + let arr,state = source |> Seq.toArray |> arrMapFold mapping state + Seq.readonly arr, state module Array = let tryHead (array : 'T[]) = From 6baebef49ce6432fbebdf4f8ea9b7d87e387a77c Mon Sep 17 00:00:00 2001 From: dongdongcai Date: Thu, 10 May 2018 13:37:06 -0400 Subject: [PATCH 02/66] Integrate tests with Equinox.Cosmos --- Equinox.sln | 10 + build.proj | 9 +- src/Equinox.Cosmos/Cosmos.fs | 89 +++-- src/Equinox.Cosmos/Equinox.Cosmos.fsproj | 5 +- tests/Equinox.Cosmos.Integration/App.config | 18 + .../CosmosIntegration.fs | 322 ++++++++++++++++++ .../Equinox.Cosmos.Integration.fsproj | 34 ++ 7 files changed, 434 insertions(+), 53 deletions(-) create mode 100644 tests/Equinox.Cosmos.Integration/App.config create mode 100644 tests/Equinox.Cosmos.Integration/CosmosIntegration.fs create mode 100644 tests/Equinox.Cosmos.Integration/Equinox.Cosmos.Integration.fsproj diff --git a/Equinox.sln b/Equinox.sln index d5c5b92ba..7a7691677 100644 --- a/Equinox.sln +++ b/Equinox.sln @@ -48,6 +48,8 @@ Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Equinox.Tool", "tools\Equin EndProject Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Equinox.Cosmos", "src\Equinox.Cosmos\Equinox.Cosmos.fsproj", "{54EA6187-9F9F-4D67-B602-163D011E43E6}" EndProject +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Equinox.Cosmos.Integration", "tests\Equinox.Cosmos.Integration\Equinox.Cosmos.Integration.fsproj", "{DE0FEBF0-72DC-4D4A-BBA7-788D875D6B4B}" +EndProject Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "TodoBackend", "samples\TodoBackend\TodoBackend.fsproj", "{EC2EC658-3D85-44F3-AD2F-52AFCAFF8871}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Samples", "Samples", "{8F3EB30C-8BA3-4CC0-8361-0EA47C19ABB9}" @@ -112,6 +114,14 @@ Global {C8992C1C-6DC5-42CD-A3D7-1C5663433FED}.Debug|Any CPU.Build.0 = Debug|Any CPU {C8992C1C-6DC5-42CD-A3D7-1C5663433FED}.Release|Any CPU.ActiveCfg = Release|Any CPU {C8992C1C-6DC5-42CD-A3D7-1C5663433FED}.Release|Any CPU.Build.0 = Release|Any CPU + {54EA6187-9F9F-4D67-B602-163D011E43E6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {54EA6187-9F9F-4D67-B602-163D011E43E6}.Debug|Any CPU.Build.0 = Debug|Any CPU + {54EA6187-9F9F-4D67-B602-163D011E43E6}.Release|Any CPU.ActiveCfg = Release|Any CPU + {54EA6187-9F9F-4D67-B602-163D011E43E6}.Release|Any CPU.Build.0 = Release|Any CPU + {DE0FEBF0-72DC-4D4A-BBA7-788D875D6B4B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {DE0FEBF0-72DC-4D4A-BBA7-788D875D6B4B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {DE0FEBF0-72DC-4D4A-BBA7-788D875D6B4B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {DE0FEBF0-72DC-4D4A-BBA7-788D875D6B4B}.Release|Any CPU.Build.0 = Release|Any CPU {EC2EC658-3D85-44F3-AD2F-52AFCAFF8871}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {EC2EC658-3D85-44F3-AD2F-52AFCAFF8871}.Debug|Any CPU.Build.0 = Debug|Any CPU {EC2EC658-3D85-44F3-AD2F-52AFCAFF8871}.Release|Any CPU.ActiveCfg = Release|Any CPU diff --git a/build.proj b/build.proj index 4ffeb8b56..d754ad9c4 100644 --- a/build.proj +++ b/build.proj @@ -15,10 +15,11 @@ - - - - + + + + + diff --git a/src/Equinox.Cosmos/Cosmos.fs b/src/Equinox.Cosmos/Cosmos.fs index 4a376dc53..0a8656d7b 100644 --- a/src/Equinox.Cosmos/Cosmos.fs +++ b/src/Equinox.Cosmos/Cosmos.fs @@ -7,36 +7,11 @@ open Microsoft.Azure.Documents open Newtonsoft.Json open Serilog open System -open TypeShape - -type SN = int64 - -[] -module SN = - - /// The first sequence number. - let [] zero : SN = 0L - - /// The last sequence number - let [] last : SN = -1L - - /// Computes the next sequence number. - let inline next (sn : SN) : SN = sn + 1L - - /// Computes the previous sequence number - let inline prev (sn: SN): SN = sn - 1L - - /// Compares two sequence numbers. - let inline compare (sn1 : SN) (sn2 : SN) : int = Operators.compare sn1 sn2 - -type StreamId = string [] module ArraySegmentExtensions = - open System.Text - - type Encoding with + type System.Text.Encoding with member x.GetString(data:ArraySegment) = x.GetString(data.Array, data.Offset, data.Count) type ByteArrayConverter() = @@ -63,6 +38,28 @@ type ByteArrayConverter() = else serializer.Serialize(writer, System.Text.Encoding.UTF8.GetString(array)) +type SN = int64 + +[] +module SN = + + /// The first sequence number. + let [] zero : SN = 0L + + /// The last sequence number + let [] last : SN = -1L + + /// Computes the next sequence number. + let inline next (sn : SN) : SN = sn + 1L + + /// Computes the previous sequence number + let inline prev (sn: SN): SN = sn - 1L + + /// Compares two sequence numbers. + let inline compare (sn1 : SN) (sn2 : SN) : int = Operators.compare sn1 sn2 + +type StreamId = string + [] /// Event data. type EventData = { @@ -406,13 +403,13 @@ module private Read = return version, events } module UnionEncoderAdapters = - let private encodedEventOfResolvedEvent (x : EquinoxEvent) : UnionEncoder.EncodedUnion = - { CaseName = x.et; Payload = x.d } - let private eventDataOfEncodedEvent (x : UnionEncoder.EncodedUnion) = - EventData.create(x.CaseName, x.Payload, [||]) - let encodeEvents (codec : UnionEncoder.IUnionEncoder<'event, byte[]>) (xs : 'event seq) : EventData[] = + let private encodedEventOfResolvedEvent (x : EquinoxEvent) : UnionCodec.EncodedUnion = + { caseName = x.et; payload = x.d } + let private eventDataOfEncodedEvent (x : UnionCodec.EncodedUnion) = + EventData.create(x.caseName, x.payload, [||]) + let encodeEvents (codec : UnionCodec.IUnionEncoder<'event, byte[]>) (xs : 'event seq) : EventData[] = xs |> Seq.map (codec.Encode >> eventDataOfEncodedEvent) |> Seq.toArray - let decodeKnownEvents (codec : UnionEncoder.IUnionEncoder<'event, byte[]>) (xs : EquinoxEvent[]) : 'event seq = + let decodeKnownEvents (codec : UnionCodec.IUnionEncoder<'event, byte[]>) (xs : EquinoxEvent[]) : 'event seq = xs |> Seq.map encodedEventOfResolvedEvent |> Seq.choose codec.TryDecode type Token = { streamVersion: int64; compactionEventNumber: int64 option } @@ -450,8 +447,8 @@ module Token = let currentVersion, newVersion = unpackGesStreamVersion current, unpackGesStreamVersion x newVersion > currentVersion -type EqxConnection(connection, ?readRetryPolicy, ?writeRetryPolicy) = - member __.Connection = connection +type EqxConnection(connection : IDocumentClient, ?readRetryPolicy, ?writeRetryPolicy) = + member __.Connection = connection, Client.UriFactory.CreateDocumentCollectionUri("test","test") member __.ReadRetryPolicy = readRetryPolicy member __.WriteRetryPolicy = writeRetryPolicy @@ -510,7 +507,7 @@ type EqxGateway(conn : EqxConnection, batching : EqxBatchingPolicy) = Token.ofPreviousStreamVersionAndCompactionEventDataIndex streamVersion compactionEventIndex encodedEvents.Length batching.BatchSize version' return GatewaySyncResult.Written token } -type EqxCategory<'event, 'state>(gateway : EqxGateway, codec : UnionEncoder.IUnionEncoder<'event, byte[]>, ?compactionStrategy) = +type EqxCategory<'event, 'state>(gateway : EqxGateway, codec : UnionCodec.IUnionEncoder<'event, byte[]>, ?compactionStrategy) = let loadAlgorithm load streamName initial log = let batched = load initial (gateway.LoadBatched streamName log None) let compacted predicate = load initial (gateway.LoadBackwardsStoppingAtCompactionEvent streamName log predicate) @@ -650,8 +647,6 @@ type Discovery = | UriAndKey of Uri * string * string * string //| ConnecionString of string * string * string -> support this later -type EquinoxConnection = IDocumentClient * Uri - type EqxConnector ( requestTimeout: TimeSpan, maxRetryAttemptsOnThrottledRequests: int, maxRetryWaitTimeInSeconds: int, ?maxConnectionLimit) = @@ -659,22 +654,24 @@ type EqxConnector let cp = Client.ConnectionPolicy.Default cp.ConnectionMode <- Client.ConnectionMode.Direct cp.ConnectionProtocol <- Client.Protocol.Tcp - cp.RetryOptions <-Client.RetryOptions(MaxRetryAttemptsOnThrottledRequests = maxRetryAttemptsOnThrottledRequests, MaxRetryWaitTimeInSeconds = maxRetryWaitTimeInSeconds) + cp.RetryOptions <- + Client.RetryOptions( + MaxRetryAttemptsOnThrottledRequests = maxRetryAttemptsOnThrottledRequests, + MaxRetryWaitTimeInSeconds = maxRetryWaitTimeInSeconds) cp.RequestTimeout <- requestTimeout - match maxConnectionLimit with | Some x -> cp.MaxConnectionLimit <- x | None -> cp.MaxConnectionLimit <- 1000 + cp.MaxConnectionLimit <- defaultArg maxConnectionLimit 1000 cp - /// Yields an IEventStoreConfiguration configured and Connect()ed to a node (or the cluster) per the requested `discovery` strategy - member __.Connect (discovery : Discovery) : Async = async { + /// Yields an connection to DocDB configured and Connect()ed to DocDB collection per the requested `discovery` strategy + member __.Connect (discovery : Discovery) : Async = async { let client (uri: Uri) (key: string) (dbId: string) (collectionName: string) = - let collectionUri = Client.UriFactory.CreateDocumentCollectionUri(dbId, collectionName) let client = new Client.DocumentClient(uri, key, connPolicy, Nullable(ConsistencyLevel.Session)) - client, collectionUri + client - let client, uri = + let client = match discovery with | Discovery.UriAndKey (uri, key, dbName, collName) -> client uri key dbName collName - do! (client.OpenAsync() |> Async.AwaitTaskCorrect) + do! client.OpenAsync() |> Async.AwaitTaskCorrect - return (client :> IDocumentClient, uri) } \ No newline at end of file + return EqxConnection(client :> IDocumentClient) } \ No newline at end of file diff --git a/src/Equinox.Cosmos/Equinox.Cosmos.fsproj b/src/Equinox.Cosmos/Equinox.Cosmos.fsproj index 4a8448ddf..8d0b3b977 100644 --- a/src/Equinox.Cosmos/Equinox.Cosmos.fsproj +++ b/src/Equinox.Cosmos/Equinox.Cosmos.fsproj @@ -1,8 +1,7 @@ - net461 - + netstandard2.0;net461 5 false true @@ -17,6 +16,7 @@ + @@ -30,7 +30,6 @@ - \ No newline at end of file diff --git a/tests/Equinox.Cosmos.Integration/App.config b/tests/Equinox.Cosmos.Integration/App.config new file mode 100644 index 000000000..2051014fa --- /dev/null +++ b/tests/Equinox.Cosmos.Integration/App.config @@ -0,0 +1,18 @@ + + + + + + + + + True + + + + + True + + + + \ No newline at end of file diff --git a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs new file mode 100644 index 000000000..17432fb22 --- /dev/null +++ b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs @@ -0,0 +1,322 @@ +module Equinox.Cosmos.Integration.EquinoxIntegration + +open Equinox.Integration.Infrastructure +open Equinox.Cosmos +open Swensen.Unquote +open System.Threading +open System + +/// Standing up an Equinox instance is complicated; to run for test purposes either: +/// - replace connection below with a connection string or Uri+Key for an initialized Equinox instance +/// - Create a local Equinox with dbName "test" and collectionName "test" using provisioning script +let connectToLocalEquinoxNode () = + EqxConnector(requestTimeout=TimeSpan.FromSeconds 3., maxRetryAttemptsOnThrottledRequests=2, maxRetryWaitTimeInSeconds=60) + .Connect(Discovery.UriAndKey(Uri "https://localhost:8081", "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==","test","test")) +let defaultBatchSize = 500 +let createEqxGateway connection batchSize = EqxGateway(connection, EqxBatchingPolicy(maxBatchSize = batchSize)) +let (|StreamArgs|) gateway = + //let databaseId, collectionId = "test", "test" + gateway//, databaseId, collectionId + +let serializationSettings = Newtonsoft.Json.Converters.FSharp.Settings.CreateCorrect() +let genCodec<'Union when 'Union :> TypeShape.UnionContract.IUnionContract>() = + Equinox.UnionCodec.JsonUtf8.Create<'Union>(serializationSettings) + +module Cart = + let fold, initial = Domain.Cart.Folds.fold, Domain.Cart.Folds.initial + let codec = genCodec() + let createServiceWithoutOptimization connection batchSize log = + let gateway = createEqxGateway connection batchSize + let resolveStream _ignoreCompactionEventTypeOption (args) = + EqxStreamBuilder(gateway, codec, fold, initial).Create(args) + Backend.Cart.Service(log, resolveStream) + let createServiceWithCompaction connection batchSize log = + let gateway = createEqxGateway connection batchSize + let resolveStream compactionEventType (args) = + EqxStreamBuilder(gateway, codec, fold, initial, compaction=CompactionStrategy.EventType compactionEventType).Create(args) + Backend.Cart.Service(log, resolveStream) + let createServiceWithCaching connection batchSize log cache = + let gateway = createEqxGateway connection batchSize + let sliding20m = CachingStrategy.SlidingWindow (cache, TimeSpan.FromMinutes 20.) + let resolveStream _ignorecompactionEventType (StreamArgs args) = EqxStreamBuilder(gateway, codec, fold, initial, caching = sliding20m).Create(args) + Backend.Cart.Service(log, resolveStream) + let createServiceWithCompactionAndCaching connection batchSize log cache = + let gateway = createEqxGateway connection batchSize + let sliding20m = CachingStrategy.SlidingWindow (cache, TimeSpan.FromMinutes 20.) + let resolveStream cet (StreamArgs args) = EqxStreamBuilder(gateway, codec, fold, initial, CompactionStrategy.EventType cet, sliding20m).Create(args) + Backend.Cart.Service(log, resolveStream) + +module ContactPreferences = + let fold, initial = Domain.ContactPreferences.Folds.fold, Domain.ContactPreferences.Folds.initial + let codec = genCodec() + let createServiceWithoutOptimization createGateway defaultBatchSize log _ignoreWindowSize _ignoreCompactionPredicate = + let gateway = createGateway defaultBatchSize + let resolveStream _windowSize _compactionPredicate (StreamArgs args) = + EqxStreamBuilder(gateway, codec, fold, initial).Create(args) + Backend.ContactPreferences.Service(log, resolveStream) + let createService createGateway log = + let resolveStream batchSize compactionPredicate (StreamArgs args) = + EqxStreamBuilder(createGateway batchSize, codec, fold, initial, CompactionStrategy.Predicate compactionPredicate).Create(args) + Backend.ContactPreferences.Service(log, resolveStream) + +#nowarn "1182" // From hereon in, we may have some 'unused' privates (the tests) + +type Tests(testOutputHelper) = + let testOutput = TestOutputAdapter testOutputHelper + + let addAndThenRemoveItems exceptTheLastOne context cartId skuId (service: Backend.Cart.Service) count = + service.FlowAsync(cartId, fun _ctx execute -> + for i in 1..count do + execute <| Domain.Cart.AddItem (context, skuId, i) + if not exceptTheLastOne || i <> count then + execute <| Domain.Cart.RemoveItem (context, skuId) ) + let addAndThenRemoveItemsManyTimes context cartId skuId service count = + addAndThenRemoveItems false context cartId skuId service count + let addAndThenRemoveItemsManyTimesExceptTheLastOne context cartId skuId service count = + addAndThenRemoveItems true context cartId skuId service count + + let createLoggerWithCapture () = + let capture = LogCaptureBuffer() + let logger = + Serilog.LoggerConfiguration() + .WriteTo.Sink(testOutput) + .WriteTo.Sink(capture) + .CreateLogger() + logger, capture + + let singleSliceForward = EsAct.SliceForward + let singleBatchForward = [EsAct.SliceForward; EsAct.BatchForward] + let batchForwardAndAppend = singleBatchForward @ [EsAct.Append] + + [] + let ``Can roundtrip against Equinox, correctly batching the reads [without any optimizations]`` context cartId skuId = Async.RunSynchronously <| async { + let log, capture = createLoggerWithCapture () + let! conn = connectToLocalEquinoxNode () + + let batchSize = 3 + let service = Cart.createServiceWithoutOptimization conn batchSize log + + // The command processing should trigger only a single read and a single write call + let addRemoveCount = 6 + do! addAndThenRemoveItemsManyTimesExceptTheLastOne context cartId skuId service addRemoveCount + test <@ batchForwardAndAppend = capture.ExternalCalls @> + + // Restart the counting + capture.Clear() + + // Validate basic operation; Key side effect: Log entries will be emitted to `capture` + let! state = service.Read cartId + let expectedEventCount = 2 * addRemoveCount - 1 + test <@ addRemoveCount = match state with { items = [{ quantity = quantity }] } -> quantity | _ -> failwith "nope" @> + + // Need to read 4 batches to read 11 events in batches of 3 + let expectedBatches = ceil(float expectedEventCount/float batchSize) |> int + test <@ List.replicate (expectedBatches-1) singleSliceForward @ singleBatchForward = capture.ExternalCalls @> + } + + [] + let ``Can roundtrip against Equinox, managing sync conflicts by retrying [without any optimizations]`` ctx initialState = Async.RunSynchronously <| async { + let log1, capture1 = createLoggerWithCapture () + let! conn = connectToLocalEquinoxNode () + // Ensure batching is included at some point in the proceedings + let batchSize = 3 + + let context, cartId, (sku11, sku12, sku21, sku22) = ctx + + // establish base stream state + let service1 = Cart.createServiceWithoutOptimization conn batchSize log1 + let! maybeInitialSku = + let (streamEmpty, skuId) = initialState + async { + if streamEmpty then return None + else + let addRemoveCount = 2 + do! addAndThenRemoveItemsManyTimesExceptTheLastOne context cartId skuId service1 addRemoveCount + return Some (skuId, addRemoveCount) } + + let act prepare (service : Backend.Cart.Service) skuId count = + service.FlowAsync(cartId, prepare = prepare, flow = fun _ctx execute -> + execute <| Domain.Cart.AddItem (context, skuId, count)) + + let eventWaitSet () = let e = new ManualResetEvent(false) in (Async.AwaitWaitHandle e |> Async.Ignore), async { e.Set() |> ignore } + let w0, s0 = eventWaitSet () + let w1, s1 = eventWaitSet () + let w2, s2 = eventWaitSet () + let w3, s3 = eventWaitSet () + let w4, s4 = eventWaitSet () + let t1 = async { + // Wait for other to have state, signal we have it, await conflict and handle + let prepare = async { + do! w0 + do! s1 + do! w2 } + do! act prepare service1 sku11 11 + // Wait for other side to load; generate conflict + let prepare = async { do! w3 } + do! act prepare service1 sku12 12 + // Signal conflict generated + do! s4 } + let log2, capture2 = createLoggerWithCapture () + let service2 = Cart.createServiceWithoutOptimization conn batchSize log2 + let t2 = async { + // Signal we have state, wait for other to do same, engineer conflict + let prepare = async { + do! s0 + do! w1 } + do! act prepare service2 sku21 21 + // Signal conflict is in place + do! s2 + // Await our conflict + let prepare = async { + do! s3 + do! w4 } + do! act prepare service2 sku22 22 } + // Act: Engineer the conflicts and applications, with logging into capture1 and capture2 + do! Async.Parallel [t1; t2] |> Async.Ignore + + // Load state + let! result = service1.Read cartId + + // Ensure correct values got persisted + let has sku qty = result.items |> List.exists (fun { skuId = s; quantity = q } -> (sku, qty) = (s, q)) + test <@ maybeInitialSku |> Option.forall (fun (skuId, quantity) -> has skuId quantity) + && has sku11 11 && has sku12 12 + && has sku21 21 && has sku22 22 @> + // Intended conflicts pertained + let hadConflict= function EsEvent (EsAction EsAct.AppendConflict) -> Some () | _ -> None + test <@ [1; 1] = [for c in [capture1; capture2] -> c.ChooseCalls hadConflict |> List.length] @> + } + + let singleBatchBackwards = [EsAct.SliceBackward; EsAct.BatchBackward] + let batchBackwardsAndAppend = singleBatchBackwards @ [EsAct.Append] + + [] + let ``Can roundtrip against Equinox, correctly compacting to avoid redundant reads`` context skuId cartId = Async.RunSynchronously <| async { + let log, capture = createLoggerWithCapture () + let! conn = connectToLocalEquinoxNode () + let batchSize = 10 + let service = Cart.createServiceWithCompaction conn batchSize log + + // Trigger 10 events, then reload + do! addAndThenRemoveItemsManyTimes context cartId skuId service 5 + let! _ = service.Read cartId + + // ... should see a single read as we are inside the batch threshold + test <@ batchBackwardsAndAppend @ singleBatchBackwards = capture.ExternalCalls @> + + // Add two more, which should push it over the threshold and hence trigger inclusion of a snapshot event (but not incurr extra roundtrips) + capture.Clear() + do! addAndThenRemoveItemsManyTimes context cartId skuId service 1 + test <@ batchBackwardsAndAppend = capture.ExternalCalls @> + + // While we now have 13 events, we should be able to read them with a single call + capture.Clear() + let! _ = service.Read cartId + test <@ singleBatchBackwards = capture.ExternalCalls @> + + // Add 8 more; total of 21 should not trigger snapshotting as Event Number 12 (the 13th one) is a shapshot + capture.Clear() + do! addAndThenRemoveItemsManyTimes context cartId skuId service 4 + test <@ batchBackwardsAndAppend = capture.ExternalCalls @> + + // While we now have 21 events, we should be able to read them with a single call + capture.Clear() + let! _ = service.Read cartId + // ... and trigger a second snapshotting (inducing a single additional read + write) + do! addAndThenRemoveItemsManyTimes context cartId skuId service 1 + // and reload the 24 events with a single read + let! _ = service.Read cartId + test <@ singleBatchBackwards @ batchBackwardsAndAppend @ singleBatchBackwards = capture.ExternalCalls @> + } + + [] + let ``Can correctly read and update against Equinox, with window size of 1 using tautological Compaction predicate`` id value = Async.RunSynchronously <| async { + let log, capture = createLoggerWithCapture () + let! conn = connectToLocalEquinoxNode () + let service = ContactPreferences.createService (createEqxGateway conn) log + + let (Domain.ContactPreferences.Id email) = id + // Feed some junk into the stream + for i in 0..11 do + let quickSurveysValue = i % 2 = 0 + do! service.Update email { value with quickSurveys = quickSurveysValue } + // Ensure there will be something to be changed by the Update below + do! service.Update email { value with quickSurveys = not value.quickSurveys } + + capture.Clear() + do! service.Update email value + + let! result = service.Read email + test <@ value = result @> + + test <@ batchBackwardsAndAppend @ singleBatchBackwards = capture.ExternalCalls @> + } + + [] + let ``Can roundtrip against Equinox, correctly caching to avoid redundant reads`` context skuId cartId = Async.RunSynchronously <| async { + let log, capture = createLoggerWithCapture () + let! conn = connectToLocalEquinoxNode () + let batchSize = 10 + let cache = Caching.Cache("cart", sizeMb = 50) + let createServiceCached () = Cart.createServiceWithCaching conn batchSize log cache + let service1, service2 = createServiceCached (), createServiceCached () + + // Trigger 10 events, then reload + do! addAndThenRemoveItemsManyTimes context cartId skuId service1 5 + let! _ = service2.Read cartId + + // ... should see a single read as we are writes are cached + test <@ batchForwardAndAppend @ singleBatchForward = capture.ExternalCalls @> + + // Add two more - the roundtrip should only incur a single read + capture.Clear() + do! addAndThenRemoveItemsManyTimes context cartId skuId service1 1 + test <@ batchForwardAndAppend = capture.ExternalCalls @> + + // While we now have 12 events, we should be able to read them with a single call + capture.Clear() + let! _ = service2.Read cartId + test <@ singleBatchForward = capture.ExternalCalls @> + } + + [] + let ``Can combine compaction with caching against Equinox`` context skuId cartId = Async.RunSynchronously <| async { + let log, capture = createLoggerWithCapture () + let! conn = connectToLocalEquinoxNode () + let batchSize = 10 + let service1 = Cart.createServiceWithCompaction conn batchSize log + let cache = Caching.Cache("cart", sizeMb = 50) + let service2 = Cart.createServiceWithCompactionAndCaching conn batchSize log cache + + // Trigger 10 events, then reload + do! addAndThenRemoveItemsManyTimes context cartId skuId service1 5 + let! _ = service2.Read cartId + + // ... should see a single read as we are inside the batch threshold + test <@ batchBackwardsAndAppend @ singleBatchBackwards = capture.ExternalCalls @> + + // Add two more, which should push it over the threshold and hence trigger inclusion of a snapshot event (but not incurr extra roundtrips) + capture.Clear() + do! addAndThenRemoveItemsManyTimes context cartId skuId service1 1 + test <@ batchBackwardsAndAppend = capture.ExternalCalls @> + + // While we now have 13 events, we whould be able to read them backwards with a single call + capture.Clear() + let! _ = service1.Read cartId + test <@ singleBatchBackwards = capture.ExternalCalls @> + + // Add 8 more; total of 21 should not trigger snapshotting as Event Number 12 (the 13th one) is a shapshot + capture.Clear() + do! addAndThenRemoveItemsManyTimes context cartId skuId service1 4 + test <@ batchBackwardsAndAppend = capture.ExternalCalls @> + + // While we now have 21 events, we should be able to read them with a single call + capture.Clear() + let! _ = service1.Read cartId + // ... and trigger a second snapshotting (inducing a single additional read + write) + do! addAndThenRemoveItemsManyTimes context cartId skuId service1 1 + // and we _could_ reload the 24 events with a single read if reading backwards. However we are using the cache, which last saw it with 10 events, which necessitates two reads + let! _ = service2.Read cartId + let suboptimalExtraSlice = [singleSliceForward] + test <@ singleBatchBackwards @ batchBackwardsAndAppend @ suboptimalExtraSlice @ singleBatchForward = capture.ExternalCalls @> + } \ No newline at end of file diff --git a/tests/Equinox.Cosmos.Integration/Equinox.Cosmos.Integration.fsproj b/tests/Equinox.Cosmos.Integration/Equinox.Cosmos.Integration.fsproj new file mode 100644 index 000000000..d1ebeda06 --- /dev/null +++ b/tests/Equinox.Cosmos.Integration/Equinox.Cosmos.Integration.fsproj @@ -0,0 +1,34 @@ + + + + netcoreapp2.1;net461 + Library + false + 5 + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file From 388e152f845468cec4f218f47c3c56b0a57a0cd7 Mon Sep 17 00:00:00 2001 From: dongdongcai Date: Mon, 14 May 2018 11:08:11 -0400 Subject: [PATCH 03/66] Check collection exist in connection creation; Set consistency level to strong for retrying --- src/Equinox.Cosmos/Cosmos.fs | 44 +++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/src/Equinox.Cosmos/Cosmos.fs b/src/Equinox.Cosmos/Cosmos.fs index 0a8656d7b..8a5866b3b 100644 --- a/src/Equinox.Cosmos/Cosmos.fs +++ b/src/Equinox.Cosmos/Cosmos.fs @@ -214,7 +214,7 @@ module private Write = // Improve this? if dce.Message.Contains "already" then - log.Information(ex, "Ges TrySync WrongExpectedVersionException") + log.Information(ex, "Eqx TrySync WrongExpectedVersionException") return EqxSyncResult.Conflict dce.RequestCharge else return raise dce @@ -258,7 +258,7 @@ module private Read = open Microsoft.Azure.Documents.Linq open System.Linq - let private getQuery ((client, collectionUri): Connection) streamId (direction: Direction) batchSize sequenceNumber = + let private getQuery ((client, collectionUri): Connection) strongConsistency streamId (direction: Direction) batchSize sequenceNumber = let sequenceNumber = match direction, sequenceNumber with @@ -268,6 +268,7 @@ module private Read = let feedOptions = new Client.FeedOptions() feedOptions.PartitionKey <- PartitionKey(streamId) feedOptions.MaxItemCount <- Nullable(batchSize) + //if (strongConsistency) then feedOptions.ConsistencyLevel <- Nullable(ConsistencyLevel.Strong) let sql = match direction with | Direction.Backward -> @@ -342,7 +343,7 @@ module private Read = "Eqx{action:l} stream={stream} count={count}/{batches} version={version} RequestCharge={ru}", action, streamName, count, batches, version, ru) - let loadForwardsFrom (log : ILogger) retryPolicy conn batchSize maxPermittedBatchReads streamName startPosition + let loadForwardsFrom (log : ILogger) retryPolicy conn batchSize maxPermittedBatchReads consistencyLevel streamName startPosition : Async = async { let mutable ru = 0.0 let mergeBatches (batches: AsyncSeq) = async { @@ -352,7 +353,7 @@ module private Read = |> AsyncSeq.concatSeq |> AsyncSeq.toArrayAsync return events, ru } - let query = getQuery conn streamName Direction.Forward batchSize startPosition + let query = getQuery conn consistencyLevel streamName Direction.Forward batchSize startPosition let call q = loggedQueryExecution streamName Direction.Forward batchSize startPosition q let retryingLoggingReadSlice q = Log.withLoggedRetries retryPolicy "readAttempt" (call q) let direction = Direction.Forward @@ -367,7 +368,7 @@ module private Read = let partitionPayloadFrom firstUsedEventNumber : EquinoxEvent[] -> int * int = let acc (tu,tr) ((EquinoxEventLen bytes) as y) = if y.sn < firstUsedEventNumber then tu, tr + bytes else tu + bytes, tr Array.fold acc (0,0) - let loadBackwardsUntilCompactionOrStart (log : ILogger) retryPolicy conn batchSize maxPermittedBatchReads streamName isCompactionEvent + let loadBackwardsUntilCompactionOrStart (log : ILogger) retryPolicy conn batchSize maxPermittedBatchReads consistencyLevel streamName isCompactionEvent : Async = async { let mergeFromCompactionPointOrStartFromBackwardsStream (log : ILogger) (batchesBackward : AsyncSeq) : Async = async { @@ -389,7 +390,7 @@ module private Read = |> AsyncSeq.toArrayAsync let eventsForward = Array.Reverse(tempBackward); tempBackward // sic - relatively cheap, in-place reverse of something we own return eventsForward, ru } - let query = getQuery conn streamName Direction.Backward batchSize SN.last + let query = getQuery conn consistencyLevel streamName Direction.Backward batchSize SN.last let call q = loggedQueryExecution streamName Direction.Backward batchSize SN.last q let retryingLoggingReadSlice q = Log.withLoggedRetries retryPolicy "readAttempt" (call q) let log = log |> Log.prop "batchSize" batchSize |> Log.prop "stream" streamName @@ -442,9 +443,9 @@ module Token = /// Use an event we are about to write to the stream to infer headroom let ofPreviousStreamVersionAndCompactionEventDataIndex prevStreamVersion compactionEventDataIndex eventsLength batchSize streamVersion' : Storage.StreamToken = ofCompactionEventNumber (Some (prevStreamVersion + 1L + int64 compactionEventDataIndex)) eventsLength batchSize streamVersion' - let private unpackGesStreamVersion (x : Storage.StreamToken) = let x : Token = unbox x.value in x.streamVersion + let private unpackEqxStreamVersion (x : Storage.StreamToken) = let x : Token = unbox x.value in x.streamVersion let supersedes current x = - let currentVersion, newVersion = unpackGesStreamVersion current, unpackGesStreamVersion x + let currentVersion, newVersion = unpackEqxStreamVersion current, unpackEqxStreamVersion x newVersion > currentVersion type EqxConnection(connection : IDocumentClient, ?readRetryPolicy, ?writeRetryPolicy) = @@ -464,7 +465,7 @@ type EqxGateway(conn : EqxConnection, batching : EqxBatchingPolicy) = let isResolvedEventEventType predicate (x:EquinoxEvent) = predicate x.et let tryIsResolvedEventEventType predicateOption = predicateOption |> Option.map isResolvedEventEventType member __.LoadBatched streamName log isCompactionEventType: Async = async { - let! version, events = Read.loadForwardsFrom log conn.ReadRetryPolicy conn.Connection batching.BatchSize batching.MaxBatches streamName 0L + let! version, events = Read.loadForwardsFrom log conn.ReadRetryPolicy conn.Connection batching.BatchSize batching.MaxBatches false streamName 0L match tryIsResolvedEventEventType isCompactionEventType with | None -> return Token.ofNonCompacting version, events | Some isCompactionEvent -> @@ -474,14 +475,14 @@ type EqxGateway(conn : EqxConnection, batching : EqxBatchingPolicy) = member __.LoadBackwardsStoppingAtCompactionEvent streamName log isCompactionEventType: Async = async { let isCompactionEvent = isResolvedEventEventType isCompactionEventType let! version, events = - Read.loadBackwardsUntilCompactionOrStart log conn.ReadRetryPolicy conn.Connection batching.BatchSize batching.MaxBatches streamName isCompactionEvent + Read.loadBackwardsUntilCompactionOrStart log conn.ReadRetryPolicy conn.Connection batching.BatchSize batching.MaxBatches false streamName isCompactionEvent match Array.tryHead events |> Option.filter isCompactionEvent with | None -> return Token.ofUncompactedVersion batching.BatchSize version, events | Some resolvedEvent -> return Token.ofCompactionResolvedEventAndVersion resolvedEvent batching.BatchSize version, events } member __.LoadFromToken streamName log (token : Storage.StreamToken) isCompactionEventType : Async = async { let streamPosition = (unbox token.value).streamVersion - let! version, events = Read.loadForwardsFrom log conn.ReadRetryPolicy conn.Connection batching.BatchSize batching.MaxBatches streamName streamPosition + let! version, events = Read.loadForwardsFrom log conn.ReadRetryPolicy conn.Connection batching.BatchSize batching.MaxBatches true streamName streamPosition match tryIsResolvedEventEventType isCompactionEventType with | None -> return Token.ofNonCompacting version, events | Some isCompactionEvent -> @@ -664,14 +665,21 @@ type EqxConnector /// Yields an connection to DocDB configured and Connect()ed to DocDB collection per the requested `discovery` strategy member __.Connect (discovery : Discovery) : Async = async { - let client (uri: Uri) (key: string) (dbId: string) (collectionName: string) = + let client (uri: Uri) (key: string) (dbId: string) (collectionName: string) = async { + let collectionUri = Client.UriFactory.CreateDocumentCollectionUri(dbId, collectionName) let client = new Client.DocumentClient(uri, key, connPolicy, Nullable(ConsistencyLevel.Session)) - client - - let client = + do! + dbId + |> Client.UriFactory.CreateDatabaseUri + |> client.ReadDatabaseAsync // check if database exists + |> Async.AwaitTaskCorrect + |> Async.bind (fun _ -> client.ReadDocumentCollectionAsync(collectionUri) |> Async.AwaitTaskCorrect) // check if collection exists + |> Async.Ignore + do! client.OpenAsync() |> Async.AwaitTaskCorrect + return client :> IDocumentClient } + + let! client = match discovery with | Discovery.UriAndKey (uri, key, dbName, collName) -> client uri key dbName collName - do! client.OpenAsync() |> Async.AwaitTaskCorrect - - return EqxConnection(client :> IDocumentClient) } \ No newline at end of file + return EqxConnection(client) } \ No newline at end of file From 926d937fe2600459493807c3e007af15305956f3 Mon Sep 17 00:00:00 2001 From: dongdongcai Date: Mon, 14 May 2018 11:47:59 -0400 Subject: [PATCH 04/66] Added Equinox to Sample/Integration --- samples/Store/Integration/CartIntegration.fs | 19 +++++++++++++++++ .../ContactPreferencesIntegration.fs | 21 +++++++++++++++++++ .../Store/Integration/CosmosIntegration.fs | 16 ++++++++++++++ .../Store/Integration/FavoritesIntegration.fs | 15 +++++++++++++ samples/Store/Integration/Integration.fsproj | 3 +++ src/Equinox.Cosmos/Cosmos.fs | 16 +++++++------- src/Equinox.Cosmos/Equinox.Cosmos.fsproj | 9 ++++---- src/Equinox.EventStore/Infrastructure.fs | 2 ++ .../CosmosIntegration.fs | 16 +++++++------- .../Equinox.Cosmos.Integration.fsproj | 19 ++++++++--------- 10 files changed, 106 insertions(+), 30 deletions(-) create mode 100644 samples/Store/Integration/CosmosIntegration.fs diff --git a/samples/Store/Integration/CartIntegration.fs b/samples/Store/Integration/CartIntegration.fs index 4bd19f57c..79cc93f92 100644 --- a/samples/Store/Integration/CartIntegration.fs +++ b/samples/Store/Integration/CartIntegration.fs @@ -1,5 +1,7 @@ module Samples.Store.Integration.CartIntegration +open Equinox.Cosmos +open Equinox.Cosmos.Integration.CosmosIntegration open Equinox.EventStore open Equinox.MemoryStore open Swensen.Unquote @@ -21,6 +23,11 @@ let resolveGesStreamWithRollingSnapshots gateway = let resolveGesStreamWithoutCustomAccessStrategy gateway = GesResolver(gateway, codec, fold, initial).Resolve +let resolveEqxStreamWithCompactionEventType gateway compactionEventType (StreamArgs args) = + EqxStreamBuilder(gateway, codec, fold, initial, Equinox.Cosmos.CompactionStrategy.EventType compactionEventType).Create(args) +let resolveEqxStreamWithoutCompactionSemantics gateway _compactionEventType (StreamArgs args) = + EqxStreamBuilder(gateway, codec, fold, initial).Create(args) + let addAndThenRemoveItemsManyTimesExceptTheLastOne context cartId skuId (service: Backend.Cart.Service) count = service.FlowAsync(cartId, fun _ctx execute -> for i in 1..count do @@ -63,3 +70,15 @@ type Tests(testOutputHelper) = let! service = arrange connectToLocalEventStoreNode createGesGateway resolveGesStreamWithRollingSnapshots do! act service args } + + [] + let ``Can roundtrip against Equinox, correctly folding the events without compaction semantics`` args = Async.RunSynchronously <| async { + let! service = arrange connectToLocalEquinoxNode createEqxGateway resolveEqxStreamWithoutCompactionSemantics + do! act service args + } + + [] + let ``Can roundtrip against Equinox, correctly folding the events with compaction`` args = Async.RunSynchronously <| async { + let! service = arrange connectToLocalEquinoxNode createEqxGateway resolveEqxStreamWithCompactionEventType + do! act service args + } \ No newline at end of file diff --git a/samples/Store/Integration/ContactPreferencesIntegration.fs b/samples/Store/Integration/ContactPreferencesIntegration.fs index feb083f15..3e5f2cc57 100644 --- a/samples/Store/Integration/ContactPreferencesIntegration.fs +++ b/samples/Store/Integration/ContactPreferencesIntegration.fs @@ -1,5 +1,7 @@ module Samples.Store.Integration.ContactPreferencesIntegration +open Equinox.Cosmos +open Equinox.Cosmos.Integration.CosmosIntegration open Equinox.EventStore open Equinox.MemoryStore open Swensen.Unquote @@ -19,6 +21,13 @@ let resolveStreamGesWithOptimizedStorageSemantics gateway = let resolveStreamGesWithoutAccessStrategy gateway = GesResolver(gateway defaultBatchSize, codec, fold, initial).Resolve +let resolveStreamEqxWithCompactionSemantics gateway = + fun predicate (StreamArgs args) -> + EqxStreamBuilder(gateway, codec, fold, initial, Equinox.Cosmos.CompactionStrategy.Predicate predicate).Create(args) +let resolveStreamEqxWithoutCompactionSemantics gateway = + fun _ignoreWindowSize _ignoreCompactionPredicate (StreamArgs args) -> + EqxStreamBuilder(gateway, codec, fold, initial).Create(args) + type Tests(testOutputHelper) = let testOutput = TestOutputAdapter testOutputHelper let createLog () = createLogger testOutput @@ -53,3 +62,15 @@ type Tests(testOutputHelper) = let! service = arrange connectToLocalEventStoreNode createGesGateway resolveStreamGesWithOptimizedStorageSemantics do! act service args } + + [] + let ``Can roundtrip against Equinox, correctly folding the events with normal semantics`` args = Async.RunSynchronously <| async { + let! service = arrangeWithoutCompaction connectToLocalEquinoxNode createEqxGateway resolveStreamEqxWithoutCompactionSemantics + do! act service args + } + + [] + let ``Can roundtrip against Equinox, correctly folding the events with compaction semantics`` args = Async.RunSynchronously <| async { + let! service = arrange connectToLocalEquinoxNode createEqxGateway resolveStreamEqxWithCompactionSemantics + do! act service args + } \ No newline at end of file diff --git a/samples/Store/Integration/CosmosIntegration.fs b/samples/Store/Integration/CosmosIntegration.fs new file mode 100644 index 000000000..fdc0a3194 --- /dev/null +++ b/samples/Store/Integration/CosmosIntegration.fs @@ -0,0 +1,16 @@ +[] +module Samples.Store.Integration.CosmosIntegration + +open Equinox.Cosmos +open System + +/// Create an Equinox is compalicated, +/// To run the test, +/// Either replace connection below with a real equinox, +/// Or create a local Equinox using script: https://jet-tfs.visualstudio.com/Jet/Marvel/_git/marvel?path=%2Fservices%2Feqx-man%2Fexample.fsx&version=GBmaster&_a=contents +/// create Equinox with dbName "test" and collectionName "test" to perform test +let connectToLocalEquinoxNode() = + EqxConnector(requestTimeout=TimeSpan.FromSeconds 3., maxRetryAttemptsOnThrottledRequests=2, maxRetryWaitTimeInSeconds=60) + .Connect(Discovery.UriAndKey((Uri "https://localhost:8081"), "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==", "test", "test")) +let defaultBatchSize = 500 +let createEqxGateway connection batchSize = EqxGateway(EqxConnection(connection), EqxBatchingPolicy(maxBatchSize = batchSize)) \ No newline at end of file diff --git a/samples/Store/Integration/FavoritesIntegration.fs b/samples/Store/Integration/FavoritesIntegration.fs index 018f9e5d2..e0395a0b6 100644 --- a/samples/Store/Integration/FavoritesIntegration.fs +++ b/samples/Store/Integration/FavoritesIntegration.fs @@ -1,5 +1,7 @@ module Samples.Store.Integration.FavoritesIntegration +open Equinox.Cosmos +open Equinox.Cosmos.Integration.CosmosIntegration open Equinox.EventStore open Equinox.MemoryStore open Swensen.Unquote @@ -19,6 +21,10 @@ let createServiceGes gateway log = let resolveStream = GesResolver(gateway, codec, fold, initial, AccessStrategy.RollingSnapshots snapshot).Resolve Backend.Favorites.Service(log, resolveStream) +let createServiceEqx gateway log = + let resolveStream cet (StreamArgs args) = EqxStreamBuilder(gateway, codec, fold, initial, Equinox.Cosmos.CompactionStrategy.EventType cet).Create(args) + Backend.Favorites.Service(log, resolveStream) + type Tests(testOutputHelper) = let testOutput = TestOutputAdapter testOutputHelper let createLog () = createLogger testOutput @@ -48,3 +54,12 @@ type Tests(testOutputHelper) = let service = createServiceGes gateway log do! act service args } + + [] + let ``Can roundtrip against Equinox, correctly folding the events`` args = Async.RunSynchronously <| async { + let log = createLog () + let! conn = connectToLocalEquinoxNode log + let gateway = createEqxGateway conn defaultBatchSize + let service = createServiceEqx gateway log + do! act service args + } \ No newline at end of file diff --git a/samples/Store/Integration/Integration.fsproj b/samples/Store/Integration/Integration.fsproj index 5cf8d2f3b..66e2ef628 100644 --- a/samples/Store/Integration/Integration.fsproj +++ b/samples/Store/Integration/Integration.fsproj @@ -10,6 +10,7 @@ + @@ -17,9 +18,11 @@ + + diff --git a/src/Equinox.Cosmos/Cosmos.fs b/src/Equinox.Cosmos/Cosmos.fs index 8a5866b3b..5dbf33526 100644 --- a/src/Equinox.Cosmos/Cosmos.fs +++ b/src/Equinox.Cosmos/Cosmos.fs @@ -258,7 +258,7 @@ module private Read = open Microsoft.Azure.Documents.Linq open System.Linq - let private getQuery ((client, collectionUri): Connection) strongConsistency streamId (direction: Direction) batchSize sequenceNumber = + let private getQuery ((client, collectionUri): Connection) streamId (direction: Direction) batchSize sequenceNumber = let sequenceNumber = match direction, sequenceNumber with @@ -343,7 +343,7 @@ module private Read = "Eqx{action:l} stream={stream} count={count}/{batches} version={version} RequestCharge={ru}", action, streamName, count, batches, version, ru) - let loadForwardsFrom (log : ILogger) retryPolicy conn batchSize maxPermittedBatchReads consistencyLevel streamName startPosition + let loadForwardsFrom (log : ILogger) retryPolicy conn batchSize maxPermittedBatchReads streamName startPosition : Async = async { let mutable ru = 0.0 let mergeBatches (batches: AsyncSeq) = async { @@ -353,7 +353,7 @@ module private Read = |> AsyncSeq.concatSeq |> AsyncSeq.toArrayAsync return events, ru } - let query = getQuery conn consistencyLevel streamName Direction.Forward batchSize startPosition + let query = getQuery conn streamName Direction.Forward batchSize startPosition let call q = loggedQueryExecution streamName Direction.Forward batchSize startPosition q let retryingLoggingReadSlice q = Log.withLoggedRetries retryPolicy "readAttempt" (call q) let direction = Direction.Forward @@ -368,7 +368,7 @@ module private Read = let partitionPayloadFrom firstUsedEventNumber : EquinoxEvent[] -> int * int = let acc (tu,tr) ((EquinoxEventLen bytes) as y) = if y.sn < firstUsedEventNumber then tu, tr + bytes else tu + bytes, tr Array.fold acc (0,0) - let loadBackwardsUntilCompactionOrStart (log : ILogger) retryPolicy conn batchSize maxPermittedBatchReads consistencyLevel streamName isCompactionEvent + let loadBackwardsUntilCompactionOrStart (log : ILogger) retryPolicy conn batchSize maxPermittedBatchReads streamName isCompactionEvent : Async = async { let mergeFromCompactionPointOrStartFromBackwardsStream (log : ILogger) (batchesBackward : AsyncSeq) : Async = async { @@ -390,7 +390,7 @@ module private Read = |> AsyncSeq.toArrayAsync let eventsForward = Array.Reverse(tempBackward); tempBackward // sic - relatively cheap, in-place reverse of something we own return eventsForward, ru } - let query = getQuery conn consistencyLevel streamName Direction.Backward batchSize SN.last + let query = getQuery conn streamName Direction.Backward batchSize SN.last let call q = loggedQueryExecution streamName Direction.Backward batchSize SN.last q let retryingLoggingReadSlice q = Log.withLoggedRetries retryPolicy "readAttempt" (call q) let log = log |> Log.prop "batchSize" batchSize |> Log.prop "stream" streamName @@ -465,7 +465,7 @@ type EqxGateway(conn : EqxConnection, batching : EqxBatchingPolicy) = let isResolvedEventEventType predicate (x:EquinoxEvent) = predicate x.et let tryIsResolvedEventEventType predicateOption = predicateOption |> Option.map isResolvedEventEventType member __.LoadBatched streamName log isCompactionEventType: Async = async { - let! version, events = Read.loadForwardsFrom log conn.ReadRetryPolicy conn.Connection batching.BatchSize batching.MaxBatches false streamName 0L + let! version, events = Read.loadForwardsFrom log conn.ReadRetryPolicy conn.Connection batching.BatchSize batching.MaxBatches streamName 0L match tryIsResolvedEventEventType isCompactionEventType with | None -> return Token.ofNonCompacting version, events | Some isCompactionEvent -> @@ -475,14 +475,14 @@ type EqxGateway(conn : EqxConnection, batching : EqxBatchingPolicy) = member __.LoadBackwardsStoppingAtCompactionEvent streamName log isCompactionEventType: Async = async { let isCompactionEvent = isResolvedEventEventType isCompactionEventType let! version, events = - Read.loadBackwardsUntilCompactionOrStart log conn.ReadRetryPolicy conn.Connection batching.BatchSize batching.MaxBatches false streamName isCompactionEvent + Read.loadBackwardsUntilCompactionOrStart log conn.ReadRetryPolicy conn.Connection batching.BatchSize batching.MaxBatches streamName isCompactionEvent match Array.tryHead events |> Option.filter isCompactionEvent with | None -> return Token.ofUncompactedVersion batching.BatchSize version, events | Some resolvedEvent -> return Token.ofCompactionResolvedEventAndVersion resolvedEvent batching.BatchSize version, events } member __.LoadFromToken streamName log (token : Storage.StreamToken) isCompactionEventType : Async = async { let streamPosition = (unbox token.value).streamVersion - let! version, events = Read.loadForwardsFrom log conn.ReadRetryPolicy conn.Connection batching.BatchSize batching.MaxBatches true streamName streamPosition + let! version, events = Read.loadForwardsFrom log conn.ReadRetryPolicy conn.Connection batching.BatchSize batching.MaxBatches streamName streamPosition match tryIsResolvedEventEventType isCompactionEventType with | None -> return Token.ofNonCompacting version, events | Some isCompactionEvent -> diff --git a/src/Equinox.Cosmos/Equinox.Cosmos.fsproj b/src/Equinox.Cosmos/Equinox.Cosmos.fsproj index 8d0b3b977..ab4066a3b 100644 --- a/src/Equinox.Cosmos/Equinox.Cosmos.fsproj +++ b/src/Equinox.Cosmos/Equinox.Cosmos.fsproj @@ -2,6 +2,7 @@ netstandard2.0;net461 + true 5 false true @@ -20,15 +21,15 @@ - - + + - + - + diff --git a/src/Equinox.EventStore/Infrastructure.fs b/src/Equinox.EventStore/Infrastructure.fs index bc00a8e32..6f68e3cb5 100644 --- a/src/Equinox.EventStore/Infrastructure.fs +++ b/src/Equinox.EventStore/Infrastructure.fs @@ -86,6 +86,8 @@ type Async with else sc ()) |> ignore) + static member inline bind (f:'a -> Async<'b>) (a:Async<'a>) : Async<'b> = async.Bind(a, f) + static member map (f:'a -> 'b) (a:Async<'a>) : Async<'b> = async.Bind(a, f >> async.Return) module AsyncSeq = /// Same as takeWhileAsync, but returns the final element too diff --git a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs index 17432fb22..7b73c925c 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs @@ -1,4 +1,4 @@ -module Equinox.Cosmos.Integration.EquinoxIntegration +module Equinox.Cosmos.Integration.CosmosIntegration open Equinox.Integration.Infrastructure open Equinox.Cosmos @@ -9,7 +9,7 @@ open System /// Standing up an Equinox instance is complicated; to run for test purposes either: /// - replace connection below with a connection string or Uri+Key for an initialized Equinox instance /// - Create a local Equinox with dbName "test" and collectionName "test" using provisioning script -let connectToLocalEquinoxNode () = +let connectToLocalEquinoxNode (_log: Serilog.ILogger) = EqxConnector(requestTimeout=TimeSpan.FromSeconds 3., maxRetryAttemptsOnThrottledRequests=2, maxRetryWaitTimeInSeconds=60) .Connect(Discovery.UriAndKey(Uri "https://localhost:8081", "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==","test","test")) let defaultBatchSize = 500 @@ -91,7 +91,7 @@ type Tests(testOutputHelper) = [] let ``Can roundtrip against Equinox, correctly batching the reads [without any optimizations]`` context cartId skuId = Async.RunSynchronously <| async { let log, capture = createLoggerWithCapture () - let! conn = connectToLocalEquinoxNode () + let! conn = connectToLocalEquinoxNode log let batchSize = 3 let service = Cart.createServiceWithoutOptimization conn batchSize log @@ -117,7 +117,7 @@ type Tests(testOutputHelper) = [] let ``Can roundtrip against Equinox, managing sync conflicts by retrying [without any optimizations]`` ctx initialState = Async.RunSynchronously <| async { let log1, capture1 = createLoggerWithCapture () - let! conn = connectToLocalEquinoxNode () + let! conn = connectToLocalEquinoxNode log1 // Ensure batching is included at some point in the proceedings let batchSize = 3 @@ -193,7 +193,7 @@ type Tests(testOutputHelper) = [] let ``Can roundtrip against Equinox, correctly compacting to avoid redundant reads`` context skuId cartId = Async.RunSynchronously <| async { let log, capture = createLoggerWithCapture () - let! conn = connectToLocalEquinoxNode () + let! conn = connectToLocalEquinoxNode log let batchSize = 10 let service = Cart.createServiceWithCompaction conn batchSize log @@ -232,7 +232,7 @@ type Tests(testOutputHelper) = [] let ``Can correctly read and update against Equinox, with window size of 1 using tautological Compaction predicate`` id value = Async.RunSynchronously <| async { let log, capture = createLoggerWithCapture () - let! conn = connectToLocalEquinoxNode () + let! conn = connectToLocalEquinoxNode log let service = ContactPreferences.createService (createEqxGateway conn) log let (Domain.ContactPreferences.Id email) = id @@ -255,7 +255,7 @@ type Tests(testOutputHelper) = [] let ``Can roundtrip against Equinox, correctly caching to avoid redundant reads`` context skuId cartId = Async.RunSynchronously <| async { let log, capture = createLoggerWithCapture () - let! conn = connectToLocalEquinoxNode () + let! conn = connectToLocalEquinoxNode log let batchSize = 10 let cache = Caching.Cache("cart", sizeMb = 50) let createServiceCached () = Cart.createServiceWithCaching conn batchSize log cache @@ -282,7 +282,7 @@ type Tests(testOutputHelper) = [] let ``Can combine compaction with caching against Equinox`` context skuId cartId = Async.RunSynchronously <| async { let log, capture = createLoggerWithCapture () - let! conn = connectToLocalEquinoxNode () + let! conn = connectToLocalEquinoxNode log let batchSize = 10 let service1 = Cart.createServiceWithCompaction conn batchSize log let cache = Caching.Cache("cart", sizeMb = 50) diff --git a/tests/Equinox.Cosmos.Integration/Equinox.Cosmos.Integration.fsproj b/tests/Equinox.Cosmos.Integration/Equinox.Cosmos.Integration.fsproj index d1ebeda06..c4c3aeeda 100644 --- a/tests/Equinox.Cosmos.Integration/Equinox.Cosmos.Integration.fsproj +++ b/tests/Equinox.Cosmos.Integration/Equinox.Cosmos.Integration.fsproj @@ -12,23 +12,22 @@ + + + + + + + + + - - - - - - - - - - \ No newline at end of file From 9835a0a3e080f9b449e52dc8b05cd80dde5653a6 Mon Sep 17 00:00:00 2001 From: dongdongcai Date: Mon, 14 May 2018 13:48:05 -0400 Subject: [PATCH 05/66] Added support for connection string --- .../Store/Integration/CosmosIntegration.fs | 2 +- src/Equinox.Cosmos/Cosmos.fs | 20 ++++++++++--------- src/Equinox.EventStore/Infrastructure.fs | 1 - 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/samples/Store/Integration/CosmosIntegration.fs b/samples/Store/Integration/CosmosIntegration.fs index fdc0a3194..3a4456423 100644 --- a/samples/Store/Integration/CosmosIntegration.fs +++ b/samples/Store/Integration/CosmosIntegration.fs @@ -11,6 +11,6 @@ open System /// create Equinox with dbName "test" and collectionName "test" to perform test let connectToLocalEquinoxNode() = EqxConnector(requestTimeout=TimeSpan.FromSeconds 3., maxRetryAttemptsOnThrottledRequests=2, maxRetryWaitTimeInSeconds=60) - .Connect(Discovery.UriAndKey((Uri "https://localhost:8081"), "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==", "test", "test")) + .Connect(Discovery.UriAndKey((Uri "https://localhost:8081", "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==", "test", "test"))) let defaultBatchSize = 500 let createEqxGateway connection batchSize = EqxGateway(EqxConnection(connection), EqxBatchingPolicy(maxBatchSize = batchSize)) \ No newline at end of file diff --git a/src/Equinox.Cosmos/Cosmos.fs b/src/Equinox.Cosmos/Cosmos.fs index 5dbf33526..53dd15df1 100644 --- a/src/Equinox.Cosmos/Cosmos.fs +++ b/src/Equinox.Cosmos/Cosmos.fs @@ -90,7 +90,7 @@ module EventData = type EquinoxEvent = { id : string s : StreamId - k : string + k : StreamId ts : DateTimeOffset sn : SN et : string @@ -167,8 +167,11 @@ module private Write = eventData |> eventDataToEquinoxEvent streamId sequenceNumber + let requestOptions = + Client.RequestOptions(PartitionKey = PartitionKey(streamId)) + let! res = - client.CreateDocumentAsync (collectionUri, equinoxEvent) + client.CreateDocumentAsync (collectionUri, equinoxEvent, requestOptions) |> Async.AwaitTaskCorrect return (sequenceNumber, res.RequestCharge) @@ -300,13 +303,12 @@ module private Read = | Some last -> last.sn let private queryExecution (query: IDocumentQuery<'T>) = - async { - let! res = query.ExecuteNextAsync<'T>() |> Async.AwaitTaskCorrect - return res.ToArray(), res.RequestCharge } + query.ExecuteNextAsync<'T>() |> Async.AwaitTaskCorrect let private loggedQueryExecution streamName direction batchSize startPos (query: IDocumentQuery) (log: ILogger) : Async = async { - let! t, (slice, ru) = queryExecution query |> Stopwatch.Time + let! t, res = queryExecution query |> Stopwatch.Time + let slice, ru = res.ToArray(), res.RequestCharge let bytes, count = slice |> Array.sumBy (|EquinoxEventLen|), slice.Length let reqMetric : Log.Measurement ={ stream = streamName; interval = t; bytes = bytes; count = count; ru = Convert.ToInt32(ru) } let evt = Log.Slice (direction, reqMetric) @@ -327,7 +329,7 @@ module private Read = let! slice = readSlice query batchLog yield slice if query.HasMoreResults then - yield! loop (batchCount + 1)} + yield! loop (batchCount + 1) } //| x -> raise <| System.ArgumentOutOfRangeException("SliceReadStatus", x, "Unknown result value") } loop 0 @@ -646,7 +648,7 @@ type EqxStreamBuilder<'event, 'state>(gateway, codec, fold, initial, ?compaction [] type Discovery = | UriAndKey of Uri * string * string * string - //| ConnecionString of string * string * string -> support this later + //| ConnectionString of string * string * string type EqxConnector ( requestTimeout: TimeSpan, maxRetryAttemptsOnThrottledRequests: int, maxRetryWaitTimeInSeconds: int, @@ -680,6 +682,6 @@ type EqxConnector let! client = match discovery with - | Discovery.UriAndKey (uri, key, dbName, collName) -> client uri key dbName collName + | Discovery.UriAndKey (uri, key, db, coll) -> client uri key db coll return EqxConnection(client) } \ No newline at end of file diff --git a/src/Equinox.EventStore/Infrastructure.fs b/src/Equinox.EventStore/Infrastructure.fs index 6f68e3cb5..914639df1 100644 --- a/src/Equinox.EventStore/Infrastructure.fs +++ b/src/Equinox.EventStore/Infrastructure.fs @@ -87,7 +87,6 @@ type Async with sc ()) |> ignore) static member inline bind (f:'a -> Async<'b>) (a:Async<'a>) : Async<'b> = async.Bind(a, f) - static member map (f:'a -> 'b) (a:Async<'a>) : Async<'b> = async.Bind(a, f >> async.Return) module AsyncSeq = /// Same as takeWhileAsync, but returns the final element too From 434b2fe9662ff7bd601c40baef4d56c7fa71fc0d Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 27 Jun 2018 04:54:48 +0100 Subject: [PATCH 06/66] Add failing test for converter embedding --- .../ByteArrayConverterTests.fs | 36 +++++++++++++++++++ .../Equinox.Cosmos.Integration.fsproj | 1 + 2 files changed, 37 insertions(+) create mode 100644 tests/Equinox.Cosmos.Integration/ByteArrayConverterTests.fs diff --git a/tests/Equinox.Cosmos.Integration/ByteArrayConverterTests.fs b/tests/Equinox.Cosmos.Integration/ByteArrayConverterTests.fs new file mode 100644 index 000000000..506fc5fcd --- /dev/null +++ b/tests/Equinox.Cosmos.Integration/ByteArrayConverterTests.fs @@ -0,0 +1,36 @@ +module Foldunk.Equinox.ByteArrayConverterTests + +open Newtonsoft.Json +open Swensen.Unquote +open System +open Xunit + +module Fixtures = + type Embedded = { embed : string } + +let serializer = new JsonSerializer() + +let inline serialize (x:'t) = + use sw = new System.IO.StringWriter() + use w = new JsonTextWriter(sw) + serializer.Serialize(w,x) + sw.ToString() + +[] +let ``ByteArrayConverter serializes properly`` () = + use sw = new System.IO.StringWriter() + use w = new JsonTextWriter(sw) + let blob = serialize ({ embed = "\"" } : Fixtures.Embedded) |> System.Text.Encoding.UTF8.GetBytes + let e : Equinox.Cosmos.EquinoxEvent = + { id = null + s = null + k = null + ts = DateTimeOffset.MinValue + sn = 0L + et = null + + d = blob + + md = null } + let res = serialize e + test <@ res.Contains """"d":{"embed":"\""}""" @> \ No newline at end of file diff --git a/tests/Equinox.Cosmos.Integration/Equinox.Cosmos.Integration.fsproj b/tests/Equinox.Cosmos.Integration/Equinox.Cosmos.Integration.fsproj index c4c3aeeda..bc2d93586 100644 --- a/tests/Equinox.Cosmos.Integration/Equinox.Cosmos.Integration.fsproj +++ b/tests/Equinox.Cosmos.Integration/Equinox.Cosmos.Integration.fsproj @@ -10,6 +10,7 @@ + From ed003fabe66c81cc3ea9d7141a3d10c21a5a6ee8 Mon Sep 17 00:00:00 2001 From: dongdongcai Date: Wed, 27 Jun 2018 11:00:48 -0400 Subject: [PATCH 07/66] Some naming changes --- .../Store/Integration/CosmosIntegration.fs | 21 +++--- src/Equinox.Cosmos/Cosmos.fs | 64 +++++++++++----- .../CosmosIntegration.fs | 20 ++--- .../Equinox.Cosmos.Integration.fsproj | 2 +- .../Infrastructure.fs | 74 +++++++++++++++++++ 5 files changed, 142 insertions(+), 39 deletions(-) create mode 100644 tests/Equinox.Cosmos.Integration/Infrastructure.fs diff --git a/samples/Store/Integration/CosmosIntegration.fs b/samples/Store/Integration/CosmosIntegration.fs index 3a4456423..f1772f0c7 100644 --- a/samples/Store/Integration/CosmosIntegration.fs +++ b/samples/Store/Integration/CosmosIntegration.fs @@ -4,13 +4,16 @@ module Samples.Store.Integration.CosmosIntegration open Equinox.Cosmos open System -/// Create an Equinox is compalicated, -/// To run the test, -/// Either replace connection below with a real equinox, -/// Or create a local Equinox using script: https://jet-tfs.visualstudio.com/Jet/Marvel/_git/marvel?path=%2Fservices%2Feqx-man%2Fexample.fsx&version=GBmaster&_a=contents -/// create Equinox with dbName "test" and collectionName "test" to perform test -let connectToLocalEquinoxNode() = - EqxConnector(requestTimeout=TimeSpan.FromSeconds 3., maxRetryAttemptsOnThrottledRequests=2, maxRetryWaitTimeInSeconds=60) - .Connect(Discovery.UriAndKey((Uri "https://localhost:8081", "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==", "test", "test"))) +/// Standing up an Equinox instance is complicated; to run for test purposes either: +/// - replace connection below with a connection string or Uri+Key for an initialized Equinox instance +/// - Create a local Equinox with dbName "test" and collectionName "test" using script: +/// /src/Equinox.Cosmos/EquinoxManager.fsx +let connectToLocalEquinoxNode log = + EqxConnector(log=log, requestTimeout=TimeSpan.FromSeconds 3., maxRetryAttemptsOnThrottledRequests=2, maxRetryWaitTimeInSeconds=60) + .Establish("equinoxStoreSampleIntegration", Discovery.UriAndKey(Uri "https://localhost:8081", "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==")) let defaultBatchSize = 500 -let createEqxGateway connection batchSize = EqxGateway(EqxConnection(connection), EqxBatchingPolicy(maxBatchSize = batchSize)) \ No newline at end of file +let createEqxGateway connection batchSize = EqxGateway(connection, EqxBatchingPolicy(maxBatchSize = batchSize)) +// Typically, one will split different categories of stream into Cosmos collections - hard coding this is thus an oversimplification +let (|StreamArgs|) streamName = + let databaseId, collectionId = "test", "test" + databaseId, collectionId, streamName \ No newline at end of file diff --git a/src/Equinox.Cosmos/Cosmos.fs b/src/Equinox.Cosmos/Cosmos.fs index 53dd15df1..954e055b1 100644 --- a/src/Equinox.Cosmos/Cosmos.fs +++ b/src/Equinox.Cosmos/Cosmos.fs @@ -14,6 +14,15 @@ module ArraySegmentExtensions = type System.Text.Encoding with member x.GetString(data:ArraySegment) = x.GetString(data.Array, data.Offset, data.Count) +[] +module Strings = + open System.Text.RegularExpressions + /// Obtains a single pattern group, if one exists + let (|RegexGroup|_|) (pattern:string) arg = + match Regex.Match(arg, pattern, RegexOptions.None, TimeSpan.FromMilliseconds(250.0)) with + | m when m.Success && m.Groups.[1].Success -> m.Groups.[1].Value |> Some + | _ -> None + type ByteArrayConverter() = inherit JsonConverter() @@ -647,12 +656,19 @@ type EqxStreamBuilder<'event, 'state>(gateway, codec, fold, initial, ?compaction [] type Discovery = - | UriAndKey of Uri * string * string * string - //| ConnectionString of string * string * string + | UriAndKey of uri:Uri * key:string + | ConnectionString of string type EqxConnector ( requestTimeout: TimeSpan, maxRetryAttemptsOnThrottledRequests: int, maxRetryWaitTimeInSeconds: int, - ?maxConnectionLimit) = + log : ILogger, + /// Connection limit (default 1000) + ?maxConnectionLimit, + ?readRetryPolicy, ?writeRetryPolicy, + /// Additional strings identifying the context of this connection; should provide enough context to disambiguate all potential connections to a cluster + /// NB as this will enter server and client logs, it should not contain sensitive information + ?tags : (string*string) seq) = + let connPolicy = let cp = Client.ConnectionPolicy.Default cp.ConnectionMode <- Client.ConnectionMode.Direct @@ -666,22 +682,32 @@ type EqxConnector cp /// Yields an connection to DocDB configured and Connect()ed to DocDB collection per the requested `discovery` strategy - member __.Connect (discovery : Discovery) : Async = async { - let client (uri: Uri) (key: string) (dbId: string) (collectionName: string) = async { - let collectionUri = Client.UriFactory.CreateDocumentCollectionUri(dbId, collectionName) - let client = new Client.DocumentClient(uri, key, connPolicy, Nullable(ConsistencyLevel.Session)) - do! - dbId - |> Client.UriFactory.CreateDatabaseUri - |> client.ReadDatabaseAsync // check if database exists - |> Async.AwaitTaskCorrect - |> Async.bind (fun _ -> client.ReadDocumentCollectionAsync(collectionUri) |> Async.AwaitTaskCorrect) // check if collection exists - |> Async.Ignore + member __.Connect + ( /// Name should be sufficient to uniquely identify this connection within a single app instance's logs + name, + discovery : Discovery) : Async = + let connect (uri: Uri, key: string) = async { + let name = String.concat ";" <| seq { + yield name + match tags with None -> () | Some tags -> for key, value in tags do yield sprintf "%s=%s" key value } + let sanitizedName = name.Replace('\'','_').Replace(':','_') // ES internally uses `:` and `'` as separators in log messages and ... people regex logs + let client = new Client.DocumentClient(uri, key, connPolicy, Nullable ConsistencyLevel.Session) + log.Information("Connected to Equinox with clientId={clientId}", sanitizedName) do! client.OpenAsync() |> Async.AwaitTaskCorrect return client :> IDocumentClient } - let! client = - match discovery with - | Discovery.UriAndKey (uri, key, db, coll) -> client uri key db coll - - return EqxConnection(client) } \ No newline at end of file + match discovery with + | Discovery.UriAndKey(uri=uri; key=key) -> + connect (uri,key) + | Discovery.ConnectionString connStr -> + let cred = + match connStr,connStr with + | Strings.RegexGroup "AccountEndpoint=(.+?);" uri, Strings.RegexGroup "AccountKey=(.+?);" key -> + System.Uri(uri), key + | _ -> failwithf "Invalid DocumentDB connection string: %s" connStr + connect cred + + /// Yields a DocDbConnection configured per the specified strategy + member __.Establish(name, discovery : Discovery) : Async = async { + let! conn = __.Connect(name, discovery) + return EqxConnection(conn, ?readRetryPolicy=readRetryPolicy, ?writeRetryPolicy=writeRetryPolicy) } \ No newline at end of file diff --git a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs index 7b73c925c..ecc8725ef 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs @@ -1,6 +1,6 @@ module Equinox.Cosmos.Integration.CosmosIntegration -open Equinox.Integration.Infrastructure +open Equinox.Cosmos.Integration.Infrastructure open Equinox.Cosmos open Swensen.Unquote open System.Threading @@ -9,9 +9,9 @@ open System /// Standing up an Equinox instance is complicated; to run for test purposes either: /// - replace connection below with a connection string or Uri+Key for an initialized Equinox instance /// - Create a local Equinox with dbName "test" and collectionName "test" using provisioning script -let connectToLocalEquinoxNode (_log: Serilog.ILogger) = - EqxConnector(requestTimeout=TimeSpan.FromSeconds 3., maxRetryAttemptsOnThrottledRequests=2, maxRetryWaitTimeInSeconds=60) - .Connect(Discovery.UriAndKey(Uri "https://localhost:8081", "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==","test","test")) +let connectToLocalEquinoxNode (log: Serilog.ILogger) = + EqxConnector(log=log, requestTimeout=TimeSpan.FromSeconds 3., maxRetryAttemptsOnThrottledRequests=2, maxRetryWaitTimeInSeconds=60) + .Establish("localDocDbSim", Discovery.UriAndKey(Uri "https://localhost:8081", "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==")) let defaultBatchSize = 500 let createEqxGateway connection batchSize = EqxGateway(connection, EqxBatchingPolicy(maxBatchSize = batchSize)) let (|StreamArgs|) gateway = @@ -84,9 +84,9 @@ type Tests(testOutputHelper) = .CreateLogger() logger, capture - let singleSliceForward = EsAct.SliceForward - let singleBatchForward = [EsAct.SliceForward; EsAct.BatchForward] - let batchForwardAndAppend = singleBatchForward @ [EsAct.Append] + let singleSliceForward = EqxAct.SliceForward + let singleBatchForward = [EqxAct.SliceForward; EqxAct.BatchForward] + let batchForwardAndAppend = singleBatchForward @ [EqxAct.Append] [] let ``Can roundtrip against Equinox, correctly batching the reads [without any optimizations]`` context cartId skuId = Async.RunSynchronously <| async { @@ -183,12 +183,12 @@ type Tests(testOutputHelper) = && has sku11 11 && has sku12 12 && has sku21 21 && has sku22 22 @> // Intended conflicts pertained - let hadConflict= function EsEvent (EsAction EsAct.AppendConflict) -> Some () | _ -> None + let hadConflict= function EqxEvent (EqxAction EqxAct.AppendConflict) -> Some () | _ -> None test <@ [1; 1] = [for c in [capture1; capture2] -> c.ChooseCalls hadConflict |> List.length] @> } - let singleBatchBackwards = [EsAct.SliceBackward; EsAct.BatchBackward] - let batchBackwardsAndAppend = singleBatchBackwards @ [EsAct.Append] + let singleBatchBackwards = [EqxAct.SliceBackward; EqxAct.BatchBackward] + let batchBackwardsAndAppend = singleBatchBackwards @ [EqxAct.Append] [] let ``Can roundtrip against Equinox, correctly compacting to avoid redundant reads`` context skuId cartId = Async.RunSynchronously <| async { diff --git a/tests/Equinox.Cosmos.Integration/Equinox.Cosmos.Integration.fsproj b/tests/Equinox.Cosmos.Integration/Equinox.Cosmos.Integration.fsproj index bc2d93586..5e00b54fa 100644 --- a/tests/Equinox.Cosmos.Integration/Equinox.Cosmos.Integration.fsproj +++ b/tests/Equinox.Cosmos.Integration/Equinox.Cosmos.Integration.fsproj @@ -8,7 +8,7 @@ - + diff --git a/tests/Equinox.Cosmos.Integration/Infrastructure.fs b/tests/Equinox.Cosmos.Integration/Infrastructure.fs new file mode 100644 index 000000000..2252f32f0 --- /dev/null +++ b/tests/Equinox.Cosmos.Integration/Infrastructure.fs @@ -0,0 +1,74 @@ +[] +module Equinox.Cosmos.Integration.Infrastructure + +open Domain +open FsCheck +open System + +type FsCheckGenerators = + static member SkuId = Arb.generate |> Gen.map SkuId |> Arb.fromGen + static member RequestId = Arb.generate |> Gen.map RequestId |> Arb.fromGen + static member ContactPreferencesId = + Arb.generate + |> Gen.map (fun x -> sprintf "%s@test.com" (x.ToString("N"))) + |> Gen.map ContactPreferences.Id + |> Arb.fromGen + +type AutoDataAttribute() = + inherit FsCheck.Xunit.PropertyAttribute(Arbitrary = [|typeof|], MaxTest = 1, QuietOnSuccess = true) + +// Derived from https://github.com/damianh/CapturingLogOutputWithXunit2AndParallelTests +// NB VS does not surface these atm, but other test runners / test reports do +type TestOutputAdapter(testOutput : Xunit.Abstractions.ITestOutputHelper) = + let formatter = Serilog.Formatting.Display.MessageTemplateTextFormatter("{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} [{Level}] {Message}{NewLine}{Exception}", null); + let writeSerilogEvent logEvent = + use writer = new System.IO.StringWriter() + formatter.Format(logEvent, writer); + writer |> string |> testOutput.WriteLine + interface Serilog.Core.ILogEventSink with member __.Emit logEvent = writeSerilogEvent logEvent + +[] +module SerilogHelpers = + open Serilog + open Serilog.Events + + let createLogger sink = + LoggerConfiguration() + .WriteTo.Sink(sink) + .CreateLogger() + + let (|SerilogScalar|_|) : Serilog.Events.LogEventPropertyValue -> obj option = function + | (:? ScalarValue as x) -> Some x.Value + | _ -> None + [] + type EqxAct = Append | AppendConflict | SliceForward | SliceBackward | BatchForward | BatchBackward + let (|EqxAction|) (evt : Equinox.Cosmos.Log.Event) = + match evt with + | Equinox.Cosmos.Log.WriteSuccess _ -> EqxAct.Append + | Equinox.Cosmos.Log.WriteConflict _ -> EqxAct.AppendConflict + | Equinox.Cosmos.Log.Slice (Equinox.Cosmos.Direction.Forward,_) -> EqxAct.SliceForward + | Equinox.Cosmos.Log.Slice (Equinox.Cosmos.Direction.Backward,_) -> EqxAct.SliceBackward + | Equinox.Cosmos.Log.Batch (Equinox.Cosmos.Direction.Forward,_,_) -> EqxAct.BatchForward + | Equinox.Cosmos.Log.Batch (Equinox.Cosmos.Direction.Backward,_,_) -> EqxAct.BatchBackward + let (|EqxEvent|_|) (logEvent : LogEvent) : Equinox.Cosmos.Log.Event option = + logEvent.Properties.Values |> Seq.tryPick (function + | SerilogScalar (:? Equinox.Cosmos.Log.Event as e) -> Some e + | _ -> None) + + let (|HasProp|_|) (name : string) (e : LogEvent) : LogEventPropertyValue option = + match e.Properties.TryGetValue name with + | true, (SerilogScalar _ as s) -> Some s | _ -> None + | _ -> None + let (|SerilogString|_|) : LogEventPropertyValue -> string option = function SerilogScalar (:? string as y) -> Some y | _ -> None + let (|SerilogBool|_|) : LogEventPropertyValue -> bool option = function SerilogScalar (:? bool as y) -> Some y | _ -> None + + type LogCaptureBuffer() = + let captured = ResizeArray() + let writeSerilogEvent (logEvent: LogEvent) = + logEvent.RenderMessage () |> System.Diagnostics.Trace.WriteLine + captured.Add logEvent + interface Serilog.Core.ILogEventSink with member __.Emit logEvent = writeSerilogEvent logEvent + member __.Clear () = captured.Clear() + member __.Entries = captured.ToArray() + member __.ChooseCalls chooser = captured |> Seq.choose chooser |> List.ofSeq + member __.ExternalCalls = __.ChooseCalls (function EqxEvent (EqxAction act) -> Some act | _ -> None) \ No newline at end of file From 3bc19ae916110ab583b1c8e7de14bc0b27d2f554 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 29 Jun 2018 08:29:29 +0100 Subject: [PATCH 08/66] rename converter to VerbatimUtf8JsonConverter --- src/Equinox.Cosmos/Cosmos.fs | 6 ++--- .../Equinox.Cosmos.Integration.fsproj | 2 +- ...s.fs => VerbatimUtf8JsonConverterTests.fs} | 23 ++++++++++--------- 3 files changed, 16 insertions(+), 15 deletions(-) rename tests/Equinox.Cosmos.Integration/{ByteArrayConverterTests.fs => VerbatimUtf8JsonConverterTests.fs} (53%) diff --git a/src/Equinox.Cosmos/Cosmos.fs b/src/Equinox.Cosmos/Cosmos.fs index 954e055b1..05051e955 100644 --- a/src/Equinox.Cosmos/Cosmos.fs +++ b/src/Equinox.Cosmos/Cosmos.fs @@ -23,7 +23,7 @@ module Strings = | m when m.Success && m.Groups.[1].Success -> m.Groups.[1].Value |> Some | _ -> None -type ByteArrayConverter() = +type VerbatimUtf8JsonConverter() = inherit JsonConverter() override this.ReadJson(reader, _, _, serializer) = @@ -104,10 +104,10 @@ type EquinoxEvent = { sn : SN et : string - [)>] + [)>] d : byte[] - [)>] + [)>] md : byte[] } type Connection = IDocumentClient * Uri diff --git a/tests/Equinox.Cosmos.Integration/Equinox.Cosmos.Integration.fsproj b/tests/Equinox.Cosmos.Integration/Equinox.Cosmos.Integration.fsproj index 5e00b54fa..04b39af40 100644 --- a/tests/Equinox.Cosmos.Integration/Equinox.Cosmos.Integration.fsproj +++ b/tests/Equinox.Cosmos.Integration/Equinox.Cosmos.Integration.fsproj @@ -10,7 +10,7 @@ - + diff --git a/tests/Equinox.Cosmos.Integration/ByteArrayConverterTests.fs b/tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs similarity index 53% rename from tests/Equinox.Cosmos.Integration/ByteArrayConverterTests.fs rename to tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs index 506fc5fcd..cebc2982f 100644 --- a/tests/Equinox.Cosmos.Integration/ByteArrayConverterTests.fs +++ b/tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs @@ -1,35 +1,36 @@ -module Foldunk.Equinox.ByteArrayConverterTests +module Equinox.Cosmos.Integration.VerbatimUtf8JsonConverterTests open Newtonsoft.Json open Swensen.Unquote open System open Xunit -module Fixtures = - type Embedded = { embed : string } - let serializer = new JsonSerializer() - let inline serialize (x:'t) = use sw = new System.IO.StringWriter() use w = new JsonTextWriter(sw) serializer.Serialize(w,x) sw.ToString() +type Embedded = { embed : string } +type Union = + | A of Embedded + | B of Embedded + interface TypeShape.UnionContract.IUnionContract + [] -let ``ByteArrayConverter serializes properly`` () = - use sw = new System.IO.StringWriter() - use w = new JsonTextWriter(sw) - let blob = serialize ({ embed = "\"" } : Fixtures.Embedded) |> System.Text.Encoding.UTF8.GetBytes +let ``VerbatimUtf8JsonConverter serializes properly`` () = + let unionEncoder = Equinox.UnionCodec.JsonUtf8.Create<_>(JsonSerializerSettings()) + let encoded = unionEncoder.Encode(A { embed = "\"" }) let e : Equinox.Cosmos.EquinoxEvent = { id = null s = null k = null ts = DateTimeOffset.MinValue sn = 0L - et = null - d = blob + et = encoded.caseName + d = encoded.payload md = null } let res = serialize e From 1a595ce88027444d004e2a17a5a7e1602479e3ee Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Mon, 2 Jul 2018 05:13:45 +0100 Subject: [PATCH 09/66] Align with master store management updates --- samples/Store/Integration/LogIntegration.fs | 40 +++++- src/Equinox.Cosmos/Cosmos.fs | 125 ++++++++++-------- .../CosmosIntegration.fs | 10 +- tools/Equinox.Tool/Program.fs | 1 + 4 files changed, 112 insertions(+), 64 deletions(-) diff --git a/samples/Store/Integration/LogIntegration.fs b/samples/Store/Integration/LogIntegration.fs index a980fcfce..6e256a47e 100644 --- a/samples/Store/Integration/LogIntegration.fs +++ b/samples/Store/Integration/LogIntegration.fs @@ -21,6 +21,22 @@ module EquinoxEsInterop = | Log.Batch (Direction.Forward,c,m) -> "LoadF", m, Some c | Log.Batch (Direction.Backward,c,m) -> "LoadB", m, Some c { action = action; stream = metric.stream; interval = metric.interval; bytes = metric.bytes; count = metric.count; batches = batches } +module EquinoxCosmosInterop = + open Equinox.Cosmos + [] + type FlatMetric = { action: string; stream: string; interval: StopwatchInterval; bytes: int; count: int; batches: int option; ru: int } with + override __.ToString() = sprintf "%s-Stream=%s %s-Elapsed=%O Ru=%O" __.action __.stream __.action __.interval.Elapsed __.ru + let flatten (evt : Log.Event) : FlatMetric = + let action, metric, batches, ru = + match evt with + | Log.WriteSuccess m -> "EqxAppendToStreamAsync", m, None, m.ru + | Log.WriteConflict m -> "EqxAppendToStreamAsync", m, None, m.ru + | Log.Slice (Direction.Forward,m) -> "EqxReadStreamEventsForwardAsync", m, None, m.ru + | Log.Slice (Direction.Backward,m) -> "EqxReadStreamEventsBackwardAsync", m, None, m.ru + | Log.Batch (Direction.Forward,c,m) -> "EqxLoadF", m, Some c, m.ru + | Log.Batch (Direction.Backward,c,m) -> "EqxLoadB", m, Some c, m.ru + { action = action; stream = metric.stream; bytes = metric.bytes; count = metric.count; batches = batches + interval = StopwatchInterval(metric.interval.StartTicks,metric.interval.EndTicks); ru = ru } type SerilogMetricsExtractor(emit : string -> unit) = let render template = @@ -37,12 +53,13 @@ type SerilogMetricsExtractor(emit : string -> unit) = let (|SerilogScalar|_|) : Serilog.Events.LogEventPropertyValue -> obj option = function | (:? Serilog.Events.ScalarValue as x) -> Some x.Value | _ -> None - let (|EsMetric|GenericMessage|) (logEvent : Serilog.Events.LogEvent) = + let (|EsMetric|CosmosMetric|GenericMessage|) (logEvent : Serilog.Events.LogEvent) = logEvent.Properties |> Seq.tryPick (function - | KeyValue (k, SerilogScalar (:? Equinox.EventStore.Log.Event as m)) -> Some <| Choice1Of2 (k,m) + | KeyValue (k, SerilogScalar (:? Equinox.EventStore.Log.Event as m)) -> Some <| Choice1Of3 (k,m) + | KeyValue (k, SerilogScalar (:? Equinox.Cosmos.Log.Event as m)) -> Some <| Choice2Of3 (k,m) | _ -> None) - |> Option.defaultValue (Choice2Of2 ()) + |> Option.defaultValue (Choice3Of3 ()) let handleLogEvent logEvent = match logEvent with | EsMetric (name, evt) as logEvent -> @@ -55,6 +72,11 @@ type SerilogMetricsExtractor(emit : string -> unit) = // 2. let rendered = logEvent.RenderMessage() in rendered.Replace("{esEvt} ","") logEvent.AddOrUpdateProperty(Serilog.Events.LogEventProperty(name, renderedMetrics)) emitEvent logEvent + | CosmosMetric (name, evt) as logEvent -> + let flat = EquinoxCosmosInterop.flatten evt + let renderedMetrics = sprintf "%s-Duration=%O" flat.action flat.interval.Elapsed |> Serilog.Events.ScalarValue + logEvent.AddOrUpdateProperty(Serilog.Events.LogEventProperty(name, renderedMetrics)) + emitEvent logEvent | GenericMessage () as logEvent -> emitEvent logEvent interface Serilog.Core.ILogEventSink with member __.Emit logEvent = handleLogEvent logEvent @@ -89,3 +111,15 @@ type Tests() = let cartId = Guid.NewGuid() |> CartId do! act buffer service itemCount context cartId skuId "ReadStreamEventsBackwardAsync-Duration" } + + [] + let ``Can roundtrip against Equinox, hooking, extracting and substituting metrics in the logging information`` context cartId skuId = Async.RunSynchronously <| async { + let buffer = ResizeArray() + let batchSize = defaultBatchSize + let (log,capture) = createLoggerWithMetricsExtraction buffer.Add + let! conn = connectToLocalEquinoxNode log + let gateway = createEqxGateway conn batchSize + let service = Backend.Cart.Service(log, CartIntegration.resolveEqxStreamWithCompactionEventType gateway) + let itemCount, cartId = batchSize / 2 + 1, cartId () + do! act buffer capture service itemCount context cartId skuId "ReadStreamEventsBackwardAsync-Duration" + } \ No newline at end of file diff --git a/src/Equinox.Cosmos/Cosmos.fs b/src/Equinox.Cosmos/Cosmos.fs index 05051e955..fce4929c1 100644 --- a/src/Equinox.Cosmos/Cosmos.fs +++ b/src/Equinox.Cosmos/Cosmos.fs @@ -10,7 +10,6 @@ open System [] module ArraySegmentExtensions = - type System.Text.Encoding with member x.GetString(data:ArraySegment) = x.GetString(data.Array, data.Offset, data.Count) @@ -26,26 +25,18 @@ module Strings = type VerbatimUtf8JsonConverter() = inherit JsonConverter() - override this.ReadJson(reader, _, _, serializer) = + override __.ReadJson(reader, _, _, serializer) = let s = serializer.Deserialize(reader, typeof) :?> string - if s = null - then - let arr: byte[] = [||] - // Why Array.Empty :> obj doesn't work here.... - arr :> obj - else - System.Text.Encoding.UTF8.GetBytes(s) :> obj + if s = null then Array.empty |> box + else reader.Value :?> string |> System.Text.Encoding.UTF8.GetBytes |> box override this.CanConvert(objectType) = typeof.Equals(objectType) override this.WriteJson(writer, value, serializer) = let array = value :?> byte[] - if Array.length array = 0 - then - serializer.Serialize(writer, null) - else - serializer.Serialize(writer, System.Text.Encoding.UTF8.GetString(array)) + if Array.length array = 0 then serializer.Serialize(writer, null) + else writer.WriteRawValue(System.Text.Encoding.UTF8.GetString(array)) type SN = int64 @@ -110,8 +101,6 @@ type EquinoxEvent = { [)>] md : byte[] } -type Connection = IDocumentClient * Uri - [] type Direction = Forward | Backward with override this.ToString() = match this with Forward -> "Forward" | Backward -> "Backward" @@ -163,12 +152,13 @@ module private Write = } let [] private multiDocInsert = "AtomicMultiDocInsert" + let [] private appendAtEnd = "appendAtEnd" let inline private sprocUri (sprocName : string) (collectionUri : Uri) = (collectionUri.ToString()) + "/sprocs/" + sprocName // TODO: do this elegantly /// Appends the single EventData using the sdk CreateDocumentAsync - let private appendSingleEvent ((client, collectionUri) : Connection) streamId sequenceNumber eventData : Async = + let private appendSingleEvent (client : IDocumentClient,collectionUri : Uri) streamId sequenceNumber eventData : Async = async { let sequenceNumber = (SN.next sequenceNumber) @@ -180,7 +170,7 @@ module private Write = Client.RequestOptions(PartitionKey = PartitionKey(streamId)) let! res = - client.CreateDocumentAsync (collectionUri, equinoxEvent, requestOptions) + client.CreateDocumentAsync(collectionUri, equinoxEvent, requestOptions) |> Async.AwaitTaskCorrect return (sequenceNumber, res.RequestCharge) @@ -188,7 +178,7 @@ module private Write = /// Appends the given EventData batch using the atomic stored procedure // This requires store procuedure in CosmosDB, is there other ways to do this? - let private appendEventBatch ((client, collectionUri) : Connection) streamId sequenceNumber eventsData : Async = + let private appendEventBatch (client : IDocumentClient,collectionUri) streamId sequenceNumber eventsData : Async = async { let sequenceNumber = (SN.next sequenceNumber) let res, sn = @@ -205,19 +195,36 @@ module private Write = return (sn - 1L), res.RequestCharge } - let private append connection streamId sequenceNumber (eventsData: EventData seq) = + let private append coll streamName sequenceNumber (eventsData: EventData seq) = match Seq.length eventsData with | l when l = 0 -> invalidArg "eventsData" "must be non-empty" | l when l = 1 -> eventsData |> Seq.head - |> appendSingleEvent connection streamId sequenceNumber - | _ -> appendEventBatch connection streamId sequenceNumber eventsData + |> appendSingleEvent coll streamName sequenceNumber + | _ -> appendEventBatch coll streamName sequenceNumber eventsData + // Add this for User Activity + let appendEventAtEnd (client : IDocumentClient,collectionUri : Uri,streamId) eventsData : Async = + async { + let res, _ = + eventsData + |> Seq.mapFold (fun sn ed -> (eventDataToEquinoxEvent streamId sn ed) |> JsonConvert.SerializeObject, SN.next sn) 0L + + let requestOptions = + Client.RequestOptions(PartitionKey = PartitionKey(streamId)) - let private writeEventsAsync (log : ILogger) (conn : Connection) (streamName : string) (version : int64) (events : EventData[]) + let! res = + client.ExecuteStoredProcedureAsync(collectionUri |> sprocUri appendAtEnd, requestOptions, res:> obj) + |> Async.AwaitTaskCorrect + + + return res.Response, res.RequestCharge + } + + let private writeEventsAsync (log : ILogger) coll streamName (version : int64) (events : EventData[]) : Async = async { try - let! wr = append conn streamName version events + let! wr = append coll streamName version events return EqxSyncResult.Written wr with ex -> // change this for store procudure @@ -242,12 +249,12 @@ module private Write = match data, metaData with Log.BlobLen bytes, Log.BlobLen metaBytes -> bytes + metaBytes events |> Array.sumBy eventDataLen - let private writeEventsLogged (conn : Connection) (streamName : string) (version : int64) (events : EventData[]) (log : ILogger) + let private writeEventsLogged coll streamName (version : int64) (events : EventData[]) (log : ILogger) : Async = async { let bytes, count = eventDataBytes events, events.Length let log = log |> Log.prop "bytes" bytes let writeLog = log |> Log.prop "stream" streamName |> Log.prop "expectedVersion" version |> Log.prop "count" count - let! t, result = writeEventsAsync writeLog conn streamName version events |> Stopwatch.Time + let! t, result = writeEventsAsync writeLog coll streamName version events |> Stopwatch.Time let reqMetric : Log.Measurement = { stream = streamName; interval = t; bytes = bytes; count = count; ru = 0} let resultLog, evt, (ru: float) = match result, reqMetric with @@ -261,16 +268,16 @@ module private Write = "Write", streamName, events.Length, version, (match evt with Log.WriteConflict _ -> true | _ -> false), ru) return result } - let writeEvents (log : ILogger) retryPolicy (conn : Connection) (streamName : string) (version : int64) (events : EventData[]) + let writeEvents (log : ILogger) retryPolicy coll (streamName : string) (version : int64) (events : EventData[]) : Async = - let call = writeEventsLogged conn streamName version events + let call = writeEventsLogged coll streamName version events Log.withLoggedRetries retryPolicy "writeAttempt" call log module private Read = open Microsoft.Azure.Documents.Linq open System.Linq - let private getQuery ((client, collectionUri): Connection) streamId (direction: Direction) batchSize sequenceNumber = + let private getQuery (client : IDocumentClient,collectionUri : Uri) streamId (direction: Direction) batchSize sequenceNumber = let sequenceNumber = match direction, sequenceNumber with @@ -379,7 +386,7 @@ module private Read = let partitionPayloadFrom firstUsedEventNumber : EquinoxEvent[] -> int * int = let acc (tu,tr) ((EquinoxEventLen bytes) as y) = if y.sn < firstUsedEventNumber then tu, tr + bytes else tu + bytes, tr Array.fold acc (0,0) - let loadBackwardsUntilCompactionOrStart (log : ILogger) retryPolicy conn batchSize maxPermittedBatchReads streamName isCompactionEvent + let loadBackwardsUntilCompactionOrStart (log : ILogger) retryPolicy coll batchSize maxPermittedBatchReads streamName isCompactionEvent : Async = async { let mergeFromCompactionPointOrStartFromBackwardsStream (log : ILogger) (batchesBackward : AsyncSeq) : Async = async { @@ -401,7 +408,7 @@ module private Read = |> AsyncSeq.toArrayAsync let eventsForward = Array.Reverse(tempBackward); tempBackward // sic - relatively cheap, in-place reverse of something we own return eventsForward, ru } - let query = getQuery conn streamName Direction.Backward batchSize SN.last + let query = getQuery coll streamName Direction.Backward batchSize SN.last let call q = loggedQueryExecution streamName Direction.Backward batchSize SN.last q let retryingLoggingReadSlice q = Log.withLoggedRetries retryPolicy "readAttempt" (call q) let log = log |> Log.prop "batchSize" batchSize |> Log.prop "stream" streamName @@ -459,8 +466,8 @@ module Token = let currentVersion, newVersion = unpackEqxStreamVersion current, unpackEqxStreamVersion x newVersion > currentVersion -type EqxConnection(connection : IDocumentClient, ?readRetryPolicy, ?writeRetryPolicy) = - member __.Connection = connection, Client.UriFactory.CreateDocumentCollectionUri("test","test") +type EqxConnection(client: IDocumentClient, ?readRetryPolicy, ?writeRetryPolicy) = + member __.Client = client member __.ReadRetryPolicy = readRetryPolicy member __.WriteRetryPolicy = writeRetryPolicy @@ -475,34 +482,35 @@ type GatewaySyncResult = Written of Storage.StreamToken | Conflict type EqxGateway(conn : EqxConnection, batching : EqxBatchingPolicy) = let isResolvedEventEventType predicate (x:EquinoxEvent) = predicate x.et let tryIsResolvedEventEventType predicateOption = predicateOption |> Option.map isResolvedEventEventType - member __.LoadBatched streamName log isCompactionEventType: Async = async { - let! version, events = Read.loadForwardsFrom log conn.ReadRetryPolicy conn.Connection batching.BatchSize batching.MaxBatches streamName 0L + let (|Coll|) (collectionUri: Uri) = conn.Client,collectionUri + member __.LoadBatched (Coll coll,streamName) log isCompactionEventType: Async = async { + let! version, events = Read.loadForwardsFrom log conn.ReadRetryPolicy coll batching.BatchSize batching.MaxBatches streamName 0L match tryIsResolvedEventEventType isCompactionEventType with | None -> return Token.ofNonCompacting version, events | Some isCompactionEvent -> match events |> Array.tryFindBack isCompactionEvent with | None -> return Token.ofUncompactedVersion batching.BatchSize version, events | Some resolvedEvent -> return Token.ofCompactionResolvedEventAndVersion resolvedEvent batching.BatchSize version, events } - member __.LoadBackwardsStoppingAtCompactionEvent streamName log isCompactionEventType: Async = async { + member __.LoadBackwardsStoppingAtCompactionEvent (Coll coll,streamName) log isCompactionEventType: Async = async { let isCompactionEvent = isResolvedEventEventType isCompactionEventType let! version, events = - Read.loadBackwardsUntilCompactionOrStart log conn.ReadRetryPolicy conn.Connection batching.BatchSize batching.MaxBatches streamName isCompactionEvent + Read.loadBackwardsUntilCompactionOrStart log conn.ReadRetryPolicy coll batching.BatchSize batching.MaxBatches streamName isCompactionEvent match Array.tryHead events |> Option.filter isCompactionEvent with | None -> return Token.ofUncompactedVersion batching.BatchSize version, events | Some resolvedEvent -> return Token.ofCompactionResolvedEventAndVersion resolvedEvent batching.BatchSize version, events } - member __.LoadFromToken streamName log (token : Storage.StreamToken) isCompactionEventType + member __.LoadFromToken (Coll coll,streamName) log (token : Storage.StreamToken) isCompactionEventType : Async = async { let streamPosition = (unbox token.value).streamVersion - let! version, events = Read.loadForwardsFrom log conn.ReadRetryPolicy conn.Connection batching.BatchSize batching.MaxBatches streamName streamPosition + let! version, events = Read.loadForwardsFrom log conn.ReadRetryPolicy coll batching.BatchSize batching.MaxBatches streamName streamPosition match tryIsResolvedEventEventType isCompactionEventType with | None -> return Token.ofNonCompacting version, events | Some isCompactionEvent -> match events |> Array.tryFindBack isCompactionEvent with | None -> return Token.ofPreviousTokenAndEventsLength token events.Length batching.BatchSize version, events | Some resolvedEvent -> return Token.ofCompactionResolvedEventAndVersion resolvedEvent batching.BatchSize version, events } - member __.TrySync streamName log (token : Storage.StreamToken) (encodedEvents: EventData array) isCompactionEventType : Async = async { + member __.TrySync (Coll coll,streamName) log (token : Storage.StreamToken) (encodedEvents: EventData array) isCompactionEventType : Async = async { let streamVersion = (unbox token.value).streamVersion - let! wr = Write.writeEvents log conn.WriteRetryPolicy conn.Connection streamName streamVersion encodedEvents + let! wr = Write.writeEvents log conn.WriteRetryPolicy coll streamName streamVersion encodedEvents match wr with | EqxSyncResult.Conflict _ -> return GatewaySyncResult.Conflict | EqxSyncResult.Written (wr, _) -> @@ -519,25 +527,30 @@ type EqxGateway(conn : EqxConnection, batching : EqxBatchingPolicy) = Token.ofPreviousStreamVersionAndCompactionEventDataIndex streamVersion compactionEventIndex encodedEvents.Length batching.BatchSize version' return GatewaySyncResult.Written token } -type EqxCategory<'event, 'state>(gateway : EqxGateway, codec : UnionCodec.IUnionEncoder<'event, byte[]>, ?compactionStrategy) = +type private Collection(gateway : EqxGateway, databaseId, collectionId) = + member __.Gateway = gateway + member __.CollectionUri = Client.UriFactory.CreateDocumentCollectionUri(databaseId, collectionId) + +type private Category<'event, 'state>(coll : Collection, codec : UnionCodec.IUnionEncoder<'event, byte[]>, ?compactionStrategy) = + let (|StreamRef|) streamName = coll.CollectionUri, streamName let loadAlgorithm load streamName initial log = - let batched = load initial (gateway.LoadBatched streamName log None) - let compacted predicate = load initial (gateway.LoadBackwardsStoppingAtCompactionEvent streamName log predicate) + let batched = load initial (coll.Gateway.LoadBatched streamName log None) + let compacted predicate = load initial (coll.Gateway.LoadBackwardsStoppingAtCompactionEvent streamName log predicate) match compactionStrategy with | Some predicate -> compacted predicate | None -> batched let load (fold: 'state -> 'event seq -> 'state) initial f = async { let! token, events = f return token, fold initial (UnionEncoderAdapters.decodeKnownEvents codec events) } - member __.Load (fold: 'state -> 'event seq -> 'state) (initial: 'state) (streamName : string) (log : ILogger) : Async = - loadAlgorithm (load fold) streamName initial log - member __.LoadFromToken (fold: 'state -> 'event seq -> 'state) (state: 'state) (streamName : string) token (log : ILogger) : Async = - (load fold) state (gateway.LoadFromToken streamName log token compactionStrategy) - member __.TrySync (fold: 'state -> 'event seq -> 'state) streamName (log : ILogger) (token, state) (events : 'event list) : Async> = async { + member __.Load (fold: 'state -> 'event seq -> 'state) (initial: 'state) (StreamRef streamRef) (log : ILogger) : Async = + loadAlgorithm (load fold) streamRef initial log + member __.LoadFromToken (fold: 'state -> 'event seq -> 'state) (state: 'state) (StreamRef streamRef) token (log : ILogger) : Async = + (load fold) state (coll.Gateway.LoadFromToken streamRef log token compactionStrategy) + member __.TrySync (fold: 'state -> 'event seq -> 'state) (StreamRef streamRef) (log : ILogger) (token, state) (events : 'event list) : Async> = async { let encodedEvents : EventData[] = UnionEncoderAdapters.encodeEvents codec (Seq.ofList events) - let! syncRes = gateway.TrySync streamName log token encodedEvents compactionStrategy + let! syncRes = coll.Gateway.TrySync streamRef log token encodedEvents compactionStrategy match syncRes with - | GatewaySyncResult.Conflict -> return Storage.SyncResult.Conflict (load fold state (gateway.LoadFromToken streamName log token compactionStrategy)) + | GatewaySyncResult.Conflict -> return Storage.SyncResult.Conflict (load fold state (coll.Gateway.LoadFromToken streamRef log token compactionStrategy)) | GatewaySyncResult.Written token' -> return Storage.SyncResult.Written (token', fold state (Seq.ofList events)) } module Caching = @@ -598,7 +611,7 @@ module Caching = let addOrUpdateSlidingExpirationCacheEntry streamName = CacheEntry >> cache.UpdateIfNewer policy (prefix + streamName) CategoryTee<'event,'state>(category, addOrUpdateSlidingExpirationCacheEntry) :> _ -type EqxFolder<'event, 'state>(category : EqxCategory<'event, 'state>, fold: 'state -> 'event seq -> 'state, initial: 'state, ?readCache) = +type private Folder<'event, 'state>(category : Category<'event, 'state>, fold: 'state -> 'event seq -> 'state, initial: 'state, ?readCache) = let loadAlgorithm streamName initial log = let batched = category.Load fold initial streamName log let cached token state = category.LoadFromToken fold state streamName token log @@ -628,21 +641,21 @@ type CachingStrategy = /// Prefix is used to distinguish multiple folds per stream | SlidingWindowPrefixed of Caching.Cache * window: TimeSpan * prefix: string -type EqxStreamBuilder<'event, 'state>(gateway, codec, fold, initial, ?compaction, ?caching) = - member __.Create streamName : Equinox.IStream<'event, 'state> = +type EqxStreamBuilder<'event, 'state>(gateway : EqxGateway, codec, fold, initial, ?compaction, ?caching) = + member __.Create (databaseId, collectionId, streamName) : Equinox.IStream<'event, 'state> = let compactionPredicateOption = match compaction with | None -> None | Some (CompactionStrategy.Predicate predicate) -> Some predicate | Some (CompactionStrategy.EventType eventType) -> Some (fun x -> x = eventType) - let eqxCategory = EqxCategory<'event, 'state>(gateway, codec, ?compactionStrategy = compactionPredicateOption) + let category = Category<'event, 'state>(Collection(gateway, databaseId, collectionId), codec, ?compactionStrategy = compactionPredicateOption) let readCacheOption = match caching with | None -> None | Some (CachingStrategy.SlidingWindow(cache, _)) -> Some(cache, null) | Some (CachingStrategy.SlidingWindowPrefixed(cache, _, prefix)) -> Some(cache, prefix) - let folder = EqxFolder<'event, 'state>(eqxCategory, fold, initial, ?readCache = readCacheOption) + let folder = Folder<'event, 'state>(category, fold, initial, ?readCache = readCacheOption) let category : ICategory<_,_> = match caching with diff --git a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs index ecc8725ef..eacebf2f4 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs @@ -14,9 +14,9 @@ let connectToLocalEquinoxNode (log: Serilog.ILogger) = .Establish("localDocDbSim", Discovery.UriAndKey(Uri "https://localhost:8081", "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==")) let defaultBatchSize = 500 let createEqxGateway connection batchSize = EqxGateway(connection, EqxBatchingPolicy(maxBatchSize = batchSize)) -let (|StreamArgs|) gateway = - //let databaseId, collectionId = "test", "test" - gateway//, databaseId, collectionId +let (|StreamArgs|) streamName = + let databaseId, collectionId = "test", "test" + databaseId, collectionId, streamName let serializationSettings = Newtonsoft.Json.Converters.FSharp.Settings.CreateCorrect() let genCodec<'Union when 'Union :> TypeShape.UnionContract.IUnionContract>() = @@ -27,12 +27,12 @@ module Cart = let codec = genCodec() let createServiceWithoutOptimization connection batchSize log = let gateway = createEqxGateway connection batchSize - let resolveStream _ignoreCompactionEventTypeOption (args) = + let resolveStream _ignoreCompactionEventTypeOption (StreamArgs args) = EqxStreamBuilder(gateway, codec, fold, initial).Create(args) Backend.Cart.Service(log, resolveStream) let createServiceWithCompaction connection batchSize log = let gateway = createEqxGateway connection batchSize - let resolveStream compactionEventType (args) = + let resolveStream compactionEventType (StreamArgs args) = EqxStreamBuilder(gateway, codec, fold, initial, compaction=CompactionStrategy.EventType compactionEventType).Create(args) Backend.Cart.Service(log, resolveStream) let createServiceWithCaching connection batchSize log cache = diff --git a/tools/Equinox.Tool/Program.fs b/tools/Equinox.Tool/Program.fs index 87b3d3eff..f61e8911c 100644 --- a/tools/Equinox.Tool/Program.fs +++ b/tools/Equinox.Tool/Program.fs @@ -11,6 +11,7 @@ open System open System.Net.Http open System.Threading + [] type Arguments = | [] Verbose From 8998a6e3e60da84db88dddf3ebcad7344a055160 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Mon, 2 Jul 2018 10:21:06 +0100 Subject: [PATCH 10/66] Sync with ES implementation --- src/Equinox.Cosmos/Cosmos.fs | 54 ++++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 21 deletions(-) diff --git a/src/Equinox.Cosmos/Cosmos.fs b/src/Equinox.Cosmos/Cosmos.fs index fce4929c1..1911b8c0d 100644 --- a/src/Equinox.Cosmos/Cosmos.fs +++ b/src/Equinox.Cosmos/Cosmos.fs @@ -115,6 +115,14 @@ module Log = | Slice of Direction * Measurement | Batch of Direction * slices: int * Measurement let prop name value (log : ILogger) = log.ForContext(name, value) + let propEvents name (kvps : System.Collections.Generic.KeyValuePair seq) (log : ILogger) = + let items = seq { for kv in kvps do yield sprintf "{\"%s\": %s}" kv.Key kv.Value } + log.ForContext(name, sprintf "[%s]" (String.concat ",\n\r" items)) + let propEventData name (events : EventData[]) (log : ILogger) = + log |> propEvents name (seq { for x in events -> Collections.Generic.KeyValuePair<_,_>(x.eventType, System.Text.Encoding.UTF8.GetString x.data)}) + let propResolvedEvents name (events : EquinoxEvent[]) (log : ILogger) = + log |> propEvents name (seq { for x in events -> Collections.Generic.KeyValuePair<_,_>(x.et, System.Text.Encoding.UTF8.GetString x.d)}) + open Serilog.Events /// Attach a property to the log context to hold the metrics // Sidestep Log.ForContext converting to a string; see https://github.com/serilog/serilog/issues/1124 @@ -135,20 +143,19 @@ module Log = type EqxSyncResult = Written of SN * float | Conflict of float module private Write = - - let private eventDataToEquinoxEvent (streamId:StreamId) (sequenceNumber:SN) (ed: EventData) = + let private eventDataToEquinoxEvent (streamId:StreamId) (sequenceNumber:SN) (ed: EventData) : EquinoxEvent = { - EquinoxEvent.et = ed.eventType - EquinoxEvent.id = (sprintf "%s-e-%d" streamId sequenceNumber) - EquinoxEvent.s = streamId - EquinoxEvent.k = streamId - EquinoxEvent.d = ed.data - EquinoxEvent.md = + et = ed.eventType + id = (sprintf "%s-e-%d" streamId sequenceNumber) + s = streamId + k = streamId + d = ed.data + md = match ed.metadata with | Some x -> x | None -> [||] - EquinoxEvent.sn = sequenceNumber - EquinoxEvent.ts = DateTimeOffset.UtcNow + sn = sequenceNumber + ts = DateTimeOffset.UtcNow } let [] private multiDocInsert = "AtomicMultiDocInsert" @@ -221,6 +228,7 @@ module private Write = return res.Response, res.RequestCharge } + /// Yields `EqxSyncResult.Written` or `EqxSyncResult.Conflict` to signify WrongExpectedVersion let private writeEventsAsync (log : ILogger) coll streamName (version : int64) (events : EventData[]) : Async = async { try @@ -233,7 +241,7 @@ module private Write = // Improve this? if dce.Message.Contains "already" then - log.Information(ex, "Eqx TrySync WrongExpectedVersionException") + log.Information(ex, "Eqx TrySync WrongExpectedVersionException writing {EventTypes}", [| for x in events -> x.eventType |]) return EqxSyncResult.Conflict dce.RequestCharge else return raise dce @@ -251,6 +259,7 @@ module private Write = let private writeEventsLogged coll streamName (version : int64) (events : EventData[]) (log : ILogger) : Async = async { + let log = if (not << log.IsEnabled) Events.LogEventLevel.Debug then log else log |> Log.propEventData "Json" events let bytes, count = eventDataBytes events, events.Length let log = log |> Log.prop "bytes" bytes let writeLog = log |> Log.prop "stream" streamName |> Log.prop "expectedVersion" version |> Log.prop "count" count @@ -277,7 +286,7 @@ module private Read = open Microsoft.Azure.Documents.Linq open System.Linq - let private getQuery (client : IDocumentClient,collectionUri : Uri) streamId (direction: Direction) batchSize sequenceNumber = + let private getQuery ((client : IDocumentClient,collectionUri : Uri),strongConsistency) streamId (direction: Direction) batchSize sequenceNumber = let sequenceNumber = match direction, sequenceNumber with @@ -287,7 +296,7 @@ module private Read = let feedOptions = new Client.FeedOptions() feedOptions.PartitionKey <- PartitionKey(streamId) feedOptions.MaxItemCount <- Nullable(batchSize) - //if (strongConsistency) then feedOptions.ConsistencyLevel <- Nullable(ConsistencyLevel.Strong) + // TODODC if (strongConsistency) then feedOptions.ConsistencyLevel <- Nullable(ConsistencyLevel.Strong) let sql = match direction with | Direction.Backward -> @@ -328,6 +337,7 @@ module private Read = let bytes, count = slice |> Array.sumBy (|EquinoxEventLen|), slice.Length let reqMetric : Log.Measurement ={ stream = streamName; interval = t; bytes = bytes; count = count; ru = Convert.ToInt32(ru) } let evt = Log.Slice (direction, reqMetric) + let log = if (not << log.IsEnabled) Events.LogEventLevel.Debug then log else log |> Log.propResolvedEvents "Json" slice (log |> Log.prop "startPos" startPos |> Log.prop "bytes" bytes |> Log.prop "ru" ru |> Log.event evt).Information( // TODO drop sliceLength, totalPayloadSize when consumption no longer requires that literal; ditto stream when literal formatting no longer required "Eqx{action:l} stream={stream} count={count} version={version} sliceLength={sliceLength} totalPayloadSize={totalPayloadSize} RequestCharge={ru}", @@ -361,7 +371,7 @@ module private Read = "Eqx{action:l} stream={stream} count={count}/{batches} version={version} RequestCharge={ru}", action, streamName, count, batches, version, ru) - let loadForwardsFrom (log : ILogger) retryPolicy conn batchSize maxPermittedBatchReads streamName startPosition + let loadForwardsFrom (log : ILogger) retryPolicy coll batchSize maxPermittedBatchReads streamName startPosition : Async = async { let mutable ru = 0.0 let mergeBatches (batches: AsyncSeq) = async { @@ -371,13 +381,14 @@ module private Read = |> AsyncSeq.concatSeq |> AsyncSeq.toArrayAsync return events, ru } - let query = getQuery conn streamName Direction.Forward batchSize startPosition + let query = getQuery coll streamName Direction.Forward batchSize startPosition let call q = loggedQueryExecution streamName Direction.Forward batchSize startPosition q let retryingLoggingReadSlice q = Log.withLoggedRetries retryPolicy "readAttempt" (call q) let direction = Direction.Forward let log = log |> Log.prop "batchSize" batchSize |> Log.prop "direction" direction |> Log.prop "stream" streamName let batches : AsyncSeq = readBatches log retryingLoggingReadSlice maxPermittedBatchReads query let! t, (events, ru) = mergeBatches batches |> Stopwatch.Time + // TODO use > (query :> IDisposable).Dispose() let version = lastSequenceNumber events log |> logBatchRead direction streamName t events batchSize version ru @@ -416,6 +427,7 @@ module private Read = let readlog = log |> Log.prop "direction" direction let batchesBackward : AsyncSeq = readBatches readlog retryingLoggingReadSlice maxPermittedBatchReads query let! t, (events, ru) = mergeFromCompactionPointOrStartFromBackwardsStream log batchesBackward |> Stopwatch.Time + // TODO use ? (query :> IDisposable).Dispose() let version = lastSequenceNumber events log |> logBatchRead direction streamName t events batchSize version ru @@ -484,7 +496,7 @@ type EqxGateway(conn : EqxConnection, batching : EqxBatchingPolicy) = let tryIsResolvedEventEventType predicateOption = predicateOption |> Option.map isResolvedEventEventType let (|Coll|) (collectionUri: Uri) = conn.Client,collectionUri member __.LoadBatched (Coll coll,streamName) log isCompactionEventType: Async = async { - let! version, events = Read.loadForwardsFrom log conn.ReadRetryPolicy coll batching.BatchSize batching.MaxBatches streamName 0L + let! version, events = Read.loadForwardsFrom log conn.ReadRetryPolicy (coll,false) batching.BatchSize batching.MaxBatches streamName 0L match tryIsResolvedEventEventType isCompactionEventType with | None -> return Token.ofNonCompacting version, events | Some isCompactionEvent -> @@ -494,14 +506,14 @@ type EqxGateway(conn : EqxConnection, batching : EqxBatchingPolicy) = member __.LoadBackwardsStoppingAtCompactionEvent (Coll coll,streamName) log isCompactionEventType: Async = async { let isCompactionEvent = isResolvedEventEventType isCompactionEventType let! version, events = - Read.loadBackwardsUntilCompactionOrStart log conn.ReadRetryPolicy coll batching.BatchSize batching.MaxBatches streamName isCompactionEvent + Read.loadBackwardsUntilCompactionOrStart log conn.ReadRetryPolicy (coll,false) batching.BatchSize batching.MaxBatches streamName isCompactionEvent match Array.tryHead events |> Option.filter isCompactionEvent with | None -> return Token.ofUncompactedVersion batching.BatchSize version, events | Some resolvedEvent -> return Token.ofCompactionResolvedEventAndVersion resolvedEvent batching.BatchSize version, events } - member __.LoadFromToken (Coll coll,streamName) log (token : Storage.StreamToken) isCompactionEventType + member __.LoadFromToken ((Coll coll,streamName),strongConsistency) log (token : Storage.StreamToken) isCompactionEventType : Async = async { let streamPosition = (unbox token.value).streamVersion - let! version, events = Read.loadForwardsFrom log conn.ReadRetryPolicy coll batching.BatchSize batching.MaxBatches streamName streamPosition + let! version, events = Read.loadForwardsFrom log conn.ReadRetryPolicy (coll,strongConsistency) batching.BatchSize batching.MaxBatches streamName streamPosition match tryIsResolvedEventEventType isCompactionEventType with | None -> return Token.ofNonCompacting version, events | Some isCompactionEvent -> @@ -545,12 +557,12 @@ type private Category<'event, 'state>(coll : Collection, codec : UnionCodec.IUni member __.Load (fold: 'state -> 'event seq -> 'state) (initial: 'state) (StreamRef streamRef) (log : ILogger) : Async = loadAlgorithm (load fold) streamRef initial log member __.LoadFromToken (fold: 'state -> 'event seq -> 'state) (state: 'state) (StreamRef streamRef) token (log : ILogger) : Async = - (load fold) state (coll.Gateway.LoadFromToken streamRef log token compactionStrategy) + (load fold) state (coll.Gateway.LoadFromToken (streamRef,false) log token compactionStrategy) member __.TrySync (fold: 'state -> 'event seq -> 'state) (StreamRef streamRef) (log : ILogger) (token, state) (events : 'event list) : Async> = async { let encodedEvents : EventData[] = UnionEncoderAdapters.encodeEvents codec (Seq.ofList events) let! syncRes = coll.Gateway.TrySync streamRef log token encodedEvents compactionStrategy match syncRes with - | GatewaySyncResult.Conflict -> return Storage.SyncResult.Conflict (load fold state (coll.Gateway.LoadFromToken streamRef log token compactionStrategy)) + | GatewaySyncResult.Conflict -> return Storage.SyncResult.Conflict (load fold state (coll.Gateway.LoadFromToken (streamRef,true) log token compactionStrategy)) | GatewaySyncResult.Written token' -> return Storage.SyncResult.Written (token', fold state (Seq.ofList events)) } module Caching = From 5051351cab7f97a155744133cf4fd328af3deca0 Mon Sep 17 00:00:00 2001 From: "Dongdong.cai" Date: Wed, 27 Jun 2018 13:35:10 -0400 Subject: [PATCH 11/66] Added script for creating docdb collection with proper indexing policy and stored procudure --- src/Equinox.Cosmos/Cosmos.fs | 48 +++++++++++++++ src/Equinox.Cosmos/EquinoxManager.fsx | 89 +++++++++++++++++++++++++++ 2 files changed, 137 insertions(+) create mode 100644 src/Equinox.Cosmos/EquinoxManager.fsx diff --git a/src/Equinox.Cosmos/Cosmos.fs b/src/Equinox.Cosmos/Cosmos.fs index 1911b8c0d..39695964f 100644 --- a/src/Equinox.Cosmos/Cosmos.fs +++ b/src/Equinox.Cosmos/Cosmos.fs @@ -679,10 +679,58 @@ type EqxStreamBuilder<'event, 'state>(gateway : EqxGateway, codec, fold, initial Equinox.Stream.create category streamName +module Initialization = + let createDatabase (client:IDocumentClient) dbName = async { + let opts = Client.RequestOptions(ConsistencyLevel = Nullable ConsistencyLevel.Session) + let! db = client.CreateDatabaseIfNotExistsAsync(Database(Id=dbName), options = opts) |> Async.AwaitTaskCorrect + return db.Resource.Id } + + let createCollection (client: IDocumentClient) (dbUri: Uri) collName ru = async { + let pkd = PartitionKeyDefinition() + pkd.Paths.Add("/k") + let colld = DocumentCollection(Id = collName, PartitionKey = pkd) + + colld.IndexingPolicy.IndexingMode <- IndexingMode.None + colld.IndexingPolicy.Automatic <- false + let! coll = client.CreateDocumentCollectionIfNotExistsAsync(dbUri, colld, Client.RequestOptions(OfferThroughput=Nullable ru)) |> Async.AwaitTaskCorrect + return coll.Resource.Id } + + let createProc (client: IDocumentClient) (collectionUri: Uri) = async { + let f ="""function multidocInsert (docs) { + var response = getContext().getResponse(); + var collection = getContext().getCollection(); + var collectionLink = collection.getSelfLink(); + if (!docs) throw new Error("docs argument is missing."); + for (i=0; i Async.AwaitTaskCorrect |> Async.Ignore } + + let initialize (client : IDocumentClient) dbName collName ru = async { + let! dbId = createDatabase client dbName + let dbUri = Client.UriFactory.CreateDatabaseUri dbId + let! collId = createCollection client dbUri collName ru + let collUri = Client.UriFactory.CreateDocumentCollectionUri (dbName, collId) + //let! _aux = createAux client dbUri collName auxRu + return! createProc client collUri + } + [] type Discovery = | UriAndKey of uri:Uri * key:string | ConnectionString of string + /// Implements connection string parsing logic curiously missing from the DocDb SDK + static member FromConnectionString (connectionString: string) = + match connectionString with + | _ when String.IsNullOrWhiteSpace connectionString -> nullArg "connectionString" + | Regex.Match "^\s*AccountEndpoint\s*=\s*([^;\s]+)\s*;\s*AccountKey\s*=\s*([^;\s]+)\s*;?\s*$" m -> + let uri = m.Groups.[1].Value + let key = m.Groups.[2].Value + UriAndKey (Uri uri, key) + | _ -> invalidArg "connectionString" "unrecognized connection string format" type EqxConnector ( requestTimeout: TimeSpan, maxRetryAttemptsOnThrottledRequests: int, maxRetryWaitTimeInSeconds: int, diff --git a/src/Equinox.Cosmos/EquinoxManager.fsx b/src/Equinox.Cosmos/EquinoxManager.fsx new file mode 100644 index 000000000..2dc38757f --- /dev/null +++ b/src/Equinox.Cosmos/EquinoxManager.fsx @@ -0,0 +1,89 @@ +// How to spin up a CosmosDB emulator locally: +// https://docs.microsoft.com/en-us/azure/cosmos-db/local-emulator#running-on-docker +// Then run the script below +#r "bin/Release/FSharp.Control.AsyncSeq.dll" +#r "bin/Release/Newtonsoft.Json.dll" +#r "bin/Release/Microsoft.Azure.Documents.Client.dll" +#load "Infrastructure.fs" + +open System +open Foldunk.Equinox +open Microsoft.Azure.Documents +open Microsoft.Azure.Documents.Client + +let URI = Uri "https://localhost:8081" +let KEY = "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==" +let DBNAME = "test" +let COLLNAME = "test" +let RU = 5000 +let AUXRU = 400 + +let client = + let connPolicy = + let cp = ConnectionPolicy.Default + cp.ConnectionMode <- ConnectionMode.Direct + cp.MaxConnectionLimit <- 200 + cp.RetryOptions <- RetryOptions(MaxRetryAttemptsOnThrottledRequests = 2, MaxRetryWaitTimeInSeconds = 10) + cp + new DocumentClient(URI, KEY, connPolicy, Nullable ConsistencyLevel.Session) + +let dburi = + let dbRequestOptions = + let o = RequestOptions () + o.ConsistencyLevel <- Nullable(ConsistencyLevel.ConsistentPrefix) + o + client.CreateDatabaseIfNotExistsAsync(Database(Id=DBNAME), options = dbRequestOptions) + |> Async.AwaitTaskCorrect + |> Async.map (fun response -> Client.UriFactory.CreateDatabaseUri (response.Resource.Id)) + |> Async.RunSynchronously + +let collectionUri = + let pkd = PartitionKeyDefinition() + pkd.Paths.Add("/k") + let coll = DocumentCollection(Id = COLLNAME, PartitionKey = pkd) + + coll.IndexingPolicy.IndexingMode <- IndexingMode.Consistent + coll.IndexingPolicy.Automatic <- true + coll.IndexingPolicy.IncludedPaths.Add(new IncludedPath (Path="/s/?")) + coll.IndexingPolicy.IncludedPaths.Add(new IncludedPath (Path="/k/?")) + coll.IndexingPolicy.IncludedPaths.Add(new IncludedPath (Path="/sn/?")) + coll.IndexingPolicy.ExcludedPaths.Add(new ExcludedPath (Path="/*")) + client.CreateDocumentCollectionIfNotExistsAsync(dburi, coll, RequestOptions(OfferThroughput=Nullable RU)) + |> Async.AwaitTaskCorrect + |> Async.map (fun response -> Client.UriFactory.CreateDocumentCollectionUri (DBNAME, response.Resource.Id)) + |> Async.RunSynchronously + +let batchSproc = + let f =""" + function multidocInsert (docs) { + var response = getContext().getResponse(); + var collection = getContext().getCollection(); + var collectionLink = collection.getSelfLink(); + + if (!docs) throw new Error("Array of events is undefined or null."); + + for (i=0; i Async.AwaitTaskCorrect + |> Async.RunSynchronously + +let auxCollectionUri = + let auxCollectionName = sprintf "%s-aux" COLLNAME + let auxColl = DocumentCollection(Id = auxCollectionName) + auxColl.IndexingPolicy.ExcludedPaths.Add(new ExcludedPath(Path="/ChangefeedPosition/*")) + auxColl.IndexingPolicy.ExcludedPaths.Add(new ExcludedPath(Path="/ProjectionsPositions/*")) + auxColl.IndexingPolicy.IncludedPaths.Add(new IncludedPath (Path="/*")) + auxColl.IndexingPolicy.IndexingMode <- IndexingMode.Lazy + auxColl.DefaultTimeToLive <- Nullable(365 * 60 * 60 * 24) + client.CreateDocumentCollectionIfNotExistsAsync(dburi, auxColl, RequestOptions(OfferThroughput=Nullable AUXRU)) + |> Async.AwaitTaskCorrect + |> Async.map (fun response -> Client.UriFactory.CreateDocumentCollectionUri (DBNAME, response.Resource.Id)) + |> Async.RunSynchronously + From 1ca6ed65cc95a3cebda2630e8f9bcb8d3a391406 Mon Sep 17 00:00:00 2001 From: "Dongdong.cai" Date: Wed, 27 Jun 2018 17:39:37 -0400 Subject: [PATCH 12/66] changes regarding reviews --- src/Equinox.Cosmos/EquinoxManager.fsx | 32 +++++++++++-------- .../CosmosIntegration.fs | 3 +- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/src/Equinox.Cosmos/EquinoxManager.fsx b/src/Equinox.Cosmos/EquinoxManager.fsx index 2dc38757f..35067d0d0 100644 --- a/src/Equinox.Cosmos/EquinoxManager.fsx +++ b/src/Equinox.Cosmos/EquinoxManager.fsx @@ -7,7 +7,7 @@ #load "Infrastructure.fs" open System -open Foldunk.Equinox +open Equinox.Cosmos open Microsoft.Azure.Documents open Microsoft.Azure.Documents.Client @@ -18,7 +18,7 @@ let COLLNAME = "test" let RU = 5000 let AUXRU = 400 -let client = +let client = let connPolicy = let cp = ConnectionPolicy.Default cp.ConnectionMode <- ConnectionMode.Direct @@ -27,17 +27,16 @@ let client = cp new DocumentClient(URI, KEY, connPolicy, Nullable ConsistencyLevel.Session) -let dburi = +let createDatabase (client:DocumentClient)= let dbRequestOptions = let o = RequestOptions () o.ConsistencyLevel <- Nullable(ConsistencyLevel.ConsistentPrefix) o - client.CreateDatabaseIfNotExistsAsync(Database(Id=DBNAME), options = dbRequestOptions) + client.CreateDatabaseIfNotExistsAsync(Database(Id=DBNAME), options = dbRequestOptions) |> Async.AwaitTaskCorrect |> Async.map (fun response -> Client.UriFactory.CreateDatabaseUri (response.Resource.Id)) - |> Async.RunSynchronously -let collectionUri = +let createCollection (client: DocumentClient) (dbUri: Uri) = let pkd = PartitionKeyDefinition() pkd.Paths.Add("/k") let coll = DocumentCollection(Id = COLLNAME, PartitionKey = pkd) @@ -48,12 +47,11 @@ let collectionUri = coll.IndexingPolicy.IncludedPaths.Add(new IncludedPath (Path="/k/?")) coll.IndexingPolicy.IncludedPaths.Add(new IncludedPath (Path="/sn/?")) coll.IndexingPolicy.ExcludedPaths.Add(new ExcludedPath (Path="/*")) - client.CreateDocumentCollectionIfNotExistsAsync(dburi, coll, RequestOptions(OfferThroughput=Nullable RU)) + client.CreateDocumentCollectionIfNotExistsAsync(dbUri, coll, RequestOptions(OfferThroughput=Nullable RU)) |> Async.AwaitTaskCorrect |> Async.map (fun response -> Client.UriFactory.CreateDocumentCollectionUri (DBNAME, response.Resource.Id)) - |> Async.RunSynchronously -let batchSproc = +let createStoreSproc (client: IDocumentClient) (collectionUri: Uri) = let f =""" function multidocInsert (docs) { var response = getContext().getResponse(); @@ -70,11 +68,11 @@ let batchSproc = }""" let batchSproc = new StoredProcedure(Id = "AtomicMultiDocInsert", Body = f) - client.CreateStoredProcedureAsync(collectionUri, batchSproc) + client.CreateStoredProcedureAsync(collectionUri, batchSproc) |> Async.AwaitTaskCorrect - |> Async.RunSynchronously + |> Async.map (fun r -> Client.UriFactory.CreateStoredProcedureUri(DBNAME, COLLNAME, r.Resource.Id)) -let auxCollectionUri = +let createAux (client: DocumentClient) (dbUri: Uri) = let auxCollectionName = sprintf "%s-aux" COLLNAME let auxColl = DocumentCollection(Id = auxCollectionName) auxColl.IndexingPolicy.ExcludedPaths.Add(new ExcludedPath(Path="/ChangefeedPosition/*")) @@ -82,8 +80,14 @@ let auxCollectionUri = auxColl.IndexingPolicy.IncludedPaths.Add(new IncludedPath (Path="/*")) auxColl.IndexingPolicy.IndexingMode <- IndexingMode.Lazy auxColl.DefaultTimeToLive <- Nullable(365 * 60 * 60 * 24) - client.CreateDocumentCollectionIfNotExistsAsync(dburi, auxColl, RequestOptions(OfferThroughput=Nullable AUXRU)) + client.CreateDocumentCollectionIfNotExistsAsync(dbUri, auxColl, RequestOptions(OfferThroughput=Nullable AUXRU)) |> Async.AwaitTaskCorrect |> Async.map (fun response -> Client.UriFactory.CreateDocumentCollectionUri (DBNAME, response.Resource.Id)) - |> Async.RunSynchronously +let go = async { + let! dbUri = createDatabase client + do! (createCollection client dbUri) |> Async.bind (createStoreSproc client) |> Async.Ignore + do! createAux client dbUri |> Async.Ignore +} + +go |> Async.RunSynchronously \ No newline at end of file diff --git a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs index eacebf2f4..e74452ea2 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs @@ -8,7 +8,8 @@ open System /// Standing up an Equinox instance is complicated; to run for test purposes either: /// - replace connection below with a connection string or Uri+Key for an initialized Equinox instance -/// - Create a local Equinox with dbName "test" and collectionName "test" using provisioning script +/// - Create a local Equinox with dbName "test" and collectionName "test" using script: +/// /src/Equinox.Cosmos/EquinoxManager.fsx let connectToLocalEquinoxNode (log: Serilog.ILogger) = EqxConnector(log=log, requestTimeout=TimeSpan.FromSeconds 3., maxRetryAttemptsOnThrottledRequests=2, maxRetryWaitTimeInSeconds=60) .Establish("localDocDbSim", Discovery.UriAndKey(Uri "https://localhost:8081", "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==")) From 32a25461a9b911e6a2e795a41622391d4471bc86 Mon Sep 17 00:00:00 2001 From: "Dongdong.cai" Date: Thu, 5 Jul 2018 12:09:44 -0400 Subject: [PATCH 13/66] Some missing changes --- src/Equinox.Cosmos/Cosmos.fs | 21 +++++++++++++------ .../VerbatimUtf8JsonConverterTests.fs | 2 ++ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/Equinox.Cosmos/Cosmos.fs b/src/Equinox.Cosmos/Cosmos.fs index 39695964f..da3bd33f9 100644 --- a/src/Equinox.Cosmos/Cosmos.fs +++ b/src/Equinox.Cosmos/Cosmos.fs @@ -5,6 +5,7 @@ open Equinox.Store open FSharp.Control open Microsoft.Azure.Documents open Newtonsoft.Json +open Newtonsoft.Json.Linq open Serilog open System @@ -25,10 +26,13 @@ module Strings = type VerbatimUtf8JsonConverter() = inherit JsonConverter() - override __.ReadJson(reader, _, _, serializer) = - let s = serializer.Deserialize(reader, typeof) :?> string - if s = null then Array.empty |> box - else reader.Value :?> string |> System.Text.Encoding.UTF8.GetBytes |> box + override __.ReadJson(reader, _, _, _) = + let token = JToken.Load(reader) + if (token.Type = JTokenType.Object) + then + token.ToString() |> System.Text.Encoding.UTF8.GetBytes |> box + else + Array.empty |> box override this.CanConvert(objectType) = typeof.Equals(objectType) @@ -94,10 +98,10 @@ type EquinoxEvent = { ts : DateTimeOffset sn : SN et : string - + df : string [)>] d : byte[] - + mdf : string [)>] md : byte[] } @@ -149,7 +153,9 @@ module private Write = id = (sprintf "%s-e-%d" streamId sequenceNumber) s = streamId k = streamId + df = "jsonbytearray" d = ed.data + mdf = "jsonbytearray" md = match ed.metadata with | Some x -> x @@ -210,6 +216,7 @@ module private Write = |> Seq.head |> appendSingleEvent coll streamName sequenceNumber | _ -> appendEventBatch coll streamName sequenceNumber eventsData + // Add this for User Activity let appendEventAtEnd (client : IDocumentClient,collectionUri : Uri,streamId) eventsData : Async = async { @@ -482,6 +489,8 @@ type EqxConnection(client: IDocumentClient, ?readRetryPolicy, ?writeRetryPolicy) member __.Client = client member __.ReadRetryPolicy = readRetryPolicy member __.WriteRetryPolicy = writeRetryPolicy + member __.Close = + (client :?> Client.DocumentClient).Dispose() type EqxBatchingPolicy(getMaxBatchSize : unit -> int, ?batchCountLimit) = new (maxBatchSize) = EqxBatchingPolicy(fun () -> maxBatchSize) diff --git a/tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs b/tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs index cebc2982f..f84f5c8c8 100644 --- a/tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs +++ b/tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs @@ -26,6 +26,8 @@ let ``VerbatimUtf8JsonConverter serializes properly`` () = { id = null s = null k = null + df = null + mdf = null ts = DateTimeOffset.MinValue sn = 0L From 5fe98e32e0aa1cb6fff461fe523090df9b8468ab Mon Sep 17 00:00:00 2001 From: "Dongdong.cai" Date: Thu, 5 Jul 2018 12:43:20 -0400 Subject: [PATCH 14/66] Fix tests --- src/Equinox.Cosmos/EquinoxManager.fsx | 2 +- .../VerbatimUtf8JsonConverterTests.fs | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/Equinox.Cosmos/EquinoxManager.fsx b/src/Equinox.Cosmos/EquinoxManager.fsx index 35067d0d0..8fdda8e1c 100644 --- a/src/Equinox.Cosmos/EquinoxManager.fsx +++ b/src/Equinox.Cosmos/EquinoxManager.fsx @@ -30,7 +30,7 @@ let client = let createDatabase (client:DocumentClient)= let dbRequestOptions = let o = RequestOptions () - o.ConsistencyLevel <- Nullable(ConsistencyLevel.ConsistentPrefix) + o.ConsistencyLevel <- Nullable(ConsistencyLevel.Session) o client.CreateDatabaseIfNotExistsAsync(Database(Id=DBNAME), options = dbRequestOptions) |> Async.AwaitTaskCorrect diff --git a/tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs b/tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs index f84f5c8c8..912cdcc1c 100644 --- a/tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs +++ b/tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs @@ -5,8 +5,8 @@ open Swensen.Unquote open System open Xunit -let serializer = new JsonSerializer() let inline serialize (x:'t) = + let serializer = new JsonSerializer() use sw = new System.IO.StringWriter() use w = new JsonTextWriter(sw) serializer.Serialize(w,x) @@ -26,14 +26,12 @@ let ``VerbatimUtf8JsonConverter serializes properly`` () = { id = null s = null k = null - df = null - mdf = null ts = DateTimeOffset.MinValue sn = 0L - + df = "jsonbytearray" et = encoded.caseName d = encoded.payload - + mdf = "jsonbytearray" md = null } let res = serialize e test <@ res.Contains """"d":{"embed":"\""}""" @> \ No newline at end of file From 5540215cbf5a0106f4740701e24b1cb0b1a3a5e1 Mon Sep 17 00:00:00 2001 From: dongdongcai Date: Thu, 25 Oct 2018 11:21:43 -0400 Subject: [PATCH 15/66] Added arg for prepare collection --- src/Equinox.Cosmos/CosmosManager.fs | 83 ++++++++++++++++++++++++ src/Equinox.Cosmos/Equinox.Cosmos.fsproj | 1 + src/Equinox.EventStore/Infrastructure.fs | 1 + 3 files changed, 85 insertions(+) create mode 100644 src/Equinox.Cosmos/CosmosManager.fs diff --git a/src/Equinox.Cosmos/CosmosManager.fs b/src/Equinox.Cosmos/CosmosManager.fs new file mode 100644 index 000000000..ddb6ea830 --- /dev/null +++ b/src/Equinox.Cosmos/CosmosManager.fs @@ -0,0 +1,83 @@ +module Equinox.Cosmos.CosmosManager + +open System +open Equinox.Cosmos +open Equinox.EventStore.Infrastructure +open Microsoft.Azure.Documents +open Microsoft.Azure.Documents.Client + +let configCosmos connStr dbName collName ru auxRu = async { + let uri, key = + match connStr,connStr with + | Strings.RegexGroup "AccountEndpoint=(.+?);" uri, Strings.RegexGroup "AccountKey=(.+?);" key -> + System.Uri(uri), key + | _ -> failwithf "Invalid DocumentDB connection string: %s" connStr + let client = + let connPolicy = + let cp = ConnectionPolicy.Default + cp.ConnectionMode <- ConnectionMode.Direct + cp.MaxConnectionLimit <- 200 + cp.RetryOptions <- RetryOptions(MaxRetryAttemptsOnThrottledRequests = 2, MaxRetryWaitTimeInSeconds = 10) + cp + new DocumentClient(uri, key, connPolicy, Nullable ConsistencyLevel.Session) + + let createDatabase (client:DocumentClient) = + let dbRequestOptions = + let o = RequestOptions () + o.ConsistencyLevel <- Nullable(ConsistencyLevel.Session) + o + client.CreateDatabaseIfNotExistsAsync(Database(Id=dbName), options = dbRequestOptions) + |> Async.AwaitTaskCorrect + |> Async.map (fun response -> Client.UriFactory.CreateDatabaseUri (response.Resource.Id)) + + let createCollection (client: DocumentClient) (dbUri: Uri) = + let pkd = PartitionKeyDefinition() + pkd.Paths.Add("/k") + let coll = DocumentCollection(Id = collName, PartitionKey = pkd) + + coll.IndexingPolicy.IndexingMode <- IndexingMode.Consistent + coll.IndexingPolicy.Automatic <- true + coll.IndexingPolicy.IncludedPaths.Add(new IncludedPath (Path="/s/?")) + coll.IndexingPolicy.IncludedPaths.Add(new IncludedPath (Path="/k/?")) + coll.IndexingPolicy.IncludedPaths.Add(new IncludedPath (Path="/sn/?")) + coll.IndexingPolicy.ExcludedPaths.Add(new ExcludedPath (Path="/*")) + client.CreateDocumentCollectionIfNotExistsAsync(dbUri, coll, RequestOptions(OfferThroughput=Nullable ru)) + |> Async.AwaitTaskCorrect + |> Async.map (fun response -> Client.UriFactory.CreateDocumentCollectionUri (dbName, response.Resource.Id)) + + let createStoreSproc (client: IDocumentClient) (collectionUri: Uri) = + let f =""" + function multidocInsert (docs) { + var response = getContext().getResponse(); + var collection = getContext().getCollection(); + var collectionLink = collection.getSelfLink(); + + if (!docs) throw new Error("Array of events is undefined or null."); + + for (i=0; i Async.AwaitTaskCorrect + |> Async.map (fun r -> Client.UriFactory.CreateStoredProcedureUri(dbName, collName, r.Resource.Id)) + + let createAux (client: DocumentClient) (dbUri: Uri) = + let auxCollectionName = sprintf "%s-aux" collName + let auxColl = DocumentCollection(Id = auxCollectionName) + auxColl.IndexingPolicy.ExcludedPaths.Add(new ExcludedPath(Path="/ChangefeedPosition/*")) + auxColl.IndexingPolicy.ExcludedPaths.Add(new ExcludedPath(Path="/ProjectionsPositions/*")) + auxColl.IndexingPolicy.IncludedPaths.Add(new IncludedPath (Path="/*")) + auxColl.IndexingPolicy.IndexingMode <- IndexingMode.Lazy + auxColl.DefaultTimeToLive <- Nullable(365 * 60 * 60 * 24) + client.CreateDocumentCollectionIfNotExistsAsync(dbUri, auxColl, RequestOptions(OfferThroughput=Nullable auxRu)) + |> Async.AwaitTaskCorrect + |> Async.map (fun response -> Client.UriFactory.CreateDocumentCollectionUri (dbName, response.Resource.Id)) + let! dbUri = createDatabase client + do! (createCollection client dbUri) |> Async.bind (createStoreSproc client) |> Async.Ignore + do! createAux client dbUri |> Async.Ignore +} diff --git a/src/Equinox.Cosmos/Equinox.Cosmos.fsproj b/src/Equinox.Cosmos/Equinox.Cosmos.fsproj index ab4066a3b..74e3230b9 100644 --- a/src/Equinox.Cosmos/Equinox.Cosmos.fsproj +++ b/src/Equinox.Cosmos/Equinox.Cosmos.fsproj @@ -13,6 +13,7 @@ + diff --git a/src/Equinox.EventStore/Infrastructure.fs b/src/Equinox.EventStore/Infrastructure.fs index 914639df1..6f68e3cb5 100644 --- a/src/Equinox.EventStore/Infrastructure.fs +++ b/src/Equinox.EventStore/Infrastructure.fs @@ -87,6 +87,7 @@ type Async with sc ()) |> ignore) static member inline bind (f:'a -> Async<'b>) (a:Async<'a>) : Async<'b> = async.Bind(a, f) + static member map (f:'a -> 'b) (a:Async<'a>) : Async<'b> = async.Bind(a, f >> async.Return) module AsyncSeq = /// Same as takeWhileAsync, but returns the final element too From b6b27b963aa6cb60b17cc39e4c2c1c4e185be3c4 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 26 Oct 2018 01:04:38 +0100 Subject: [PATCH 16/66] Minor tidying --- src/Equinox.Cosmos/CosmosManager.fs | 39 +++++++++++------------- src/Equinox.EventStore/Infrastructure.fs | 1 - 2 files changed, 17 insertions(+), 23 deletions(-) diff --git a/src/Equinox.Cosmos/CosmosManager.fs b/src/Equinox.Cosmos/CosmosManager.fs index ddb6ea830..7df62f87d 100644 --- a/src/Equinox.Cosmos/CosmosManager.fs +++ b/src/Equinox.Cosmos/CosmosManager.fs @@ -21,16 +21,12 @@ let configCosmos connStr dbName collName ru auxRu = async { cp new DocumentClient(uri, key, connPolicy, Nullable ConsistencyLevel.Session) - let createDatabase (client:DocumentClient) = - let dbRequestOptions = - let o = RequestOptions () - o.ConsistencyLevel <- Nullable(ConsistencyLevel.Session) - o - client.CreateDatabaseIfNotExistsAsync(Database(Id=dbName), options = dbRequestOptions) - |> Async.AwaitTaskCorrect - |> Async.map (fun response -> Client.UriFactory.CreateDatabaseUri (response.Resource.Id)) + let createDatabase (client:DocumentClient) = async { + let dbRequestOptions = RequestOptions(ConsistencyLevel = Nullable ConsistencyLevel.Session) + let! db = client.CreateDatabaseIfNotExistsAsync(Database(Id=dbName), options = dbRequestOptions) |> Async.AwaitTaskCorrect + return Client.UriFactory.CreateDatabaseUri (db.Resource.Id) } - let createCollection (client: DocumentClient) (dbUri: Uri) = + let createCollection (client: DocumentClient) (dbUri: Uri) = async { let pkd = PartitionKeyDefinition() pkd.Paths.Add("/k") let coll = DocumentCollection(Id = collName, PartitionKey = pkd) @@ -41,11 +37,10 @@ let configCosmos connStr dbName collName ru auxRu = async { coll.IndexingPolicy.IncludedPaths.Add(new IncludedPath (Path="/k/?")) coll.IndexingPolicy.IncludedPaths.Add(new IncludedPath (Path="/sn/?")) coll.IndexingPolicy.ExcludedPaths.Add(new ExcludedPath (Path="/*")) - client.CreateDocumentCollectionIfNotExistsAsync(dbUri, coll, RequestOptions(OfferThroughput=Nullable ru)) - |> Async.AwaitTaskCorrect - |> Async.map (fun response -> Client.UriFactory.CreateDocumentCollectionUri (dbName, response.Resource.Id)) + let! dc = client.CreateDocumentCollectionIfNotExistsAsync(dbUri, coll, RequestOptions(OfferThroughput=Nullable ru)) |> Async.AwaitTaskCorrect + return Client.UriFactory.CreateDocumentCollectionUri (dbName, dc.Resource.Id) } - let createStoreSproc (client: IDocumentClient) (collectionUri: Uri) = + let createStoreSproc (client: IDocumentClient) (collectionUri: Uri) = async { let f =""" function multidocInsert (docs) { var response = getContext().getResponse(); @@ -62,11 +57,10 @@ let configCosmos connStr dbName collName ru auxRu = async { }""" let batchSproc = new StoredProcedure(Id = "AtomicMultiDocInsert", Body = f) - client.CreateStoredProcedureAsync(collectionUri, batchSproc) - |> Async.AwaitTaskCorrect - |> Async.map (fun r -> Client.UriFactory.CreateStoredProcedureUri(dbName, collName, r.Resource.Id)) + let! sp = client.CreateStoredProcedureAsync(collectionUri, batchSproc) |> Async.AwaitTaskCorrect + return Client.UriFactory.CreateStoredProcedureUri(dbName, collName, sp.Resource.Id) } - let createAux (client: DocumentClient) (dbUri: Uri) = + let createAux (client: DocumentClient) (dbUri: Uri) = async { let auxCollectionName = sprintf "%s-aux" collName let auxColl = DocumentCollection(Id = auxCollectionName) auxColl.IndexingPolicy.ExcludedPaths.Add(new ExcludedPath(Path="/ChangefeedPosition/*")) @@ -74,10 +68,11 @@ let configCosmos connStr dbName collName ru auxRu = async { auxColl.IndexingPolicy.IncludedPaths.Add(new IncludedPath (Path="/*")) auxColl.IndexingPolicy.IndexingMode <- IndexingMode.Lazy auxColl.DefaultTimeToLive <- Nullable(365 * 60 * 60 * 24) - client.CreateDocumentCollectionIfNotExistsAsync(dbUri, auxColl, RequestOptions(OfferThroughput=Nullable auxRu)) - |> Async.AwaitTaskCorrect - |> Async.map (fun response -> Client.UriFactory.CreateDocumentCollectionUri (dbName, response.Resource.Id)) + let! dc = client.CreateDocumentCollectionIfNotExistsAsync(dbUri, auxColl, RequestOptions(OfferThroughput=Nullable auxRu)) |> Async.AwaitTaskCorrect + return Client.UriFactory.CreateDocumentCollectionUri (dbName, dc.Resource.Id) } let! dbUri = createDatabase client - do! (createCollection client dbUri) |> Async.bind (createStoreSproc client) |> Async.Ignore - do! createAux client dbUri |> Async.Ignore + let! coll = createCollection client dbUri + let! _sp = createStoreSproc client coll + let! _aux = createAux client dbUri + do () } diff --git a/src/Equinox.EventStore/Infrastructure.fs b/src/Equinox.EventStore/Infrastructure.fs index 6f68e3cb5..914639df1 100644 --- a/src/Equinox.EventStore/Infrastructure.fs +++ b/src/Equinox.EventStore/Infrastructure.fs @@ -87,7 +87,6 @@ type Async with sc ()) |> ignore) static member inline bind (f:'a -> Async<'b>) (a:Async<'a>) : Async<'b> = async.Bind(a, f) - static member map (f:'a -> 'b) (a:Async<'a>) : Async<'b> = async.Bind(a, f >> async.Return) module AsyncSeq = /// Same as takeWhileAsync, but returns the final element too From ac5d2e2677bfd5777dd7601c76a08cd2c8e3dd25 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sun, 28 Oct 2018 04:45:13 +0000 Subject: [PATCH 17/66] Reorg to match ambient style --- samples/Store/Integration/LogIntegration.fs | 2 +- src/Equinox.Cosmos/Cosmos.fs | 304 ++++++-------------- src/Equinox.EventStore/Infrastructure.fs | 15 - 3 files changed, 83 insertions(+), 238 deletions(-) diff --git a/samples/Store/Integration/LogIntegration.fs b/samples/Store/Integration/LogIntegration.fs index 6e256a47e..41f73076a 100644 --- a/samples/Store/Integration/LogIntegration.fs +++ b/samples/Store/Integration/LogIntegration.fs @@ -24,7 +24,7 @@ module EquinoxEsInterop = module EquinoxCosmosInterop = open Equinox.Cosmos [] - type FlatMetric = { action: string; stream: string; interval: StopwatchInterval; bytes: int; count: int; batches: int option; ru: int } with + type FlatMetric = { action: string; stream: string; interval: StopwatchInterval; bytes: int; count: int; batches: int option; ru: float } with override __.ToString() = sprintf "%s-Stream=%s %s-Elapsed=%O Ru=%O" __.action __.stream __.action __.interval.Elapsed __.ru let flatten (evt : Log.Event) : FlatMetric = let action, metric, batches, ru = diff --git a/src/Equinox.Cosmos/Cosmos.fs b/src/Equinox.Cosmos/Cosmos.fs index da3bd33f9..0b675dc6f 100644 --- a/src/Equinox.Cosmos/Cosmos.fs +++ b/src/Equinox.Cosmos/Cosmos.fs @@ -4,6 +4,7 @@ open Equinox open Equinox.Store open FSharp.Control open Microsoft.Azure.Documents +open Microsoft.Azure.Documents.Linq open Newtonsoft.Json open Newtonsoft.Json.Linq open Serilog @@ -28,67 +29,27 @@ type VerbatimUtf8JsonConverter() = override __.ReadJson(reader, _, _, _) = let token = JToken.Load(reader) - if (token.Type = JTokenType.Object) - then - token.ToString() |> System.Text.Encoding.UTF8.GetBytes |> box - else - Array.empty |> box + if token.Type = JTokenType.Object then token.ToString() |> System.Text.Encoding.UTF8.GetBytes |> box + else Array.empty |> box - override this.CanConvert(objectType) = + override __.CanConvert(objectType) = typeof.Equals(objectType) - override this.WriteJson(writer, value, serializer) = + override __.WriteJson(writer, value, serializer) = let array = value :?> byte[] - if Array.length array = 0 then serializer.Serialize(writer, null) + if array = null || Array.length array = 0 then serializer.Serialize(writer, null) else writer.WriteRawValue(System.Text.Encoding.UTF8.GetString(array)) -type SN = int64 - -[] -module SN = - - /// The first sequence number. - let [] zero : SN = 0L - - /// The last sequence number - let [] last : SN = -1L - - /// Computes the next sequence number. - let inline next (sn : SN) : SN = sn + 1L - - /// Computes the previous sequence number - let inline prev (sn: SN): SN = sn - 1L - - /// Compares two sequence numbers. - let inline compare (sn1 : SN) (sn2 : SN) : int = Operators.compare sn1 sn2 +/// 0-based Event Index in stream +type EventIndex = int64 type StreamId = string [] -/// Event data. -type EventData = { - eventType : string - data : byte[] - metadata : byte[] option } - with - - static member create (eventType: string, data: byte[], ?metadata: byte[]) = - { - eventType = eventType - data = data - metadata = - match metadata with - | None -> None - | Some md -> md |> Some } - -/// Operations on event data. -[] -[] -module EventData = - - let eventType (ed:EventData) = ed.eventType - let data (ed:EventData) = ed.data - let metadata (ed:EventData) = ed.metadata +type EventData = + { eventType : string + data : byte[] + metadata : byte[] } [] type EquinoxEvent = { @@ -96,7 +57,7 @@ type EquinoxEvent = { s : StreamId k : StreamId ts : DateTimeOffset - sn : SN + sn : EventIndex et : string df : string [)>] @@ -111,7 +72,7 @@ type Direction = Forward | Backward with module Log = [] - type Measurement = { stream: string; interval: StopwatchInterval; bytes: int; count: int; ru: int } + type Measurement = { stream: string; interval: StopwatchInterval; bytes: int; count: int; ru: float } [] type Event = | WriteSuccess of Measurement @@ -131,7 +92,7 @@ module Log = /// Attach a property to the log context to hold the metrics // Sidestep Log.ForContext converting to a string; see https://github.com/serilog/serilog/issues/1124 let event (value : Event) (log : ILogger) = - let enrich (e : LogEvent) = e.AddPropertyIfAbsent(LogEventProperty("eqxEvt", ScalarValue(value))) + let enrich (e : LogEvent) = e.AddPropertyIfAbsent(LogEventProperty("cosmosEvt", ScalarValue(value))) log.ForContext({ new Serilog.Core.ILogEventEnricher with member __.Enrich(evt,_) = enrich evt }) let withLoggedRetries<'t> retryPolicy (contextLabel : string) (f : ILogger -> Async<'t>) log: Async<'t> = match retryPolicy with @@ -144,124 +105,65 @@ module Log = let (|BlobLen|) = function null -> 0 | (x : byte[]) -> x.Length [] -type EqxSyncResult = Written of SN * float | Conflict of float +type EqxSyncResult = Written of EventIndex * requestCharge: float | Conflict of requestCharge: float module private Write = - let private eventDataToEquinoxEvent (streamId:StreamId) (sequenceNumber:SN) (ed: EventData) : EquinoxEvent = - { - et = ed.eventType - id = (sprintf "%s-e-%d" streamId sequenceNumber) + let private eventDataToEquinoxEvent (streamId:StreamId) (index: EventIndex) (ed: EventData) : EquinoxEvent = + { et = ed.eventType + id = sprintf "%s-e-%d" streamId index s = streamId k = streamId df = "jsonbytearray" d = ed.data mdf = "jsonbytearray" - md = - match ed.metadata with - | Some x -> x - | None -> [||] - sn = sequenceNumber - ts = DateTimeOffset.UtcNow - } - - let [] private multiDocInsert = "AtomicMultiDocInsert" - let [] private appendAtEnd = "appendAtEnd" - - let inline private sprocUri (sprocName : string) (collectionUri : Uri) = - (collectionUri.ToString()) + "/sprocs/" + sprocName // TODO: do this elegantly + md = ed.metadata + sn = index + ts = DateTimeOffset.UtcNow } /// Appends the single EventData using the sdk CreateDocumentAsync - let private appendSingleEvent (client : IDocumentClient,collectionUri : Uri) streamId sequenceNumber eventData : Async = - async { - let sequenceNumber = (SN.next sequenceNumber) + let private appendSingleEvent (client : IDocumentClient,collectionUri : Uri) streamId version eventData : Async = async { + let index = version + 1L + let equinoxEvent = eventData |> eventDataToEquinoxEvent streamId index - let equinoxEvent = - eventData - |> eventDataToEquinoxEvent streamId sequenceNumber + let requestOptions = Client.RequestOptions(PartitionKey = PartitionKey(streamId)) + let! res = client.CreateDocumentAsync(collectionUri, equinoxEvent, requestOptions) |> Async.AwaitTaskCorrect - let requestOptions = - Client.RequestOptions(PartitionKey = PartitionKey(streamId)) - - let! res = - client.CreateDocumentAsync(collectionUri, equinoxEvent, requestOptions) - |> Async.AwaitTaskCorrect - - return (sequenceNumber, res.RequestCharge) - } + return index, res.RequestCharge } /// Appends the given EventData batch using the atomic stored procedure - // This requires store procuedure in CosmosDB, is there other ways to do this? - let private appendEventBatch (client : IDocumentClient,collectionUri) streamId sequenceNumber eventsData : Async = - async { - let sequenceNumber = (SN.next sequenceNumber) - let res, sn = - eventsData - |> Seq.mapFold (fun sn ed -> (eventDataToEquinoxEvent streamId sn ed) |> JsonConvert.SerializeObject, SN.next sn) sequenceNumber - - let requestOptions = - Client.RequestOptions(PartitionKey = PartitionKey(streamId)) + let private appendEventBatch (client : IDocumentClient,collectionUri) streamId version eventsData : Async = async { + let events = + eventsData |> Seq.mapi (fun i ed -> + let index = version + int64 (i+1) + eventDataToEquinoxEvent streamId index ed + |> JsonConvert.SerializeObject) + |> Seq.toArray - let! res = - client.ExecuteStoredProcedureAsync(collectionUri |> sprocUri multiDocInsert, requestOptions, res:> obj) - |> Async.AwaitTaskCorrect + let requestOptions = Client.RequestOptions(PartitionKey = PartitionKey(streamId)) + let sprocUri = sprintf "%O/sprocs/AtomicMultiDocInsert" collectionUri + let! ct = Async.CancellationToken + let! res = client.ExecuteStoredProcedureAsync(sprocUri, requestOptions, ct, box events) |> Async.AwaitTaskCorrect - return (sn - 1L), res.RequestCharge - } + return version + int64 events.Length, res.RequestCharge } let private append coll streamName sequenceNumber (eventsData: EventData seq) = match Seq.length eventsData with | l when l = 0 -> invalidArg "eventsData" "must be non-empty" - | l when l = 1 -> - eventsData - |> Seq.head - |> appendSingleEvent coll streamName sequenceNumber + | l when l = 1 -> eventsData |> Seq.exactlyOne |> appendSingleEvent coll streamName sequenceNumber | _ -> appendEventBatch coll streamName sequenceNumber eventsData - // Add this for User Activity - let appendEventAtEnd (client : IDocumentClient,collectionUri : Uri,streamId) eventsData : Async = - async { - let res, _ = - eventsData - |> Seq.mapFold (fun sn ed -> (eventDataToEquinoxEvent streamId sn ed) |> JsonConvert.SerializeObject, SN.next sn) 0L - - let requestOptions = - Client.RequestOptions(PartitionKey = PartitionKey(streamId)) - - let! res = - client.ExecuteStoredProcedureAsync(collectionUri |> sprocUri appendAtEnd, requestOptions, res:> obj) - |> Async.AwaitTaskCorrect - - - return res.Response, res.RequestCharge - } - /// Yields `EqxSyncResult.Written` or `EqxSyncResult.Conflict` to signify WrongExpectedVersion let private writeEventsAsync (log : ILogger) coll streamName (version : int64) (events : EventData[]) : Async = async { try let! wr = append coll streamName version events return EqxSyncResult.Written wr - with ex -> - // change this for store procudure - match ex with - | :? DocumentClientException as dce -> - // Improve this? - if dce.Message.Contains "already" - then - log.Information(ex, "Eqx TrySync WrongExpectedVersionException writing {EventTypes}", [| for x in events -> x.eventType |]) - return EqxSyncResult.Conflict dce.RequestCharge - else - return raise dce - | e -> return raise e } + with :? DocumentClientException as ex when ex.Message.Contains "already" -> // TODO improve check, handle SP variant + log.Information(ex, "Eqx TrySync WrongExpectedVersionException writing {EventTypes}", [| for x in events -> x.eventType |]) + return EqxSyncResult.Conflict ex.RequestCharge } let eventDataBytes events = - let eventDataLen (x : EventData) = - let data = x.data - let metaData = - match x.metadata with - | None -> [||] - | Some x -> x - match data, metaData with Log.BlobLen bytes, Log.BlobLen metaBytes -> bytes + metaBytes + let eventDataLen { data = Log.BlobLen bytes; metadata = Log.BlobLen metaBytes } = bytes + metaBytes events |> Array.sumBy eventDataLen let private writeEventsLogged coll streamName (version : int64) (events : EventData[]) (log : ILogger) @@ -271,17 +173,12 @@ module private Write = let log = log |> Log.prop "bytes" bytes let writeLog = log |> Log.prop "stream" streamName |> Log.prop "expectedVersion" version |> Log.prop "count" count let! t, result = writeEventsAsync writeLog coll streamName version events |> Stopwatch.Time - let reqMetric : Log.Measurement = { stream = streamName; interval = t; bytes = bytes; count = count; ru = 0} - let resultLog, evt, (ru: float) = - match result, reqMetric with - | EqxSyncResult.Conflict ru, m -> log, Log.WriteConflict { m with ru = Convert.ToInt32(ru) }, ru - | EqxSyncResult.Written (x, ru), m -> - log |> Log.prop "nextExpectedVersion" x |> Log.prop "ru" ru, - Log.WriteSuccess { m with ru = Convert.ToInt32(ru) }, - ru - // TODO drop expectedVersion when consumption no longer requires that literal; ditto stream when literal formatting no longer required - (resultLog |> Log.event evt).Information("Eqx{action:l} stream={stream} count={count} expectedVersion={expectedVersion} conflict={conflict}, RequestCharge={ru}", - "Write", streamName, events.Length, version, (match evt with Log.WriteConflict _ -> true | _ -> false), ru) + let conflict, (ru: float), resultLog = + let mkMetric ru : Log.Measurement = { stream = streamName; interval = t; bytes = bytes; count = count; ru = ru } + match result with + | EqxSyncResult.Conflict ru -> true, ru, log |> Log.event (Log.WriteConflict (mkMetric ru)) + | EqxSyncResult.Written (x, ru) -> false, ru, log |> Log.event (Log.WriteSuccess (mkMetric ru)) |> Log.prop "nextExpectedVersion" x + resultLog.Information("Eqx{action:l} count={count} conflict={conflict}, RequestCharge={ru}", "Write", events.Length, conflict, ru) return result } let writeEvents (log : ILogger) retryPolicy coll (streamName : string) (version : int64) (events : EventData[]) @@ -290,65 +187,32 @@ module private Write = Log.withLoggedRetries retryPolicy "writeAttempt" call log module private Read = - open Microsoft.Azure.Documents.Linq - open System.Linq - - let private getQuery ((client : IDocumentClient,collectionUri : Uri),strongConsistency) streamId (direction: Direction) batchSize sequenceNumber = - - let sequenceNumber = - match direction, sequenceNumber with - | Direction.Backward, SN.last -> Int64.MaxValue - | _ -> sequenceNumber - - let feedOptions = new Client.FeedOptions() - feedOptions.PartitionKey <- PartitionKey(streamId) - feedOptions.MaxItemCount <- Nullable(batchSize) - // TODODC if (strongConsistency) then feedOptions.ConsistencyLevel <- Nullable(ConsistencyLevel.Strong) - let sql = - match direction with - | Direction.Backward -> - let query = """ - SELECT * FROM c - WHERE c.s = @streamId - AND c.sn <= @sequenceNumber - ORDER BY c.sn DESC""" - SqlQuerySpec query - | Direction.Forward -> - let query = """ - SELECT * FROM c - WHERE c.s = @streamId - AND c.sn >= @sequenceNumber - ORDER BY c.sn ASC """ - SqlQuerySpec query - sql.Parameters <- SqlParameterCollection - [| - SqlParameter("@streamId", streamId) - SqlParameter("@sequenceNumber", sequenceNumber) - |] - client.CreateDocumentQuery(collectionUri, sql, feedOptions).AsDocumentQuery() + let private getQuery ((client : IDocumentClient,collectionUri : Uri),strongConsistency) streamId (direction: Direction) batchSize (version: int64) = + let querySpec = + // TODODC if (strongConsistency) then feedOptions.ConsistencyLevel <- Nullable(ConsistencyLevel.Strong) + let filter = if direction = Direction.Backward then "c.sn <= @version ORDER BY c.sn DESC" else "c.sn >= @version ORDER BY c.sn ASC" + let prms = [| SqlParameter("@streamId", streamId); SqlParameter("@version", version) |] + SqlQuerySpec("SELECT * FROM c WHERE c.s = @streamId AND " + filter, SqlParameterCollection prms) + let feedOptions = new Client.FeedOptions(PartitionKey=PartitionKey streamId, MaxItemCount=Nullable batchSize) + client.CreateDocumentQuery(collectionUri, querySpec, feedOptions).AsDocumentQuery() let (|EquinoxEventLen|) (x : EquinoxEvent) = match x.d, x.md with Log.BlobLen bytes, Log.BlobLen metaBytes -> bytes + metaBytes - let private lastSequenceNumber (xs:EquinoxEvent seq) : SN = + let private lastSequenceNumber (xs:EquinoxEvent seq) : EventIndex = match xs |> Seq.tryLast with - | None -> SN.last + | None -> -1L | Some last -> last.sn - let private queryExecution (query: IDocumentQuery<'T>) = - query.ExecuteNextAsync<'T>() |> Async.AwaitTaskCorrect - - let private loggedQueryExecution streamName direction batchSize startPos (query: IDocumentQuery) (log: ILogger) + let private loggedQueryExecution streamName direction startPos (query: IDocumentQuery) (log: ILogger) : Async = async { - let! t, res = queryExecution query |> Stopwatch.Time - let slice, ru = res.ToArray(), res.RequestCharge + let! t, (res : Client.FeedResponse) = query.ExecuteNextAsync() |> Async.AwaitTaskCorrect |> Stopwatch.Time + let slice, ru = Array.ofSeq res, res.RequestCharge let bytes, count = slice |> Array.sumBy (|EquinoxEventLen|), slice.Length - let reqMetric : Log.Measurement ={ stream = streamName; interval = t; bytes = bytes; count = count; ru = Convert.ToInt32(ru) } + let reqMetric : Log.Measurement = { stream = streamName; interval = t; bytes = bytes; count = count; ru = ru } let evt = Log.Slice (direction, reqMetric) let log = if (not << log.IsEnabled) Events.LogEventLevel.Debug then log else log |> Log.propResolvedEvents "Json" slice - (log |> Log.prop "startPos" startPos |> Log.prop "bytes" bytes |> Log.prop "ru" ru |> Log.event evt).Information( - // TODO drop sliceLength, totalPayloadSize when consumption no longer requires that literal; ditto stream when literal formatting no longer required - "Eqx{action:l} stream={stream} count={count} version={version} sliceLength={sliceLength} totalPayloadSize={totalPayloadSize} RequestCharge={ru}", - "Read", streamName, count, (lastSequenceNumber slice), batchSize, bytes, ru) + (log |> Log.prop "startPos" startPos |> Log.prop "bytes" bytes |> Log.prop "ru" ru |> Log.event evt) + .Information("Eqx{action:l} count={count} version={sliceVersion} RequestCharge={ru}", "Read", count, lastSequenceNumber slice, ru) return slice, ru } let private readBatches (log : ILogger) (readSlice: IDocumentQuery -> ILogger -> Async) (maxPermittedBatchReads: int option) (query: IDocumentQuery) @@ -363,14 +227,13 @@ module private Read = yield slice if query.HasMoreResults then yield! loop (batchCount + 1) } - //| x -> raise <| System.ArgumentOutOfRangeException("SliceReadStatus", x, "Unknown result value") } loop 0 let equinoxEventBytes events = events |> Array.sumBy (|EquinoxEventLen|) let logBatchRead direction streamName t events batchSize version (ru: float) (log : ILogger) = let bytes, count = equinoxEventBytes events, events.Length - let reqMetric : Log.Measurement = { stream = streamName; interval = t; bytes = bytes; count = count; ru = Convert.ToInt32(ru) } + let reqMetric : Log.Measurement = { stream = streamName; interval = t; bytes = bytes; count = count; ru = ru } let batches = (events.Length - 1)/batchSize + 1 let action = match direction with Direction.Forward -> "LoadF" | Direction.Backward -> "LoadB" let evt = Log.Event.Batch (direction, batches, reqMetric) @@ -388,15 +251,14 @@ module private Read = |> AsyncSeq.concatSeq |> AsyncSeq.toArrayAsync return events, ru } - let query = getQuery coll streamName Direction.Forward batchSize startPosition - let call q = loggedQueryExecution streamName Direction.Forward batchSize startPosition q + use query = getQuery coll streamName Direction.Forward batchSize startPosition + let call q = loggedQueryExecution streamName Direction.Forward startPosition q let retryingLoggingReadSlice q = Log.withLoggedRetries retryPolicy "readAttempt" (call q) let direction = Direction.Forward let log = log |> Log.prop "batchSize" batchSize |> Log.prop "direction" direction |> Log.prop "stream" streamName let batches : AsyncSeq = readBatches log retryingLoggingReadSlice maxPermittedBatchReads query let! t, (events, ru) = mergeBatches batches |> Stopwatch.Time - // TODO use > - (query :> IDisposable).Dispose() + query.Dispose() let version = lastSequenceNumber events log |> logBatchRead direction streamName t events batchSize version ru return version, events } @@ -426,29 +288,28 @@ module private Read = |> AsyncSeq.toArrayAsync let eventsForward = Array.Reverse(tempBackward); tempBackward // sic - relatively cheap, in-place reverse of something we own return eventsForward, ru } - let query = getQuery coll streamName Direction.Backward batchSize SN.last - let call q = loggedQueryExecution streamName Direction.Backward batchSize SN.last q + use query = getQuery coll streamName Direction.Backward batchSize EventIndex.MaxValue + let call q = loggedQueryExecution streamName Direction.Backward EventIndex.MaxValue q let retryingLoggingReadSlice q = Log.withLoggedRetries retryPolicy "readAttempt" (call q) let log = log |> Log.prop "batchSize" batchSize |> Log.prop "stream" streamName let direction = Direction.Backward let readlog = log |> Log.prop "direction" direction let batchesBackward : AsyncSeq = readBatches readlog retryingLoggingReadSlice maxPermittedBatchReads query let! t, (events, ru) = mergeFromCompactionPointOrStartFromBackwardsStream log batchesBackward |> Stopwatch.Time - // TODO use ? - (query :> IDisposable).Dispose() + query.Dispose() let version = lastSequenceNumber events log |> logBatchRead direction streamName t events batchSize version ru return version, events } module UnionEncoderAdapters = - let private encodedEventOfResolvedEvent (x : EquinoxEvent) : UnionCodec.EncodedUnion = + let private encodedEventOfStoredEvent (x : EquinoxEvent) : UnionCodec.EncodedUnion = { caseName = x.et; payload = x.d } - let private eventDataOfEncodedEvent (x : UnionCodec.EncodedUnion) = - EventData.create(x.caseName, x.payload, [||]) + let private eventDataOfEncodedEvent (x : UnionCodec.EncodedUnion) : EventData = + { eventType = x.caseName; data = x.payload; metadata = null } let encodeEvents (codec : UnionCodec.IUnionEncoder<'event, byte[]>) (xs : 'event seq) : EventData[] = xs |> Seq.map (codec.Encode >> eventDataOfEncodedEvent) |> Seq.toArray let decodeKnownEvents (codec : UnionCodec.IUnionEncoder<'event, byte[]>) (xs : EquinoxEvent[]) : 'event seq = - xs |> Seq.map encodedEventOfResolvedEvent |> Seq.choose codec.TryDecode + xs |> Seq.map encodedEventOfStoredEvent |> Seq.choose codec.TryDecode type Token = { streamVersion: int64; compactionEventNumber: int64 option } @@ -489,8 +350,7 @@ type EqxConnection(client: IDocumentClient, ?readRetryPolicy, ?writeRetryPolicy) member __.Client = client member __.ReadRetryPolicy = readRetryPolicy member __.WriteRetryPolicy = writeRetryPolicy - member __.Close = - (client :?> Client.DocumentClient).Dispose() + member __.Close = (client :?> Client.DocumentClient).Dispose() type EqxBatchingPolicy(getMaxBatchSize : unit -> int, ?batchCountLimit) = new (maxBatchSize) = EqxBatchingPolicy(fun () -> maxBatchSize) @@ -763,7 +623,7 @@ type EqxConnector cp.MaxConnectionLimit <- defaultArg maxConnectionLimit 1000 cp - /// Yields an connection to DocDB configured and Connect()ed to DocDB collection per the requested `discovery` strategy + /// Yields an IDocumentClient configured and Connect()ed to a given DocDB collection per the requested `discovery` strategy member __.Connect ( /// Name should be sufficient to uniquely identify this connection within a single app instance's logs name, @@ -772,9 +632,9 @@ type EqxConnector let name = String.concat ";" <| seq { yield name match tags with None -> () | Some tags -> for key, value in tags do yield sprintf "%s=%s" key value } - let sanitizedName = name.Replace('\'','_').Replace(':','_') // ES internally uses `:` and `'` as separators in log messages and ... people regex logs + let sanitizedName = name.Replace('\'','_').Replace(':','_') // sic; Align with logging for ES Adapter let client = new Client.DocumentClient(uri, key, connPolicy, Nullable ConsistencyLevel.Session) - log.Information("Connected to Equinox with clientId={clientId}", sanitizedName) + log.Information("Connected to Cosmos with clientId={clientId}", sanitizedName) do! client.OpenAsync() |> Async.AwaitTaskCorrect return client :> IDocumentClient } diff --git a/src/Equinox.EventStore/Infrastructure.fs b/src/Equinox.EventStore/Infrastructure.fs index 914639df1..7aefae65a 100644 --- a/src/Equinox.EventStore/Infrastructure.fs +++ b/src/Equinox.EventStore/Infrastructure.fs @@ -15,21 +15,6 @@ module Seq = Some res else None - let arrMapFold f acc (array: _[]) = - match array.Length with - | 0 -> [| |], acc - | len -> - let f = OptimizedClosures.FSharpFunc<_,_,_>.Adapt(f) - let mutable acc = acc - let res = Array.zeroCreate len - for i = 0 to array.Length-1 do - let h',s' = f.Invoke(acc,array.[i]) - res.[i] <- h' - acc <- s' - res, acc - let mapFold<'T,'State,'Result> (mapping: 'State -> 'T -> 'Result * 'State) state source = - let arr,state = source |> Seq.toArray |> arrMapFold mapping state - Seq.readonly arr, state module Array = let tryHead (array : 'T[]) = From 666ae57af9263a53ad94114a628227b8999ca85f Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Mon, 29 Oct 2018 22:14:52 +0000 Subject: [PATCH 18/66] Tidy --- src/Equinox.Cosmos/Cosmos.fs | 31 +++---- src/Equinox.Cosmos/Equinox.Cosmos.fsproj | 2 +- src/Equinox.Cosmos/EquinoxManager.fsx | 93 ------------------- .../VerbatimUtf8JsonConverterTests.fs | 3 +- 4 files changed, 15 insertions(+), 114 deletions(-) delete mode 100644 src/Equinox.Cosmos/EquinoxManager.fsx diff --git a/src/Equinox.Cosmos/Cosmos.fs b/src/Equinox.Cosmos/Cosmos.fs index 0b675dc6f..b1fdb838d 100644 --- a/src/Equinox.Cosmos/Cosmos.fs +++ b/src/Equinox.Cosmos/Cosmos.fs @@ -108,22 +108,10 @@ module Log = type EqxSyncResult = Written of EventIndex * requestCharge: float | Conflict of requestCharge: float module private Write = - let private eventDataToEquinoxEvent (streamId:StreamId) (index: EventIndex) (ed: EventData) : EquinoxEvent = - { et = ed.eventType - id = sprintf "%s-e-%d" streamId index - s = streamId - k = streamId - df = "jsonbytearray" - d = ed.data - mdf = "jsonbytearray" - md = ed.metadata - sn = index - ts = DateTimeOffset.UtcNow } - /// Appends the single EventData using the sdk CreateDocumentAsync let private appendSingleEvent (client : IDocumentClient,collectionUri : Uri) streamId version eventData : Async = async { let index = version + 1L - let equinoxEvent = eventData |> eventDataToEquinoxEvent streamId index + let equinoxEvent = eventData |> EquinoxEvent.mk streamId index let requestOptions = Client.RequestOptions(PartitionKey = PartitionKey(streamId)) let! res = client.CreateDocumentAsync(collectionUri, equinoxEvent, requestOptions) |> Async.AwaitTaskCorrect @@ -135,7 +123,7 @@ module private Write = let events = eventsData |> Seq.mapi (fun i ed -> let index = version + int64 (i+1) - eventDataToEquinoxEvent streamId index ed + EquinoxEvent.mk streamId index ed |> JsonConvert.SerializeObject) |> Seq.toArray @@ -420,8 +408,8 @@ type private Category<'event, 'state>(coll : Collection, codec : UnionCodec.IUni match compactionStrategy with | Some predicate -> compacted predicate | None -> batched - let load (fold: 'state -> 'event seq -> 'state) initial f = async { - let! token, events = f + let load (fold: 'state -> 'event seq -> 'state) initial loadF = async { + let! token, events = loadF return token, fold initial (UnionEncoderAdapters.decodeKnownEvents codec events) } member __.Load (fold: 'state -> 'event seq -> 'state) (initial: 'state) (StreamRef streamRef) (log : ILogger) : Async = loadAlgorithm (load fold) streamRef initial log @@ -607,14 +595,19 @@ type EqxConnector /// Connection limit (default 1000) ?maxConnectionLimit, ?readRetryPolicy, ?writeRetryPolicy, + /// Connection mode (default: Gateway mode, Https) + ?mode, ?defaultConsistencyLevel, + /// Additional strings identifying the context of this connection; should provide enough context to disambiguate all potential connections to a cluster /// NB as this will enter server and client logs, it should not contain sensitive information ?tags : (string*string) seq) = let connPolicy = let cp = Client.ConnectionPolicy.Default - cp.ConnectionMode <- Client.ConnectionMode.Direct - cp.ConnectionProtocol <- Client.Protocol.Tcp + match mode with + | None | Some Gateway -> cp.ConnectionMode <- Client.ConnectionMode.Gateway // default; only supports Https + | Some DirectHttps -> cp.ConnectionMode <- Client.ConnectionMode.Direct; cp.ConnectionProtocol <- Client.Protocol.Https // Https is default when using Direct + | Some DirectTcp -> cp.ConnectionMode <- Client.ConnectionMode.Direct; cp.ConnectionProtocol <- Client.Protocol.Tcp cp.RetryOptions <- Client.RetryOptions( MaxRetryAttemptsOnThrottledRequests = maxRetryAttemptsOnThrottledRequests, @@ -633,7 +626,7 @@ type EqxConnector yield name match tags with None -> () | Some tags -> for key, value in tags do yield sprintf "%s=%s" key value } let sanitizedName = name.Replace('\'','_').Replace(':','_') // sic; Align with logging for ES Adapter - let client = new Client.DocumentClient(uri, key, connPolicy, Nullable ConsistencyLevel.Session) + let client = new Client.DocumentClient(uri, key, connPolicy, Nullable(defaultArg defaultConsistencyLevel ConsistencyLevel.Session)) log.Information("Connected to Cosmos with clientId={clientId}", sanitizedName) do! client.OpenAsync() |> Async.AwaitTaskCorrect return client :> IDocumentClient } diff --git a/src/Equinox.Cosmos/Equinox.Cosmos.fsproj b/src/Equinox.Cosmos/Equinox.Cosmos.fsproj index 74e3230b9..c234fd531 100644 --- a/src/Equinox.Cosmos/Equinox.Cosmos.fsproj +++ b/src/Equinox.Cosmos/Equinox.Cosmos.fsproj @@ -26,7 +26,7 @@ - + diff --git a/src/Equinox.Cosmos/EquinoxManager.fsx b/src/Equinox.Cosmos/EquinoxManager.fsx deleted file mode 100644 index 8fdda8e1c..000000000 --- a/src/Equinox.Cosmos/EquinoxManager.fsx +++ /dev/null @@ -1,93 +0,0 @@ -// How to spin up a CosmosDB emulator locally: -// https://docs.microsoft.com/en-us/azure/cosmos-db/local-emulator#running-on-docker -// Then run the script below -#r "bin/Release/FSharp.Control.AsyncSeq.dll" -#r "bin/Release/Newtonsoft.Json.dll" -#r "bin/Release/Microsoft.Azure.Documents.Client.dll" -#load "Infrastructure.fs" - -open System -open Equinox.Cosmos -open Microsoft.Azure.Documents -open Microsoft.Azure.Documents.Client - -let URI = Uri "https://localhost:8081" -let KEY = "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==" -let DBNAME = "test" -let COLLNAME = "test" -let RU = 5000 -let AUXRU = 400 - -let client = - let connPolicy = - let cp = ConnectionPolicy.Default - cp.ConnectionMode <- ConnectionMode.Direct - cp.MaxConnectionLimit <- 200 - cp.RetryOptions <- RetryOptions(MaxRetryAttemptsOnThrottledRequests = 2, MaxRetryWaitTimeInSeconds = 10) - cp - new DocumentClient(URI, KEY, connPolicy, Nullable ConsistencyLevel.Session) - -let createDatabase (client:DocumentClient)= - let dbRequestOptions = - let o = RequestOptions () - o.ConsistencyLevel <- Nullable(ConsistencyLevel.Session) - o - client.CreateDatabaseIfNotExistsAsync(Database(Id=DBNAME), options = dbRequestOptions) - |> Async.AwaitTaskCorrect - |> Async.map (fun response -> Client.UriFactory.CreateDatabaseUri (response.Resource.Id)) - -let createCollection (client: DocumentClient) (dbUri: Uri) = - let pkd = PartitionKeyDefinition() - pkd.Paths.Add("/k") - let coll = DocumentCollection(Id = COLLNAME, PartitionKey = pkd) - - coll.IndexingPolicy.IndexingMode <- IndexingMode.Consistent - coll.IndexingPolicy.Automatic <- true - coll.IndexingPolicy.IncludedPaths.Add(new IncludedPath (Path="/s/?")) - coll.IndexingPolicy.IncludedPaths.Add(new IncludedPath (Path="/k/?")) - coll.IndexingPolicy.IncludedPaths.Add(new IncludedPath (Path="/sn/?")) - coll.IndexingPolicy.ExcludedPaths.Add(new ExcludedPath (Path="/*")) - client.CreateDocumentCollectionIfNotExistsAsync(dbUri, coll, RequestOptions(OfferThroughput=Nullable RU)) - |> Async.AwaitTaskCorrect - |> Async.map (fun response -> Client.UriFactory.CreateDocumentCollectionUri (DBNAME, response.Resource.Id)) - -let createStoreSproc (client: IDocumentClient) (collectionUri: Uri) = - let f =""" - function multidocInsert (docs) { - var response = getContext().getResponse(); - var collection = getContext().getCollection(); - var collectionLink = collection.getSelfLink(); - - if (!docs) throw new Error("Array of events is undefined or null."); - - for (i=0; i Async.AwaitTaskCorrect - |> Async.map (fun r -> Client.UriFactory.CreateStoredProcedureUri(DBNAME, COLLNAME, r.Resource.Id)) - -let createAux (client: DocumentClient) (dbUri: Uri) = - let auxCollectionName = sprintf "%s-aux" COLLNAME - let auxColl = DocumentCollection(Id = auxCollectionName) - auxColl.IndexingPolicy.ExcludedPaths.Add(new ExcludedPath(Path="/ChangefeedPosition/*")) - auxColl.IndexingPolicy.ExcludedPaths.Add(new ExcludedPath(Path="/ProjectionsPositions/*")) - auxColl.IndexingPolicy.IncludedPaths.Add(new IncludedPath (Path="/*")) - auxColl.IndexingPolicy.IndexingMode <- IndexingMode.Lazy - auxColl.DefaultTimeToLive <- Nullable(365 * 60 * 60 * 24) - client.CreateDocumentCollectionIfNotExistsAsync(dbUri, auxColl, RequestOptions(OfferThroughput=Nullable AUXRU)) - |> Async.AwaitTaskCorrect - |> Async.map (fun response -> Client.UriFactory.CreateDocumentCollectionUri (DBNAME, response.Resource.Id)) - -let go = async { - let! dbUri = createDatabase client - do! (createCollection client dbUri) |> Async.bind (createStoreSproc client) |> Async.Ignore - do! createAux client dbUri |> Async.Ignore -} - -go |> Async.RunSynchronously \ No newline at end of file diff --git a/tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs b/tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs index 912cdcc1c..88c74fc26 100644 --- a/tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs +++ b/tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs @@ -1,5 +1,6 @@ module Equinox.Cosmos.Integration.VerbatimUtf8JsonConverterTests +open Equinox.Cosmos open Newtonsoft.Json open Swensen.Unquote open System @@ -22,7 +23,7 @@ type Union = let ``VerbatimUtf8JsonConverter serializes properly`` () = let unionEncoder = Equinox.UnionCodec.JsonUtf8.Create<_>(JsonSerializerSettings()) let encoded = unionEncoder.Encode(A { embed = "\"" }) - let e : Equinox.Cosmos.EquinoxEvent = + let e : EquinoxEvent = { id = null s = null k = null From 3beec28d8179d0e72e891cd7abc5b8daddb073dc Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 30 Oct 2018 15:37:05 +0000 Subject: [PATCH 19/66] Change file structure and scripts --- build.ps1 | 8 +- samples/Store/Integration/CartIntegration.fs | 12 +- .../ContactPreferencesIntegration.fs | 12 +- .../Store/Integration/CosmosIntegration.fs | 2 +- .../Store/Integration/FavoritesIntegration.fs | 6 +- samples/Store/Integration/LogIntegration.fs | 6 +- src/Equinox.Cosmos/Cosmos.fs | 456 +++++++++--------- src/Equinox.Cosmos/CosmosManager.fs | 32 +- .../CosmosIntegration.fs | 49 +- .../Infrastructure.fs | 8 + .../VerbatimUtf8JsonConverterTests.fs | 7 +- 11 files changed, 304 insertions(+), 294 deletions(-) diff --git a/build.ps1 b/build.ps1 index 1af6d6e42..9fab92f4d 100644 --- a/build.ps1 +++ b/build.ps1 @@ -2,6 +2,7 @@ param( [string] $verbosity="m", [Alias("s")][switch][bool] $skipStores=$false, [Alias("se")][switch][bool] $skipEs=$skipStores, + [Alias("sc")][switch][bool] $skipCosmos=$skipStores, [string] $additionalMsBuildArgs="-t:Build" ) @@ -13,10 +14,13 @@ function warn ($msg) { Write-Host "$msg" -BackgroundColor DarkGreen } $env:EQUINOX_INTEGRATION_SKIP_EVENTSTORE=[string]$skipEs if ($skipEs) { warn "Skipping EventStore tests" } -warn "RUNNING: dotnet msbuild $args" +$env:EQUINOX_INTEGRATION_SKIP_COSMOS=[string]$skipCosmos +if ($skipCosmos) { warn "Skipping Cosmos tests" } + +Write-Host "dotnet msbuild $args" . dotnet msbuild build.proj @args if( $LASTEXITCODE -ne 0) { warn "open msbuild.log for error info or rebuild with -v n/d/diag for more detail, or open msbuild.binlog using https://github.com/KirillOsenkov/MSBuildStructuredLog/releases/download/v2.0.40/MSBuildStructuredLogSetup.exe" exit $LASTEXITCODE -} +} \ No newline at end of file diff --git a/samples/Store/Integration/CartIntegration.fs b/samples/Store/Integration/CartIntegration.fs index 79cc93f92..25c2e65fa 100644 --- a/samples/Store/Integration/CartIntegration.fs +++ b/samples/Store/Integration/CartIntegration.fs @@ -71,14 +71,14 @@ type Tests(testOutputHelper) = do! act service args } - [] - let ``Can roundtrip against Equinox, correctly folding the events without compaction semantics`` args = Async.RunSynchronously <| async { - let! service = arrange connectToLocalEquinoxNode createEqxGateway resolveEqxStreamWithoutCompactionSemantics + [] + let ``Can roundtrip against Cosmos, correctly folding the events without compaction semantics`` args = Async.RunSynchronously <| async { + let! service = arrange connectToSpecifiedCosmosOrSimulator createEqxGateway resolveEqxStreamWithoutCompactionSemantics do! act service args } - [] - let ``Can roundtrip against Equinox, correctly folding the events with compaction`` args = Async.RunSynchronously <| async { - let! service = arrange connectToLocalEquinoxNode createEqxGateway resolveEqxStreamWithCompactionEventType + [] + let ``Can roundtrip against Cosmos, correctly folding the events with compaction`` args = Async.RunSynchronously <| async { + let! service = arrange connectToSpecifiedCosmosOrSimulator createEqxGateway resolveEqxStreamWithCompactionEventType do! act service args } \ No newline at end of file diff --git a/samples/Store/Integration/ContactPreferencesIntegration.fs b/samples/Store/Integration/ContactPreferencesIntegration.fs index 3e5f2cc57..989014836 100644 --- a/samples/Store/Integration/ContactPreferencesIntegration.fs +++ b/samples/Store/Integration/ContactPreferencesIntegration.fs @@ -63,14 +63,14 @@ type Tests(testOutputHelper) = do! act service args } - [] - let ``Can roundtrip against Equinox, correctly folding the events with normal semantics`` args = Async.RunSynchronously <| async { - let! service = arrangeWithoutCompaction connectToLocalEquinoxNode createEqxGateway resolveStreamEqxWithoutCompactionSemantics + [] + let ``Can roundtrip against Cosmos, correctly folding the events with normal semantics`` args = Async.RunSynchronously <| async { + let! service = arrangeWithoutCompaction connectToSpecifiedCosmosOrSimulator createEqxGateway resolveStreamEqxWithoutCompactionSemantics do! act service args } - [] - let ``Can roundtrip against Equinox, correctly folding the events with compaction semantics`` args = Async.RunSynchronously <| async { - let! service = arrange connectToLocalEquinoxNode createEqxGateway resolveStreamEqxWithCompactionSemantics + [] + let ``Can roundtrip against Cosmos, correctly folding the events with compaction semantics`` args = Async.RunSynchronously <| async { + let! service = arrange connectToSpecifiedCosmosOrSimulator createEqxGateway resolveStreamEqxWithCompactionSemantics do! act service args } \ No newline at end of file diff --git a/samples/Store/Integration/CosmosIntegration.fs b/samples/Store/Integration/CosmosIntegration.fs index f1772f0c7..eca9e107d 100644 --- a/samples/Store/Integration/CosmosIntegration.fs +++ b/samples/Store/Integration/CosmosIntegration.fs @@ -8,7 +8,7 @@ open System /// - replace connection below with a connection string or Uri+Key for an initialized Equinox instance /// - Create a local Equinox with dbName "test" and collectionName "test" using script: /// /src/Equinox.Cosmos/EquinoxManager.fsx -let connectToLocalEquinoxNode log = +let connectToCosmos log = EqxConnector(log=log, requestTimeout=TimeSpan.FromSeconds 3., maxRetryAttemptsOnThrottledRequests=2, maxRetryWaitTimeInSeconds=60) .Establish("equinoxStoreSampleIntegration", Discovery.UriAndKey(Uri "https://localhost:8081", "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==")) let defaultBatchSize = 500 diff --git a/samples/Store/Integration/FavoritesIntegration.fs b/samples/Store/Integration/FavoritesIntegration.fs index e0395a0b6..da50a7099 100644 --- a/samples/Store/Integration/FavoritesIntegration.fs +++ b/samples/Store/Integration/FavoritesIntegration.fs @@ -55,10 +55,10 @@ type Tests(testOutputHelper) = do! act service args } - [] - let ``Can roundtrip against Equinox, correctly folding the events`` args = Async.RunSynchronously <| async { + [] + let ``Can roundtrip against Cosmos, correctly folding the events`` args = Async.RunSynchronously <| async { let log = createLog () - let! conn = connectToLocalEquinoxNode log + let! conn = connectToSpecifiedCosmosOrSimulator log let gateway = createEqxGateway conn defaultBatchSize let service = createServiceEqx gateway log do! act service args diff --git a/samples/Store/Integration/LogIntegration.fs b/samples/Store/Integration/LogIntegration.fs index 41f73076a..82a2968b9 100644 --- a/samples/Store/Integration/LogIntegration.fs +++ b/samples/Store/Integration/LogIntegration.fs @@ -112,12 +112,12 @@ type Tests() = do! act buffer service itemCount context cartId skuId "ReadStreamEventsBackwardAsync-Duration" } - [] - let ``Can roundtrip against Equinox, hooking, extracting and substituting metrics in the logging information`` context cartId skuId = Async.RunSynchronously <| async { + [] + let ``Can roundtrip against Cosmos, hooking, extracting and substituting metrics in the logging information`` context cartId skuId = Async.RunSynchronously <| async { let buffer = ResizeArray() let batchSize = defaultBatchSize let (log,capture) = createLoggerWithMetricsExtraction buffer.Add - let! conn = connectToLocalEquinoxNode log + let! conn = connectToCosmos log let gateway = createEqxGateway conn batchSize let service = Backend.Cart.Service(log, CartIntegration.resolveEqxStreamWithCompactionEventType gateway) let itemCount, cartId = batchSize / 2 + 1, cartId () diff --git a/src/Equinox.Cosmos/Cosmos.fs b/src/Equinox.Cosmos/Cosmos.fs index b1fdb838d..913edf55e 100644 --- a/src/Equinox.Cosmos/Cosmos.fs +++ b/src/Equinox.Cosmos/Cosmos.fs @@ -10,61 +10,65 @@ open Newtonsoft.Json.Linq open Serilog open System -[] -module ArraySegmentExtensions = - type System.Text.Encoding with - member x.GetString(data:ArraySegment) = x.GetString(data.Array, data.Offset, data.Count) - -[] -module Strings = - open System.Text.RegularExpressions - /// Obtains a single pattern group, if one exists - let (|RegexGroup|_|) (pattern:string) arg = - match Regex.Match(arg, pattern, RegexOptions.None, TimeSpan.FromMilliseconds(250.0)) with - | m when m.Success && m.Groups.[1].Success -> m.Groups.[1].Value |> Some - | _ -> None - -type VerbatimUtf8JsonConverter() = - inherit JsonConverter() - - override __.ReadJson(reader, _, _, _) = - let token = JToken.Load(reader) - if token.Type = JTokenType.Object then token.ToString() |> System.Text.Encoding.UTF8.GetBytes |> box - else Array.empty |> box - - override __.CanConvert(objectType) = - typeof.Equals(objectType) - - override __.WriteJson(writer, value, serializer) = - let array = value :?> byte[] - if array = null || Array.length array = 0 then serializer.Serialize(writer, null) - else writer.WriteRawValue(System.Text.Encoding.UTF8.GetString(array)) - -/// 0-based Event Index in stream -type EventIndex = int64 - -type StreamId = string - -[] -type EventData = - { eventType : string - data : byte[] - metadata : byte[] } - -[] -type EquinoxEvent = { - id : string - s : StreamId - k : StreamId - ts : DateTimeOffset - sn : EventIndex - et : string - df : string - [)>] - d : byte[] - mdf : string - [)>] - md : byte[] } +module Store = + type [] Position = + | Simple of collectionUri: Uri * streamName: string * version: int64 option + | Complex of collectionUri: Uri * partitionKey: string * streamName: string * version: int64 option + member __.CollectionUri : Uri = __ |> function + | Simple (collectionUri,_,_) | Complex (collectionUri,_,_,_) -> collectionUri + member __.PartitionKey : PartitionKey = __ |> function + | Simple (_,streamName,_) -> streamName |> PartitionKey + | Complex (_,partitionKey,_,_) -> partitionKey |> PartitionKey + member __.StreamName : string = __ |> function + | Simple (_,streamName,_) | Complex (_,_,streamName,_) -> streamName + member __.StreamVersion : int64 = __ |> function + | Simple (_,_,version) | Complex (_,_,_,version) -> defaultArg version -1L + member __.GenerateId offset : obj = __ |> function + | Simple _ -> __.IncrementVersion offset |> box + | Complex _ -> sprintf "%s-%d" __.StreamName (__.IncrementVersion offset) |> box + member __.WithVersion (streamVersion: int64) : Position = __ |> function + | Simple (collectionUri,streamName,_) -> + Simple (collectionUri,streamName,Some streamVersion) + | Complex (collectionUri,partitionKey,streamName,_) -> + Complex(collectionUri,partitionKey,streamName,Some streamVersion) + member __.IncrementVersion (offset: int) : int64 = __ |> function + | Simple (_,_,Some version) | Complex (_,_,_,Some version) -> (version+int64 offset) + | _ -> failwithf "Cannot IncrementVersion %A" __ + + type EventData = { eventType: string; data: byte[]; metadata: byte[] } + type [] Event = + { id: obj // Unique key within partition (where many streams in same partition e.g. "{Category}-{guid}-{index}", otherwise "{index}" + s: string // "{Category}-{guid}" bit when >1 stream in same partition + i: Nullable // {index} where >1 stream in same partition + ts: DateTimeOffset // ISO 8601 + et: string // required + [)>] + d: byte[] // required + [)>] + md: byte[] } // optional + static member Create (pos: Position) offset (ed: EventData) : Event = + let id,sid,index = pos |> function + | Position.Simple (_,_streamName,_version) -> pos.GenerateId offset, null, Nullable () + | Position.Complex (_,_,_streamName,_version) -> pos.GenerateId offset, pos.StreamName, Nullable (pos.IncrementVersion offset) + { id = id; s = sid; i = index + ts = DateTimeOffset.UtcNow + et = ed.eventType; d = ed.data; md = ed.metadata } + member __.StreamVersion = if __.i.HasValue then __.i.Value else unbox __.id + and VerbatimUtf8JsonConverter() = + inherit JsonConverter() + + override __.ReadJson(reader, _, _, _) = + let token = JToken.Load(reader) + if token.Type = JTokenType.Object then token.ToString() |> System.Text.Encoding.UTF8.GetBytes |> box + else Array.empty |> box + + override __.CanConvert(objectType) = + typeof.Equals(objectType) + + override __.WriteJson(writer, value, serializer) = + let array = value :?> byte[] + if array = null || Array.length array = 0 then serializer.Serialize(writer, null) + else writer.WriteRawValue(System.Text.Encoding.UTF8.GetString(array)) [] type Direction = Forward | Backward with @@ -83,9 +87,9 @@ module Log = let propEvents name (kvps : System.Collections.Generic.KeyValuePair seq) (log : ILogger) = let items = seq { for kv in kvps do yield sprintf "{\"%s\": %s}" kv.Key kv.Value } log.ForContext(name, sprintf "[%s]" (String.concat ",\n\r" items)) - let propEventData name (events : EventData[]) (log : ILogger) = + let propEventData name (events : Store.EventData[]) (log : ILogger) = log |> propEvents name (seq { for x in events -> Collections.Generic.KeyValuePair<_,_>(x.eventType, System.Text.Encoding.UTF8.GetString x.data)}) - let propResolvedEvents name (events : EquinoxEvent[]) (log : ILogger) = + let propResolvedEvents name (events : Store.Event[]) (log : ILogger) = log |> propEvents name (seq { for x in events -> Collections.Generic.KeyValuePair<_,_>(x.et, System.Text.Encoding.UTF8.GetString x.d)}) open Serilog.Events @@ -105,107 +109,110 @@ module Log = let (|BlobLen|) = function null -> 0 | (x : byte[]) -> x.Length [] -type EqxSyncResult = Written of EventIndex * requestCharge: float | Conflict of requestCharge: float +type EqxSyncResult = Written of Store.Position * requestCharge: float | Conflict of requestCharge: float module private Write = /// Appends the single EventData using the sdk CreateDocumentAsync - let private appendSingleEvent (client : IDocumentClient,collectionUri : Uri) streamId version eventData : Async = async { - let index = version + 1L - let equinoxEvent = eventData |> EquinoxEvent.mk streamId index - - let requestOptions = Client.RequestOptions(PartitionKey = PartitionKey(streamId)) - let! res = client.CreateDocumentAsync(collectionUri, equinoxEvent, requestOptions) |> Async.AwaitTaskCorrect - - return index, res.RequestCharge } + let private appendSingleEvent (client: IDocumentClient) (pos: Store.Position) eventData + : Async = async { + let evnt = Store.Event.Create pos 1 eventData + let! res = client.CreateDocumentAsync(pos.CollectionUri, evnt, Client.RequestOptions(PartitionKey=pos.PartitionKey)) |> Async.AwaitTaskCorrect + return pos.WithVersion(pos.IncrementVersion 1), res.RequestCharge } /// Appends the given EventData batch using the atomic stored procedure - let private appendEventBatch (client : IDocumentClient,collectionUri) streamId version eventsData : Async = async { - let events = - eventsData |> Seq.mapi (fun i ed -> - let index = version + int64 (i+1) - EquinoxEvent.mk streamId index ed - |> JsonConvert.SerializeObject) - |> Seq.toArray - - let requestOptions = Client.RequestOptions(PartitionKey = PartitionKey(streamId)) - let sprocUri = sprintf "%O/sprocs/AtomicMultiDocInsert" collectionUri + let private appendEventBatch (client: IDocumentClient) (pos: Store.Position) eventsData + : Async = async { + let sprocUri = sprintf "%O/sprocs/AtomicMultiDocInsert" pos.CollectionUri + let requestOptions = Client.RequestOptions(PartitionKey = pos.PartitionKey) + let events = eventsData |> Seq.mapi (fun i ed -> Store.Event.Create pos (i+1) ed |> JsonConvert.SerializeObject) |> Seq.toArray let! ct = Async.CancellationToken let! res = client.ExecuteStoredProcedureAsync(sprocUri, requestOptions, ct, box events) |> Async.AwaitTaskCorrect + return pos.WithVersion(pos.IncrementVersion events.Length), res.RequestCharge } - return version + int64 events.Length, res.RequestCharge } - - let private append coll streamName sequenceNumber (eventsData: EventData seq) = + let private append client pk (eventsData: Store.EventData seq) = match Seq.length eventsData with | l when l = 0 -> invalidArg "eventsData" "must be non-empty" - | l when l = 1 -> eventsData |> Seq.exactlyOne |> appendSingleEvent coll streamName sequenceNumber - | _ -> appendEventBatch coll streamName sequenceNumber eventsData + | l when l = 1 -> eventsData |> Seq.exactlyOne |> appendSingleEvent client pk + | _ -> appendEventBatch client pk eventsData /// Yields `EqxSyncResult.Written` or `EqxSyncResult.Conflict` to signify WrongExpectedVersion - let private writeEventsAsync (log : ILogger) coll streamName (version : int64) (events : EventData[]) - : Async = async { + let private writeEventsAsync (log : ILogger) client pk (events : Store.EventData[]): Async = async { try - let! wr = append coll streamName version events + let! wr = append client pk events return EqxSyncResult.Written wr with :? DocumentClientException as ex when ex.Message.Contains "already" -> // TODO improve check, handle SP variant log.Information(ex, "Eqx TrySync WrongExpectedVersionException writing {EventTypes}", [| for x in events -> x.eventType |]) return EqxSyncResult.Conflict ex.RequestCharge } - let eventDataBytes events = - let eventDataLen { data = Log.BlobLen bytes; metadata = Log.BlobLen metaBytes } = bytes + metaBytes + let bytes events = + let eventDataLen ({ data = Log.BlobLen bytes; metadata = Log.BlobLen metaBytes } : Store.EventData) = bytes + metaBytes events |> Array.sumBy eventDataLen - let private writeEventsLogged coll streamName (version : int64) (events : EventData[]) (log : ILogger) - : Async = async { + let private writeEventsLogged client (pos : Store.Position) (events : Store.EventData[]) (log : ILogger): Async = async { let log = if (not << log.IsEnabled) Events.LogEventLevel.Debug then log else log |> Log.propEventData "Json" events - let bytes, count = eventDataBytes events, events.Length + let bytes, count = bytes events, events.Length let log = log |> Log.prop "bytes" bytes - let writeLog = log |> Log.prop "stream" streamName |> Log.prop "expectedVersion" version |> Log.prop "count" count - let! t, result = writeEventsAsync writeLog coll streamName version events |> Stopwatch.Time + let writeLog = log |> Log.prop "stream" pos.StreamName |> Log.prop "expectedVersion" pos.StreamVersion |> Log.prop "count" count + let! t, result = writeEventsAsync writeLog client pos events |> Stopwatch.Time let conflict, (ru: float), resultLog = - let mkMetric ru : Log.Measurement = { stream = streamName; interval = t; bytes = bytes; count = count; ru = ru } + let mkMetric ru : Log.Measurement = { stream = pos.StreamName; interval = t; bytes = bytes; count = count; ru = ru } match result with | EqxSyncResult.Conflict ru -> true, ru, log |> Log.event (Log.WriteConflict (mkMetric ru)) | EqxSyncResult.Written (x, ru) -> false, ru, log |> Log.event (Log.WriteSuccess (mkMetric ru)) |> Log.prop "nextExpectedVersion" x resultLog.Information("Eqx{action:l} count={count} conflict={conflict}, RequestCharge={ru}", "Write", events.Length, conflict, ru) return result } - let writeEvents (log : ILogger) retryPolicy coll (streamName : string) (version : int64) (events : EventData[]) - : Async = - let call = writeEventsLogged coll streamName version events + let writeEvents (log : ILogger) retryPolicy client pk (events : Store.EventData[]): Async = + let call = writeEventsLogged client pk events Log.withLoggedRetries retryPolicy "writeAttempt" call log module private Read = - let private getQuery ((client : IDocumentClient,collectionUri : Uri),strongConsistency) streamId (direction: Direction) batchSize (version: int64) = - let querySpec = - // TODODC if (strongConsistency) then feedOptions.ConsistencyLevel <- Nullable(ConsistencyLevel.Strong) - let filter = if direction = Direction.Backward then "c.sn <= @version ORDER BY c.sn DESC" else "c.sn >= @version ORDER BY c.sn ASC" - let prms = [| SqlParameter("@streamId", streamId); SqlParameter("@version", version) |] - SqlQuerySpec("SELECT * FROM c WHERE c.s = @streamId AND " + filter, SqlParameterCollection prms) - let feedOptions = new Client.FeedOptions(PartitionKey=PartitionKey streamId, MaxItemCount=Nullable batchSize) - client.CreateDocumentQuery(collectionUri, querySpec, feedOptions).AsDocumentQuery() - - let (|EquinoxEventLen|) (x : EquinoxEvent) = match x.d, x.md with Log.BlobLen bytes, Log.BlobLen metaBytes -> bytes + metaBytes - - let private lastSequenceNumber (xs:EquinoxEvent seq) : EventIndex = + let mkSingletonQuery query arg value = SqlQuerySpec(query, SqlParameterCollection (Seq.singleton (SqlParameter(arg, value)))) + let mkIndexQuery query (index:int64) = mkSingletonQuery query "@index" index + let private getQuery (client : IDocumentClient) (pos:Store.Position) (direction: Direction) batchSize = + let collectionUri,querySpec = + match pos with + | Store.Position.Simple (collectionUri, _, None) -> + if direction = Direction.Forward then invalidOp "Cannot read forward from None" + else collectionUri, SqlQuerySpec("SELECT * FROM c ORDER BY c.id DESC") + | Store.Position.Simple (collectionUri, _, Some index) -> + let filter = + if direction = Direction.Forward then "c.id >= @index ORDER BY c.id ASC" + else "c.id < @index ORDER BY c.id DESC" + collectionUri, mkIndexQuery("SELECT * FROM c WHERE " + filter) index + | Store.Position.Complex (collectionUri, _partitionKey, streamName, None) -> + if direction = Direction.Forward then invalidOp "Cannot read forward from None" + else collectionUri, mkSingletonQuery "SELECT * FROM c WHERE c.s = @streamId ORDER BY c.id DESC" "@streamId" streamName + | Store.Position.Complex (collectionUri, _partitionKey, streamName, Some index) -> + let filter = if direction = Direction.Forward then "c.i >= @index ORDER BY c.id ASC" else "c.i <= @index ORDER BY c.id DESC" + let prms = [| SqlParameter("@streamId", streamName); SqlParameter("@index", index) |] + collectionUri, SqlQuerySpec("SELECT * FROM c WHERE c.s = @streamId AND " + filter, SqlParameterCollection prms) + let feedOptions = new Client.FeedOptions(PartitionKey=pos.PartitionKey, MaxItemCount=Nullable batchSize) + client.CreateDocumentQuery(collectionUri, querySpec, feedOptions).AsDocumentQuery() + + let (|EventLen|) (x : Store.Event) = match x.d, x.md with Log.BlobLen bytes, Log.BlobLen metaBytes -> bytes + metaBytes + + let private lastStreamVersion (xs:Store.Event seq) : int64 = match xs |> Seq.tryLast with | None -> -1L - | Some last -> last.sn + | Some last -> last.StreamVersion - let private loggedQueryExecution streamName direction startPos (query: IDocumentQuery) (log: ILogger) - : Async = async { - let! t, (res : Client.FeedResponse) = query.ExecuteNextAsync() |> Async.AwaitTaskCorrect |> Stopwatch.Time + let private loggedQueryExecution (pos:Store.Position) direction (query: IDocumentQuery) (log: ILogger): Async = async { + let! t, (res : Client.FeedResponse) = query.ExecuteNextAsync() |> Async.AwaitTaskCorrect |> Stopwatch.Time let slice, ru = Array.ofSeq res, res.RequestCharge - let bytes, count = slice |> Array.sumBy (|EquinoxEventLen|), slice.Length - let reqMetric : Log.Measurement = { stream = streamName; interval = t; bytes = bytes; count = count; ru = ru } + let bytes, count = slice |> Array.sumBy (|EventLen|), slice.Length + let reqMetric : Log.Measurement = { stream = pos.StreamName; interval = t; bytes = bytes; count = count; ru = ru } let evt = Log.Slice (direction, reqMetric) let log = if (not << log.IsEnabled) Events.LogEventLevel.Debug then log else log |> Log.propResolvedEvents "Json" slice - (log |> Log.prop "startPos" startPos |> Log.prop "bytes" bytes |> Log.prop "ru" ru |> Log.event evt) - .Information("Eqx{action:l} count={count} version={sliceVersion} RequestCharge={ru}", "Read", count, lastSequenceNumber slice, ru) + (log |> Log.prop "startPos" pos.StreamVersion |> Log.prop "bytes" bytes |> Log.prop "ru" ru |> Log.event evt) + .Information("Eqx{action:l} count={count} version={sliceVersion} RequestCharge={ru}", "Read", count, lastStreamVersion slice, ru) return slice, ru } - let private readBatches (log : ILogger) (readSlice: IDocumentQuery -> ILogger -> Async) (maxPermittedBatchReads: int option) (query: IDocumentQuery) - : AsyncSeq = - let rec loop batchCount : AsyncSeq = asyncSeq { + let private readBatches (log : ILogger) (readSlice: IDocumentQuery -> ILogger -> Async) + (maxPermittedBatchReads: int option) + (query: IDocumentQuery) + : AsyncSeq = + let rec loop batchCount : AsyncSeq = asyncSeq { match maxPermittedBatchReads with | Some mpbr when batchCount >= mpbr -> log.Information "batch Limit exceeded"; invalidOp "batch Limit exceeded" | _ -> () @@ -217,11 +224,11 @@ module private Read = yield! loop (batchCount + 1) } loop 0 - let equinoxEventBytes events = events |> Array.sumBy (|EquinoxEventLen|) + let bytes events = events |> Array.sumBy (|EventLen|) - let logBatchRead direction streamName t events batchSize version (ru: float) (log : ILogger) = - let bytes, count = equinoxEventBytes events, events.Length - let reqMetric : Log.Measurement = { stream = streamName; interval = t; bytes = bytes; count = count; ru = ru } + let logBatchRead direction streamName interval events batchSize version (ru: float) (log : ILogger) = + let bytes, count = bytes events, events.Length + let reqMetric : Log.Measurement = { stream = streamName; interval = interval; bytes = bytes; count = count; ru = ru } let batches = (events.Length - 1)/batchSize + 1 let action = match direction with Direction.Forward -> "LoadF" | Direction.Backward -> "LoadB" let evt = Log.Event.Batch (direction, batches, reqMetric) @@ -229,35 +236,34 @@ module private Read = "Eqx{action:l} stream={stream} count={count}/{batches} version={version} RequestCharge={ru}", action, streamName, count, batches, version, ru) - let loadForwardsFrom (log : ILogger) retryPolicy coll batchSize maxPermittedBatchReads streamName startPosition - : Async = async { + let loadForwardsFrom (log : ILogger) retryPolicy client batchSize maxPermittedBatchReads (pos,_strongConsistency): Async = async { let mutable ru = 0.0 - let mergeBatches (batches: AsyncSeq) = async { - let! (events : EquinoxEvent[]) = + let mergeBatches (batches: AsyncSeq) = async { + let! (events : Store.Event[]) = batches |> AsyncSeq.map (fun (events, r) -> ru <- ru + r; events) |> AsyncSeq.concatSeq |> AsyncSeq.toArrayAsync return events, ru } - use query = getQuery coll streamName Direction.Forward batchSize startPosition - let call q = loggedQueryExecution streamName Direction.Forward startPosition q + use query = getQuery client pos Direction.Forward batchSize + let call q = loggedQueryExecution pos Direction.Forward q let retryingLoggingReadSlice q = Log.withLoggedRetries retryPolicy "readAttempt" (call q) let direction = Direction.Forward - let log = log |> Log.prop "batchSize" batchSize |> Log.prop "direction" direction |> Log.prop "stream" streamName - let batches : AsyncSeq = readBatches log retryingLoggingReadSlice maxPermittedBatchReads query + let log = log |> Log.prop "batchSize" batchSize |> Log.prop "direction" direction |> Log.prop "stream" pos.StreamName + let batches : AsyncSeq = readBatches log retryingLoggingReadSlice maxPermittedBatchReads query let! t, (events, ru) = mergeBatches batches |> Stopwatch.Time query.Dispose() - let version = lastSequenceNumber events - log |> logBatchRead direction streamName t events batchSize version ru - return version, events } + let version = lastStreamVersion events + log |> logBatchRead direction pos.StreamName t events batchSize version ru + return pos.WithVersion version, events } - let partitionPayloadFrom firstUsedEventNumber : EquinoxEvent[] -> int * int = - let acc (tu,tr) ((EquinoxEventLen bytes) as y) = if y.sn < firstUsedEventNumber then tu, tr + bytes else tu + bytes, tr + let partitionPayloadFrom firstUsedEventNumber : Store.Event[] -> int * int = + let acc (tu,tr) ((EventLen bytes) as y) = if y.StreamVersion < firstUsedEventNumber then tu, tr + bytes else tu + bytes, tr Array.fold acc (0,0) - let loadBackwardsUntilCompactionOrStart (log : ILogger) retryPolicy coll batchSize maxPermittedBatchReads streamName isCompactionEvent - : Async = async { - let mergeFromCompactionPointOrStartFromBackwardsStream (log : ILogger) (batchesBackward : AsyncSeq) - : Async = async { + let loadBackwardsUntilCompactionOrStart (log : ILogger) retryPolicy client batchSize maxPermittedBatchReads isCompactionEvent (pos : Store.Position) + : Async = async { + let mergeFromCompactionPointOrStartFromBackwardsStream (log : ILogger) (batchesBackward : AsyncSeq) + : Async = async { let lastBatch = ref None let mutable ru = 0.0 let! tempBackward = @@ -268,68 +274,68 @@ module private Read = if not (isCompactionEvent x) then true // continue the search else match !lastBatch with - | None -> log.Information("EqxStop stream={stream} at={eventNumber}", streamName, x.sn) + | None -> log.Information("EqxStop stream={stream} at={eventNumber}", pos.StreamName, x.StreamVersion) | Some batch -> - let used, residual = batch |> partitionPayloadFrom x.sn - log.Information("EqxStop stream={stream} at={eventNumber} used={used} residual={residual}", streamName, x.sn, used, residual) + let used, residual = batch |> partitionPayloadFrom x.StreamVersion + log.Information("EqxStop stream={stream} at={eventNumber} used={used} residual={residual}", pos.StreamName, x.StreamVersion, used, residual) false) |> AsyncSeq.toArrayAsync let eventsForward = Array.Reverse(tempBackward); tempBackward // sic - relatively cheap, in-place reverse of something we own return eventsForward, ru } - use query = getQuery coll streamName Direction.Backward batchSize EventIndex.MaxValue - let call q = loggedQueryExecution streamName Direction.Backward EventIndex.MaxValue q + use query = getQuery client pos Direction.Backward batchSize + let call q = loggedQueryExecution pos Direction.Backward q let retryingLoggingReadSlice q = Log.withLoggedRetries retryPolicy "readAttempt" (call q) - let log = log |> Log.prop "batchSize" batchSize |> Log.prop "stream" streamName + let log = log |> Log.prop "batchSize" batchSize |> Log.prop "stream" pos.StreamName let direction = Direction.Backward let readlog = log |> Log.prop "direction" direction - let batchesBackward : AsyncSeq = readBatches readlog retryingLoggingReadSlice maxPermittedBatchReads query + let batchesBackward : AsyncSeq = readBatches readlog retryingLoggingReadSlice maxPermittedBatchReads query let! t, (events, ru) = mergeFromCompactionPointOrStartFromBackwardsStream log batchesBackward |> Stopwatch.Time query.Dispose() - let version = lastSequenceNumber events - log |> logBatchRead direction streamName t events batchSize version ru - return version, events } + let version = lastStreamVersion events + log |> logBatchRead direction pos.StreamName t events batchSize version ru + return pos.WithVersion version, events } module UnionEncoderAdapters = - let private encodedEventOfStoredEvent (x : EquinoxEvent) : UnionCodec.EncodedUnion = + let private encodedEventOfStoredEvent (x : Store.Event) : UnionCodec.EncodedUnion = { caseName = x.et; payload = x.d } - let private eventDataOfEncodedEvent (x : UnionCodec.EncodedUnion) : EventData = + let private eventDataOfEncodedEvent (x : UnionCodec.EncodedUnion) : Store.EventData = { eventType = x.caseName; data = x.payload; metadata = null } - let encodeEvents (codec : UnionCodec.IUnionEncoder<'event, byte[]>) (xs : 'event seq) : EventData[] = + let encodeEvents (codec : UnionCodec.IUnionEncoder<'event, byte[]>) (xs : 'event seq) : Store.EventData[] = xs |> Seq.map (codec.Encode >> eventDataOfEncodedEvent) |> Seq.toArray - let decodeKnownEvents (codec : UnionCodec.IUnionEncoder<'event, byte[]>) (xs : EquinoxEvent[]) : 'event seq = + let decodeKnownEvents (codec : UnionCodec.IUnionEncoder<'event, byte[]>) (xs : Store.Event[]) : 'event seq = xs |> Seq.map encodedEventOfStoredEvent |> Seq.choose codec.TryDecode -type Token = { streamVersion: int64; compactionEventNumber: int64 option } +type []Token = { pos: Store.Position; compactionEventNumber: int64 option } [] module Token = - let private create compactionEventNumber batchCapacityLimit streamVersion : Storage.StreamToken = - { value = box { streamVersion = streamVersion; compactionEventNumber = compactionEventNumber }; batchCapacityLimit = batchCapacityLimit } + let private create compactionEventNumber batchCapacityLimit pos : Storage.StreamToken = + { value = box { pos = pos; compactionEventNumber = compactionEventNumber }; batchCapacityLimit = batchCapacityLimit } /// No batching / compaction; we only need to retain the StreamVersion - let ofNonCompacting streamVersion : Storage.StreamToken = - create None None streamVersion + let ofNonCompacting (pos : Store.Position) : Storage.StreamToken = + create None None pos // headroom before compaction is necessary given the stated knowledge of the last (if known) `compactionEventNumberOption` let private batchCapacityLimit compactedEventNumberOption unstoredEventsPending (batchSize : int) (streamVersion : int64) : int = match compactedEventNumberOption with | Some (compactionEventNumber : int64) -> (batchSize - unstoredEventsPending) - int (streamVersion - compactionEventNumber + 1L) |> max 0 | None -> (batchSize - unstoredEventsPending) - (int streamVersion + 1) - 1 |> max 0 - let (*private*) ofCompactionEventNumber compactedEventNumberOption unstoredEventsPending batchSize streamVersion : Storage.StreamToken = - let batchCapacityLimit = batchCapacityLimit compactedEventNumberOption unstoredEventsPending batchSize streamVersion - create compactedEventNumberOption (Some batchCapacityLimit) streamVersion + let (*private*) ofCompactionEventNumber compactedEventNumberOption unstoredEventsPending batchSize (pos : Store.Position) : Storage.StreamToken = + let batchCapacityLimit = batchCapacityLimit compactedEventNumberOption unstoredEventsPending batchSize pos.StreamVersion + create compactedEventNumberOption (Some batchCapacityLimit) pos /// Assume we have not seen any compaction events; use the batchSize and version to infer headroom - let ofUncompactedVersion batchSize streamVersion : Storage.StreamToken = - ofCompactionEventNumber None 0 batchSize streamVersion + let ofUncompactedVersion batchSize pos : Storage.StreamToken = + ofCompactionEventNumber None 0 batchSize pos /// Use previousToken plus the data we are adding and the position we are adding it to infer a headroom - let ofPreviousTokenAndEventsLength (previousToken : Storage.StreamToken) eventsLength batchSize streamVersion : Storage.StreamToken = + let ofPreviousTokenAndEventsLength (previousToken : Storage.StreamToken) eventsLength batchSize pos : Storage.StreamToken = let compactedEventNumber = (unbox previousToken.value).compactionEventNumber - ofCompactionEventNumber compactedEventNumber eventsLength batchSize streamVersion + ofCompactionEventNumber compactedEventNumber eventsLength batchSize pos /// Use an event just read from the stream to infer headroom - let ofCompactionResolvedEventAndVersion (compactionEvent: EquinoxEvent) batchSize streamVersion : Storage.StreamToken = - ofCompactionEventNumber (Some compactionEvent.sn) 0 batchSize streamVersion + let ofCompactionResolvedEventAndVersion (compactionEvent: Store.Event) batchSize pos : Storage.StreamToken = + ofCompactionEventNumber (Some compactionEvent.StreamVersion) 0 batchSize pos /// Use an event we are about to write to the stream to infer headroom let ofPreviousStreamVersionAndCompactionEventDataIndex prevStreamVersion compactionEventDataIndex eventsLength batchSize streamVersion' : Storage.StreamToken = ofCompactionEventNumber (Some (prevStreamVersion + 1L + int64 compactionEventDataIndex)) eventsLength batchSize streamVersion' - let private unpackEqxStreamVersion (x : Storage.StreamToken) = let x : Token = unbox x.value in x.streamVersion + let private unpackEqxStreamVersion (x : Storage.StreamToken) = let x : Token = unbox x.value in x.pos.StreamVersion let supersedes current x = let currentVersion, newVersion = unpackEqxStreamVersion current, unpackEqxStreamVersion x newVersion > currentVersion @@ -349,37 +355,34 @@ type EqxBatchingPolicy(getMaxBatchSize : unit -> int, ?batchCountLimit) = type GatewaySyncResult = Written of Storage.StreamToken | Conflict type EqxGateway(conn : EqxConnection, batching : EqxBatchingPolicy) = - let isResolvedEventEventType predicate (x:EquinoxEvent) = predicate x.et + let isResolvedEventEventType predicate (x:Store.Event) = predicate x.et let tryIsResolvedEventEventType predicateOption = predicateOption |> Option.map isResolvedEventEventType - let (|Coll|) (collectionUri: Uri) = conn.Client,collectionUri - member __.LoadBatched (Coll coll,streamName) log isCompactionEventType: Async = async { - let! version, events = Read.loadForwardsFrom log conn.ReadRetryPolicy (coll,false) batching.BatchSize batching.MaxBatches streamName 0L + let (|Pos|) (token: Storage.StreamToken) : Store.Position = (unbox token.value).pos + member __.LoadBatched log isCompactionEventType (pos : Store.Position): Async = async { + let! pos, events = Read.loadForwardsFrom log conn.ReadRetryPolicy conn.Client batching.BatchSize batching.MaxBatches (pos,false) match tryIsResolvedEventEventType isCompactionEventType with - | None -> return Token.ofNonCompacting version, events + | None -> return Token.ofNonCompacting pos, events | Some isCompactionEvent -> match events |> Array.tryFindBack isCompactionEvent with - | None -> return Token.ofUncompactedVersion batching.BatchSize version, events - | Some resolvedEvent -> return Token.ofCompactionResolvedEventAndVersion resolvedEvent batching.BatchSize version, events } - member __.LoadBackwardsStoppingAtCompactionEvent (Coll coll,streamName) log isCompactionEventType: Async = async { + | None -> return Token.ofUncompactedVersion batching.BatchSize pos, events + | Some resolvedEvent -> return Token.ofCompactionResolvedEventAndVersion resolvedEvent batching.BatchSize pos, events } + member __.LoadBackwardsStoppingAtCompactionEvent log isCompactionEventType pos: Async = async { let isCompactionEvent = isResolvedEventEventType isCompactionEventType - let! version, events = - Read.loadBackwardsUntilCompactionOrStart log conn.ReadRetryPolicy (coll,false) batching.BatchSize batching.MaxBatches streamName isCompactionEvent + let! pos, events = + Read.loadBackwardsUntilCompactionOrStart log conn.ReadRetryPolicy conn.Client batching.BatchSize batching.MaxBatches isCompactionEvent pos match Array.tryHead events |> Option.filter isCompactionEvent with - | None -> return Token.ofUncompactedVersion batching.BatchSize version, events - | Some resolvedEvent -> return Token.ofCompactionResolvedEventAndVersion resolvedEvent batching.BatchSize version, events } - member __.LoadFromToken ((Coll coll,streamName),strongConsistency) log (token : Storage.StreamToken) isCompactionEventType - : Async = async { - let streamPosition = (unbox token.value).streamVersion - let! version, events = Read.loadForwardsFrom log conn.ReadRetryPolicy (coll,strongConsistency) batching.BatchSize batching.MaxBatches streamName streamPosition + | None -> return Token.ofUncompactedVersion batching.BatchSize pos, events + | Some resolvedEvent -> return Token.ofCompactionResolvedEventAndVersion resolvedEvent batching.BatchSize pos, events } + member __.LoadFromToken log (Pos pos as token) isCompactionEventType synchronized: Async = async { + let! pos, events = Read.loadForwardsFrom log conn.ReadRetryPolicy conn.Client batching.BatchSize batching.MaxBatches (pos,synchronized) match tryIsResolvedEventEventType isCompactionEventType with - | None -> return Token.ofNonCompacting version, events + | None -> return Token.ofNonCompacting pos, events | Some isCompactionEvent -> match events |> Array.tryFindBack isCompactionEvent with - | None -> return Token.ofPreviousTokenAndEventsLength token events.Length batching.BatchSize version, events - | Some resolvedEvent -> return Token.ofCompactionResolvedEventAndVersion resolvedEvent batching.BatchSize version, events } - member __.TrySync (Coll coll,streamName) log (token : Storage.StreamToken) (encodedEvents: EventData array) isCompactionEventType : Async = async { - let streamVersion = (unbox token.value).streamVersion - let! wr = Write.writeEvents log conn.WriteRetryPolicy coll streamName streamVersion encodedEvents + | None -> return Token.ofPreviousTokenAndEventsLength token events.Length batching.BatchSize pos, events + | Some resolvedEvent -> return Token.ofCompactionResolvedEventAndVersion resolvedEvent batching.BatchSize pos, events } + member __.TrySync log (Pos pos as token) (encodedEvents: Store.EventData[]) isCompactionEventType: Async = async { + let! wr = Write.writeEvents log conn.WriteRetryPolicy conn.Client pos encodedEvents match wr with | EqxSyncResult.Conflict _ -> return GatewaySyncResult.Conflict | EqxSyncResult.Written (wr, _) -> @@ -389,11 +392,11 @@ type EqxGateway(conn : EqxConnection, batching : EqxBatchingPolicy) = match isCompactionEventType with | None -> Token.ofNonCompacting version' | Some isCompactionEvent -> - let isEventDataEventType predicate (x:EventData) = predicate x.eventType + let isEventDataEventType predicate (x:Store.EventData) = predicate x.eventType match encodedEvents |> Array.tryFindIndexBack (isEventDataEventType isCompactionEvent) with | None -> Token.ofPreviousTokenAndEventsLength token encodedEvents.Length batching.BatchSize version' | Some compactionEventIndex -> - Token.ofPreviousStreamVersionAndCompactionEventDataIndex streamVersion compactionEventIndex encodedEvents.Length batching.BatchSize version' + Token.ofPreviousStreamVersionAndCompactionEventDataIndex pos.StreamVersion compactionEventIndex encodedEvents.Length batching.BatchSize version' return GatewaySyncResult.Written token } type private Collection(gateway : EqxGateway, databaseId, collectionId) = @@ -401,25 +404,25 @@ type private Collection(gateway : EqxGateway, databaseId, collectionId) = member __.CollectionUri = Client.UriFactory.CreateDocumentCollectionUri(databaseId, collectionId) type private Category<'event, 'state>(coll : Collection, codec : UnionCodec.IUnionEncoder<'event, byte[]>, ?compactionStrategy) = - let (|StreamRef|) streamName = coll.CollectionUri, streamName - let loadAlgorithm load streamName initial log = - let batched = load initial (coll.Gateway.LoadBatched streamName log None) - let compacted predicate = load initial (coll.Gateway.LoadBackwardsStoppingAtCompactionEvent streamName log predicate) + let (|Pos|) streamName = Store.Position.Simple (coll.CollectionUri, streamName, None) + let loadAlgorithm load (Pos pos) initial log = + let batched = load initial (coll.Gateway.LoadBatched log None pos) + let compacted predicate = load initial (coll.Gateway.LoadBackwardsStoppingAtCompactionEvent log predicate pos) match compactionStrategy with | Some predicate -> compacted predicate | None -> batched let load (fold: 'state -> 'event seq -> 'state) initial loadF = async { let! token, events = loadF return token, fold initial (UnionEncoderAdapters.decodeKnownEvents codec events) } - member __.Load (fold: 'state -> 'event seq -> 'state) (initial: 'state) (StreamRef streamRef) (log : ILogger) : Async = - loadAlgorithm (load fold) streamRef initial log - member __.LoadFromToken (fold: 'state -> 'event seq -> 'state) (state: 'state) (StreamRef streamRef) token (log : ILogger) : Async = - (load fold) state (coll.Gateway.LoadFromToken (streamRef,false) log token compactionStrategy) - member __.TrySync (fold: 'state -> 'event seq -> 'state) (StreamRef streamRef) (log : ILogger) (token, state) (events : 'event list) : Async> = async { - let encodedEvents : EventData[] = UnionEncoderAdapters.encodeEvents codec (Seq.ofList events) - let! syncRes = coll.Gateway.TrySync streamRef log token encodedEvents compactionStrategy + member __.Load (fold: 'state -> 'event seq -> 'state) (initial: 'state) streamName (log : ILogger) : Async = + loadAlgorithm (load fold) streamName initial log + member __.LoadFromToken (fold: 'state -> 'event seq -> 'state) (state: 'state) token (log : ILogger) : Async = + (load fold) state (coll.Gateway.LoadFromToken log token compactionStrategy false) + member __.TrySync (fold: 'state -> 'event seq -> 'state) (log : ILogger) (token, state) (events : 'event list) : Async> = async { + let encodedEvents : Store.EventData[] = UnionEncoderAdapters.encodeEvents codec (Seq.ofList events) + let! syncRes = coll.Gateway.TrySync log token encodedEvents compactionStrategy match syncRes with - | GatewaySyncResult.Conflict -> return Storage.SyncResult.Conflict (load fold state (coll.Gateway.LoadFromToken (streamRef,true) log token compactionStrategy)) + | GatewaySyncResult.Conflict -> return Storage.SyncResult.Conflict (load fold state (coll.Gateway.LoadFromToken log token compactionStrategy true)) | GatewaySyncResult.Written token' -> return Storage.SyncResult.Written (token', fold state (Seq.ofList events)) } module Caching = @@ -483,7 +486,7 @@ module Caching = type private Folder<'event, 'state>(category : Category<'event, 'state>, fold: 'state -> 'event seq -> 'state, initial: 'state, ?readCache) = let loadAlgorithm streamName initial log = let batched = category.Load fold initial streamName log - let cached token state = category.LoadFromToken fold state streamName token log + let cached token state = category.LoadFromToken fold state token log match readCache with | None -> batched | Some (cache : Caching.Cache, prefix : string) -> @@ -493,8 +496,8 @@ type private Folder<'event, 'state>(category : Category<'event, 'state>, fold: ' interface ICategory<'event, 'state> with member __.Load (streamName : string) (log : ILogger) : Async = loadAlgorithm streamName initial log - member __.TrySync streamName (log : ILogger) (token, state) (events : 'event list) : Async> = async { - let! syncRes = category.TrySync fold streamName log (token, state) events + member __.TrySync _streamName(* TODO remove from main interface *) (log : ILogger) (token, state) (events : 'event list) : Async> = async { + let! syncRes = category.TrySync fold log (token, state) events match syncRes with | Storage.SyncResult.Conflict resync -> return Storage.SyncResult.Conflict resync | Storage.SyncResult.Written (token',state') -> return Storage.SyncResult.Written (token',state') } @@ -577,8 +580,7 @@ module Initialization = [] type Discovery = - | UriAndKey of uri:Uri * key:string - | ConnectionString of string + | UriAndKey of databaseUri:Uri * key:string /// Implements connection string parsing logic curiously missing from the DocDb SDK static member FromConnectionString (connectionString: string) = match connectionString with @@ -589,15 +591,28 @@ type Discovery = UriAndKey (Uri uri, key) | _ -> invalidArg "connectionString" "unrecognized connection string format" +type ConnectionMode = + /// Default mode, uses Https - inefficient as uses a double hop + | Gateway + /// Most efficient, but requires direct connectivity + | DirectTcp + // More efficient than Gateway, but suboptimal + | DirectHttps + type EqxConnector ( requestTimeout: TimeSpan, maxRetryAttemptsOnThrottledRequests: int, maxRetryWaitTimeInSeconds: int, log : ILogger, /// Connection limit (default 1000) ?maxConnectionLimit, - ?readRetryPolicy, ?writeRetryPolicy, - /// Connection mode (default: Gateway mode, Https) - ?mode, ?defaultConsistencyLevel, - + /// Connection mode (default: ConnectionMode.Gateway (lowest perf, least trouble)) + ?mode : ConnectionMode, + /// consistency mode (default: ConsistencyLevel.Session) + ?defaultConsistencyLevel : ConsistencyLevel, + + /// Retries for read requests, over and above those defined by the mandatory policies + ?readRetryPolicy, + /// Retries for write requests, over and above those defined by the mandatory policies + ?writeRetryPolicy, /// Additional strings identifying the context of this connection; should provide enough context to disambiguate all potential connections to a cluster /// NB as this will enter server and client logs, it should not contain sensitive information ?tags : (string*string) seq) = @@ -627,22 +642,13 @@ type EqxConnector match tags with None -> () | Some tags -> for key, value in tags do yield sprintf "%s=%s" key value } let sanitizedName = name.Replace('\'','_').Replace(':','_') // sic; Align with logging for ES Adapter let client = new Client.DocumentClient(uri, key, connPolicy, Nullable(defaultArg defaultConsistencyLevel ConsistencyLevel.Session)) - log.Information("Connected to Cosmos with clientId={clientId}", sanitizedName) + log.Information("Connecting to Cosmos with clientId={clientId}", sanitizedName) do! client.OpenAsync() |> Async.AwaitTaskCorrect return client :> IDocumentClient } - match discovery with - | Discovery.UriAndKey(uri=uri; key=key) -> - connect (uri,key) - | Discovery.ConnectionString connStr -> - let cred = - match connStr,connStr with - | Strings.RegexGroup "AccountEndpoint=(.+?);" uri, Strings.RegexGroup "AccountKey=(.+?);" key -> - System.Uri(uri), key - | _ -> failwithf "Invalid DocumentDB connection string: %s" connStr - connect cred + match discovery with Discovery.UriAndKey(databaseUri=uri; key=key) -> connect (uri,key) /// Yields a DocDbConnection configured per the specified strategy member __.Establish(name, discovery : Discovery) : Async = async { let! conn = __.Connect(name, discovery) - return EqxConnection(conn, ?readRetryPolicy=readRetryPolicy, ?writeRetryPolicy=writeRetryPolicy) } \ No newline at end of file + return EqxConnection(conn, ?readRetryPolicy=readRetryPolicy, ?writeRetryPolicy=writeRetryPolicy) } diff --git a/src/Equinox.Cosmos/CosmosManager.fs b/src/Equinox.Cosmos/CosmosManager.fs index 7df62f87d..ea11a4a3f 100644 --- a/src/Equinox.Cosmos/CosmosManager.fs +++ b/src/Equinox.Cosmos/CosmosManager.fs @@ -2,31 +2,18 @@ open System open Equinox.Cosmos -open Equinox.EventStore.Infrastructure +open Equinox.Store.Infrastructure open Microsoft.Azure.Documents open Microsoft.Azure.Documents.Client -let configCosmos connStr dbName collName ru auxRu = async { - let uri, key = - match connStr,connStr with - | Strings.RegexGroup "AccountEndpoint=(.+?);" uri, Strings.RegexGroup "AccountKey=(.+?);" key -> - System.Uri(uri), key - | _ -> failwithf "Invalid DocumentDB connection string: %s" connStr - let client = - let connPolicy = - let cp = ConnectionPolicy.Default - cp.ConnectionMode <- ConnectionMode.Direct - cp.MaxConnectionLimit <- 200 - cp.RetryOptions <- RetryOptions(MaxRetryAttemptsOnThrottledRequests = 2, MaxRetryWaitTimeInSeconds = 10) - cp - new DocumentClient(uri, key, connPolicy, Nullable ConsistencyLevel.Session) +let configCosmos (client : IDocumentClient) dbName collName ru auxRu = async { - let createDatabase (client:DocumentClient) = async { + let createDatabase (client:IDocumentClient) = async { let dbRequestOptions = RequestOptions(ConsistencyLevel = Nullable ConsistencyLevel.Session) let! db = client.CreateDatabaseIfNotExistsAsync(Database(Id=dbName), options = dbRequestOptions) |> Async.AwaitTaskCorrect return Client.UriFactory.CreateDatabaseUri (db.Resource.Id) } - let createCollection (client: DocumentClient) (dbUri: Uri) = async { + let createCollection (client: IDocumentClient) (dbUri: Uri) = async { let pkd = PartitionKeyDefinition() pkd.Paths.Add("/k") let coll = DocumentCollection(Id = collName, PartitionKey = pkd) @@ -41,8 +28,7 @@ let configCosmos connStr dbName collName ru auxRu = async { return Client.UriFactory.CreateDocumentCollectionUri (dbName, dc.Resource.Id) } let createStoreSproc (client: IDocumentClient) (collectionUri: Uri) = async { - let f =""" - function multidocInsert (docs) { + let f ="""function multidocInsert (docs) { var response = getContext().getResponse(); var collection = getContext().getCollection(); var collectionLink = collection.getSelfLink(); @@ -50,17 +36,17 @@ let configCosmos connStr dbName collName ru auxRu = async { if (!docs) throw new Error("Array of events is undefined or null."); for (i=0; i Async.AwaitTaskCorrect return Client.UriFactory.CreateStoredProcedureUri(dbName, collName, sp.Resource.Id) } - let createAux (client: DocumentClient) (dbUri: Uri) = async { + let createAux (client: IDocumentClient) (dbUri: Uri) = async { let auxCollectionName = sprintf "%s-aux" collName let auxColl = DocumentCollection(Id = auxCollectionName) auxColl.IndexingPolicy.ExcludedPaths.Add(new ExcludedPath(Path="/ChangefeedPosition/*")) @@ -75,4 +61,4 @@ let configCosmos connStr dbName collName ru auxRu = async { let! _sp = createStoreSproc client coll let! _aux = createAux client dbUri do () -} +} \ No newline at end of file diff --git a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs index e74452ea2..446b110b7 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs @@ -10,9 +10,18 @@ open System /// - replace connection below with a connection string or Uri+Key for an initialized Equinox instance /// - Create a local Equinox with dbName "test" and collectionName "test" using script: /// /src/Equinox.Cosmos/EquinoxManager.fsx -let connectToLocalEquinoxNode (log: Serilog.ILogger) = +let connectToCosmos (log: Serilog.ILogger) name discovery = EqxConnector(log=log, requestTimeout=TimeSpan.FromSeconds 3., maxRetryAttemptsOnThrottledRequests=2, maxRetryWaitTimeInSeconds=60) - .Establish("localDocDbSim", Discovery.UriAndKey(Uri "https://localhost:8081", "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==")) + .Establish(name, discovery) +let connectToSpecifiedCosmosOrSimulator (log: Serilog.ILogger) = + match Environment.GetEnvironmentVariable "EQUINOX_COSMOS_CONNECTION" |> Option.ofObj with + | None -> + Discovery.UriAndKey(Uri "https://localhost:8081", "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==") + |> connectToCosmos log "localDocDbSim" + | Some connectionString -> + Discovery.FromConnectionString connectionString + |> connectToCosmos log "specified" + let defaultBatchSize = 500 let createEqxGateway connection batchSize = EqxGateway(connection, EqxBatchingPolicy(maxBatchSize = batchSize)) let (|StreamArgs|) streamName = @@ -89,10 +98,10 @@ type Tests(testOutputHelper) = let singleBatchForward = [EqxAct.SliceForward; EqxAct.BatchForward] let batchForwardAndAppend = singleBatchForward @ [EqxAct.Append] - [] - let ``Can roundtrip against Equinox, correctly batching the reads [without any optimizations]`` context cartId skuId = Async.RunSynchronously <| async { + [] + let ``Can roundtrip against Cosmos, correctly batching the reads [without any optimizations]`` context cartId skuId = Async.RunSynchronously <| async { let log, capture = createLoggerWithCapture () - let! conn = connectToLocalEquinoxNode log + let! conn = connectToSpecifiedCosmosOrSimulator log let batchSize = 3 let service = Cart.createServiceWithoutOptimization conn batchSize log @@ -115,10 +124,10 @@ type Tests(testOutputHelper) = test <@ List.replicate (expectedBatches-1) singleSliceForward @ singleBatchForward = capture.ExternalCalls @> } - [] - let ``Can roundtrip against Equinox, managing sync conflicts by retrying [without any optimizations]`` ctx initialState = Async.RunSynchronously <| async { + [] + let ``Can roundtrip against Cosmos, managing sync conflicts by retrying [without any optimizations]`` ctx initialState = Async.RunSynchronously <| async { let log1, capture1 = createLoggerWithCapture () - let! conn = connectToLocalEquinoxNode log1 + let! conn = connectToSpecifiedCosmosOrSimulator log1 // Ensure batching is included at some point in the proceedings let batchSize = 3 @@ -191,10 +200,10 @@ type Tests(testOutputHelper) = let singleBatchBackwards = [EqxAct.SliceBackward; EqxAct.BatchBackward] let batchBackwardsAndAppend = singleBatchBackwards @ [EqxAct.Append] - [] - let ``Can roundtrip against Equinox, correctly compacting to avoid redundant reads`` context skuId cartId = Async.RunSynchronously <| async { + [] + let ``Can roundtrip against Cosmos, correctly compacting to avoid redundant reads`` context skuId cartId = Async.RunSynchronously <| async { let log, capture = createLoggerWithCapture () - let! conn = connectToLocalEquinoxNode log + let! conn = connectToSpecifiedCosmosOrSimulator log let batchSize = 10 let service = Cart.createServiceWithCompaction conn batchSize log @@ -230,10 +239,10 @@ type Tests(testOutputHelper) = test <@ singleBatchBackwards @ batchBackwardsAndAppend @ singleBatchBackwards = capture.ExternalCalls @> } - [] - let ``Can correctly read and update against Equinox, with window size of 1 using tautological Compaction predicate`` id value = Async.RunSynchronously <| async { + [] + let ``Can correctly read and update against Cosmos, with window size of 1 using tautological Compaction predicate`` id value = Async.RunSynchronously <| async { let log, capture = createLoggerWithCapture () - let! conn = connectToLocalEquinoxNode log + let! conn = connectToSpecifiedCosmosOrSimulator log let service = ContactPreferences.createService (createEqxGateway conn) log let (Domain.ContactPreferences.Id email) = id @@ -253,10 +262,10 @@ type Tests(testOutputHelper) = test <@ batchBackwardsAndAppend @ singleBatchBackwards = capture.ExternalCalls @> } - [] - let ``Can roundtrip against Equinox, correctly caching to avoid redundant reads`` context skuId cartId = Async.RunSynchronously <| async { + [] + let ``Can roundtrip against Cosmos, correctly caching to avoid redundant reads`` context skuId cartId = Async.RunSynchronously <| async { let log, capture = createLoggerWithCapture () - let! conn = connectToLocalEquinoxNode log + let! conn = connectToSpecifiedCosmosOrSimulator log let batchSize = 10 let cache = Caching.Cache("cart", sizeMb = 50) let createServiceCached () = Cart.createServiceWithCaching conn batchSize log cache @@ -280,10 +289,10 @@ type Tests(testOutputHelper) = test <@ singleBatchForward = capture.ExternalCalls @> } - [] - let ``Can combine compaction with caching against Equinox`` context skuId cartId = Async.RunSynchronously <| async { + [] + let ``Can combine compaction with caching against Cosmos`` context skuId cartId = Async.RunSynchronously <| async { let log, capture = createLoggerWithCapture () - let! conn = connectToLocalEquinoxNode log + let! conn = connectToSpecifiedCosmosOrSimulator log let batchSize = 10 let service1 = Cart.createServiceWithCompaction conn batchSize log let cache = Caching.Cache("cart", sizeMb = 50) diff --git a/tests/Equinox.Cosmos.Integration/Infrastructure.fs b/tests/Equinox.Cosmos.Integration/Infrastructure.fs index 2252f32f0..1d4ce5d1e 100644 --- a/tests/Equinox.Cosmos.Integration/Infrastructure.fs +++ b/tests/Equinox.Cosmos.Integration/Infrastructure.fs @@ -17,6 +17,14 @@ type FsCheckGenerators = type AutoDataAttribute() = inherit FsCheck.Xunit.PropertyAttribute(Arbitrary = [|typeof|], MaxTest = 1, QuietOnSuccess = true) + member val SkipIfRequestedViaEnvironmentVariable : string = null with get, set + + override __.Skip = + match Option.ofObj __.SkipIfRequestedViaEnvironmentVariable |> Option.map Environment.GetEnvironmentVariable |> Option.bind Option.ofObj with + | Some value when value.Equals(bool.TrueString, StringComparison.OrdinalIgnoreCase) -> + sprintf "Skipped as requested via %s" __.SkipIfRequestedViaEnvironmentVariable + | _ -> null + // Derived from https://github.com/damianh/CapturingLogOutputWithXunit2AndParallelTests // NB VS does not surface these atm, but other test runners / test reports do type TestOutputAdapter(testOutput : Xunit.Abstractions.ITestOutputHelper) = diff --git a/tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs b/tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs index 88c74fc26..326545c8c 100644 --- a/tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs +++ b/tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs @@ -23,16 +23,13 @@ type Union = let ``VerbatimUtf8JsonConverter serializes properly`` () = let unionEncoder = Equinox.UnionCodec.JsonUtf8.Create<_>(JsonSerializerSettings()) let encoded = unionEncoder.Encode(A { embed = "\"" }) - let e : EquinoxEvent = + let e : Store.Event = { id = null s = null - k = null ts = DateTimeOffset.MinValue - sn = 0L - df = "jsonbytearray" + i = Nullable 0L et = encoded.caseName d = encoded.payload - mdf = "jsonbytearray" md = null } let res = serialize e test <@ res.Contains """"d":{"embed":"\""}""" @> \ No newline at end of file From c8259abc2257f4fec18db258e1e223a83545ecdb Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 31 Oct 2018 12:26:23 +0000 Subject: [PATCH 20/66] Rework representation in index and bench app --- .../Store/Integration/CosmosIntegration.fs | 2 +- src/Equinox.Cosmos/Cosmos.fs | 179 ++++++++++-------- src/Equinox.Cosmos/CosmosManager.fs | 64 ------- src/Equinox.Cosmos/Equinox.Cosmos.fsproj | 1 - .../CosmosIntegration.fs | 2 +- .../VerbatimUtf8JsonConverterTests.fs | 1 + 6 files changed, 100 insertions(+), 149 deletions(-) delete mode 100644 src/Equinox.Cosmos/CosmosManager.fs diff --git a/samples/Store/Integration/CosmosIntegration.fs b/samples/Store/Integration/CosmosIntegration.fs index eca9e107d..620a7f97d 100644 --- a/samples/Store/Integration/CosmosIntegration.fs +++ b/samples/Store/Integration/CosmosIntegration.fs @@ -10,7 +10,7 @@ open System /// /src/Equinox.Cosmos/EquinoxManager.fsx let connectToCosmos log = EqxConnector(log=log, requestTimeout=TimeSpan.FromSeconds 3., maxRetryAttemptsOnThrottledRequests=2, maxRetryWaitTimeInSeconds=60) - .Establish("equinoxStoreSampleIntegration", Discovery.UriAndKey(Uri "https://localhost:8081", "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==")) + .Connect("equinoxStoreSampleIntegration", Discovery.UriAndKey(Uri "https://localhost:8081", "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==")) let defaultBatchSize = 500 let createEqxGateway connection batchSize = EqxGateway(connection, EqxBatchingPolicy(maxBatchSize = batchSize)) // Typically, one will split different categories of stream into Cosmos collections - hard coding this is thus an oversimplification diff --git a/src/Equinox.Cosmos/Cosmos.fs b/src/Equinox.Cosmos/Cosmos.fs index 913edf55e..14ddddc45 100644 --- a/src/Equinox.Cosmos/Cosmos.fs +++ b/src/Equinox.Cosmos/Cosmos.fs @@ -12,48 +12,74 @@ open System module Store = type [] Position = - | Simple of collectionUri: Uri * streamName: string * version: int64 option - | Complex of collectionUri: Uri * partitionKey: string * streamName: string * version: int64 option + | Complex of collectionUri: Uri * customPartitionKey: string option * streamName: string * index: int64 option + | Simple of collectionUri: Uri * streamName: string * index: int64 option member __.CollectionUri : Uri = __ |> function | Simple (collectionUri,_,_) | Complex (collectionUri,_,_,_) -> collectionUri - member __.PartitionKey : PartitionKey = __ |> function - | Simple (_,streamName,_) -> streamName |> PartitionKey - | Complex (_,partitionKey,_,_) -> partitionKey |> PartitionKey + member __.PartitionKey : string = __ |> function + | Simple (_,streamName,_) -> streamName + | Complex (_,customPartitionKey,streamName,_) -> defaultArg customPartitionKey streamName member __.StreamName : string = __ |> function | Simple (_,streamName,_) | Complex (_,_,streamName,_) -> streamName - member __.StreamVersion : int64 = __ |> function - | Simple (_,_,version) | Complex (_,_,_,version) -> defaultArg version -1L + member __.Index : int64 = __ |> function + | Simple (_,_,index) | Complex (_,_,_,index) -> defaultArg index -1L + member __.IndexRel (offset: int) : int64 = __ |> function + | Simple (_,_,Some index) | Complex (_,_,_,Some index) -> (index+int64 offset) + | _ -> failwithf "Cannot IndexRel %A" __ member __.GenerateId offset : obj = __ |> function - | Simple _ -> __.IncrementVersion offset |> box - | Complex _ -> sprintf "%s-%d" __.StreamName (__.IncrementVersion offset) |> box - member __.WithVersion (streamVersion: int64) : Position = __ |> function + | Simple _ -> __.IndexRel offset |> box + | Complex _ -> sprintf "%s-%d" __.StreamName (__.IndexRel offset) |> box + member __.WithIndex (index: int64) : Position = __ |> function | Simple (collectionUri,streamName,_) -> - Simple (collectionUri,streamName,Some streamVersion) + Simple (collectionUri,streamName,Some index) | Complex (collectionUri,partitionKey,streamName,_) -> - Complex(collectionUri,partitionKey,streamName,Some streamVersion) - member __.IncrementVersion (offset: int) : int64 = __ |> function - | Simple (_,_,Some version) | Complex (_,_,_,Some version) -> (version+int64 offset) - | _ -> failwithf "Cannot IncrementVersion %A" __ + Complex(collectionUri,partitionKey,streamName,Some index) type EventData = { eventType: string; data: byte[]; metadata: byte[] } - type [] Event = - { id: obj // Unique key within partition (where many streams in same partition e.g. "{Category}-{guid}-{index}", otherwise "{index}" - s: string // "{Category}-{guid}" bit when >1 stream in same partition - i: Nullable // {index} where >1 stream in same partition + type [] Event = + { (* DocDb-mandated essential elements *) + + // DocDb-mandated Partition Key, must be maintained within the document + // Not actually required if running in single partition mode, but for simplicity, we always write it + // Some users generate a custom Partition Key to colocate multiple streams to enable colocating and querying across multiple streams + k: string // Complex: "{customPartitionKey}" or (default:) "{streamName}"; Simple: "{streamName}" + + // DocDb-mandated unique key; needs to be unique within a partition + // Could use {index} as an int64/number here, if not for the fact that we allow the app to provide a custom `k` to enable streams to colocate + id: obj // Complex: "{streamName}-{index:020d}"; Simple: {index} (int64, written as number) + + (* Indexing/routing properties - separated from `id` to enable indexing/routing used within Equinox.Cosmos *) + + /// Stream name + [] + s: string // Complex: "{streamName}"; Simple: omitted + + /// Index of event within Stream + i: Nullable // Complex: {index}; Simple: omitted + + (* Event payload elements *) + + /// Creation date (as opposed to sytem-defined _lastUpdated which is rewritten by triggers adnd/or replication) ts: DateTimeOffset // ISO 8601 + + /// The Event Type, used to drive deserialization et: string // required + + /// Event body, as UTF-8 encoded json ready to be injected into the Json being rendered for DocDb [)>] d: byte[] // required - [)>] + + /// Optional metadata (null, or same as d) + [); JsonProperty(Required=Required.DisallowNull)>] md: byte[] } // optional static member Create (pos: Position) offset (ed: EventData) : Event = - let id,sid,index = pos |> function + let id,s,i = pos |> function | Position.Simple (_,_streamName,_version) -> pos.GenerateId offset, null, Nullable () - | Position.Complex (_,_,_streamName,_version) -> pos.GenerateId offset, pos.StreamName, Nullable (pos.IncrementVersion offset) - { id = id; s = sid; i = index + | Position.Complex (_,_,_streamName,_version) -> pos.GenerateId offset, pos.StreamName, Nullable (pos.IndexRel offset) + { k = pos.PartitionKey; id = id; s = s; i = i ts = DateTimeOffset.UtcNow et = ed.eventType; d = ed.data; md = ed.metadata } - member __.StreamVersion = if __.i.HasValue then __.i.Value else unbox __.id + member __.Index = if __.i.HasValue then __.i.Value else unbox __.id and VerbatimUtf8JsonConverter() = inherit JsonConverter() @@ -112,35 +138,21 @@ module Log = type EqxSyncResult = Written of Store.Position * requestCharge: float | Conflict of requestCharge: float module private Write = - /// Appends the single EventData using the sdk CreateDocumentAsync - let private appendSingleEvent (client: IDocumentClient) (pos: Store.Position) eventData - : Async = async { - let evnt = Store.Event.Create pos 1 eventData - let! res = client.CreateDocumentAsync(pos.CollectionUri, evnt, Client.RequestOptions(PartitionKey=pos.PartitionKey)) |> Async.AwaitTaskCorrect - return pos.WithVersion(pos.IncrementVersion 1), res.RequestCharge } - - /// Appends the given EventData batch using the atomic stored procedure - let private appendEventBatch (client: IDocumentClient) (pos: Store.Position) eventsData - : Async = async { + let append (client: IDocumentClient) (pos: Store.Position) (eventsData: Store.EventData seq): Async = async { let sprocUri = sprintf "%O/sprocs/AtomicMultiDocInsert" pos.CollectionUri - let requestOptions = Client.RequestOptions(PartitionKey = pos.PartitionKey) - let events = eventsData |> Seq.mapi (fun i ed -> Store.Event.Create pos (i+1) ed |> JsonConvert.SerializeObject) |> Seq.toArray + let opts = Client.RequestOptions(PartitionKey=PartitionKey pos.PartitionKey) let! ct = Async.CancellationToken - let! res = client.ExecuteStoredProcedureAsync(sprocUri, requestOptions, ct, box events) |> Async.AwaitTaskCorrect - return pos.WithVersion(pos.IncrementVersion events.Length), res.RequestCharge } - - let private append client pk (eventsData: Store.EventData seq) = - match Seq.length eventsData with - | l when l = 0 -> invalidArg "eventsData" "must be non-empty" - | l when l = 1 -> eventsData |> Seq.exactlyOne |> appendSingleEvent client pk - | _ -> appendEventBatch client pk eventsData + let events = eventsData |> Seq.mapi (fun i ed -> Store.Event.Create pos (i+1) ed |> JsonConvert.SerializeObject) |> Seq.toArray + if events.Length = 0 then invalidArg "eventsData" "must be non-empty" + let! res = client.ExecuteStoredProcedureAsync(sprocUri, opts, ct, box events) |> Async.AwaitTaskCorrect + return pos.WithIndex(pos.IndexRel events.Length), res.RequestCharge } - /// Yields `EqxSyncResult.Written` or `EqxSyncResult.Conflict` to signify WrongExpectedVersion + /// Yields `EqxSyncResult.Written`, or `EqxSyncResult.Conflict` to signify WrongExpectedVersion let private writeEventsAsync (log : ILogger) client pk (events : Store.EventData[]): Async = async { try let! wr = append client pk events return EqxSyncResult.Written wr - with :? DocumentClientException as ex when ex.Message.Contains "already" -> // TODO improve check, handle SP variant + with :? DocumentClientException as ex when ex.Message.Contains "already" -> // TODO this does not work for the SP log.Information(ex, "Eqx TrySync WrongExpectedVersionException writing {EventTypes}", [| for x in events -> x.eventType |]) return EqxSyncResult.Conflict ex.RequestCharge } @@ -152,14 +164,14 @@ module private Write = let log = if (not << log.IsEnabled) Events.LogEventLevel.Debug then log else log |> Log.propEventData "Json" events let bytes, count = bytes events, events.Length let log = log |> Log.prop "bytes" bytes - let writeLog = log |> Log.prop "stream" pos.StreamName |> Log.prop "expectedVersion" pos.StreamVersion |> Log.prop "count" count + let writeLog = log |> Log.prop "stream" pos.StreamName |> Log.prop "expectedVersion" pos.Index |> Log.prop "count" count let! t, result = writeEventsAsync writeLog client pos events |> Stopwatch.Time let conflict, (ru: float), resultLog = let mkMetric ru : Log.Measurement = { stream = pos.StreamName; interval = t; bytes = bytes; count = count; ru = ru } match result with | EqxSyncResult.Conflict ru -> true, ru, log |> Log.event (Log.WriteConflict (mkMetric ru)) | EqxSyncResult.Written (x, ru) -> false, ru, log |> Log.event (Log.WriteSuccess (mkMetric ru)) |> Log.prop "nextExpectedVersion" x - resultLog.Information("Eqx{action:l} count={count} conflict={conflict}, RequestCharge={ru}", "Write", events.Length, conflict, ru) + resultLog.Information("Eqx{action:l} count={count} conflict={conflict}, rus={ru}", "Write", events.Length, conflict, ru) return result } let writeEvents (log : ILogger) retryPolicy client pk (events : Store.EventData[]): Async = @@ -168,7 +180,7 @@ module private Write = module private Read = let mkSingletonQuery query arg value = SqlQuerySpec(query, SqlParameterCollection (Seq.singleton (SqlParameter(arg, value)))) - let mkIndexQuery query (index:int64) = mkSingletonQuery query "@index" index + let mkIdQuery query transform (index:int64) = mkSingletonQuery query "@id" (transform index) let private getQuery (client : IDocumentClient) (pos:Store.Position) (direction: Direction) batchSize = let collectionUri,querySpec = match pos with @@ -177,26 +189,23 @@ module private Read = else collectionUri, SqlQuerySpec("SELECT * FROM c ORDER BY c.id DESC") | Store.Position.Simple (collectionUri, _, Some index) -> let filter = - if direction = Direction.Forward then "c.id >= @index ORDER BY c.id ASC" - else "c.id < @index ORDER BY c.id DESC" - collectionUri, mkIndexQuery("SELECT * FROM c WHERE " + filter) index + if direction = Direction.Forward then "c.id >= @id ORDER BY c.id ASC" + else "c.id < @id ORDER BY c.id DESC" + collectionUri, mkIdQuery("SELECT * FROM c WHERE " + filter) id index | Store.Position.Complex (collectionUri, _partitionKey, streamName, None) -> if direction = Direction.Forward then invalidOp "Cannot read forward from None" - else collectionUri, mkSingletonQuery "SELECT * FROM c WHERE c.s = @streamId ORDER BY c.id DESC" "@streamId" streamName + else collectionUri, mkIdQuery "SELECT * FROM c WHERE STARTSWITH(c.id, @id) ORDER BY c.id DESC" (fun _index -> streamName) -1L | Store.Position.Complex (collectionUri, _partitionKey, streamName, Some index) -> - let filter = if direction = Direction.Forward then "c.i >= @index ORDER BY c.id ASC" else "c.i <= @index ORDER BY c.id DESC" - let prms = [| SqlParameter("@streamId", streamName); SqlParameter("@index", index) |] - collectionUri, SqlQuerySpec("SELECT * FROM c WHERE c.s = @streamId AND " + filter, SqlParameterCollection prms) - let feedOptions = new Client.FeedOptions(PartitionKey=pos.PartitionKey, MaxItemCount=Nullable batchSize) + let filter = + if direction = Direction.Forward then "c.id >= @id ORDER BY c.id ASC" + else "c.id <= @id ORDER BY c.id DESC" + let mkComplexIndex streamName index = sprintf "%s-%020d" streamName index + collectionUri, mkIdQuery ("SELECT * FROM c WHERE " + filter) (mkComplexIndex streamName) index + let feedOptions = new Client.FeedOptions(PartitionKey=PartitionKey pos.PartitionKey, MaxItemCount=Nullable batchSize) client.CreateDocumentQuery(collectionUri, querySpec, feedOptions).AsDocumentQuery() let (|EventLen|) (x : Store.Event) = match x.d, x.md with Log.BlobLen bytes, Log.BlobLen metaBytes -> bytes + metaBytes - let private lastStreamVersion (xs:Store.Event seq) : int64 = - match xs |> Seq.tryLast with - | None -> -1L - | Some last -> last.StreamVersion - let private loggedQueryExecution (pos:Store.Position) direction (query: IDocumentQuery) (log: ILogger): Async = async { let! t, (res : Client.FeedResponse) = query.ExecuteNextAsync() |> Async.AwaitTaskCorrect |> Stopwatch.Time let slice, ru = Array.ofSeq res, res.RequestCharge @@ -204,8 +213,9 @@ module private Read = let reqMetric : Log.Measurement = { stream = pos.StreamName; interval = t; bytes = bytes; count = count; ru = ru } let evt = Log.Slice (direction, reqMetric) let log = if (not << log.IsEnabled) Events.LogEventLevel.Debug then log else log |> Log.propResolvedEvents "Json" slice - (log |> Log.prop "startPos" pos.StreamVersion |> Log.prop "bytes" bytes |> Log.prop "ru" ru |> Log.event evt) - .Information("Eqx{action:l} count={count} version={sliceVersion} RequestCharge={ru}", "Read", count, lastStreamVersion slice, ru) + let index = match slice |> Array.tryHead with Some head -> Nullable head.Index | None -> Nullable () + (log |> Log.prop "startIndex" pos.Index |> Log.prop "bytes" bytes |> Log.event evt) + .Information("Eqx{action:l} count={count} index={index} rus={ru}", "Read", count, index, ru) return slice, ru } let private readBatches (log : ILogger) (readSlice: IDocumentQuery -> ILogger -> Async) @@ -233,9 +243,14 @@ module private Read = let action = match direction with Direction.Forward -> "LoadF" | Direction.Backward -> "LoadB" let evt = Log.Event.Batch (direction, batches, reqMetric) (log |> Log.prop "bytes" bytes |> Log.event evt).Information( - "Eqx{action:l} stream={stream} count={count}/{batches} version={version} RequestCharge={ru}", + "Eqx{action:l} stream={stream} count={count}/{batches} index={index} rus={ru}", action, streamName, count, batches, version, ru) + let private lastEventIndex (xs:Store.Event seq) : int64 = + match xs |> Seq.tryLast with + | None -> -1L + | Some last -> last.Index + let loadForwardsFrom (log : ILogger) retryPolicy client batchSize maxPermittedBatchReads (pos,_strongConsistency): Async = async { let mutable ru = 0.0 let mergeBatches (batches: AsyncSeq) = async { @@ -253,12 +268,12 @@ module private Read = let batches : AsyncSeq = readBatches log retryingLoggingReadSlice maxPermittedBatchReads query let! t, (events, ru) = mergeBatches batches |> Stopwatch.Time query.Dispose() - let version = lastStreamVersion events + let version = lastEventIndex events log |> logBatchRead direction pos.StreamName t events batchSize version ru - return pos.WithVersion version, events } + return pos.WithIndex version, events } let partitionPayloadFrom firstUsedEventNumber : Store.Event[] -> int * int = - let acc (tu,tr) ((EventLen bytes) as y) = if y.StreamVersion < firstUsedEventNumber then tu, tr + bytes else tu + bytes, tr + let acc (tu,tr) ((EventLen bytes) as y) = if y.Index < firstUsedEventNumber then tu, tr + bytes else tu + bytes, tr Array.fold acc (0,0) let loadBackwardsUntilCompactionOrStart (log : ILogger) retryPolicy client batchSize maxPermittedBatchReads isCompactionEvent (pos : Store.Position) : Async = async { @@ -274,10 +289,10 @@ module private Read = if not (isCompactionEvent x) then true // continue the search else match !lastBatch with - | None -> log.Information("EqxStop stream={stream} at={eventNumber}", pos.StreamName, x.StreamVersion) + | None -> log.Information("EqxStop stream={stream} at={eventNumber}", pos.StreamName, x.Index) | Some batch -> - let used, residual = batch |> partitionPayloadFrom x.StreamVersion - log.Information("EqxStop stream={stream} at={eventNumber} used={used} residual={residual}", pos.StreamName, x.StreamVersion, used, residual) + let used, residual = batch |> partitionPayloadFrom x.Index + log.Information("EqxStop stream={stream} at={eventNumber} used={used} residual={residual}", pos.StreamName, x.Index, used, residual) false) |> AsyncSeq.toArrayAsync let eventsForward = Array.Reverse(tempBackward); tempBackward // sic - relatively cheap, in-place reverse of something we own @@ -291,9 +306,9 @@ module private Read = let batchesBackward : AsyncSeq = readBatches readlog retryingLoggingReadSlice maxPermittedBatchReads query let! t, (events, ru) = mergeFromCompactionPointOrStartFromBackwardsStream log batchesBackward |> Stopwatch.Time query.Dispose() - let version = lastStreamVersion events + let version = lastEventIndex events log |> logBatchRead direction pos.StreamName t events batchSize version ru - return pos.WithVersion version, events } + return pos.WithIndex version, events } module UnionEncoderAdapters = let private encodedEventOfStoredEvent (x : Store.Event) : UnionCodec.EncodedUnion = @@ -320,7 +335,7 @@ module Token = | Some (compactionEventNumber : int64) -> (batchSize - unstoredEventsPending) - int (streamVersion - compactionEventNumber + 1L) |> max 0 | None -> (batchSize - unstoredEventsPending) - (int streamVersion + 1) - 1 |> max 0 let (*private*) ofCompactionEventNumber compactedEventNumberOption unstoredEventsPending batchSize (pos : Store.Position) : Storage.StreamToken = - let batchCapacityLimit = batchCapacityLimit compactedEventNumberOption unstoredEventsPending batchSize pos.StreamVersion + let batchCapacityLimit = batchCapacityLimit compactedEventNumberOption unstoredEventsPending batchSize pos.Index create compactedEventNumberOption (Some batchCapacityLimit) pos /// Assume we have not seen any compaction events; use the batchSize and version to infer headroom let ofUncompactedVersion batchSize pos : Storage.StreamToken = @@ -331,11 +346,11 @@ module Token = ofCompactionEventNumber compactedEventNumber eventsLength batchSize pos /// Use an event just read from the stream to infer headroom let ofCompactionResolvedEventAndVersion (compactionEvent: Store.Event) batchSize pos : Storage.StreamToken = - ofCompactionEventNumber (Some compactionEvent.StreamVersion) 0 batchSize pos + ofCompactionEventNumber (Some compactionEvent.Index) 0 batchSize pos /// Use an event we are about to write to the stream to infer headroom let ofPreviousStreamVersionAndCompactionEventDataIndex prevStreamVersion compactionEventDataIndex eventsLength batchSize streamVersion' : Storage.StreamToken = ofCompactionEventNumber (Some (prevStreamVersion + 1L + int64 compactionEventDataIndex)) eventsLength batchSize streamVersion' - let private unpackEqxStreamVersion (x : Storage.StreamToken) = let x : Token = unbox x.value in x.pos.StreamVersion + let private unpackEqxStreamVersion (x : Storage.StreamToken) = let x : Token = unbox x.value in x.pos.Index let supersedes current x = let currentVersion, newVersion = unpackEqxStreamVersion current, unpackEqxStreamVersion x newVersion > currentVersion @@ -396,7 +411,7 @@ type EqxGateway(conn : EqxConnection, batching : EqxBatchingPolicy) = match encodedEvents |> Array.tryFindIndexBack (isEventDataEventType isCompactionEvent) with | None -> Token.ofPreviousTokenAndEventsLength token encodedEvents.Length batching.BatchSize version' | Some compactionEventIndex -> - Token.ofPreviousStreamVersionAndCompactionEventDataIndex pos.StreamVersion compactionEventIndex encodedEvents.Length batching.BatchSize version' + Token.ofPreviousStreamVersionAndCompactionEventDataIndex pos.Index compactionEventIndex encodedEvents.Length batching.BatchSize version' return GatewaySyncResult.Written token } type private Collection(gateway : EqxGateway, databaseId, collectionId) = @@ -632,7 +647,7 @@ type EqxConnector cp /// Yields an IDocumentClient configured and Connect()ed to a given DocDB collection per the requested `discovery` strategy - member __.Connect + let connect ( /// Name should be sufficient to uniquely identify this connection within a single app instance's logs name, discovery : Discovery) : Async = @@ -642,13 +657,13 @@ type EqxConnector match tags with None -> () | Some tags -> for key, value in tags do yield sprintf "%s=%s" key value } let sanitizedName = name.Replace('\'','_').Replace(':','_') // sic; Align with logging for ES Adapter let client = new Client.DocumentClient(uri, key, connPolicy, Nullable(defaultArg defaultConsistencyLevel ConsistencyLevel.Session)) - log.Information("Connecting to Cosmos with clientId={clientId}", sanitizedName) + log.Information("Connecting to Cosmos with Connection Name {connectionName}", sanitizedName) do! client.OpenAsync() |> Async.AwaitTaskCorrect return client :> IDocumentClient } match discovery with Discovery.UriAndKey(databaseUri=uri; key=key) -> connect (uri,key) /// Yields a DocDbConnection configured per the specified strategy - member __.Establish(name, discovery : Discovery) : Async = async { - let! conn = __.Connect(name, discovery) - return EqxConnection(conn, ?readRetryPolicy=readRetryPolicy, ?writeRetryPolicy=writeRetryPolicy) } + member __.Connect(name, discovery : Discovery) : Async = async { + let! conn = connect(name, discovery) + return EqxConnection(conn, ?readRetryPolicy=readRetryPolicy, ?writeRetryPolicy=writeRetryPolicy) } \ No newline at end of file diff --git a/src/Equinox.Cosmos/CosmosManager.fs b/src/Equinox.Cosmos/CosmosManager.fs deleted file mode 100644 index ea11a4a3f..000000000 --- a/src/Equinox.Cosmos/CosmosManager.fs +++ /dev/null @@ -1,64 +0,0 @@ -module Equinox.Cosmos.CosmosManager - -open System -open Equinox.Cosmos -open Equinox.Store.Infrastructure -open Microsoft.Azure.Documents -open Microsoft.Azure.Documents.Client - -let configCosmos (client : IDocumentClient) dbName collName ru auxRu = async { - - let createDatabase (client:IDocumentClient) = async { - let dbRequestOptions = RequestOptions(ConsistencyLevel = Nullable ConsistencyLevel.Session) - let! db = client.CreateDatabaseIfNotExistsAsync(Database(Id=dbName), options = dbRequestOptions) |> Async.AwaitTaskCorrect - return Client.UriFactory.CreateDatabaseUri (db.Resource.Id) } - - let createCollection (client: IDocumentClient) (dbUri: Uri) = async { - let pkd = PartitionKeyDefinition() - pkd.Paths.Add("/k") - let coll = DocumentCollection(Id = collName, PartitionKey = pkd) - - coll.IndexingPolicy.IndexingMode <- IndexingMode.Consistent - coll.IndexingPolicy.Automatic <- true - coll.IndexingPolicy.IncludedPaths.Add(new IncludedPath (Path="/s/?")) - coll.IndexingPolicy.IncludedPaths.Add(new IncludedPath (Path="/k/?")) - coll.IndexingPolicy.IncludedPaths.Add(new IncludedPath (Path="/sn/?")) - coll.IndexingPolicy.ExcludedPaths.Add(new ExcludedPath (Path="/*")) - let! dc = client.CreateDocumentCollectionIfNotExistsAsync(dbUri, coll, RequestOptions(OfferThroughput=Nullable ru)) |> Async.AwaitTaskCorrect - return Client.UriFactory.CreateDocumentCollectionUri (dbName, dc.Resource.Id) } - - let createStoreSproc (client: IDocumentClient) (collectionUri: Uri) = async { - let f ="""function multidocInsert (docs) { - var response = getContext().getResponse(); - var collection = getContext().getCollection(); - var collectionLink = collection.getSelfLink(); - - if (!docs) throw new Error("Array of events is undefined or null."); - - for (i=0; i Async.AwaitTaskCorrect - return Client.UriFactory.CreateStoredProcedureUri(dbName, collName, sp.Resource.Id) } - - let createAux (client: IDocumentClient) (dbUri: Uri) = async { - let auxCollectionName = sprintf "%s-aux" collName - let auxColl = DocumentCollection(Id = auxCollectionName) - auxColl.IndexingPolicy.ExcludedPaths.Add(new ExcludedPath(Path="/ChangefeedPosition/*")) - auxColl.IndexingPolicy.ExcludedPaths.Add(new ExcludedPath(Path="/ProjectionsPositions/*")) - auxColl.IndexingPolicy.IncludedPaths.Add(new IncludedPath (Path="/*")) - auxColl.IndexingPolicy.IndexingMode <- IndexingMode.Lazy - auxColl.DefaultTimeToLive <- Nullable(365 * 60 * 60 * 24) - let! dc = client.CreateDocumentCollectionIfNotExistsAsync(dbUri, auxColl, RequestOptions(OfferThroughput=Nullable auxRu)) |> Async.AwaitTaskCorrect - return Client.UriFactory.CreateDocumentCollectionUri (dbName, dc.Resource.Id) } - let! dbUri = createDatabase client - let! coll = createCollection client dbUri - let! _sp = createStoreSproc client coll - let! _aux = createAux client dbUri - do () -} \ No newline at end of file diff --git a/src/Equinox.Cosmos/Equinox.Cosmos.fsproj b/src/Equinox.Cosmos/Equinox.Cosmos.fsproj index c234fd531..c577971a0 100644 --- a/src/Equinox.Cosmos/Equinox.Cosmos.fsproj +++ b/src/Equinox.Cosmos/Equinox.Cosmos.fsproj @@ -13,7 +13,6 @@ - diff --git a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs index 446b110b7..94e066ba8 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs @@ -12,7 +12,7 @@ open System /// /src/Equinox.Cosmos/EquinoxManager.fsx let connectToCosmos (log: Serilog.ILogger) name discovery = EqxConnector(log=log, requestTimeout=TimeSpan.FromSeconds 3., maxRetryAttemptsOnThrottledRequests=2, maxRetryWaitTimeInSeconds=60) - .Establish(name, discovery) + .Connect(name, discovery) let connectToSpecifiedCosmosOrSimulator (log: Serilog.ILogger) = match Environment.GetEnvironmentVariable "EQUINOX_COSMOS_CONNECTION" |> Option.ofObj with | None -> diff --git a/tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs b/tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs index 326545c8c..27bbad3fc 100644 --- a/tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs +++ b/tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs @@ -25,6 +25,7 @@ let ``VerbatimUtf8JsonConverter serializes properly`` () = let encoded = unionEncoder.Encode(A { embed = "\"" }) let e : Store.Event = { id = null + k = null s = null ts = DateTimeOffset.MinValue i = Nullable 0L From 5b03f9e23dde005d9a275abb0d14923ed1ad2882 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 31 Oct 2018 16:42:27 +0000 Subject: [PATCH 21/66] Final field renames --- src/Equinox.Cosmos/Cosmos.fs | 22 +++++++++---------- .../VerbatimUtf8JsonConverterTests.fs | 8 +++---- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/Equinox.Cosmos/Cosmos.fs b/src/Equinox.Cosmos/Cosmos.fs index 14ddddc45..4518e4ad2 100644 --- a/src/Equinox.Cosmos/Cosmos.fs +++ b/src/Equinox.Cosmos/Cosmos.fs @@ -42,7 +42,7 @@ module Store = // DocDb-mandated Partition Key, must be maintained within the document // Not actually required if running in single partition mode, but for simplicity, we always write it // Some users generate a custom Partition Key to colocate multiple streams to enable colocating and querying across multiple streams - k: string // Complex: "{customPartitionKey}" or (default:) "{streamName}"; Simple: "{streamName}" + p: string // Complex: "{customPartitionKey}" or (default:) "{streamName}"; Simple: "{streamName}" // DocDb-mandated unique key; needs to be unique within a partition // Could use {index} as an int64/number here, if not for the fact that we allow the app to provide a custom `k` to enable streams to colocate @@ -60,10 +60,10 @@ module Store = (* Event payload elements *) /// Creation date (as opposed to sytem-defined _lastUpdated which is rewritten by triggers adnd/or replication) - ts: DateTimeOffset // ISO 8601 + c: DateTimeOffset // ISO 8601 /// The Event Type, used to drive deserialization - et: string // required + t: string // required /// Event body, as UTF-8 encoded json ready to be injected into the Json being rendered for DocDb [)>] @@ -71,14 +71,14 @@ module Store = /// Optional metadata (null, or same as d) [); JsonProperty(Required=Required.DisallowNull)>] - md: byte[] } // optional + m: byte[] } // optional static member Create (pos: Position) offset (ed: EventData) : Event = let id,s,i = pos |> function | Position.Simple (_,_streamName,_version) -> pos.GenerateId offset, null, Nullable () | Position.Complex (_,_,_streamName,_version) -> pos.GenerateId offset, pos.StreamName, Nullable (pos.IndexRel offset) - { k = pos.PartitionKey; id = id; s = s; i = i - ts = DateTimeOffset.UtcNow - et = ed.eventType; d = ed.data; md = ed.metadata } + { p = pos.PartitionKey; id = id; s = s; i = i + c = DateTimeOffset.UtcNow + t = ed.eventType; d = ed.data; m = ed.metadata } member __.Index = if __.i.HasValue then __.i.Value else unbox __.id and VerbatimUtf8JsonConverter() = inherit JsonConverter() @@ -116,7 +116,7 @@ module Log = let propEventData name (events : Store.EventData[]) (log : ILogger) = log |> propEvents name (seq { for x in events -> Collections.Generic.KeyValuePair<_,_>(x.eventType, System.Text.Encoding.UTF8.GetString x.data)}) let propResolvedEvents name (events : Store.Event[]) (log : ILogger) = - log |> propEvents name (seq { for x in events -> Collections.Generic.KeyValuePair<_,_>(x.et, System.Text.Encoding.UTF8.GetString x.d)}) + log |> propEvents name (seq { for x in events -> Collections.Generic.KeyValuePair<_,_>(x.t, System.Text.Encoding.UTF8.GetString x.d)}) open Serilog.Events /// Attach a property to the log context to hold the metrics @@ -204,7 +204,7 @@ module private Read = let feedOptions = new Client.FeedOptions(PartitionKey=PartitionKey pos.PartitionKey, MaxItemCount=Nullable batchSize) client.CreateDocumentQuery(collectionUri, querySpec, feedOptions).AsDocumentQuery() - let (|EventLen|) (x : Store.Event) = match x.d, x.md with Log.BlobLen bytes, Log.BlobLen metaBytes -> bytes + metaBytes + let (|EventLen|) (x : Store.Event) = match x.d, x.m with Log.BlobLen bytes, Log.BlobLen metaBytes -> bytes + metaBytes let private loggedQueryExecution (pos:Store.Position) direction (query: IDocumentQuery) (log: ILogger): Async = async { let! t, (res : Client.FeedResponse) = query.ExecuteNextAsync() |> Async.AwaitTaskCorrect |> Stopwatch.Time @@ -312,7 +312,7 @@ module private Read = module UnionEncoderAdapters = let private encodedEventOfStoredEvent (x : Store.Event) : UnionCodec.EncodedUnion = - { caseName = x.et; payload = x.d } + { caseName = x.t; payload = x.d } let private eventDataOfEncodedEvent (x : UnionCodec.EncodedUnion) : Store.EventData = { eventType = x.caseName; data = x.payload; metadata = null } let encodeEvents (codec : UnionCodec.IUnionEncoder<'event, byte[]>) (xs : 'event seq) : Store.EventData[] = @@ -370,7 +370,7 @@ type EqxBatchingPolicy(getMaxBatchSize : unit -> int, ?batchCountLimit) = type GatewaySyncResult = Written of Storage.StreamToken | Conflict type EqxGateway(conn : EqxConnection, batching : EqxBatchingPolicy) = - let isResolvedEventEventType predicate (x:Store.Event) = predicate x.et + let isResolvedEventEventType predicate (x:Store.Event) = predicate x.t let tryIsResolvedEventEventType predicateOption = predicateOption |> Option.map isResolvedEventEventType let (|Pos|) (token: Storage.StreamToken) : Store.Position = (unbox token.value).pos member __.LoadBatched log isCompactionEventType (pos : Store.Position): Async = async { diff --git a/tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs b/tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs index 27bbad3fc..5f0a64435 100644 --- a/tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs +++ b/tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs @@ -25,12 +25,12 @@ let ``VerbatimUtf8JsonConverter serializes properly`` () = let encoded = unionEncoder.Encode(A { embed = "\"" }) let e : Store.Event = { id = null - k = null + p = null s = null - ts = DateTimeOffset.MinValue i = Nullable 0L - et = encoded.caseName + c = DateTimeOffset.MinValue + t = encoded.caseName d = encoded.payload - md = null } + m = null } let res = serialize e test <@ res.Contains """"d":{"embed":"\""}""" @> \ No newline at end of file From 5b5cd5837e3ffd5c878b3c658d75fd13c7cd67e2 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 1 Nov 2018 03:17:23 +0000 Subject: [PATCH 22/66] Rejig Bench, readme etc - NB broken --- build.ps1 | 17 ++++++++++- samples/Store/Integration/CartIntegration.fs | 4 +-- .../ContactPreferencesIntegration.fs | 2 +- .../Store/Integration/CosmosIntegration.fs | 19 ------------ .../Store/Integration/FavoritesIntegration.fs | 2 +- samples/Store/Integration/Integration.fsproj | 1 - samples/Store/Integration/LogIntegration.fs | 3 +- .../CosmosFixtures.fs | 30 +++++++++++++++++++ ...ure.fs => CosmosFixturesInfrastructure.fs} | 0 .../CosmosIntegration.fs | 22 -------------- .../Equinox.Cosmos.Integration.fsproj | 3 +- 11 files changed, 54 insertions(+), 49 deletions(-) delete mode 100644 samples/Store/Integration/CosmosIntegration.fs create mode 100644 tests/Equinox.Cosmos.Integration/CosmosFixtures.fs rename tests/Equinox.Cosmos.Integration/{Infrastructure.fs => CosmosFixturesInfrastructure.fs} (100%) diff --git a/build.ps1 b/build.ps1 index 9fab92f4d..f6a6d2c78 100644 --- a/build.ps1 +++ b/build.ps1 @@ -3,6 +3,8 @@ param( [Alias("s")][switch][bool] $skipStores=$false, [Alias("se")][switch][bool] $skipEs=$skipStores, [Alias("sc")][switch][bool] $skipCosmos=$skipStores, + [Alias("scp")][switch][bool] $skipProvisionCosmos=$false, + [Alias("scd")][switch][bool] $skipDeprovisionCosmos=$false, [string] $additionalMsBuildArgs="-t:Build" ) @@ -14,8 +16,16 @@ function warn ($msg) { Write-Host "$msg" -BackgroundColor DarkGreen } $env:EQUINOX_INTEGRATION_SKIP_EVENTSTORE=[string]$skipEs if ($skipEs) { warn "Skipping EventStore tests" } +if ($skipCosmos) { + warn "Skipping Cosmos tests" as requested +} elseif ($skipProvisionCosmos -or -not $env:EQUINOX_COSMOS_CONNECTION -or -not $env:EQUINOX_COSMOS_COLLECTION) { + warn "Skipping Provisioning Cosmos" +} else { + warn "Provisioning cosmos..." + $collection=[guid]::NewGuid() + cli/Equinox.Cli/bin/Release/net461/Equinox.Cli cosmos -s $env:EQUINOX_COSMOS_CONNECTION -d test -c $env:EQUINOX_COSMOS_COLLECTION provision -ru 10000 +} $env:EQUINOX_INTEGRATION_SKIP_COSMOS=[string]$skipCosmos -if ($skipCosmos) { warn "Skipping Cosmos tests" } Write-Host "dotnet msbuild $args" . dotnet msbuild build.proj @args @@ -23,4 +33,9 @@ Write-Host "dotnet msbuild $args" if( $LASTEXITCODE -ne 0) { warn "open msbuild.log for error info or rebuild with -v n/d/diag for more detail, or open msbuild.binlog using https://github.com/KirillOsenkov/MSBuildStructuredLog/releases/download/v2.0.40/MSBuildStructuredLogSetup.exe" exit $LASTEXITCODE +} + +if (-not $skipCosmos -and -not $skipDeprovisionCosmos) { + warn "Deprovisioning Cosmos" + cli/Equinox.Cli/bin/Release/net461/Equinox.Cli cosmos -s $env:EQUINOX_COSMOS_CONNECTION -d test -c $env:EQUINOX_COSMOS_COLLECTION provision -ru 0 } \ No newline at end of file diff --git a/samples/Store/Integration/CartIntegration.fs b/samples/Store/Integration/CartIntegration.fs index 25c2e65fa..24f3ed93d 100644 --- a/samples/Store/Integration/CartIntegration.fs +++ b/samples/Store/Integration/CartIntegration.fs @@ -1,7 +1,7 @@ module Samples.Store.Integration.CartIntegration open Equinox.Cosmos -open Equinox.Cosmos.Integration.CosmosIntegration +open Equinox.Cosmos.Integration open Equinox.EventStore open Equinox.MemoryStore open Swensen.Unquote @@ -23,7 +23,7 @@ let resolveGesStreamWithRollingSnapshots gateway = let resolveGesStreamWithoutCustomAccessStrategy gateway = GesResolver(gateway, codec, fold, initial).Resolve -let resolveEqxStreamWithCompactionEventType gateway compactionEventType (StreamArgs args) = +let resolveEqxStreamWithCompactionEventType gateway compactionEventType (StreamArgs args) = EqxStreamBuilder(gateway, codec, fold, initial, Equinox.Cosmos.CompactionStrategy.EventType compactionEventType).Create(args) let resolveEqxStreamWithoutCompactionSemantics gateway _compactionEventType (StreamArgs args) = EqxStreamBuilder(gateway, codec, fold, initial).Create(args) diff --git a/samples/Store/Integration/ContactPreferencesIntegration.fs b/samples/Store/Integration/ContactPreferencesIntegration.fs index 989014836..4f14c3833 100644 --- a/samples/Store/Integration/ContactPreferencesIntegration.fs +++ b/samples/Store/Integration/ContactPreferencesIntegration.fs @@ -1,7 +1,7 @@ module Samples.Store.Integration.ContactPreferencesIntegration open Equinox.Cosmos -open Equinox.Cosmos.Integration.CosmosIntegration +open Equinox.Cosmos.Integration open Equinox.EventStore open Equinox.MemoryStore open Swensen.Unquote diff --git a/samples/Store/Integration/CosmosIntegration.fs b/samples/Store/Integration/CosmosIntegration.fs deleted file mode 100644 index 620a7f97d..000000000 --- a/samples/Store/Integration/CosmosIntegration.fs +++ /dev/null @@ -1,19 +0,0 @@ -[] -module Samples.Store.Integration.CosmosIntegration - -open Equinox.Cosmos -open System - -/// Standing up an Equinox instance is complicated; to run for test purposes either: -/// - replace connection below with a connection string or Uri+Key for an initialized Equinox instance -/// - Create a local Equinox with dbName "test" and collectionName "test" using script: -/// /src/Equinox.Cosmos/EquinoxManager.fsx -let connectToCosmos log = - EqxConnector(log=log, requestTimeout=TimeSpan.FromSeconds 3., maxRetryAttemptsOnThrottledRequests=2, maxRetryWaitTimeInSeconds=60) - .Connect("equinoxStoreSampleIntegration", Discovery.UriAndKey(Uri "https://localhost:8081", "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==")) -let defaultBatchSize = 500 -let createEqxGateway connection batchSize = EqxGateway(connection, EqxBatchingPolicy(maxBatchSize = batchSize)) -// Typically, one will split different categories of stream into Cosmos collections - hard coding this is thus an oversimplification -let (|StreamArgs|) streamName = - let databaseId, collectionId = "test", "test" - databaseId, collectionId, streamName \ No newline at end of file diff --git a/samples/Store/Integration/FavoritesIntegration.fs b/samples/Store/Integration/FavoritesIntegration.fs index da50a7099..bf3e79728 100644 --- a/samples/Store/Integration/FavoritesIntegration.fs +++ b/samples/Store/Integration/FavoritesIntegration.fs @@ -1,7 +1,7 @@ module Samples.Store.Integration.FavoritesIntegration open Equinox.Cosmos -open Equinox.Cosmos.Integration.CosmosIntegration +open Equinox.Cosmos.Integration open Equinox.EventStore open Equinox.MemoryStore open Swensen.Unquote diff --git a/samples/Store/Integration/Integration.fsproj b/samples/Store/Integration/Integration.fsproj index 66e2ef628..c6b78c16c 100644 --- a/samples/Store/Integration/Integration.fsproj +++ b/samples/Store/Integration/Integration.fsproj @@ -10,7 +10,6 @@ - diff --git a/samples/Store/Integration/LogIntegration.fs b/samples/Store/Integration/LogIntegration.fs index 82a2968b9..6f99d0899 100644 --- a/samples/Store/Integration/LogIntegration.fs +++ b/samples/Store/Integration/LogIntegration.fs @@ -2,6 +2,7 @@ open Domain open Equinox.Store +open Equinox.Cosmos.Integration open Swensen.Unquote open System open System.Collections.Concurrent @@ -117,7 +118,7 @@ type Tests() = let buffer = ResizeArray() let batchSize = defaultBatchSize let (log,capture) = createLoggerWithMetricsExtraction buffer.Add - let! conn = connectToCosmos log + let! conn = connectToSpecifiedCosmosOrSimulator log let gateway = createEqxGateway conn batchSize let service = Backend.Cart.Service(log, CartIntegration.resolveEqxStreamWithCompactionEventType gateway) let itemCount, cartId = batchSize / 2 + 1, cartId () diff --git a/tests/Equinox.Cosmos.Integration/CosmosFixtures.fs b/tests/Equinox.Cosmos.Integration/CosmosFixtures.fs new file mode 100644 index 000000000..021b59ddf --- /dev/null +++ b/tests/Equinox.Cosmos.Integration/CosmosFixtures.fs @@ -0,0 +1,30 @@ +[] +module Equinox.Cosmos.Integration.CosmosFixtures + +open Equinox.Cosmos +open System + +/// Standing up an Equinox instance is necessary to run for test purposes; either: +/// - replace connection below with a connection string or Uri+Key for an initialized Equinox instance +/// - Create a local Equinox via cli/Equinox.Cli/bin/Release/net461/Equinox.Cli with args: +/// cosmos -s $env:EQUINOX_COSMOS_CONNECTION -d test -c $env:EQUINOX_COSMOS_COLLECTION provision -ru 10000 +let private connectToCosmos (log: Serilog.ILogger) name discovery = + EqxConnector(log=log, requestTimeout=TimeSpan.FromSeconds 3., maxRetryAttemptsOnThrottledRequests=2, maxRetryWaitTimeInSeconds=60) + .Connect(name, discovery) +let private read env = Environment.GetEnvironmentVariable env |> Option.ofObj + +let connectToSpecifiedCosmosOrSimulator (log: Serilog.ILogger) = + match read "EQUINOX_COSMOS_CONNECTION" with + | None -> + Discovery.UriAndKey(Uri "https://localhost:8081", "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==") + |> connectToCosmos log "localDocDbSim" + | Some connectionString -> + Discovery.FromConnectionString connectionString + |> connectToCosmos log "EQUINOX_COSMOS_CONNECTION" + +let (|StreamArgs|) streamName = + let databaseId, collectionId = defaultArg (read "EQUINOX_COSMOS_DATABASE") "test", defaultArg (read "EQUINOX_COSMOS_COLLECTION") "test" + databaseId, collectionId, streamName + +let defaultBatchSize = 500 +let createEqxGateway connection batchSize = EqxGateway(connection, EqxBatchingPolicy(maxBatchSize = batchSize)) \ No newline at end of file diff --git a/tests/Equinox.Cosmos.Integration/Infrastructure.fs b/tests/Equinox.Cosmos.Integration/CosmosFixturesInfrastructure.fs similarity index 100% rename from tests/Equinox.Cosmos.Integration/Infrastructure.fs rename to tests/Equinox.Cosmos.Integration/CosmosFixturesInfrastructure.fs diff --git a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs index 94e066ba8..d0e34da74 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs @@ -6,28 +6,6 @@ open Swensen.Unquote open System.Threading open System -/// Standing up an Equinox instance is complicated; to run for test purposes either: -/// - replace connection below with a connection string or Uri+Key for an initialized Equinox instance -/// - Create a local Equinox with dbName "test" and collectionName "test" using script: -/// /src/Equinox.Cosmos/EquinoxManager.fsx -let connectToCosmos (log: Serilog.ILogger) name discovery = - EqxConnector(log=log, requestTimeout=TimeSpan.FromSeconds 3., maxRetryAttemptsOnThrottledRequests=2, maxRetryWaitTimeInSeconds=60) - .Connect(name, discovery) -let connectToSpecifiedCosmosOrSimulator (log: Serilog.ILogger) = - match Environment.GetEnvironmentVariable "EQUINOX_COSMOS_CONNECTION" |> Option.ofObj with - | None -> - Discovery.UriAndKey(Uri "https://localhost:8081", "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==") - |> connectToCosmos log "localDocDbSim" - | Some connectionString -> - Discovery.FromConnectionString connectionString - |> connectToCosmos log "specified" - -let defaultBatchSize = 500 -let createEqxGateway connection batchSize = EqxGateway(connection, EqxBatchingPolicy(maxBatchSize = batchSize)) -let (|StreamArgs|) streamName = - let databaseId, collectionId = "test", "test" - databaseId, collectionId, streamName - let serializationSettings = Newtonsoft.Json.Converters.FSharp.Settings.CreateCorrect() let genCodec<'Union when 'Union :> TypeShape.UnionContract.IUnionContract>() = Equinox.UnionCodec.JsonUtf8.Create<'Union>(serializationSettings) diff --git a/tests/Equinox.Cosmos.Integration/Equinox.Cosmos.Integration.fsproj b/tests/Equinox.Cosmos.Integration/Equinox.Cosmos.Integration.fsproj index 04b39af40..62f071fb0 100644 --- a/tests/Equinox.Cosmos.Integration/Equinox.Cosmos.Integration.fsproj +++ b/tests/Equinox.Cosmos.Integration/Equinox.Cosmos.Integration.fsproj @@ -8,7 +8,8 @@ - + + From 11158ec8e56673f31a2aa9bbab21b1d00232f65f Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 1 Nov 2018 10:43:34 +0000 Subject: [PATCH 23/66] More CLI parsing polish --- src/Equinox.Cosmos/Cosmos.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Equinox.Cosmos/Cosmos.fs b/src/Equinox.Cosmos/Cosmos.fs index 4518e4ad2..5f39cf10a 100644 --- a/src/Equinox.Cosmos/Cosmos.fs +++ b/src/Equinox.Cosmos/Cosmos.fs @@ -139,7 +139,7 @@ type EqxSyncResult = Written of Store.Position * requestCharge: float | Conflict module private Write = let append (client: IDocumentClient) (pos: Store.Position) (eventsData: Store.EventData seq): Async = async { - let sprocUri = sprintf "%O/sprocs/AtomicMultiDocInsert" pos.CollectionUri + let sprocUri = sprintf "%O/sprocs/multidocInsert" pos.CollectionUri let opts = Client.RequestOptions(PartitionKey=PartitionKey pos.PartitionKey) let! ct = Async.CancellationToken let events = eventsData |> Seq.mapi (fun i ed -> Store.Event.Create pos (i+1) ed |> JsonConvert.SerializeObject) |> Seq.toArray From 329f24d172946a6f3516805ca889f902363dbf78 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 1 Nov 2018 20:54:34 +0000 Subject: [PATCH 24/66] Fix to adjust for new Cli name --- build.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.ps1 b/build.ps1 index f6a6d2c78..f550b7d6c 100644 --- a/build.ps1 +++ b/build.ps1 @@ -23,7 +23,7 @@ if ($skipCosmos) { } else { warn "Provisioning cosmos..." $collection=[guid]::NewGuid() - cli/Equinox.Cli/bin/Release/net461/Equinox.Cli cosmos -s $env:EQUINOX_COSMOS_CONNECTION -d test -c $env:EQUINOX_COSMOS_COLLECTION provision -ru 10000 + dotnet run cli/Equinox.Cli cosmos -s $env:EQUINOX_COSMOS_CONNECTION -d test -c $env:EQUINOX_COSMOS_COLLECTION provision -ru 10000 } $env:EQUINOX_INTEGRATION_SKIP_COSMOS=[string]$skipCosmos @@ -37,5 +37,5 @@ if( $LASTEXITCODE -ne 0) { if (-not $skipCosmos -and -not $skipDeprovisionCosmos) { warn "Deprovisioning Cosmos" - cli/Equinox.Cli/bin/Release/net461/Equinox.Cli cosmos -s $env:EQUINOX_COSMOS_CONNECTION -d test -c $env:EQUINOX_COSMOS_COLLECTION provision -ru 0 + dotnet run cli/Equinox.Cli cosmos -s $env:EQUINOX_COSMOS_CONNECTION -d test -c $env:EQUINOX_COSMOS_COLLECTION provision -ru 0 } \ No newline at end of file From 16535d4111323d511db20cd50b53694c4fe3c71c Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 1 Nov 2018 21:42:20 +0000 Subject: [PATCH 25/66] Remove Complex partitioning scheme --- src/Equinox.Cosmos/Cosmos.fs | 129 ++++++------------ .../VerbatimUtf8JsonConverterTests.fs | 6 +- 2 files changed, 47 insertions(+), 88 deletions(-) diff --git a/src/Equinox.Cosmos/Cosmos.fs b/src/Equinox.Cosmos/Cosmos.fs index 5f39cf10a..fee18f1a7 100644 --- a/src/Equinox.Cosmos/Cosmos.fs +++ b/src/Equinox.Cosmos/Cosmos.fs @@ -11,51 +11,28 @@ open Serilog open System module Store = - type [] Position = - | Complex of collectionUri: Uri * customPartitionKey: string option * streamName: string * index: int64 option - | Simple of collectionUri: Uri * streamName: string * index: int64 option - member __.CollectionUri : Uri = __ |> function - | Simple (collectionUri,_,_) | Complex (collectionUri,_,_,_) -> collectionUri - member __.PartitionKey : string = __ |> function - | Simple (_,streamName,_) -> streamName - | Complex (_,customPartitionKey,streamName,_) -> defaultArg customPartitionKey streamName - member __.StreamName : string = __ |> function - | Simple (_,streamName,_) | Complex (_,_,streamName,_) -> streamName - member __.Index : int64 = __ |> function - | Simple (_,_,index) | Complex (_,_,_,index) -> defaultArg index -1L - member __.IndexRel (offset: int) : int64 = __ |> function - | Simple (_,_,Some index) | Complex (_,_,_,Some index) -> (index+int64 offset) - | _ -> failwithf "Cannot IndexRel %A" __ - member __.GenerateId offset : obj = __ |> function - | Simple _ -> __.IndexRel offset |> box - | Complex _ -> sprintf "%s-%d" __.StreamName (__.IndexRel offset) |> box - member __.WithIndex (index: int64) : Position = __ |> function - | Simple (collectionUri,streamName,_) -> - Simple (collectionUri,streamName,Some index) - | Complex (collectionUri,partitionKey,streamName,_) -> - Complex(collectionUri,partitionKey,streamName,Some index) + [] + type Position = + { collectionUri: Uri; streamName: string; index: int64 option } + member __.PartitionKey : PartitionKey = __.streamName |> PartitionKey + member __.Index : int64 = defaultArg __.index -1L + member __.IndexRel (offset: int) : int64 = __.index |> function + | Some index -> index+int64 offset + | None -> failwithf "Cannot IndexRel %A" __ type EventData = { eventType: string; data: byte[]; metadata: byte[] } - type [] Event = + + [] + type Event = { (* DocDb-mandated essential elements *) // DocDb-mandated Partition Key, must be maintained within the document // Not actually required if running in single partition mode, but for simplicity, we always write it - // Some users generate a custom Partition Key to colocate multiple streams to enable colocating and querying across multiple streams - p: string // Complex: "{customPartitionKey}" or (default:) "{streamName}"; Simple: "{streamName}" - - // DocDb-mandated unique key; needs to be unique within a partition - // Could use {index} as an int64/number here, if not for the fact that we allow the app to provide a custom `k` to enable streams to colocate - id: obj // Complex: "{streamName}-{index:020d}"; Simple: {index} (int64, written as number) + p: string // {streamName} - (* Indexing/routing properties - separated from `id` to enable indexing/routing used within Equinox.Cosmos *) - - /// Stream name - [] - s: string // Complex: "{streamName}"; Simple: omitted - - /// Index of event within Stream - i: Nullable // Complex: {index}; Simple: omitted + // DocDb-mandated unique key; needs to be unique within any partition it is maintained + // Also defines the ordering of the items + id: int64 // {index} (* Event payload elements *) @@ -73,13 +50,9 @@ module Store = [); JsonProperty(Required=Required.DisallowNull)>] m: byte[] } // optional static member Create (pos: Position) offset (ed: EventData) : Event = - let id,s,i = pos |> function - | Position.Simple (_,_streamName,_version) -> pos.GenerateId offset, null, Nullable () - | Position.Complex (_,_,_streamName,_version) -> pos.GenerateId offset, pos.StreamName, Nullable (pos.IndexRel offset) - { p = pos.PartitionKey; id = id; s = s; i = i + { p = pos.streamName; id = pos.IndexRel offset c = DateTimeOffset.UtcNow t = ed.eventType; d = ed.data; m = ed.metadata } - member __.Index = if __.i.HasValue then __.i.Value else unbox __.id and VerbatimUtf8JsonConverter() = inherit JsonConverter() @@ -139,13 +112,13 @@ type EqxSyncResult = Written of Store.Position * requestCharge: float | Conflict module private Write = let append (client: IDocumentClient) (pos: Store.Position) (eventsData: Store.EventData seq): Async = async { - let sprocUri = sprintf "%O/sprocs/multidocInsert" pos.CollectionUri - let opts = Client.RequestOptions(PartitionKey=PartitionKey pos.PartitionKey) + let sprocUri = sprintf "%O/sprocs/multidocInsert" pos.collectionUri + let opts = Client.RequestOptions(PartitionKey=pos.PartitionKey) let! ct = Async.CancellationToken let events = eventsData |> Seq.mapi (fun i ed -> Store.Event.Create pos (i+1) ed |> JsonConvert.SerializeObject) |> Seq.toArray if events.Length = 0 then invalidArg "eventsData" "must be non-empty" let! res = client.ExecuteStoredProcedureAsync(sprocUri, opts, ct, box events) |> Async.AwaitTaskCorrect - return pos.WithIndex(pos.IndexRel events.Length), res.RequestCharge } + return { pos with index = Some (pos.IndexRel events.Length) }, res.RequestCharge } /// Yields `EqxSyncResult.Written`, or `EqxSyncResult.Conflict` to signify WrongExpectedVersion let private writeEventsAsync (log : ILogger) client pk (events : Store.EventData[]): Async = async { @@ -164,10 +137,10 @@ module private Write = let log = if (not << log.IsEnabled) Events.LogEventLevel.Debug then log else log |> Log.propEventData "Json" events let bytes, count = bytes events, events.Length let log = log |> Log.prop "bytes" bytes - let writeLog = log |> Log.prop "stream" pos.StreamName |> Log.prop "expectedVersion" pos.Index |> Log.prop "count" count + let writeLog = log |> Log.prop "stream" pos.streamName |> Log.prop "expectedVersion" pos.Index |> Log.prop "count" count let! t, result = writeEventsAsync writeLog client pos events |> Stopwatch.Time let conflict, (ru: float), resultLog = - let mkMetric ru : Log.Measurement = { stream = pos.StreamName; interval = t; bytes = bytes; count = count; ru = ru } + let mkMetric ru : Log.Measurement = { stream = pos.streamName; interval = t; bytes = bytes; count = count; ru = ru } match result with | EqxSyncResult.Conflict ru -> true, ru, log |> Log.event (Log.WriteConflict (mkMetric ru)) | EqxSyncResult.Written (x, ru) -> false, ru, log |> Log.event (Log.WriteSuccess (mkMetric ru)) |> Log.prop "nextExpectedVersion" x @@ -179,30 +152,18 @@ module private Write = Log.withLoggedRetries retryPolicy "writeAttempt" call log module private Read = - let mkSingletonQuery query arg value = SqlQuerySpec(query, SqlParameterCollection (Seq.singleton (SqlParameter(arg, value)))) - let mkIdQuery query transform (index:int64) = mkSingletonQuery query "@id" (transform index) let private getQuery (client : IDocumentClient) (pos:Store.Position) (direction: Direction) batchSize = - let collectionUri,querySpec = - match pos with - | Store.Position.Simple (collectionUri, _, None) -> - if direction = Direction.Forward then invalidOp "Cannot read forward from None" - else collectionUri, SqlQuerySpec("SELECT * FROM c ORDER BY c.id DESC") - | Store.Position.Simple (collectionUri, _, Some index) -> - let filter = - if direction = Direction.Forward then "c.id >= @id ORDER BY c.id ASC" - else "c.id < @id ORDER BY c.id DESC" - collectionUri, mkIdQuery("SELECT * FROM c WHERE " + filter) id index - | Store.Position.Complex (collectionUri, _partitionKey, streamName, None) -> + let querySpec = + match pos.index with + | None -> if direction = Direction.Forward then invalidOp "Cannot read forward from None" - else collectionUri, mkIdQuery "SELECT * FROM c WHERE STARTSWITH(c.id, @id) ORDER BY c.id DESC" (fun _index -> streamName) -1L - | Store.Position.Complex (collectionUri, _partitionKey, streamName, Some index) -> - let filter = - if direction = Direction.Forward then "c.id >= @id ORDER BY c.id ASC" - else "c.id <= @id ORDER BY c.id DESC" - let mkComplexIndex streamName index = sprintf "%s-%020d" streamName index - collectionUri, mkIdQuery ("SELECT * FROM c WHERE " + filter) (mkComplexIndex streamName) index + else SqlQuerySpec "SELECT * FROM c ORDER BY c.id DESC" + | Some index -> + SqlQuerySpec( "SELECT * FROM c WHERE " + + (if direction = Direction.Forward then "c.id >= @id ORDER BY c.id ASC" else "c.id < @id ORDER BY c.id DESC"), + SqlParameterCollection (Seq.singleton (SqlParameter("@id", index)))) let feedOptions = new Client.FeedOptions(PartitionKey=PartitionKey pos.PartitionKey, MaxItemCount=Nullable batchSize) - client.CreateDocumentQuery(collectionUri, querySpec, feedOptions).AsDocumentQuery() + client.CreateDocumentQuery(pos.collectionUri, querySpec, feedOptions).AsDocumentQuery() let (|EventLen|) (x : Store.Event) = match x.d, x.m with Log.BlobLen bytes, Log.BlobLen metaBytes -> bytes + metaBytes @@ -210,10 +171,10 @@ module private Read = let! t, (res : Client.FeedResponse) = query.ExecuteNextAsync() |> Async.AwaitTaskCorrect |> Stopwatch.Time let slice, ru = Array.ofSeq res, res.RequestCharge let bytes, count = slice |> Array.sumBy (|EventLen|), slice.Length - let reqMetric : Log.Measurement = { stream = pos.StreamName; interval = t; bytes = bytes; count = count; ru = ru } + let reqMetric : Log.Measurement = { stream = pos.streamName; interval = t; bytes = bytes; count = count; ru = ru } let evt = Log.Slice (direction, reqMetric) let log = if (not << log.IsEnabled) Events.LogEventLevel.Debug then log else log |> Log.propResolvedEvents "Json" slice - let index = match slice |> Array.tryHead with Some head -> Nullable head.Index | None -> Nullable () + let index = match slice |> Array.tryHead with Some head -> Nullable head.id | None -> Nullable () (log |> Log.prop "startIndex" pos.Index |> Log.prop "bytes" bytes |> Log.event evt) .Information("Eqx{action:l} count={count} index={index} rus={ru}", "Read", count, index, ru) return slice, ru } @@ -249,7 +210,7 @@ module private Read = let private lastEventIndex (xs:Store.Event seq) : int64 = match xs |> Seq.tryLast with | None -> -1L - | Some last -> last.Index + | Some last -> last.id let loadForwardsFrom (log : ILogger) retryPolicy client batchSize maxPermittedBatchReads (pos,_strongConsistency): Async = async { let mutable ru = 0.0 @@ -264,16 +225,16 @@ module private Read = let call q = loggedQueryExecution pos Direction.Forward q let retryingLoggingReadSlice q = Log.withLoggedRetries retryPolicy "readAttempt" (call q) let direction = Direction.Forward - let log = log |> Log.prop "batchSize" batchSize |> Log.prop "direction" direction |> Log.prop "stream" pos.StreamName + let log = log |> Log.prop "batchSize" batchSize |> Log.prop "direction" direction |> Log.prop "stream" pos.streamName let batches : AsyncSeq = readBatches log retryingLoggingReadSlice maxPermittedBatchReads query let! t, (events, ru) = mergeBatches batches |> Stopwatch.Time query.Dispose() let version = lastEventIndex events - log |> logBatchRead direction pos.StreamName t events batchSize version ru - return pos.WithIndex version, events } + log |> logBatchRead direction pos.streamName t events batchSize version ru + return { pos with index = Some version }, events } let partitionPayloadFrom firstUsedEventNumber : Store.Event[] -> int * int = - let acc (tu,tr) ((EventLen bytes) as y) = if y.Index < firstUsedEventNumber then tu, tr + bytes else tu + bytes, tr + let acc (tu,tr) ((EventLen bytes) as y) = if y.id < firstUsedEventNumber then tu, tr + bytes else tu + bytes, tr Array.fold acc (0,0) let loadBackwardsUntilCompactionOrStart (log : ILogger) retryPolicy client batchSize maxPermittedBatchReads isCompactionEvent (pos : Store.Position) : Async = async { @@ -289,10 +250,10 @@ module private Read = if not (isCompactionEvent x) then true // continue the search else match !lastBatch with - | None -> log.Information("EqxStop stream={stream} at={eventNumber}", pos.StreamName, x.Index) + | None -> log.Information("EqxStop stream={stream} at={eventNumber}", pos.streamName, x.id) | Some batch -> - let used, residual = batch |> partitionPayloadFrom x.Index - log.Information("EqxStop stream={stream} at={eventNumber} used={used} residual={residual}", pos.StreamName, x.Index, used, residual) + let used, residual = batch |> partitionPayloadFrom x.id + log.Information("EqxStop stream={stream} at={eventNumber} used={used} residual={residual}", pos.streamName, x.id, used, residual) false) |> AsyncSeq.toArrayAsync let eventsForward = Array.Reverse(tempBackward); tempBackward // sic - relatively cheap, in-place reverse of something we own @@ -300,15 +261,15 @@ module private Read = use query = getQuery client pos Direction.Backward batchSize let call q = loggedQueryExecution pos Direction.Backward q let retryingLoggingReadSlice q = Log.withLoggedRetries retryPolicy "readAttempt" (call q) - let log = log |> Log.prop "batchSize" batchSize |> Log.prop "stream" pos.StreamName + let log = log |> Log.prop "batchSize" batchSize |> Log.prop "stream" pos.streamName let direction = Direction.Backward let readlog = log |> Log.prop "direction" direction let batchesBackward : AsyncSeq = readBatches readlog retryingLoggingReadSlice maxPermittedBatchReads query let! t, (events, ru) = mergeFromCompactionPointOrStartFromBackwardsStream log batchesBackward |> Stopwatch.Time query.Dispose() let version = lastEventIndex events - log |> logBatchRead direction pos.StreamName t events batchSize version ru - return pos.WithIndex version, events } + log |> logBatchRead direction pos.streamName t events batchSize version ru + return { pos with index = Some version } , events } module UnionEncoderAdapters = let private encodedEventOfStoredEvent (x : Store.Event) : UnionCodec.EncodedUnion = @@ -346,7 +307,7 @@ module Token = ofCompactionEventNumber compactedEventNumber eventsLength batchSize pos /// Use an event just read from the stream to infer headroom let ofCompactionResolvedEventAndVersion (compactionEvent: Store.Event) batchSize pos : Storage.StreamToken = - ofCompactionEventNumber (Some compactionEvent.Index) 0 batchSize pos + ofCompactionEventNumber (Some compactionEvent.id) 0 batchSize pos /// Use an event we are about to write to the stream to infer headroom let ofPreviousStreamVersionAndCompactionEventDataIndex prevStreamVersion compactionEventDataIndex eventsLength batchSize streamVersion' : Storage.StreamToken = ofCompactionEventNumber (Some (prevStreamVersion + 1L + int64 compactionEventDataIndex)) eventsLength batchSize streamVersion' @@ -419,7 +380,7 @@ type private Collection(gateway : EqxGateway, databaseId, collectionId) = member __.CollectionUri = Client.UriFactory.CreateDocumentCollectionUri(databaseId, collectionId) type private Category<'event, 'state>(coll : Collection, codec : UnionCodec.IUnionEncoder<'event, byte[]>, ?compactionStrategy) = - let (|Pos|) streamName = Store.Position.Simple (coll.CollectionUri, streamName, None) + let (|Pos|) streamName : Store.Position = { collectionUri = coll.CollectionUri; streamName = streamName; index = None } let loadAlgorithm load (Pos pos) initial log = let batched = load initial (coll.Gateway.LoadBatched log None pos) let compacted predicate = load initial (coll.Gateway.LoadBackwardsStoppingAtCompactionEvent log predicate pos) diff --git a/tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs b/tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs index 5f0a64435..9e73f2672 100644 --- a/tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs +++ b/tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs @@ -24,10 +24,8 @@ let ``VerbatimUtf8JsonConverter serializes properly`` () = let unionEncoder = Equinox.UnionCodec.JsonUtf8.Create<_>(JsonSerializerSettings()) let encoded = unionEncoder.Encode(A { embed = "\"" }) let e : Store.Event = - { id = null - p = null - s = null - i = Nullable 0L + { id = 0L + p = "streamName" c = DateTimeOffset.MinValue t = encoded.caseName d = encoded.payload From d85fdd462f372e55c593126415c523a2cd5fa960 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 1 Nov 2018 21:51:03 +0000 Subject: [PATCH 26/66] Remove extranneous operators --- src/Equinox.EventStore/Infrastructure.fs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Equinox.EventStore/Infrastructure.fs b/src/Equinox.EventStore/Infrastructure.fs index 7aefae65a..22ce54431 100644 --- a/src/Equinox.EventStore/Infrastructure.fs +++ b/src/Equinox.EventStore/Infrastructure.fs @@ -71,7 +71,6 @@ type Async with else sc ()) |> ignore) - static member inline bind (f:'a -> Async<'b>) (a:Async<'a>) : Async<'b> = async.Bind(a, f) module AsyncSeq = /// Same as takeWhileAsync, but returns the final element too From 767d34b2d742c75bc5a967f4d7cea323feab1914 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 1 Nov 2018 22:09:28 +0000 Subject: [PATCH 27/66] remove dead file --- tests/Equinox.Cosmos.Integration/App.config | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100644 tests/Equinox.Cosmos.Integration/App.config diff --git a/tests/Equinox.Cosmos.Integration/App.config b/tests/Equinox.Cosmos.Integration/App.config deleted file mode 100644 index 2051014fa..000000000 --- a/tests/Equinox.Cosmos.Integration/App.config +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - - - - True - - - - - True - - - - \ No newline at end of file From 3b1175d711343b61e7223098d5c2f575899e7aa4 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 1 Nov 2018 22:09:56 +0000 Subject: [PATCH 28/66] fix names --- tests/Equinox.Cosmos.Integration/CosmosFixtures.fs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/Equinox.Cosmos.Integration/CosmosFixtures.fs b/tests/Equinox.Cosmos.Integration/CosmosFixtures.fs index 021b59ddf..fd5a85323 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosFixtures.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosFixtures.fs @@ -6,8 +6,7 @@ open System /// Standing up an Equinox instance is necessary to run for test purposes; either: /// - replace connection below with a connection string or Uri+Key for an initialized Equinox instance -/// - Create a local Equinox via cli/Equinox.Cli/bin/Release/net461/Equinox.Cli with args: -/// cosmos -s $env:EQUINOX_COSMOS_CONNECTION -d test -c $env:EQUINOX_COSMOS_COLLECTION provision -ru 10000 +/// - Create a local Equinox via dotnet run cli/Equinox.Cli cosmos -s $env:EQUINOX_COSMOS_CONNECTION -d test -c $env:EQUINOX_COSMOS_COLLECTION provision -ru 10000 let private connectToCosmos (log: Serilog.ILogger) name discovery = EqxConnector(log=log, requestTimeout=TimeSpan.FromSeconds 3., maxRetryAttemptsOnThrottledRequests=2, maxRetryWaitTimeInSeconds=60) .Connect(name, discovery) From accc67ce4c6c0bd585112fe2cc7dcc447ec1f49a Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 2 Nov 2018 07:23:45 +0000 Subject: [PATCH 29/66] Fix all the things --- build.ps1 | 31 +++++---- src/Equinox.Cosmos/Cosmos.fs | 63 +++++++++++-------- .../CosmosFixtures.fs | 4 +- .../VerbatimUtf8JsonConverterTests.fs | 3 +- 4 files changed, 60 insertions(+), 41 deletions(-) diff --git a/build.ps1 b/build.ps1 index f550b7d6c..5e9dbde31 100644 --- a/build.ps1 +++ b/build.ps1 @@ -3,8 +3,12 @@ param( [Alias("s")][switch][bool] $skipStores=$false, [Alias("se")][switch][bool] $skipEs=$skipStores, [Alias("sc")][switch][bool] $skipCosmos=$skipStores, - [Alias("scp")][switch][bool] $skipProvisionCosmos=$false, - [Alias("scd")][switch][bool] $skipDeprovisionCosmos=$false, + [Alias("cs")][string] $cosmosServer=$env:EQUINOX_COSMOS_CONNECTION, + [Alias("cd")][string] $cosmosDatabase=$env:EQUINOX_COSMOS_DATABASE, + [Alias("cc")][string] $cosmosCollection=$env:EQUINOX_COSMOS_COLLECTION, + [Alias("scp")][switch][bool] $skipProvisionCosmos=$skipCosmos -or -not $cosmosServer -or -not $cosmosDatabase -or -not $cosmosCollection, + [Alias("scd")][switch][bool] $skipDeprovisionCosmos=$skipProvisionCosmos, + [string] $additionalMsBuildArgs [string] $additionalMsBuildArgs="-t:Build" ) @@ -16,18 +20,23 @@ function warn ($msg) { Write-Host "$msg" -BackgroundColor DarkGreen } $env:EQUINOX_INTEGRATION_SKIP_EVENTSTORE=[string]$skipEs if ($skipEs) { warn "Skipping EventStore tests" } +function cliCosmos($arghs) { + Write-Host "dotnet run cli/Equinox.Cli cosmos -s $cosmosServer -d $cosmosDatabase -c $cosmosCollection $arghs" + dotnet run cli/Equinox.Cli cosmos -s $cosmosServer -d $cosmosDatabase -c $cosmosCollection @arghs +} + if ($skipCosmos) { - warn "Skipping Cosmos tests" as requested -} elseif ($skipProvisionCosmos -or -not $env:EQUINOX_COSMOS_CONNECTION -or -not $env:EQUINOX_COSMOS_COLLECTION) { - warn "Skipping Provisioning Cosmos" + warn "Skipping Cosmos tests as requested" +} elseif ($skipProvisionCosmos) { + warn "Skipping Provisioning Cosmos" } else { warn "Provisioning cosmos..." - $collection=[guid]::NewGuid() - dotnet run cli/Equinox.Cli cosmos -s $env:EQUINOX_COSMOS_CONNECTION -d test -c $env:EQUINOX_COSMOS_COLLECTION provision -ru 10000 + dotnet run cli/Equinox.Cli cosmos $cosmosServer -d $cosmosDatabase -c $cosmosCollection provision -ru 10000 + $deprovisionCosmos=$true } $env:EQUINOX_INTEGRATION_SKIP_COSMOS=[string]$skipCosmos -Write-Host "dotnet msbuild $args" +warn "RUNNING: dotnet msbuild $args" . dotnet msbuild build.proj @args if( $LASTEXITCODE -ne 0) { @@ -35,7 +44,7 @@ if( $LASTEXITCODE -ne 0) { exit $LASTEXITCODE } -if (-not $skipCosmos -and -not $skipDeprovisionCosmos) { - warn "Deprovisioning Cosmos" - dotnet run cli/Equinox.Cli cosmos -s $env:EQUINOX_COSMOS_CONNECTION -d test -c $env:EQUINOX_COSMOS_COLLECTION provision -ru 0 +if (-not $skipDeprovisionCosmos) { + warn "Deprovisioning Cosmos" + throw "Deprovisioning step not implemented yet - please deallocate your resources using the Azure Portal" } \ No newline at end of file diff --git a/src/Equinox.Cosmos/Cosmos.fs b/src/Equinox.Cosmos/Cosmos.fs index fee18f1a7..a886ce779 100644 --- a/src/Equinox.Cosmos/Cosmos.fs +++ b/src/Equinox.Cosmos/Cosmos.fs @@ -14,7 +14,6 @@ module Store = [] type Position = { collectionUri: Uri; streamName: string; index: int64 option } - member __.PartitionKey : PartitionKey = __.streamName |> PartitionKey member __.Index : int64 = defaultArg __.index -1L member __.IndexRel (offset: int) : int64 = __.index |> function | Some index -> index+int64 offset @@ -28,11 +27,14 @@ module Store = // DocDb-mandated Partition Key, must be maintained within the document // Not actually required if running in single partition mode, but for simplicity, we always write it - p: string // {streamName} + p: string // "{streamName}" - // DocDb-mandated unique key; needs to be unique within any partition it is maintained - // Also defines the ordering of the items - id: int64 // {index} + // DocDb-mandated unique row key; needs to be unique within any partition it is maintained; must be a string + // At the present time, one can't perform an ORDER BY on this field, hence we also have i, which is identical + id: string // "{index}" + + // Same as `id`; necessitated by fact that it's not presently possible to do an ORDER BY on the row key + i: int64 // {index} (* Event payload elements *) @@ -46,13 +48,19 @@ module Store = [)>] d: byte[] // required - /// Optional metadata (null, or same as d) - [); JsonProperty(Required=Required.DisallowNull)>] + /// Optional metadata (null, or same as d, not written if missing) + [); JsonProperty(Required=Required.Default, NullValueHandling=NullValueHandling.Ignore)>] m: byte[] } // optional + /// Unless running in single partion mode (which would restrict us to 10GB per collection) + /// we need to nominate a partition key that will be in every document + static member PartitionKeyField = "p" + /// As one cannot sort by the implicit `id` field, we have an indexed `i` field which we use for sort and range query purporses + static member IndexedFields = [Event.PartitionKeyField; "i"] static member Create (pos: Position) offset (ed: EventData) : Event = - { p = pos.streamName; id = pos.IndexRel offset + { p = pos.streamName; id = string (pos.IndexRel offset); i = pos.IndexRel offset c = DateTimeOffset.UtcNow t = ed.eventType; d = ed.data; m = ed.metadata } + /// Manages injecting prepared json into the data being submitted to DocDb as-is, on the basis we can trust it to be valid json as DocDb will need it to be and VerbatimUtf8JsonConverter() = inherit JsonConverter() @@ -111,9 +119,10 @@ module Log = type EqxSyncResult = Written of Store.Position * requestCharge: float | Conflict of requestCharge: float module private Write = + let [] sprocName = "AtomicMultiDocInsert" let append (client: IDocumentClient) (pos: Store.Position) (eventsData: Store.EventData seq): Async = async { - let sprocUri = sprintf "%O/sprocs/multidocInsert" pos.collectionUri - let opts = Client.RequestOptions(PartitionKey=pos.PartitionKey) + let sprocUri = sprintf "%O/sprocs/%s" pos.collectionUri sprocName + let opts = Client.RequestOptions(PartitionKey=PartitionKey(pos.streamName)) let! ct = Async.CancellationToken let events = eventsData |> Seq.mapi (fun i ed -> Store.Event.Create pos (i+1) ed |> JsonConvert.SerializeObject) |> Seq.toArray if events.Length = 0 then invalidArg "eventsData" "must be non-empty" @@ -155,14 +164,11 @@ module private Read = let private getQuery (client : IDocumentClient) (pos:Store.Position) (direction: Direction) batchSize = let querySpec = match pos.index with - | None -> - if direction = Direction.Forward then invalidOp "Cannot read forward from None" - else SqlQuerySpec "SELECT * FROM c ORDER BY c.id DESC" + | None -> SqlQuerySpec(if direction = Direction.Forward then "SELECT * FROM c ORDER BY c.i ASC" else "SELECT * FROM c ORDER BY c.i DESC") | Some index -> - SqlQuerySpec( "SELECT * FROM c WHERE " + - (if direction = Direction.Forward then "c.id >= @id ORDER BY c.id ASC" else "c.id < @id ORDER BY c.id DESC"), - SqlParameterCollection (Seq.singleton (SqlParameter("@id", index)))) - let feedOptions = new Client.FeedOptions(PartitionKey=PartitionKey pos.PartitionKey, MaxItemCount=Nullable batchSize) + let f = if direction = Direction.Forward then "c.i >= @id ORDER BY c.i ASC" else "c.i < @id ORDER BY c.i DESC" + SqlQuerySpec( "SELECT * FROM c WHERE " + f, SqlParameterCollection (Seq.singleton (SqlParameter("@id", index)))) + let feedOptions = new Client.FeedOptions(PartitionKey=PartitionKey(pos.streamName), MaxItemCount=Nullable batchSize) client.CreateDocumentQuery(pos.collectionUri, querySpec, feedOptions).AsDocumentQuery() let (|EventLen|) (x : Store.Event) = match x.d, x.m with Log.BlobLen bytes, Log.BlobLen metaBytes -> bytes + metaBytes @@ -174,7 +180,7 @@ module private Read = let reqMetric : Log.Measurement = { stream = pos.streamName; interval = t; bytes = bytes; count = count; ru = ru } let evt = Log.Slice (direction, reqMetric) let log = if (not << log.IsEnabled) Events.LogEventLevel.Debug then log else log |> Log.propResolvedEvents "Json" slice - let index = match slice |> Array.tryHead with Some head -> Nullable head.id | None -> Nullable () + let index = match slice |> Array.tryHead with Some head -> head.id | None -> null (log |> Log.prop "startIndex" pos.Index |> Log.prop "bytes" bytes |> Log.event evt) .Information("Eqx{action:l} count={count} index={index} rus={ru}", "Read", count, index, ru) return slice, ru } @@ -210,7 +216,7 @@ module private Read = let private lastEventIndex (xs:Store.Event seq) : int64 = match xs |> Seq.tryLast with | None -> -1L - | Some last -> last.id + | Some last -> int64 last.id let loadForwardsFrom (log : ILogger) retryPolicy client batchSize maxPermittedBatchReads (pos,_strongConsistency): Async = async { let mutable ru = 0.0 @@ -307,7 +313,7 @@ module Token = ofCompactionEventNumber compactedEventNumber eventsLength batchSize pos /// Use an event just read from the stream to infer headroom let ofCompactionResolvedEventAndVersion (compactionEvent: Store.Event) batchSize pos : Storage.StreamToken = - ofCompactionEventNumber (Some compactionEvent.id) 0 batchSize pos + ofCompactionEventNumber (Some (int64 compactionEvent.id)) 0 batchSize pos /// Use an event we are about to write to the stream to infer headroom let ofPreviousStreamVersionAndCompactionEventDataIndex prevStreamVersion compactionEventDataIndex eventsLength batchSize streamVersion' : Storage.StreamToken = ofCompactionEventNumber (Some (prevStreamVersion + 1L + int64 compactionEventDataIndex)) eventsLength batchSize streamVersion' @@ -523,26 +529,31 @@ module Initialization = let createCollection (client: IDocumentClient) (dbUri: Uri) collName ru = async { let pkd = PartitionKeyDefinition() - pkd.Paths.Add("/k") + pkd.Paths.Add(sprintf "/%s" Store.Event.PartitionKeyField) let colld = DocumentCollection(Id = collName, PartitionKey = pkd) - colld.IndexingPolicy.IndexingMode <- IndexingMode.None - colld.IndexingPolicy.Automatic <- false + colld.IndexingPolicy.IndexingMode <- IndexingMode.Consistent + colld.IndexingPolicy.Automatic <- true + // Can either do a blacklist or a whitelist + // Given how long and variable the blacklist would be, we whitelist instead + colld.IndexingPolicy.ExcludedPaths <- System.Collections.ObjectModel.Collection [|ExcludedPath(Path="/*")|] + // NB its critical to index the nominated PartitionKey field defined above or there will be runtime errors + colld.IndexingPolicy.IncludedPaths <- System.Collections.ObjectModel.Collection [| for k in Store.Event.IndexedFields -> IncludedPath(Path=sprintf "/%s/?" k) |] let! coll = client.CreateDocumentCollectionIfNotExistsAsync(dbUri, colld, Client.RequestOptions(OfferThroughput=Nullable ru)) |> Async.AwaitTaskCorrect return coll.Resource.Id } let createProc (client: IDocumentClient) (collectionUri: Uri) = async { - let f ="""function multidocInsert (docs) { + let f ="""function multidocInsert(docs) { var response = getContext().getResponse(); var collection = getContext().getCollection(); var collectionLink = collection.getSelfLink(); if (!docs) throw new Error("docs argument is missing."); - for (i=0; i Async.AwaitTaskCorrect |> Async.Ignore } let initialize (client : IDocumentClient) dbName collName ru = async { diff --git a/tests/Equinox.Cosmos.Integration/CosmosFixtures.fs b/tests/Equinox.Cosmos.Integration/CosmosFixtures.fs index fd5a85323..7a3192c20 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosFixtures.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosFixtures.fs @@ -6,7 +6,7 @@ open System /// Standing up an Equinox instance is necessary to run for test purposes; either: /// - replace connection below with a connection string or Uri+Key for an initialized Equinox instance -/// - Create a local Equinox via dotnet run cli/Equinox.Cli cosmos -s $env:EQUINOX_COSMOS_CONNECTION -d test -c $env:EQUINOX_COSMOS_COLLECTION provision -ru 10000 +/// - Create a local Equinox via dotnet run cli/Equinox.cli -s $env:EQUINOX_COSMOS_CONNECTION -d test -c $env:EQUINOX_COSMOS_COLLECTION provision -ru 10000 let private connectToCosmos (log: Serilog.ILogger) name discovery = EqxConnector(log=log, requestTimeout=TimeSpan.FromSeconds 3., maxRetryAttemptsOnThrottledRequests=2, maxRetryWaitTimeInSeconds=60) .Connect(name, discovery) @@ -22,7 +22,7 @@ let connectToSpecifiedCosmosOrSimulator (log: Serilog.ILogger) = |> connectToCosmos log "EQUINOX_COSMOS_CONNECTION" let (|StreamArgs|) streamName = - let databaseId, collectionId = defaultArg (read "EQUINOX_COSMOS_DATABASE") "test", defaultArg (read "EQUINOX_COSMOS_COLLECTION") "test" + let databaseId, collectionId = defaultArg (read "EQUINOX_COSMOS_DATABASE") "equinox-test", defaultArg (read "EQUINOX_COSMOS_COLLECTION") "equinox-test" databaseId, collectionId, streamName let defaultBatchSize = 500 diff --git a/tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs b/tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs index 9e73f2672..76122ed69 100644 --- a/tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs +++ b/tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs @@ -24,8 +24,7 @@ let ``VerbatimUtf8JsonConverter serializes properly`` () = let unionEncoder = Equinox.UnionCodec.JsonUtf8.Create<_>(JsonSerializerSettings()) let encoded = unionEncoder.Encode(A { embed = "\"" }) let e : Store.Event = - { id = 0L - p = "streamName" + { p = "streamName"; id = string 0; i = 0L c = DateTimeOffset.MinValue t = encoded.caseName d = encoded.payload From 793b7a1f6356aeb896b5d3f0981e8415cd464e5f Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 6 Nov 2018 14:45:27 +0000 Subject: [PATCH 30/66] Add StoredProcedure.js --- src/StoredProcedure.js | 148 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 src/StoredProcedure.js diff --git a/src/StoredProcedure.js b/src/StoredProcedure.js new file mode 100644 index 000000000..ae86473b9 --- /dev/null +++ b/src/StoredProcedure.js @@ -0,0 +1,148 @@ + +function write (partitionkey, events, expectedVersion, pendingEvents, projections) { + + if (events === undefined || events==null) events = []; + if (expectedVersion === undefined) expectedVersion = -2; + if (pendingEvents === undefined) pendingEvents = null; + if (projections === undefined || projections==null) projections = {}; + + var response = getContext().getResponse(); + var collection = getContext().getCollection(); + var collectionLink = collection.getSelfLink(); + + tryQueryAndUpdate(); + + // Recursively queries for a document by id w/ support for continuation tokens. + // Calls tryUpdate(document) as soon as the query returns a document. + function tryQueryAndUpdate(continuation) { + var query = {query: "select * from root r where r.id = @id and r.p = @p", parameters: [{name: "@id", value: "-1"},{name: "@p", value: partitionkey}]}; + var requestOptions = {continuation: continuation}; + + var isAccepted = collection.queryDocuments(collectionLink, query, requestOptions, function (err, documents, responseOptions) { + if (err) throw err; + + if (documents.length > 0) { + // If the document is found, update it. + // There is no need to check for a continuation token since we are querying for a single document. + tryUpdate(documents[0], false); + } else if (responseOptions.continuation) { + // Else if the query came back empty, but with a continuation token; repeat the query w/ the token. + // It is highly unlikely for this to happen when performing a query by id; but is included to serve as an example for larger queries. + tryQueryAndUpdate(responseOptions.continuation); + } else { + // Else the snapshot does not exist; create snapshot + var doc = {p:partitionkey, id:"-1", latest:-1, projections:{"_lowWatermark":{"base":-1}}}; + tryUpdate(doc, true); + } + }); + + // If we hit execution bounds - throw an exception. + // This is highly unlikely given that this is a query by id; but is included to serve as an example for larger queries. + if (!isAccepted) { + throw new Error("The stored procedure timed out."); + } + } + + function insertEvents() + { + for (i=0; i0) { + if (doc.pendingEvents==null) + doc.pendingEvents = {"base": parseInt(events[0].id)-1, "value": events}; + else + Array.prototype.push.apply(doc.pendingEvents.value, events); + } + } + + // The kernel function + function tryUpdate(doc, isCreate) { + + // DocumentDB supports optimistic concurrency control via HTTP ETag. + var requestOptions = {etag: doc._etag}; + + // Step 1: Insert events to DB + if (expectedVersion==-2) { + // thor mode + var i; + for (i=0; i parseInt(value.id)>newBase); + doc.pendingEvents.base = newBase; + if (doc.pendingEvents.value.length==0) + delete doc["pendingEvents"]; + } + } + + // Step 6: Replace existing snapshot document or create the first snapshot document for this partition key + if (!isCreate) + { + var isAccepted = collection.replaceDocument(doc._self, doc, requestOptions, function (err, updatedDocument, responseOptions) { + if (err) throw err; + }); + + // If we hit execution bounds - throw an exception. + if (!isAccepted) { + throw new Error("The stored procedure timed out."); + } + } + else { + try { + collection.createDocument(collectionLink, doc); + } catch (err) { + throw new Error ("Create doc " + JSON.stringify(docs) + " failed"); + } + } + } + + response.setBody(true); + } \ No newline at end of file From b6076ae59b3037b201de2c5600e4a1374cf57c64 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 7 Nov 2018 16:30:01 +0000 Subject: [PATCH 31/66] Push compaction down into Stores, resolves #23 --- build.ps1 | 7 +- samples/Store/Integration/CartIntegration.fs | 6 +- .../ContactPreferencesIntegration.fs | 12 +-- .../Store/Integration/FavoritesIntegration.fs | 2 +- src/Equinox.Cosmos/Cosmos.fs | 93 ++++++++++++++----- .../CosmosIntegration.fs | 18 ++-- 6 files changed, 90 insertions(+), 48 deletions(-) diff --git a/build.ps1 b/build.ps1 index 5e9dbde31..3b27d0148 100644 --- a/build.ps1 +++ b/build.ps1 @@ -8,7 +8,6 @@ param( [Alias("cc")][string] $cosmosCollection=$env:EQUINOX_COSMOS_COLLECTION, [Alias("scp")][switch][bool] $skipProvisionCosmos=$skipCosmos -or -not $cosmosServer -or -not $cosmosDatabase -or -not $cosmosCollection, [Alias("scd")][switch][bool] $skipDeprovisionCosmos=$skipProvisionCosmos, - [string] $additionalMsBuildArgs [string] $additionalMsBuildArgs="-t:Build" ) @@ -21,8 +20,8 @@ $env:EQUINOX_INTEGRATION_SKIP_EVENTSTORE=[string]$skipEs if ($skipEs) { warn "Skipping EventStore tests" } function cliCosmos($arghs) { - Write-Host "dotnet run cli/Equinox.Cli cosmos -s $cosmosServer -d $cosmosDatabase -c $cosmosCollection $arghs" - dotnet run cli/Equinox.Cli cosmos -s $cosmosServer -d $cosmosDatabase -c $cosmosCollection @arghs + Write-Host "dotnet run cli/Equinox.Cli cosmos -s -d $cosmosDatabase -c $cosmosCollection $arghs" + dotnet run -p cli/Equinox.Cli -f netcoreapp2.1 cosmos -s $cosmosServer -d $cosmosDatabase -c $cosmosCollection @arghs } if ($skipCosmos) { @@ -31,7 +30,7 @@ if ($skipCosmos) { warn "Skipping Provisioning Cosmos" } else { warn "Provisioning cosmos..." - dotnet run cli/Equinox.Cli cosmos $cosmosServer -d $cosmosDatabase -c $cosmosCollection provision -ru 10000 + cliCosmos @("provision", "-ru", "1000") $deprovisionCosmos=$true } $env:EQUINOX_INTEGRATION_SKIP_COSMOS=[string]$skipCosmos diff --git a/samples/Store/Integration/CartIntegration.fs b/samples/Store/Integration/CartIntegration.fs index 24f3ed93d..7bc68178a 100644 --- a/samples/Store/Integration/CartIntegration.fs +++ b/samples/Store/Integration/CartIntegration.fs @@ -23,9 +23,9 @@ let resolveGesStreamWithRollingSnapshots gateway = let resolveGesStreamWithoutCustomAccessStrategy gateway = GesResolver(gateway, codec, fold, initial).Resolve -let resolveEqxStreamWithCompactionEventType gateway compactionEventType (StreamArgs args) = - EqxStreamBuilder(gateway, codec, fold, initial, Equinox.Cosmos.CompactionStrategy.EventType compactionEventType).Create(args) -let resolveEqxStreamWithoutCompactionSemantics gateway _compactionEventType (StreamArgs args) = +let resolveEqxStreamWithCompactionEventType gateway (StreamArgs args) = + EqxStreamBuilder(gateway, codec, fold, initial, Equinox.Cosmos.AccessStrategy.RollingSnapshots compact).Create(args) +let resolveEqxStreamWithoutCompactionSemantics gateway (StreamArgs args) = EqxStreamBuilder(gateway, codec, fold, initial).Create(args) let addAndThenRemoveItemsManyTimesExceptTheLastOne context cartId skuId (service: Backend.Cart.Service) count = diff --git a/samples/Store/Integration/ContactPreferencesIntegration.fs b/samples/Store/Integration/ContactPreferencesIntegration.fs index 4f14c3833..e2714eae4 100644 --- a/samples/Store/Integration/ContactPreferencesIntegration.fs +++ b/samples/Store/Integration/ContactPreferencesIntegration.fs @@ -21,12 +21,10 @@ let resolveStreamGesWithOptimizedStorageSemantics gateway = let resolveStreamGesWithoutAccessStrategy gateway = GesResolver(gateway defaultBatchSize, codec, fold, initial).Resolve -let resolveStreamEqxWithCompactionSemantics gateway = - fun predicate (StreamArgs args) -> - EqxStreamBuilder(gateway, codec, fold, initial, Equinox.Cosmos.CompactionStrategy.Predicate predicate).Create(args) -let resolveStreamEqxWithoutCompactionSemantics gateway = - fun _ignoreWindowSize _ignoreCompactionPredicate (StreamArgs args) -> - EqxStreamBuilder(gateway, codec, fold, initial).Create(args) +let resolveStreamEqxWithCompactionSemantics gateway (StreamArgs args) = + EqxStreamBuilder(gateway 1, codec, fold, initial, Equinox.Cosmos.AccessStrategy.EventsAreState).Create(args) +let resolveStreamEqxWithoutCompactionSemantics gateway (StreamArgs args) = + EqxStreamBuilder(gateway defaultBatchSize, codec, fold, initial).Create(args) type Tests(testOutputHelper) = let testOutput = TestOutputAdapter testOutputHelper @@ -65,7 +63,7 @@ type Tests(testOutputHelper) = [] let ``Can roundtrip against Cosmos, correctly folding the events with normal semantics`` args = Async.RunSynchronously <| async { - let! service = arrangeWithoutCompaction connectToSpecifiedCosmosOrSimulator createEqxGateway resolveStreamEqxWithoutCompactionSemantics + let! service = arrange connectToSpecifiedCosmosOrSimulator createEqxGateway resolveStreamEqxWithoutCompactionSemantics do! act service args } diff --git a/samples/Store/Integration/FavoritesIntegration.fs b/samples/Store/Integration/FavoritesIntegration.fs index bf3e79728..b7c0fce7e 100644 --- a/samples/Store/Integration/FavoritesIntegration.fs +++ b/samples/Store/Integration/FavoritesIntegration.fs @@ -22,7 +22,7 @@ let createServiceGes gateway log = Backend.Favorites.Service(log, resolveStream) let createServiceEqx gateway log = - let resolveStream cet (StreamArgs args) = EqxStreamBuilder(gateway, codec, fold, initial, Equinox.Cosmos.CompactionStrategy.EventType cet).Create(args) + let resolveStream (StreamArgs args) = EqxStreamBuilder(gateway, codec, fold, initial, Equinox.Cosmos.AccessStrategy.RollingSnapshots compact).Create(args) Backend.Favorites.Service(log, resolveStream) type Tests(testOutputHelper) = diff --git a/src/Equinox.Cosmos/Cosmos.fs b/src/Equinox.Cosmos/Cosmos.fs index a886ce779..49763b3d2 100644 --- a/src/Equinox.Cosmos/Cosmos.fs +++ b/src/Equinox.Cosmos/Cosmos.fs @@ -77,6 +77,41 @@ module Store = if array = null || Array.length array = 0 then serializer.Serialize(writer, null) else writer.WriteRawValue(System.Text.Encoding.UTF8.GetString(array)) + [] + type IndexEvent = + { p: string // "{streamName}" + id: string // "{-1}" + + w: int64 // 100: window size + /// last index/i value + m: int64 // {index} + + (* "x": [ + { "i":0, + "c":"ISO 8601" + "e":[ + [{"t":"added","d":"..."},{"t":"compacted/1","d":"..."}], + [{"t":"removed","d":"..."}], + ] + } + ] *) + x: JObject[][] } + + (* Pseudocode: + function sync(p, expectedVersion, windowSize, events) { + if (i == 0) then { + coll.insert(p,0,{ p:p, id:-1, w:windowSize, m:flatLen(events)}) + } else { + const i = doc.find(p=p && id=-1) + if(i.m <> expectedVersion) then emit from expectedVersion else + i.x.append(events) + for (var (i, c, e: [ {e1}, ...]) in events) { + coll.insert({p:p, id:i, i:i, c:c, e:e1) + } + // trim i.x to w total items in i.[e] + coll.update(p,id,i) + } + } *) [] type Direction = Forward | Backward with override this.ToString() = match this with Forward -> "Forward" | Backward -> "Backward" @@ -289,7 +324,6 @@ module UnionEncoderAdapters = type []Token = { pos: Store.Position; compactionEventNumber: int64 option } -[] module Token = let private create compactionEventNumber batchCapacityLimit pos : Storage.StreamToken = { value = box { pos = pos; compactionEventNumber = compactionEventNumber }; batchCapacityLimit = batchCapacityLimit } @@ -385,26 +419,49 @@ type private Collection(gateway : EqxGateway, databaseId, collectionId) = member __.Gateway = gateway member __.CollectionUri = Client.UriFactory.CreateDocumentCollectionUri(databaseId, collectionId) -type private Category<'event, 'state>(coll : Collection, codec : UnionCodec.IUnionEncoder<'event, byte[]>, ?compactionStrategy) = +[] +type AccessStrategy<'event,'state> = + | EventsAreState + | RollingSnapshots of eventType: string * compact: ('state -> 'event) + +type private CompactionContext(eventsLen : int, capacityBeforeCompaction : int) = + /// Determines whether writing a Compaction event is warranted (based on the existing state and the current `Accumulated` changes) + member __.IsCompactionDue = eventsLen > capacityBeforeCompaction + +type private Category<'event, 'state>(coll : Collection, codec : UnionCodec.IUnionEncoder<'event, byte[]>, ?access : AccessStrategy<'event,'state>) = let (|Pos|) streamName : Store.Position = { collectionUri = coll.CollectionUri; streamName = streamName; index = None } + let compactionPredicate = + match access with + | None -> None + | Some AccessStrategy.EventsAreState -> Some (fun _ -> true) + | Some (AccessStrategy.RollingSnapshots (et,_)) -> Some ((=) et) let loadAlgorithm load (Pos pos) initial log = let batched = load initial (coll.Gateway.LoadBatched log None pos) let compacted predicate = load initial (coll.Gateway.LoadBackwardsStoppingAtCompactionEvent log predicate pos) - match compactionStrategy with - | Some predicate -> compacted predicate + match access with | None -> batched + | Some AccessStrategy.EventsAreState -> compacted (fun _ -> true) + | Some (AccessStrategy.RollingSnapshots (et,_)) -> compacted ((=) et) let load (fold: 'state -> 'event seq -> 'state) initial loadF = async { let! token, events = loadF return token, fold initial (UnionEncoderAdapters.decodeKnownEvents codec events) } member __.Load (fold: 'state -> 'event seq -> 'state) (initial: 'state) streamName (log : ILogger) : Async = loadAlgorithm (load fold) streamName initial log member __.LoadFromToken (fold: 'state -> 'event seq -> 'state) (state: 'state) token (log : ILogger) : Async = - (load fold) state (coll.Gateway.LoadFromToken log token compactionStrategy false) - member __.TrySync (fold: 'state -> 'event seq -> 'state) (log : ILogger) (token, state) (events : 'event list) : Async> = async { + (load fold) state (coll.Gateway.LoadFromToken log token compactionPredicate false) + member __.TrySync (fold: 'state -> 'event seq -> 'state) (log : ILogger) + (token : Storage.StreamToken, state : 'state) + (events : 'event list, state' : 'state) : Async> = async { + let events = + match access with + | None | Some AccessStrategy.EventsAreState -> events + | Some (AccessStrategy.RollingSnapshots (_,f)) -> + let cc = CompactionContext(List.length events, token.batchCapacityLimit.Value) + if cc.IsCompactionDue then events @ [f state'] else events let encodedEvents : Store.EventData[] = UnionEncoderAdapters.encodeEvents codec (Seq.ofList events) - let! syncRes = coll.Gateway.TrySync log token encodedEvents compactionStrategy + let! syncRes = coll.Gateway.TrySync log token encodedEvents compactionPredicate match syncRes with - | GatewaySyncResult.Conflict -> return Storage.SyncResult.Conflict (load fold state (coll.Gateway.LoadFromToken log token compactionStrategy true)) + | GatewaySyncResult.Conflict -> return Storage.SyncResult.Conflict (load fold state (coll.Gateway.LoadFromToken log token compactionPredicate true)) | GatewaySyncResult.Written token' -> return Storage.SyncResult.Written (token', fold state (Seq.ofList events)) } module Caching = @@ -449,8 +506,8 @@ module Caching = interface ICategory<'event, 'state> with member __.Load (streamName : string) (log : ILogger) : Async = interceptAsync (inner.Load streamName log) streamName - member __.TrySync streamName (log : ILogger) (token, state) (events : 'event list) : Async> = async { - let! syncRes = inner.TrySync streamName log (token, state) events + member __.TrySync streamName (log : ILogger) (token, state) (events : 'event list, state' : 'state) : Async> = async { + let! syncRes = inner.TrySync streamName log (token, state) (events,state') match syncRes with | Storage.SyncResult.Conflict resync -> return Storage.SyncResult.Conflict (interceptAsync resync streamName) | Storage.SyncResult.Written (token', state') -> return Storage.SyncResult.Written (token', state') } @@ -478,17 +535,12 @@ type private Folder<'event, 'state>(category : Category<'event, 'state>, fold: ' interface ICategory<'event, 'state> with member __.Load (streamName : string) (log : ILogger) : Async = loadAlgorithm streamName initial log - member __.TrySync _streamName(* TODO remove from main interface *) (log : ILogger) (token, state) (events : 'event list) : Async> = async { - let! syncRes = category.TrySync fold log (token, state) events + member __.TrySync _streamName(* TODO remove from main interface *) (log : ILogger) (token, state) (events : 'event list, state': 'state) : Async> = async { + let! syncRes = category.TrySync fold log (token, state) (events,state') match syncRes with | Storage.SyncResult.Conflict resync -> return Storage.SyncResult.Conflict resync | Storage.SyncResult.Written (token',state') -> return Storage.SyncResult.Written (token',state') } -[] -type CompactionStrategy = - | EventType of string - | Predicate of (string -> bool) - [] type CachingStrategy = | SlidingWindow of Caching.Cache * window: TimeSpan @@ -497,12 +549,7 @@ type CachingStrategy = type EqxStreamBuilder<'event, 'state>(gateway : EqxGateway, codec, fold, initial, ?compaction, ?caching) = member __.Create (databaseId, collectionId, streamName) : Equinox.IStream<'event, 'state> = - let compactionPredicateOption = - match compaction with - | None -> None - | Some (CompactionStrategy.Predicate predicate) -> Some predicate - | Some (CompactionStrategy.EventType eventType) -> Some (fun x -> x = eventType) - let category = Category<'event, 'state>(Collection(gateway, databaseId, collectionId), codec, ?compactionStrategy = compactionPredicateOption) + let category = Category<'event, 'state>(Collection(gateway, databaseId, collectionId), codec, ?access = compaction) let readCacheOption = match caching with diff --git a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs index d0e34da74..e0ed98f5a 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs @@ -11,27 +11,27 @@ let genCodec<'Union when 'Union :> TypeShape.UnionContract.IUnionContract>() = Equinox.UnionCodec.JsonUtf8.Create<'Union>(serializationSettings) module Cart = - let fold, initial = Domain.Cart.Folds.fold, Domain.Cart.Folds.initial + let fold, initial, compact = Domain.Cart.Folds.fold, Domain.Cart.Folds.initial, Domain.Cart.Folds.compact let codec = genCodec() let createServiceWithoutOptimization connection batchSize log = let gateway = createEqxGateway connection batchSize - let resolveStream _ignoreCompactionEventTypeOption (StreamArgs args) = + let resolveStream (StreamArgs args) = EqxStreamBuilder(gateway, codec, fold, initial).Create(args) Backend.Cart.Service(log, resolveStream) let createServiceWithCompaction connection batchSize log = let gateway = createEqxGateway connection batchSize - let resolveStream compactionEventType (StreamArgs args) = - EqxStreamBuilder(gateway, codec, fold, initial, compaction=CompactionStrategy.EventType compactionEventType).Create(args) + let resolveStream (StreamArgs args) = + EqxStreamBuilder(gateway, codec, fold, initial, AccessStrategy.RollingSnapshots compact).Create(args) Backend.Cart.Service(log, resolveStream) let createServiceWithCaching connection batchSize log cache = let gateway = createEqxGateway connection batchSize let sliding20m = CachingStrategy.SlidingWindow (cache, TimeSpan.FromMinutes 20.) - let resolveStream _ignorecompactionEventType (StreamArgs args) = EqxStreamBuilder(gateway, codec, fold, initial, caching = sliding20m).Create(args) + let resolveStream (StreamArgs args) = EqxStreamBuilder(gateway, codec, fold, initial, caching = sliding20m).Create(args) Backend.Cart.Service(log, resolveStream) let createServiceWithCompactionAndCaching connection batchSize log cache = let gateway = createEqxGateway connection batchSize let sliding20m = CachingStrategy.SlidingWindow (cache, TimeSpan.FromMinutes 20.) - let resolveStream cet (StreamArgs args) = EqxStreamBuilder(gateway, codec, fold, initial, CompactionStrategy.EventType cet, sliding20m).Create(args) + let resolveStream (StreamArgs args) = EqxStreamBuilder(gateway, codec, fold, initial, AccessStrategy.RollingSnapshots compact, sliding20m).Create(args) Backend.Cart.Service(log, resolveStream) module ContactPreferences = @@ -39,12 +39,10 @@ module ContactPreferences = let codec = genCodec() let createServiceWithoutOptimization createGateway defaultBatchSize log _ignoreWindowSize _ignoreCompactionPredicate = let gateway = createGateway defaultBatchSize - let resolveStream _windowSize _compactionPredicate (StreamArgs args) = - EqxStreamBuilder(gateway, codec, fold, initial).Create(args) + let resolveStream (StreamArgs args) = EqxStreamBuilder(gateway, codec, fold, initial).Create(args) Backend.ContactPreferences.Service(log, resolveStream) let createService createGateway log = - let resolveStream batchSize compactionPredicate (StreamArgs args) = - EqxStreamBuilder(createGateway batchSize, codec, fold, initial, CompactionStrategy.Predicate compactionPredicate).Create(args) + let resolveStream (StreamArgs args) = EqxStreamBuilder(createGateway 1, codec, fold, initial, AccessStrategy.EventsAreState).Create(args) Backend.ContactPreferences.Service(log, resolveStream) #nowarn "1182" // From hereon in, we may have some 'unused' privates (the tests) From ff0910003e38921cd639d43679409c3b8cb3977b Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 7 Nov 2018 21:39:10 +0000 Subject: [PATCH 32/66] Fix Memory test --- tools/Equinox.Tool/Program.fs | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/Equinox.Tool/Program.fs b/tools/Equinox.Tool/Program.fs index f61e8911c..87b3d3eff 100644 --- a/tools/Equinox.Tool/Program.fs +++ b/tools/Equinox.Tool/Program.fs @@ -11,7 +11,6 @@ open System open System.Net.Http open System.Threading - [] type Arguments = | [] Verbose From a0126c204f297fde357b8f337aa6704ed66dd64f Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 7 Nov 2018 23:57:34 +0000 Subject: [PATCH 33/66] Add Cosmos Ru counts to CLI --- src/Equinox.Cosmos/Cosmos.fs | 14 +++++++------- tools/Equinox.Tool/Program.fs | 3 ++- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/Equinox.Cosmos/Cosmos.fs b/src/Equinox.Cosmos/Cosmos.fs index 49763b3d2..c18efa105 100644 --- a/src/Equinox.Cosmos/Cosmos.fs +++ b/src/Equinox.Cosmos/Cosmos.fs @@ -183,12 +183,12 @@ module private Write = let log = log |> Log.prop "bytes" bytes let writeLog = log |> Log.prop "stream" pos.streamName |> Log.prop "expectedVersion" pos.Index |> Log.prop "count" count let! t, result = writeEventsAsync writeLog client pos events |> Stopwatch.Time - let conflict, (ru: float), resultLog = + let (ru: float), resultLog = let mkMetric ru : Log.Measurement = { stream = pos.streamName; interval = t; bytes = bytes; count = count; ru = ru } match result with - | EqxSyncResult.Conflict ru -> true, ru, log |> Log.event (Log.WriteConflict (mkMetric ru)) - | EqxSyncResult.Written (x, ru) -> false, ru, log |> Log.event (Log.WriteSuccess (mkMetric ru)) |> Log.prop "nextExpectedVersion" x - resultLog.Information("Eqx{action:l} count={count} conflict={conflict}, rus={ru}", "Write", events.Length, conflict, ru) + | EqxSyncResult.Conflict ru -> ru, log |> Log.event (Log.WriteConflict (mkMetric ru)) |> Log.prop "conflict" true + | EqxSyncResult.Written (x, ru) -> ru, log |> Log.event (Log.WriteSuccess (mkMetric ru)) |> Log.prop "nextExpectedVersion" x + resultLog.Information("Eqx {action:l} {count} {ms}ms rc={ru}", "Write", events.Length, (let e = t.Elapsed in e.TotalMilliseconds), ru) return result } let writeEvents (log : ILogger) retryPolicy client pk (events : Store.EventData[]): Async = @@ -217,7 +217,7 @@ module private Read = let log = if (not << log.IsEnabled) Events.LogEventLevel.Debug then log else log |> Log.propResolvedEvents "Json" slice let index = match slice |> Array.tryHead with Some head -> head.id | None -> null (log |> Log.prop "startIndex" pos.Index |> Log.prop "bytes" bytes |> Log.event evt) - .Information("Eqx{action:l} count={count} index={index} rus={ru}", "Read", count, index, ru) + .Information("Eqx {action:l} {count} {ms}ms i={index} rc={ru}", "Read", count, (let e = t.Elapsed in e.TotalMilliseconds), index, ru) return slice, ru } let private readBatches (log : ILogger) (readSlice: IDocumentQuery -> ILogger -> Async) @@ -245,8 +245,8 @@ module private Read = let action = match direction with Direction.Forward -> "LoadF" | Direction.Backward -> "LoadB" let evt = Log.Event.Batch (direction, batches, reqMetric) (log |> Log.prop "bytes" bytes |> Log.event evt).Information( - "Eqx{action:l} stream={stream} count={count}/{batches} index={index} rus={ru}", - action, streamName, count, batches, version, ru) + "Eqx {action:l} stream={stream} {count}/{batches} {ms}ms i={index} rc={ru}", + action, streamName, count, batches, (let e = interval.Elapsed in e.TotalMilliseconds), version, ru) let private lastEventIndex (xs:Store.Event seq) : int64 = match xs |> Seq.tryLast with diff --git a/tools/Equinox.Tool/Program.fs b/tools/Equinox.Tool/Program.fs index 87b3d3eff..9aea3b18e 100644 --- a/tools/Equinox.Tool/Program.fs +++ b/tools/Equinox.Tool/Program.fs @@ -140,7 +140,8 @@ module LoadTest = let createDomainLog verbose verboseConsole maybeSeqEndpoint = let c = LoggerConfiguration().Destructure.FSharpTypes().Enrich.FromLogContext() let c = if verbose then c.MinimumLevel.Debug() else c - let c = c.WriteTo.Console((if verboseConsole then LogEventLevel.Debug else LogEventLevel.Information), theme = Sinks.SystemConsole.Themes.AnsiConsoleTheme.Code) + let c = c.WriteTo.Sink(RuCounterSink()) + let c = c.WriteTo.Console((if verboseConsole then LogEventLevel.Debug else LogEventLevel.Warning), theme = Sinks.SystemConsole.Themes.AnsiConsoleTheme.Code) let c = match maybeSeqEndpoint with None -> c | Some endpoint -> c.WriteTo.Seq(endpoint) c.CreateLogger() From 3534989b6126dc1b9ebe66b742dd7925de46d239 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 8 Nov 2018 17:29:48 +0000 Subject: [PATCH 34/66] WIP --- samples/Store/Integration/LogIntegration.fs | 1 + src/Equinox.Cosmos/Cosmos.fs | 228 +++++++++++++++--- .../CosmosFixturesInfrastructure.fs | 3 +- .../CosmosIntegration.fs | 39 ++- 4 files changed, 236 insertions(+), 35 deletions(-) diff --git a/samples/Store/Integration/LogIntegration.fs b/samples/Store/Integration/LogIntegration.fs index 6f99d0899..5310f5341 100644 --- a/samples/Store/Integration/LogIntegration.fs +++ b/samples/Store/Integration/LogIntegration.fs @@ -36,6 +36,7 @@ module EquinoxCosmosInterop = | Log.Slice (Direction.Backward,m) -> "EqxReadStreamEventsBackwardAsync", m, None, m.ru | Log.Batch (Direction.Forward,c,m) -> "EqxLoadF", m, Some c, m.ru | Log.Batch (Direction.Backward,c,m) -> "EqxLoadB", m, Some c, m.ru + | Log.Index m -> "EqxLoadI", m, None, m.ru { action = action; stream = metric.stream; bytes = metric.bytes; count = metric.count; batches = batches interval = StopwatchInterval(metric.interval.StartTicks,metric.interval.EndTicks); ru = ru } diff --git a/src/Equinox.Cosmos/Cosmos.fs b/src/Equinox.Cosmos/Cosmos.fs index c18efa105..f22bad9f7 100644 --- a/src/Equinox.Cosmos/Cosmos.fs +++ b/src/Equinox.Cosmos/Cosmos.fs @@ -10,6 +10,48 @@ open Newtonsoft.Json.Linq open Serilog open System + +[] +module DocDbExtensions = + type Client.RequestOptions with + /// Simplified ETag precondition builder + member options.ETag + with get () = + match options.AccessCondition with + | null -> null + | ac -> ac.Condition + + and set etag = + if String.IsNullOrEmpty etag then () else + options.AccessCondition <- Client.AccessCondition(Type = Client.AccessConditionType.IfMatch, Condition = etag) + + /// Extracts the innermost exception from a nested hierarchy of Aggregate Exceptions + let (|AggregateException|) (exn : exn) = + let rec aux (e : exn) = + match e with + | :? AggregateException as agg when agg.InnerExceptions.Count = 1 -> + aux agg.InnerExceptions.[0] + | _ -> e + + aux exn + + /// DocumentDB Error HttpStatusCode extractor + let (|DocDbStatusCode|_|) (e : exn) = + match e with + | AggregateException (:? DocumentClientException as dce) -> match dce.StatusCode with v when v.HasValue -> Some v.Value | _ -> None + | _ -> None + + type DocDbCollection(client : IDocumentClient, collectionUri) = + member __.TryReadDocument(documentId : string, ?options : Client.RequestOptions) = async { + let! ct = Async.CancellationToken + let options = defaultArg options null + let docLink = sprintf "%O/docs/%s" collectionUri documentId + try let! document = async { return! client.ReadDocumentAsync<'T>(docLink, options = options, cancellationToken = ct) |> Async.AwaitTaskCorrect } + return Some document + with DocDbStatusCode System.Net.HttpStatusCode.NotFound -> + // TODO // dont drop RUs + return None } + module Store = [] type Position = @@ -20,6 +62,13 @@ module Store = | None -> failwithf "Cannot IndexRel %A" __ type EventData = { eventType: string; data: byte[]; metadata: byte[] } + type IEventData = + /// The Event Type, used to drive deserialization + abstract member EventType : string + /// Event body, as UTF-8 encoded json ready to be injected into the Json being rendered for DocDb + abstract member DataUtf8 : byte[] + /// Optional metadata (null, or same as d, not written if missing) + abstract member MetaUtf8 : byte[] [] type Event = @@ -60,6 +109,11 @@ module Store = { p = pos.streamName; id = string (pos.IndexRel offset); i = pos.IndexRel offset c = DateTimeOffset.UtcNow t = ed.eventType; d = ed.data; m = ed.metadata } + interface IEventData with + member __.EventType = __.t + member __.DataUtf8 = __.d + member __.MetaUtf8 = __.m + /// Manages injecting prepared json into the data being submitted to DocDb as-is, on the basis we can trust it to be valid json as DocDb will need it to be and VerbatimUtf8JsonConverter() = inherit JsonConverter() @@ -82,11 +136,17 @@ module Store = { p: string // "{streamName}" id: string // "{-1}" - w: int64 // 100: window size + //w: int64 // 100: window size /// last index/i value m: int64 // {index} - (* "x": [ + /// Compacted projections based on version identified by `m` + c: IndexProjection[] + + (*// Potential schema to manage Pending Events together with compaction events based on each one + // This scheme is more complete than the simple `c` encoding, which relies on every writer being able to write all salient snapshots + // For instance, in the case of blue/green deploys, older versions need to be able to coexist without destroying the perf for eachother + "x": [ { "i":0, "c":"ISO 8601" "e":[ @@ -95,8 +155,27 @@ module Store = ] } ] *) - x: JObject[][] } + //x: JObject[][] + } + static member IdConstant = "-1" + static member Create (pos: Position) eventCount (eds: EventData[]) : IndexEvent = + { p = pos.streamName; id = IndexEvent.IdConstant; m = pos.IndexRel eventCount + c = [| for ed in eds -> { t = ed.eventType; d = ed.data; m = ed.metadata } |] } + and IndexProjection = + { /// The Event Type, used to drive deserialization + t: string // required + + /// Event body, as UTF-8 encoded json ready to be injected into the Json being rendered for DocDb + [)>] + d: byte[] // required + /// Optional metadata (null, or same as d, not written if missing) + [); JsonProperty(Required=Required.Default, NullValueHandling=NullValueHandling.Ignore)>] + m: byte[] } // optional + interface IEventData with + member __.EventType = __.t + member __.DataUtf8 = __.d + member __.MetaUtf8 = __.m (* Pseudocode: function sync(p, expectedVersion, windowSize, events) { if (i == 0) then { @@ -123,7 +202,11 @@ module Log = type Event = | WriteSuccess of Measurement | WriteConflict of Measurement + /// Individual read request in a Batch | Slice of Direction * Measurement + /// Individual read request for the Index + | Index of Measurement + /// Summarizes a set of Slices read together | Batch of Direction * slices: int * Measurement let prop name value (log : ILogger) = log.ForContext(name, value) let propEvents name (kvps : System.Collections.Generic.KeyValuePair seq) (log : ILogger) = @@ -133,6 +216,8 @@ module Log = log |> propEvents name (seq { for x in events -> Collections.Generic.KeyValuePair<_,_>(x.eventType, System.Text.Encoding.UTF8.GetString x.data)}) let propResolvedEvents name (events : Store.Event[]) (log : ILogger) = log |> propEvents name (seq { for x in events -> Collections.Generic.KeyValuePair<_,_>(x.t, System.Text.Encoding.UTF8.GetString x.d)}) + let propProjectionEvents name (events : Store.IndexProjection[]) (log : ILogger) = + log |> propEvents name (seq { for x in events -> Collections.Generic.KeyValuePair<_,_>(x.t, System.Text.Encoding.UTF8.GetString x.d)}) open Serilog.Events /// Attach a property to the log context to hold the metrics @@ -155,19 +240,23 @@ type EqxSyncResult = Written of Store.Position * requestCharge: float | Conflict module private Write = let [] sprocName = "AtomicMultiDocInsert" - let append (client: IDocumentClient) (pos: Store.Position) (eventsData: Store.EventData seq): Async = async { + let append (client: IDocumentClient) (pos: Store.Position) (eventsData: Store.EventData seq,maybeIndexEvents): Async = async { let sprocUri = sprintf "%O/sprocs/%s" pos.collectionUri sprocName let opts = Client.RequestOptions(PartitionKey=PartitionKey(pos.streamName)) let! ct = Async.CancellationToken let events = eventsData |> Seq.mapi (fun i ed -> Store.Event.Create pos (i+1) ed |> JsonConvert.SerializeObject) |> Seq.toArray if events.Length = 0 then invalidArg "eventsData" "must be non-empty" - let! res = client.ExecuteStoredProcedureAsync(sprocUri, opts, ct, box events) |> Async.AwaitTaskCorrect + let index : Store.IndexEvent = + match maybeIndexEvents with + | None | Some [||] -> Unchecked.defaultof<_> + | Some eds -> Store.IndexEvent.Create pos (events.Length) eds + let! res = client.ExecuteStoredProcedureAsync(sprocUri, opts, ct, box events, box pos.Index, box index) |> Async.AwaitTaskCorrect return { pos with index = Some (pos.IndexRel events.Length) }, res.RequestCharge } /// Yields `EqxSyncResult.Written`, or `EqxSyncResult.Conflict` to signify WrongExpectedVersion - let private writeEventsAsync (log : ILogger) client pk (events : Store.EventData[]): Async = async { + let private writeEventsAsync (log : ILogger) client pk (events : Store.EventData[],maybeIndexEvents): Async = async { try - let! wr = append client pk events + let! wr = append client pk (events,maybeIndexEvents) return EqxSyncResult.Written wr with :? DocumentClientException as ex when ex.Message.Contains "already" -> // TODO this does not work for the SP log.Information(ex, "Eqx TrySync WrongExpectedVersionException writing {EventTypes}", [| for x in events -> x.eventType |]) @@ -177,12 +266,12 @@ module private Write = let eventDataLen ({ data = Log.BlobLen bytes; metadata = Log.BlobLen metaBytes } : Store.EventData) = bytes + metaBytes events |> Array.sumBy eventDataLen - let private writeEventsLogged client (pos : Store.Position) (events : Store.EventData[]) (log : ILogger): Async = async { + let private writeEventsLogged client (pos : Store.Position) (events : Store.EventData[], maybeIndexEvents) (log : ILogger): Async = async { let log = if (not << log.IsEnabled) Events.LogEventLevel.Debug then log else log |> Log.propEventData "Json" events let bytes, count = bytes events, events.Length let log = log |> Log.prop "bytes" bytes let writeLog = log |> Log.prop "stream" pos.streamName |> Log.prop "expectedVersion" pos.Index |> Log.prop "count" count - let! t, result = writeEventsAsync writeLog client pos events |> Stopwatch.Time + let! t, result = writeEventsAsync writeLog client pos (events,maybeIndexEvents) |> Stopwatch.Time let (ru: float), resultLog = let mkMetric ru : Log.Measurement = { stream = pos.streamName; interval = t; bytes = bytes; count = count; ru = ru } match result with @@ -191,11 +280,41 @@ module private Write = resultLog.Information("Eqx {action:l} {count} {ms}ms rc={ru}", "Write", events.Length, (let e = t.Elapsed in e.TotalMilliseconds), ru) return result } - let writeEvents (log : ILogger) retryPolicy client pk (events : Store.EventData[]): Async = - let call = writeEventsLogged client pk events + let writeEvents (log : ILogger) retryPolicy client pk (events : Store.EventData[],maybeIndexEvents): Async = + let call = writeEventsLogged client pk (events,maybeIndexEvents) Log.withLoggedRetries retryPolicy "writeAttempt" call log module private Read = + let private getIndex (client : IDocumentClient) (pos:Store.Position) (log: ILogger) = async { + // TODO cancellation token, use cache if available + let! t, (res : Client.DocumentResponse option) = + let coll = DocDbCollection(client, pos.collectionUri) + let ac = pos.index |> Option.map (fun i -> Client.AccessCondition(Type=Client.AccessConditionType.IfNoneMatch,Condition=string i)) + let ro = Client.RequestOptions(PartitionKey=PartitionKey(pos.streamName), AccessCondition = match ac with Some ac -> ac | None -> null) + coll.TryReadDocument(Store.IndexEvent.IdConstant, ro) + |> Stopwatch.Time + + match res with + | None -> return None, 0. + | Some res -> + + let doc, ru = res.Document, res.RequestCharge + let (|EventLen|) (x : Store.IndexProjection) = match x.d, x.m with Log.BlobLen bytes, Log.BlobLen metaBytes -> bytes + metaBytes + let bytes, count = doc.c |> Array.sumBy (|EventLen|), doc.c.Length + let evt = Log.Index { stream = pos.streamName; interval = t; bytes = bytes; count = count; ru = ru } + let log = if (not << log.IsEnabled) Events.LogEventLevel.Debug then log else log |> Log.propProjectionEvents "Json" doc.c + (log |> Log.prop "index" pos.Index |> Log.prop "bytes" bytes |> Log.event evt) + .Information("Eqx {action:l} {ms}ms rc={ru}", "Index", (let e = t.Elapsed in e.TotalMilliseconds), ru) + return Some doc, ru } + + type [] IndexResult = NotFound | Found of Store.Position * Store.IndexProjection[] + let loadIndex (log : ILogger) retryPolicy client (pos : Store.Position): Async = async { + let log = log |> Log.prop "stream" pos.streamName + let! res, _rc = Log.withLoggedRetries retryPolicy "readAttempt" (getIndex client pos) log + match res with + | None -> return IndexResult.NotFound + | Some index -> return IndexResult.Found ({ pos with index = Some index.m }, index.c) } + let private getQuery (client : IDocumentClient) (pos:Store.Position) (direction: Direction) batchSize = let querySpec = match pos.index with @@ -204,6 +323,7 @@ module private Read = let f = if direction = Direction.Forward then "c.i >= @id ORDER BY c.i ASC" else "c.i < @id ORDER BY c.i DESC" SqlQuerySpec( "SELECT * FROM c WHERE " + f, SqlParameterCollection (Seq.singleton (SqlParameter("@id", index)))) let feedOptions = new Client.FeedOptions(PartitionKey=PartitionKey(pos.streamName), MaxItemCount=Nullable batchSize) + // TODO cancellation token client.CreateDocumentQuery(pos.collectionUri, querySpec, feedOptions).AsDocumentQuery() let (|EventLen|) (x : Store.Event) = match x.d, x.m with Log.BlobLen bytes, Log.BlobLen metaBytes -> bytes + metaBytes @@ -291,10 +411,10 @@ module private Read = if not (isCompactionEvent x) then true // continue the search else match !lastBatch with - | None -> log.Information("EqxStop stream={stream} at={eventNumber}", pos.streamName, x.id) + | None -> log.Information("Eqx Stop stream={stream} at={eventNumber}", pos.streamName, x.id) | Some batch -> let used, residual = batch |> partitionPayloadFrom x.id - log.Information("EqxStop stream={stream} at={eventNumber} used={used} residual={residual}", pos.streamName, x.id, used, residual) + log.Information("Eqx Stop stream={stream} at={eventNumber} used={used} residual={residual}", pos.streamName, x.id, used, residual) false) |> AsyncSeq.toArrayAsync let eventsForward = Array.Reverse(tempBackward); tempBackward // sic - relatively cheap, in-place reverse of something we own @@ -315,11 +435,15 @@ module private Read = module UnionEncoderAdapters = let private encodedEventOfStoredEvent (x : Store.Event) : UnionCodec.EncodedUnion = { caseName = x.t; payload = x.d } + let private encodedEventOfStoredEventI (x : Store.IEventData) : UnionCodec.EncodedUnion = + { caseName = x.EventType; payload = x.DataUtf8 } let private eventDataOfEncodedEvent (x : UnionCodec.EncodedUnion) : Store.EventData = { eventType = x.caseName; data = x.payload; metadata = null } let encodeEvents (codec : UnionCodec.IUnionEncoder<'event, byte[]>) (xs : 'event seq) : Store.EventData[] = xs |> Seq.map (codec.Encode >> eventDataOfEncodedEvent) |> Seq.toArray - let decodeKnownEvents (codec : UnionCodec.IUnionEncoder<'event, byte[]>) (xs : Store.Event[]) : 'event seq = + let decodeKnownEventsI (codec : UnionCodec.IUnionEncoder<'event, byte[]>) (xs : Store.IEventData seq) : 'event seq = + xs |> Seq.map encodedEventOfStoredEventI |> Seq.choose codec.TryDecode + let decodeKnownEvents (codec : UnionCodec.IUnionEncoder<'event, byte[]>) (xs : Store.Event seq) : 'event seq = xs |> Seq.map encodedEventOfStoredEvent |> Seq.choose codec.TryDecode type []Token = { pos: Store.Position; compactionEventNumber: int64 option } @@ -356,7 +480,7 @@ module Token = let currentVersion, newVersion = unpackEqxStreamVersion current, unpackEqxStreamVersion x newVersion > currentVersion -type EqxConnection(client: IDocumentClient, ?readRetryPolicy, ?writeRetryPolicy) = +type EqxConnection(client: IDocumentClient, ?readRetryPolicy (*: (int -> Async<'T>) -> Async<'T>*), ?writeRetryPolicy) = member __.Client = client member __.ReadRetryPolicy = readRetryPolicy member __.WriteRetryPolicy = writeRetryPolicy @@ -389,6 +513,14 @@ type EqxGateway(conn : EqxConnection, batching : EqxBatchingPolicy) = match Array.tryHead events |> Option.filter isCompactionEvent with | None -> return Token.ofUncompactedVersion batching.BatchSize pos, events | Some resolvedEvent -> return Token.ofCompactionResolvedEventAndVersion resolvedEvent batching.BatchSize pos, events } + member __.IndexedOrBatched log isCompactionEventType pos: Async = async { + let! res = Read.loadIndex log None(* TODO conn.ReadRetryPolicy*) conn.Client pos + match res with + | Read.IndexResult.Found (pos, projectionsAndEvents) when projectionsAndEvents |> Array.exists (fun x -> isCompactionEventType x.t) -> + return Token.ofNonCompacting pos, projectionsAndEvents |> Seq.cast |> Array.ofSeq + | _ -> + let! streamToken, events = __.LoadBackwardsStoppingAtCompactionEvent log isCompactionEventType pos + return streamToken, events |> Seq.cast |> Array.ofSeq } member __.LoadFromToken log (Pos pos as token) isCompactionEventType synchronized: Async = async { let! pos, events = Read.loadForwardsFrom log conn.ReadRetryPolicy conn.Client batching.BatchSize batching.MaxBatches (pos,synchronized) match tryIsResolvedEventEventType isCompactionEventType with @@ -397,8 +529,8 @@ type EqxGateway(conn : EqxConnection, batching : EqxBatchingPolicy) = match events |> Array.tryFindBack isCompactionEvent with | None -> return Token.ofPreviousTokenAndEventsLength token events.Length batching.BatchSize pos, events | Some resolvedEvent -> return Token.ofCompactionResolvedEventAndVersion resolvedEvent batching.BatchSize pos, events } - member __.TrySync log (Pos pos as token) (encodedEvents: Store.EventData[]) isCompactionEventType: Async = async { - let! wr = Write.writeEvents log conn.WriteRetryPolicy conn.Client pos encodedEvents + member __.TrySync log (Pos pos as token) (encodedEvents: Store.EventData[],maybeIndexEvents) isCompactionEventType: Async = async { + let! wr = Write.writeEvents log conn.WriteRetryPolicy conn.Client pos (encodedEvents,maybeIndexEvents) match wr with | EqxSyncResult.Conflict _ -> return GatewaySyncResult.Conflict | EqxSyncResult.Written (wr, _) -> @@ -419,47 +551,69 @@ type private Collection(gateway : EqxGateway, databaseId, collectionId) = member __.Gateway = gateway member __.CollectionUri = Client.UriFactory.CreateDocumentCollectionUri(databaseId, collectionId) +[] +type SearchStrategy<'event> = + | EventType of string + | Predicate of ('event -> bool) + [] type AccessStrategy<'event,'state> = | EventsAreState - | RollingSnapshots of eventType: string * compact: ('state -> 'event) + | //[] + RollingSnapshots of eventType: string * compact: ('state -> 'event) + | IndexedSearch of (string -> bool) * index: ('state -> 'event seq) type private CompactionContext(eventsLen : int, capacityBeforeCompaction : int) = - /// Determines whether writing a Compaction event is warranted (based on the existing state and the current `Accumulated` changes) + /// Determines whether writing a Compaction event is warranted (based on the existing state and the current `Accumulated` changes) member __.IsCompactionDue = eventsLen > capacityBeforeCompaction type private Category<'event, 'state>(coll : Collection, codec : UnionCodec.IUnionEncoder<'event, byte[]>, ?access : AccessStrategy<'event,'state>) = let (|Pos|) streamName : Store.Position = { collectionUri = coll.CollectionUri; streamName = streamName; index = None } let compactionPredicate = match access with + | Some (AccessStrategy.IndexedSearch _) | None -> None | Some AccessStrategy.EventsAreState -> Some (fun _ -> true) | Some (AccessStrategy.RollingSnapshots (et,_)) -> Some ((=) et) - let loadAlgorithm load (Pos pos) initial log = - let batched = load initial (coll.Gateway.LoadBatched log None pos) - let compacted predicate = load initial (coll.Gateway.LoadBackwardsStoppingAtCompactionEvent log predicate pos) + //let searchPredicate = + // match access with + // | None -> None + // | Some AccessStrategy.EventsAreState -> Some (SearchStrategy.Predicate (fun _ -> true)) + // | Some (AccessStrategy.IndexedSearch (ep,_)) -> Some (SearchStrategy.Predicate ep) + let load (fold: 'state -> 'event seq -> 'state) initial loadF = async { + let! token, events = loadF + return token, fold initial (UnionEncoderAdapters.decodeKnownEvents codec events) } + let loadI (fold: 'state -> 'event seq -> 'state) initial loadF = async { + let! token, events = loadF + return token, fold initial (UnionEncoderAdapters.decodeKnownEventsI codec events) } + let loadAlgorithm fold (Pos pos) initial log = + let batched = load fold initial (coll.Gateway.LoadBatched log None pos) + let compacted predicate = load fold initial (coll.Gateway.LoadBackwardsStoppingAtCompactionEvent log predicate pos) + let indexed predicate = loadI fold initial (coll.Gateway.IndexedOrBatched log predicate pos) match access with + | Some (AccessStrategy.IndexedSearch (predicate,_)) -> indexed predicate | None -> batched | Some AccessStrategy.EventsAreState -> compacted (fun _ -> true) | Some (AccessStrategy.RollingSnapshots (et,_)) -> compacted ((=) et) - let load (fold: 'state -> 'event seq -> 'state) initial loadF = async { - let! token, events = loadF - return token, fold initial (UnionEncoderAdapters.decodeKnownEvents codec events) } member __.Load (fold: 'state -> 'event seq -> 'state) (initial: 'state) streamName (log : ILogger) : Async = - loadAlgorithm (load fold) streamName initial log + loadAlgorithm fold streamName initial log member __.LoadFromToken (fold: 'state -> 'event seq -> 'state) (state: 'state) token (log : ILogger) : Async = (load fold) state (coll.Gateway.LoadFromToken log token compactionPredicate false) member __.TrySync (fold: 'state -> 'event seq -> 'state) (log : ILogger) (token : Storage.StreamToken, state : 'state) (events : 'event list, state' : 'state) : Async> = async { - let events = + let events, index = match access with - | None | Some AccessStrategy.EventsAreState -> events + | None | Some AccessStrategy.EventsAreState -> + events, None | Some (AccessStrategy.RollingSnapshots (_,f)) -> let cc = CompactionContext(List.length events, token.batchCapacityLimit.Value) - if cc.IsCompactionDue then events @ [f state'] else events + (if cc.IsCompactionDue then events @ [f state'] else events), None + | Some (AccessStrategy.IndexedSearch (_,index)) -> + events, Some (index state') let encodedEvents : Store.EventData[] = UnionEncoderAdapters.encodeEvents codec (Seq.ofList events) - let! syncRes = coll.Gateway.TrySync log token encodedEvents compactionPredicate + let maybeIndexEvents : Store.EventData[] option = index |> Option.map (UnionEncoderAdapters.encodeEvents codec) + let! syncRes = coll.Gateway.TrySync log token (encodedEvents,maybeIndexEvents) compactionPredicate match syncRes with | GatewaySyncResult.Conflict -> return Storage.SyncResult.Conflict (load fold state (coll.Gateway.LoadFromToken log token compactionPredicate true)) | GatewaySyncResult.Written token' -> return Storage.SyncResult.Written (token', fold state (Seq.ofList events)) } @@ -590,7 +744,7 @@ module Initialization = return coll.Resource.Id } let createProc (client: IDocumentClient) (collectionUri: Uri) = async { - let f ="""function multidocInsert(docs) { + let f = """function multidocInsert(docs,expectedVersion,index) { var response = getContext().getResponse(); var collection = getContext().getCollection(); var collectionLink = collection.getSelfLink(); @@ -598,6 +752,16 @@ module Initialization = for (var i=0; i () | Some tags -> for key, value in tags do yield sprintf "%s=%s" key value } let sanitizedName = name.Replace('\'','_').Replace(':','_') // sic; Align with logging for ES Adapter let client = new Client.DocumentClient(uri, key, connPolicy, Nullable(defaultArg defaultConsistencyLevel ConsistencyLevel.Session)) - log.Information("Connecting to Cosmos with Connection Name {connectionName}", sanitizedName) + log.Information("Eqx connecting to Cosmos with Connection Name {connectionName}", sanitizedName) do! client.OpenAsync() |> Async.AwaitTaskCorrect return client :> IDocumentClient } diff --git a/tests/Equinox.Cosmos.Integration/CosmosFixturesInfrastructure.fs b/tests/Equinox.Cosmos.Integration/CosmosFixturesInfrastructure.fs index 1d4ce5d1e..48dd68c50 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosFixturesInfrastructure.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosFixturesInfrastructure.fs @@ -49,7 +49,7 @@ module SerilogHelpers = | (:? ScalarValue as x) -> Some x.Value | _ -> None [] - type EqxAct = Append | AppendConflict | SliceForward | SliceBackward | BatchForward | BatchBackward + type EqxAct = Append | AppendConflict | SliceForward | SliceBackward | BatchForward | BatchBackward | Indexed let (|EqxAction|) (evt : Equinox.Cosmos.Log.Event) = match evt with | Equinox.Cosmos.Log.WriteSuccess _ -> EqxAct.Append @@ -58,6 +58,7 @@ module SerilogHelpers = | Equinox.Cosmos.Log.Slice (Equinox.Cosmos.Direction.Backward,_) -> EqxAct.SliceBackward | Equinox.Cosmos.Log.Batch (Equinox.Cosmos.Direction.Forward,_,_) -> EqxAct.BatchForward | Equinox.Cosmos.Log.Batch (Equinox.Cosmos.Direction.Backward,_,_) -> EqxAct.BatchBackward + | Equinox.Cosmos.Log.Index _ -> EqxAct.Indexed let (|EqxEvent|_|) (logEvent : LogEvent) : Equinox.Cosmos.Log.Event option = logEvent.Properties.Values |> Seq.tryPick (function | SerilogScalar (:? Equinox.Cosmos.Log.Event as e) -> Some e diff --git a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs index e0ed98f5a..ce582bb79 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs @@ -11,7 +11,7 @@ let genCodec<'Union when 'Union :> TypeShape.UnionContract.IUnionContract>() = Equinox.UnionCodec.JsonUtf8.Create<'Union>(serializationSettings) module Cart = - let fold, initial, compact = Domain.Cart.Folds.fold, Domain.Cart.Folds.initial, Domain.Cart.Folds.compact + let fold, initial, compact, index = Domain.Cart.Folds.fold, Domain.Cart.Folds.initial, Domain.Cart.Folds.compact, Domain.Cart.Folds.index let codec = genCodec() let createServiceWithoutOptimization connection batchSize log = let gateway = createEqxGateway connection batchSize @@ -28,6 +28,11 @@ module Cart = let sliding20m = CachingStrategy.SlidingWindow (cache, TimeSpan.FromMinutes 20.) let resolveStream (StreamArgs args) = EqxStreamBuilder(gateway, codec, fold, initial, caching = sliding20m).Create(args) Backend.Cart.Service(log, resolveStream) + let createServiceWithCachingIndexed connection batchSize log cache = + let gateway = createEqxGateway connection batchSize + let _sliding20m = CachingStrategy.SlidingWindow (cache, TimeSpan.FromMinutes 20.) // TODO + let resolveStream (StreamArgs args) = EqxStreamBuilder(gateway, codec, fold, initial, AccessStrategy.IndexedSearch index).Create(args) + Backend.Cart.Service(log, resolveStream) let createServiceWithCompactionAndCaching connection batchSize log cache = let gateway = createEqxGateway connection batchSize let sliding20m = CachingStrategy.SlidingWindow (cache, TimeSpan.FromMinutes 20.) @@ -216,7 +221,7 @@ type Tests(testOutputHelper) = } [] - let ``Can correctly read and update against Cosmos, with window size of 1 using tautological Compaction predicate`` id value = Async.RunSynchronously <| async { + let ``Can correctly read and update against Cosmos with EventsAreState Access Strategy`` id value = Async.RunSynchronously <| async { let log, capture = createLoggerWithCapture () let! conn = connectToSpecifiedCosmosOrSimulator log let service = ContactPreferences.createService (createEqxGateway conn) log @@ -265,6 +270,36 @@ type Tests(testOutputHelper) = test <@ singleBatchForward = capture.ExternalCalls @> } + let singleIndexed = [EqxAct.Indexed] + let indexedReadAndAppend = singleIndexed @ [EqxAct.Append] + + [] + let ``Can roundtrip against Cosmos, correctly using the index to avoid redundant reads`` context skuId cartId = Async.RunSynchronously <| async { + let log, capture = createLoggerWithCapture () + let! conn = connectToSpecifiedCosmosOrSimulator log + let batchSize = 10 + let cache = Caching.Cache("cart", sizeMb = 50) + let createServiceCached () = Cart.createServiceWithCachingIndexed conn batchSize log cache + let service1, service2 = createServiceCached (), createServiceCached () + + // Trigger 10 events, then reload + do! addAndThenRemoveItemsManyTimes context cartId skuId service1 5 + let! _ = service2.Read cartId + + // ... should see a single read as we are writes are cached + test <@ indexedReadAndAppend @ singleIndexed = capture.ExternalCalls @> + + // Add two more - the roundtrip should only incur a single read + capture.Clear() + do! addAndThenRemoveItemsManyTimes context cartId skuId service1 1 + test <@ indexedReadAndAppend = capture.ExternalCalls @> + + // While we now have 12 events, we should be able to read them with a single call + capture.Clear() + let! _ = service2.Read cartId + test <@ singleIndexed = capture.ExternalCalls @> + } + [] let ``Can combine compaction with caching against Cosmos`` context skuId cartId = Async.RunSynchronously <| async { let log, capture = createLoggerWithCapture () From 4d4378f287cc4d2486b000d6087fc84a627593a7 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 13 Nov 2018 14:46:47 +0000 Subject: [PATCH 35/66] Retain etags (to no end as yet) --- samples/Store/Integration/LogIntegration.fs | 2 + src/Equinox.Cosmos/Cosmos.fs | 276 +++++++++++------- .../CosmosFixturesInfrastructure.fs | 4 +- .../CosmosIntegration.fs | 47 ++- 4 files changed, 212 insertions(+), 117 deletions(-) diff --git a/samples/Store/Integration/LogIntegration.fs b/samples/Store/Integration/LogIntegration.fs index 5310f5341..15b6e4537 100644 --- a/samples/Store/Integration/LogIntegration.fs +++ b/samples/Store/Integration/LogIntegration.fs @@ -37,6 +37,8 @@ module EquinoxCosmosInterop = | Log.Batch (Direction.Forward,c,m) -> "EqxLoadF", m, Some c, m.ru | Log.Batch (Direction.Backward,c,m) -> "EqxLoadB", m, Some c, m.ru | Log.Index m -> "EqxLoadI", m, None, m.ru + | Log.IndexNotFound m -> "EqxLoadIN", m, None, m.ru + | Log.IndexCached m -> "EqxLoadIC", m, None, m.ru { action = action; stream = metric.stream; bytes = metric.bytes; count = metric.count; batches = batches interval = StopwatchInterval(metric.interval.StartTicks,metric.interval.EndTicks); ru = ru } diff --git a/src/Equinox.Cosmos/Cosmos.fs b/src/Equinox.Cosmos/Cosmos.fs index f22bad9f7..46b1021e5 100644 --- a/src/Equinox.Cosmos/Cosmos.fs +++ b/src/Equinox.Cosmos/Cosmos.fs @@ -36,26 +36,34 @@ module DocDbExtensions = aux exn /// DocumentDB Error HttpStatusCode extractor - let (|DocDbStatusCode|_|) (e : exn) = + let (|DocDbException|_|) (e : exn) = match e with - | AggregateException (:? DocumentClientException as dce) -> match dce.StatusCode with v when v.HasValue -> Some v.Value | _ -> None + | AggregateException (:? DocumentClientException as dce) -> Some dce + | _ -> None + /// Map Nullable to Option + let (|HasValue|_|) (x:Nullable<_>) = if x.HasValue then Some x.Value else None + /// DocumentDB Error HttpStatusCode extractor + let (|DocDbStatusCode|_|) (e : DocumentClientException) = + match e.StatusCode with + | HasValue x -> Some x | _ -> None + type ReadResult<'T> = Found of 'T | NotFound | PreconditionFailed type DocDbCollection(client : IDocumentClient, collectionUri) = - member __.TryReadDocument(documentId : string, ?options : Client.RequestOptions) = async { + member __.TryReadDocument(documentId : string, ?options : Client.RequestOptions): Async> = async { let! ct = Async.CancellationToken let options = defaultArg options null let docLink = sprintf "%O/docs/%s" collectionUri documentId try let! document = async { return! client.ReadDocumentAsync<'T>(docLink, options = options, cancellationToken = ct) |> Async.AwaitTaskCorrect } - return Some document - with DocDbStatusCode System.Net.HttpStatusCode.NotFound -> - // TODO // dont drop RUs - return None } + return document.RequestCharge, Found document.Document + with + | DocDbException (DocDbStatusCode System.Net.HttpStatusCode.NotFound as e) -> return e.RequestCharge, NotFound + | DocDbException (DocDbStatusCode System.Net.HttpStatusCode.PreconditionFailed as e) -> return e.RequestCharge, PreconditionFailed } module Store = [] type Position = - { collectionUri: Uri; streamName: string; index: int64 option } + { collectionUri: Uri; streamName: string; index: int64 option; etag: string option } member __.Index : int64 = defaultArg __.index -1L member __.IndexRel (offset: int) : int64 = __.index |> function | Some index -> index+int64 offset @@ -206,6 +214,10 @@ module Log = | Slice of Direction * Measurement /// Individual read request for the Index | Index of Measurement + /// Individual read request for the Index, not found + | IndexNotFound of Measurement + /// Index read with Single RU Request Charge due to correct use of etag in cache + | IndexCached of Measurement /// Summarizes a set of Slices read together | Batch of Direction * slices: int * Measurement let prop name value (log : ILogger) = log.ForContext(name, value) @@ -216,6 +228,8 @@ module Log = log |> propEvents name (seq { for x in events -> Collections.Generic.KeyValuePair<_,_>(x.eventType, System.Text.Encoding.UTF8.GetString x.data)}) let propResolvedEvents name (events : Store.Event[]) (log : ILogger) = log |> propEvents name (seq { for x in events -> Collections.Generic.KeyValuePair<_,_>(x.t, System.Text.Encoding.UTF8.GetString x.d)}) + let propIEventDatas name (events : Store.IEventData[]) (log : ILogger) = + log |> propEvents name (seq { for x in events -> Collections.Generic.KeyValuePair<_,_>(x.EventType, System.Text.Encoding.UTF8.GetString x.DataUtf8)}) let propProjectionEvents name (events : Store.IndexProjection[]) (log : ILogger) = log |> propEvents name (seq { for x in events -> Collections.Generic.KeyValuePair<_,_>(x.t, System.Text.Encoding.UTF8.GetString x.d)}) @@ -236,31 +250,34 @@ module Log = let (|BlobLen|) = function null -> 0 | (x : byte[]) -> x.Length [] -type EqxSyncResult = Written of Store.Position * requestCharge: float | Conflict of requestCharge: float +type EqxSyncResult = + | Written of Store.Position + | ConflictUnknown of Store.Position + | Conflict of Store.Position * events: Store.IEventData[] module private Write = + [] + type WriteResponse = { etag: string; conflicts: Store.IndexProjection[] } let [] sprocName = "AtomicMultiDocInsert" - let append (client: IDocumentClient) (pos: Store.Position) (eventsData: Store.EventData seq,maybeIndexEvents): Async = async { + + let private writeEventsAsync (client: IDocumentClient) (pos: Store.Position) (events: Store.EventData seq,maybeIndexEvents): Async = async { let sprocUri = sprintf "%O/sprocs/%s" pos.collectionUri sprocName let opts = Client.RequestOptions(PartitionKey=PartitionKey(pos.streamName)) let! ct = Async.CancellationToken - let events = eventsData |> Seq.mapi (fun i ed -> Store.Event.Create pos (i+1) ed |> JsonConvert.SerializeObject) |> Seq.toArray + let events = events |> Seq.mapi (fun i ed -> Store.Event.Create pos (i+1) ed |> JsonConvert.SerializeObject) |> Seq.toArray if events.Length = 0 then invalidArg "eventsData" "must be non-empty" let index : Store.IndexEvent = match maybeIndexEvents with | None | Some [||] -> Unchecked.defaultof<_> | Some eds -> Store.IndexEvent.Create pos (events.Length) eds - let! res = client.ExecuteStoredProcedureAsync(sprocUri, opts, ct, box events, box pos.Index, box index) |> Async.AwaitTaskCorrect - return { pos with index = Some (pos.IndexRel events.Length) }, res.RequestCharge } - - /// Yields `EqxSyncResult.Written`, or `EqxSyncResult.Conflict` to signify WrongExpectedVersion - let private writeEventsAsync (log : ILogger) client pk (events : Store.EventData[],maybeIndexEvents): Async = async { try - let! wr = append client pk (events,maybeIndexEvents) - return EqxSyncResult.Written wr - with :? DocumentClientException as ex when ex.Message.Contains "already" -> // TODO this does not work for the SP - log.Information(ex, "Eqx TrySync WrongExpectedVersionException writing {EventTypes}", [| for x in events -> x.eventType |]) - return EqxSyncResult.Conflict ex.RequestCharge } + let! (res : Client.StoredProcedureResponse) = client.ExecuteStoredProcedureAsync(sprocUri, opts, ct, box events, box pos.Index, box pos.etag, box index) |> Async.AwaitTaskCorrect + match res.RequestCharge, (match res.Response.etag with null -> None | x -> Some x), res.Response.conflicts with + | rc,e,null -> return rc, EqxSyncResult.Written { pos with index = Some (pos.IndexRel events.Length); etag=e } + | rc,e,[||] -> return rc, EqxSyncResult.ConflictUnknown { pos with etag=e } + | rc,e, xs -> return rc, EqxSyncResult.Conflict ({ pos with index = Some (pos.IndexRel xs.Length); etag=e }, Array.map (fun x -> x :> _) xs) + with DocDbException ex when ex.Message.Contains "already" -> // TODO this does not work for the SP + return ex.RequestCharge, EqxSyncResult.ConflictUnknown { pos with etag=None } } let bytes events = let eventDataLen ({ data = Log.BlobLen bytes; metadata = Log.BlobLen metaBytes } : Store.EventData) = bytes + metaBytes @@ -271,12 +288,20 @@ module private Write = let bytes, count = bytes events, events.Length let log = log |> Log.prop "bytes" bytes let writeLog = log |> Log.prop "stream" pos.streamName |> Log.prop "expectedVersion" pos.Index |> Log.prop "count" count - let! t, result = writeEventsAsync writeLog client pos (events,maybeIndexEvents) |> Stopwatch.Time - let (ru: float), resultLog = + let! t, (ru,result) = writeEventsAsync client pos (events,maybeIndexEvents) |> Stopwatch.Time + let resultLog = let mkMetric ru : Log.Measurement = { stream = pos.streamName; interval = t; bytes = bytes; count = count; ru = ru } + let logConflict () = writeLog.Information("Eqx TrySync WrongExpectedVersion writing {EventTypes}", [| for x in events -> x.eventType |]) match result with - | EqxSyncResult.Conflict ru -> ru, log |> Log.event (Log.WriteConflict (mkMetric ru)) |> Log.prop "conflict" true - | EqxSyncResult.Written (x, ru) -> ru, log |> Log.event (Log.WriteSuccess (mkMetric ru)) |> Log.prop "nextExpectedVersion" x + | EqxSyncResult.Written pos -> + log |> Log.event (Log.WriteSuccess (mkMetric ru)) |> Log.prop "nextExpectedVersion" pos + | EqxSyncResult.ConflictUnknown pos -> + logConflict () + log |> Log.event (Log.WriteConflict (mkMetric ru)) |> Log.prop "nextExpectedVersion" pos |> Log.prop "conflict" true + | EqxSyncResult.Conflict (pos, xs) -> + logConflict () + let log = if (not << log.IsEnabled) Events.LogEventLevel.Debug then log else log |> Log.prop "nextExpectedVersion" pos |> Log.propIEventDatas "conflictJson" xs + log |> Log.event (Log.WriteConflict (mkMetric ru)) |> Log.prop "conflict" true resultLog.Information("Eqx {action:l} {count} {ms}ms rc={ru}", "Write", events.Length, (let e = t.Elapsed in e.TotalMilliseconds), ru) return result } @@ -285,35 +310,37 @@ module private Write = Log.withLoggedRetries retryPolicy "writeAttempt" call log module private Read = - let private getIndex (client : IDocumentClient) (pos:Store.Position) (log: ILogger) = async { - // TODO cancellation token, use cache if available - let! t, (res : Client.DocumentResponse option) = - let coll = DocDbCollection(client, pos.collectionUri) - let ac = pos.index |> Option.map (fun i -> Client.AccessCondition(Type=Client.AccessConditionType.IfNoneMatch,Condition=string i)) - let ro = Client.RequestOptions(PartitionKey=PartitionKey(pos.streamName), AccessCondition = match ac with Some ac -> ac | None -> null) - coll.TryReadDocument(Store.IndexEvent.IdConstant, ro) - |> Stopwatch.Time - + let private getIndex (client : IDocumentClient) (pos:Store.Position) = + let coll = DocDbCollection(client, pos.collectionUri) + let ac = match pos.etag with None -> null | Some etag-> Client.AccessCondition(Type=Client.AccessConditionType.IfNoneMatch, Condition=etag) + let ro = Client.RequestOptions(PartitionKey=PartitionKey(pos.streamName), AccessCondition = ac) + coll.TryReadDocument(Store.IndexEvent.IdConstant, ro) + let private loggedGetIndex (getIndex : Store.Position -> Async<_>) (pos:Store.Position) (log: ILogger) = async { + let log = log |> Log.prop "stream" pos.streamName + let! t, (ru, res : ReadResult) = getIndex pos |> Stopwatch.Time + let log count bytes (f : Log.Measurement -> _) = log |> Log.event (f { stream = pos.streamName; interval = t; bytes = bytes; count = count; ru = ru }) match res with - | None -> return None, 0. - | Some res -> - - let doc, ru = res.Document, res.RequestCharge - let (|EventLen|) (x : Store.IndexProjection) = match x.d, x.m with Log.BlobLen bytes, Log.BlobLen metaBytes -> bytes + metaBytes - let bytes, count = doc.c |> Array.sumBy (|EventLen|), doc.c.Length - let evt = Log.Index { stream = pos.streamName; interval = t; bytes = bytes; count = count; ru = ru } - let log = if (not << log.IsEnabled) Events.LogEventLevel.Debug then log else log |> Log.propProjectionEvents "Json" doc.c - (log |> Log.prop "index" pos.Index |> Log.prop "bytes" bytes |> Log.event evt) - .Information("Eqx {action:l} {ms}ms rc={ru}", "Index", (let e = t.Elapsed in e.TotalMilliseconds), ru) - return Some doc, ru } - - type [] IndexResult = NotFound | Found of Store.Position * Store.IndexProjection[] + | ReadResult.PreconditionFailed -> + (log 0 0 Log.IndexCached).Information("Eqx {action:l} {ms}ms rc={ru}", "IndexCached", (let e = t.Elapsed in e.TotalMilliseconds), ru) + | ReadResult.NotFound -> + (log 0 0 Log.IndexNotFound).Information("Eqx {action:l} {ms}ms rc={ru}", "IndexNotFound", (let e = t.Elapsed in e.TotalMilliseconds), ru) + | ReadResult.Found doc -> + let log = + let (|EventLen|) (x : Store.IndexProjection) = match x.d, x.m with Log.BlobLen bytes, Log.BlobLen metaBytes -> bytes + metaBytes + let bytes, count = doc.c |> Array.sumBy (|EventLen|), doc.c.Length + log bytes count Log.Index + let log = if (not << log.IsEnabled) Events.LogEventLevel.Debug then log else log |> Log.propProjectionEvents "Json" doc.c + log.Information("Eqx {action:l} {ms}ms rc={ru}", "Index", (let e = t.Elapsed in e.TotalMilliseconds), ru) + return ru, res } + type [] IndexResult = Unchanged | NotFound | Found of Store.Position * Store.IndexProjection[] + /// `pos` being Some implies that the caller holds a cached value and hence is ready to deal with IndexResult.UnChanged let loadIndex (log : ILogger) retryPolicy client (pos : Store.Position): Async = async { - let log = log |> Log.prop "stream" pos.streamName - let! res, _rc = Log.withLoggedRetries retryPolicy "readAttempt" (getIndex client pos) log + let getIndex = getIndex client + let! _rc, res = Log.withLoggedRetries retryPolicy "readAttempt" (loggedGetIndex getIndex pos) log match res with - | None -> return IndexResult.NotFound - | Some index -> return IndexResult.Found ({ pos with index = Some index.m }, index.c) } + | ReadResult.PreconditionFailed -> return IndexResult.Unchanged + | ReadResult.NotFound -> return IndexResult.NotFound + | ReadResult.Found index -> return IndexResult.Found ({ pos with index = Some index.m }, index.c) } let private getQuery (client : IDocumentClient) (pos:Store.Position) (direction: Direction) batchSize = let querySpec = @@ -323,15 +350,16 @@ module private Read = let f = if direction = Direction.Forward then "c.i >= @id ORDER BY c.i ASC" else "c.i < @id ORDER BY c.i DESC" SqlQuerySpec( "SELECT * FROM c WHERE " + f, SqlParameterCollection (Seq.singleton (SqlParameter("@id", index)))) let feedOptions = new Client.FeedOptions(PartitionKey=PartitionKey(pos.streamName), MaxItemCount=Nullable batchSize) - // TODO cancellation token client.CreateDocumentQuery(pos.collectionUri, querySpec, feedOptions).AsDocumentQuery() let (|EventLen|) (x : Store.Event) = match x.d, x.m with Log.BlobLen bytes, Log.BlobLen metaBytes -> bytes + metaBytes + let bytes events = events |> Array.sumBy (|EventLen|) let private loggedQueryExecution (pos:Store.Position) direction (query: IDocumentQuery) (log: ILogger): Async = async { - let! t, (res : Client.FeedResponse) = query.ExecuteNextAsync() |> Async.AwaitTaskCorrect |> Stopwatch.Time + let! ct = Async.CancellationToken + let! t, (res : Client.FeedResponse) = query.ExecuteNextAsync(ct) |> Async.AwaitTaskCorrect |> Stopwatch.Time let slice, ru = Array.ofSeq res, res.RequestCharge - let bytes, count = slice |> Array.sumBy (|EventLen|), slice.Length + let bytes, count = bytes slice, slice.Length let reqMetric : Log.Measurement = { stream = pos.streamName; interval = t; bytes = bytes; count = count; ru = ru } let evt = Log.Slice (direction, reqMetric) let log = if (not << log.IsEnabled) Events.LogEventLevel.Debug then log else log |> Log.propResolvedEvents "Json" slice @@ -356,8 +384,6 @@ module private Read = yield! loop (batchCount + 1) } loop 0 - let bytes events = events |> Array.sumBy (|EventLen|) - let logBatchRead direction streamName interval events batchSize version (ru: float) (log : ILogger) = let bytes, count = bytes events, events.Length let reqMetric : Log.Measurement = { stream = streamName; interval = interval; bytes = bytes; count = count; ru = ru } @@ -373,7 +399,7 @@ module private Read = | None -> -1L | Some last -> int64 last.id - let loadForwardsFrom (log : ILogger) retryPolicy client batchSize maxPermittedBatchReads (pos,_strongConsistency): Async = async { + let loadForwardsFrom (log : ILogger) retryPolicy client batchSize maxPermittedBatchReads (pos): Async = async { let mutable ru = 0.0 let mergeBatches (batches: AsyncSeq) = async { let! (events : Store.Event[]) = @@ -469,6 +495,9 @@ module Token = let ofPreviousTokenAndEventsLength (previousToken : Storage.StreamToken) eventsLength batchSize pos : Storage.StreamToken = let compactedEventNumber = (unbox previousToken.value).compactionEventNumber ofCompactionEventNumber compactedEventNumber eventsLength batchSize pos + let ofPreviousTokenWithUpdatedPosition (previousToken : Storage.StreamToken) batchSize pos : Storage.StreamToken = + let compactedEventNumber = (unbox previousToken.value).compactionEventNumber + ofCompactionEventNumber compactedEventNumber 0 batchSize pos /// Use an event just read from the stream to infer headroom let ofCompactionResolvedEventAndVersion (compactionEvent: Store.Event) batchSize pos : Storage.StreamToken = ofCompactionEventNumber (Some (int64 compactionEvent.id)) 0 batchSize pos @@ -492,14 +521,28 @@ type EqxBatchingPolicy(getMaxBatchSize : unit -> int, ?batchCountLimit) = member __.MaxBatches = batchCountLimit [] -type GatewaySyncResult = Written of Storage.StreamToken | Conflict +type GatewaySyncResult = Written of Storage.StreamToken | ConflictUnknown of Storage.StreamToken | Conflict of Storage.StreamToken * Store.IEventData[] + +[] +type LoadFromTokenResult = Unchanged | Found of Storage.StreamToken * Store.IEventData[] type EqxGateway(conn : EqxConnection, batching : EqxBatchingPolicy) = let isResolvedEventEventType predicate (x:Store.Event) = predicate x.t let tryIsResolvedEventEventType predicateOption = predicateOption |> Option.map isResolvedEventEventType + //let isResolvedEventDataEventType predicate (x:Store.Event) = predicate x.t + //let tryIsEventDataEventType predicateOption = predicateOption |> Option.map isResolvedEventDataEventType let (|Pos|) (token: Storage.StreamToken) : Store.Position = (unbox token.value).pos + let (|IEventDataArray|) events = [| for e in events -> e :> Store.IEventData |] + member private __.InterpretIndexOrFallback log isCompactionEventType pos res: Async = async { + match res with + | Read.IndexResult.Found (pos, projectionsAndEvents) when projectionsAndEvents |> Array.exists (fun x -> isCompactionEventType x.t) -> + return Token.ofNonCompacting pos, projectionsAndEvents |> Seq.cast |> Array.ofSeq + | Read.IndexResult.Unchanged -> return invalidOp "Not handled" + | _ -> + let! streamToken, events = __.LoadBackwardsStoppingAtCompactionEvent log isCompactionEventType pos + return streamToken, events |> Seq.cast |> Array.ofSeq } member __.LoadBatched log isCompactionEventType (pos : Store.Position): Async = async { - let! pos, events = Read.loadForwardsFrom log conn.ReadRetryPolicy conn.Client batching.BatchSize batching.MaxBatches (pos,false) + let! pos, events = Read.loadForwardsFrom log conn.ReadRetryPolicy conn.Client batching.BatchSize batching.MaxBatches pos match tryIsResolvedEventEventType isCompactionEventType with | None -> return Token.ofNonCompacting pos, events | Some isCompactionEvent -> @@ -515,36 +558,44 @@ type EqxGateway(conn : EqxConnection, batching : EqxBatchingPolicy) = | Some resolvedEvent -> return Token.ofCompactionResolvedEventAndVersion resolvedEvent batching.BatchSize pos, events } member __.IndexedOrBatched log isCompactionEventType pos: Async = async { let! res = Read.loadIndex log None(* TODO conn.ReadRetryPolicy*) conn.Client pos - match res with - | Read.IndexResult.Found (pos, projectionsAndEvents) when projectionsAndEvents |> Array.exists (fun x -> isCompactionEventType x.t) -> - return Token.ofNonCompacting pos, projectionsAndEvents |> Seq.cast |> Array.ofSeq - | _ -> - let! streamToken, events = __.LoadBackwardsStoppingAtCompactionEvent log isCompactionEventType pos - return streamToken, events |> Seq.cast |> Array.ofSeq } - member __.LoadFromToken log (Pos pos as token) isCompactionEventType synchronized: Async = async { - let! pos, events = Read.loadForwardsFrom log conn.ReadRetryPolicy conn.Client batching.BatchSize batching.MaxBatches (pos,synchronized) - match tryIsResolvedEventEventType isCompactionEventType with - | None -> return Token.ofNonCompacting pos, events - | Some isCompactionEvent -> - match events |> Array.tryFindBack isCompactionEvent with - | None -> return Token.ofPreviousTokenAndEventsLength token events.Length batching.BatchSize pos, events - | Some resolvedEvent -> return Token.ofCompactionResolvedEventAndVersion resolvedEvent batching.BatchSize pos, events } + return! __.InterpretIndexOrFallback log isCompactionEventType pos res } + member __.LoadFromToken log (Pos pos as token) isCompactionEventType tryIndex: Async = async { + let ok r = LoadFromTokenResult.Found r + if not tryIndex then + let! pos, ((IEventDataArray xs) as events) = Read.loadForwardsFrom log conn.ReadRetryPolicy conn.Client batching.BatchSize batching.MaxBatches pos + let ok t = ok (t,xs) + match tryIsResolvedEventEventType isCompactionEventType with + | None -> return ok (Token.ofNonCompacting pos) + | Some isCompactionEvent -> + match events |> Array.tryFindBack isCompactionEvent with + | None -> return ok (Token.ofPreviousTokenAndEventsLength token events.Length batching.BatchSize pos) + | Some resolvedEvent -> return ok (Token.ofCompactionResolvedEventAndVersion resolvedEvent batching.BatchSize pos) + else + let! res = Read.loadIndex log None(* TODO conn.ReadRetryPolicy*) conn.Client pos + match res with + | Read.IndexResult.Unchanged -> + return LoadFromTokenResult.Unchanged + | _ -> + let! loaded = __.InterpretIndexOrFallback log isCompactionEventType.Value pos res + return ok loaded } member __.TrySync log (Pos pos as token) (encodedEvents: Store.EventData[],maybeIndexEvents) isCompactionEventType: Async = async { let! wr = Write.writeEvents log conn.WriteRetryPolicy conn.Client pos (encodedEvents,maybeIndexEvents) match wr with - | EqxSyncResult.Conflict _ -> return GatewaySyncResult.Conflict - | EqxSyncResult.Written (wr, _) -> + | EqxSyncResult.Conflict (pos',events) -> + return GatewaySyncResult.Conflict (Token.ofPreviousTokenAndEventsLength token events.Length batching.BatchSize pos',events) + | EqxSyncResult.ConflictUnknown pos' -> + return GatewaySyncResult.ConflictUnknown (Token.ofPreviousTokenWithUpdatedPosition token batching.BatchSize pos') + | EqxSyncResult.Written pos' -> - let version' = wr let token = match isCompactionEventType with - | None -> Token.ofNonCompacting version' + | None -> Token.ofNonCompacting pos' | Some isCompactionEvent -> let isEventDataEventType predicate (x:Store.EventData) = predicate x.eventType match encodedEvents |> Array.tryFindIndexBack (isEventDataEventType isCompactionEvent) with - | None -> Token.ofPreviousTokenAndEventsLength token encodedEvents.Length batching.BatchSize version' + | None -> Token.ofPreviousTokenAndEventsLength token encodedEvents.Length batching.BatchSize pos' | Some compactionEventIndex -> - Token.ofPreviousStreamVersionAndCompactionEventDataIndex pos.Index compactionEventIndex encodedEvents.Length batching.BatchSize version' + Token.ofPreviousStreamVersionAndCompactionEventDataIndex pos.Index compactionEventIndex encodedEvents.Length batching.BatchSize pos' return GatewaySyncResult.Written token } type private Collection(gateway : EqxGateway, databaseId, collectionId) = @@ -568,25 +619,22 @@ type private CompactionContext(eventsLen : int, capacityBeforeCompaction : int) member __.IsCompactionDue = eventsLen > capacityBeforeCompaction type private Category<'event, 'state>(coll : Collection, codec : UnionCodec.IUnionEncoder<'event, byte[]>, ?access : AccessStrategy<'event,'state>) = - let (|Pos|) streamName : Store.Position = { collectionUri = coll.CollectionUri; streamName = streamName; index = None } + let (|Pos|) streamName : Store.Position = { collectionUri = coll.CollectionUri; streamName = streamName; index = None; etag = None } let compactionPredicate = match access with - | Some (AccessStrategy.IndexedSearch _) | None -> None + | Some (AccessStrategy.IndexedSearch (predicate,_)) -> Some predicate | Some AccessStrategy.EventsAreState -> Some (fun _ -> true) | Some (AccessStrategy.RollingSnapshots (et,_)) -> Some ((=) et) - //let searchPredicate = - // match access with - // | None -> None - // | Some AccessStrategy.EventsAreState -> Some (SearchStrategy.Predicate (fun _ -> true)) - // | Some (AccessStrategy.IndexedSearch (ep,_)) -> Some (SearchStrategy.Predicate ep) let load (fold: 'state -> 'event seq -> 'state) initial loadF = async { let! token, events = loadF return token, fold initial (UnionEncoderAdapters.decodeKnownEvents codec events) } + let foldI (fold: 'state -> 'event seq -> 'state) initial token events = + token, fold initial (UnionEncoderAdapters.decodeKnownEventsI codec events) let loadI (fold: 'state -> 'event seq -> 'state) initial loadF = async { let! token, events = loadF - return token, fold initial (UnionEncoderAdapters.decodeKnownEventsI codec events) } - let loadAlgorithm fold (Pos pos) initial log = + return foldI fold initial token events } + let loadAlgorithm fold (pos : Store.Position) initial log = let batched = load fold initial (coll.Gateway.LoadBatched log None pos) let compacted predicate = load fold initial (coll.Gateway.LoadBackwardsStoppingAtCompactionEvent log predicate pos) let indexed predicate = loadI fold initial (coll.Gateway.IndexedOrBatched log predicate pos) @@ -595,11 +643,16 @@ type private Category<'event, 'state>(coll : Collection, codec : UnionCodec.IUni | None -> batched | Some AccessStrategy.EventsAreState -> compacted (fun _ -> true) | Some (AccessStrategy.RollingSnapshots (et,_)) -> compacted ((=) et) - member __.Load (fold: 'state -> 'event seq -> 'state) (initial: 'state) streamName (log : ILogger) : Async = - loadAlgorithm fold streamName initial log - member __.LoadFromToken (fold: 'state -> 'event seq -> 'state) (state: 'state) token (log : ILogger) : Async = - (load fold) state (coll.Gateway.LoadFromToken log token compactionPredicate false) - member __.TrySync (fold: 'state -> 'event seq -> 'state) (log : ILogger) + member __.Load (fold: 'state -> 'event seq -> 'state) (initial: 'state) (Pos pos) (log : ILogger) : Async = + loadAlgorithm fold pos initial log + member __.LoadFromToken (fold: 'state -> 'event seq -> 'state) (initial: 'state) (state: 'state) token (log : ILogger) + : Async = async { + let indexed = match access with Some (AccessStrategy.IndexedSearch _) -> true | _ -> false + let! res = coll.Gateway.LoadFromToken log token compactionPredicate indexed + match res with + | LoadFromTokenResult.Unchanged -> return token, state + | LoadFromTokenResult.Found (token,events ) -> return foldI fold initial token events } + member __.TrySync (fold: 'state -> 'event seq -> 'state) initial (log : ILogger) (token : Storage.StreamToken, state : 'state) (events : 'event list, state' : 'state) : Async> = async { let events, index = @@ -615,8 +668,9 @@ type private Category<'event, 'state>(coll : Collection, codec : UnionCodec.IUni let maybeIndexEvents : Store.EventData[] option = index |> Option.map (UnionEncoderAdapters.encodeEvents codec) let! syncRes = coll.Gateway.TrySync log token (encodedEvents,maybeIndexEvents) compactionPredicate match syncRes with - | GatewaySyncResult.Conflict -> return Storage.SyncResult.Conflict (load fold state (coll.Gateway.LoadFromToken log token compactionPredicate true)) - | GatewaySyncResult.Written token' -> return Storage.SyncResult.Written (token', fold state (Seq.ofList events)) } + | GatewaySyncResult.Conflict (token',events) -> return Storage.SyncResult.Conflict (async { return foldI fold initial token' events }) + | GatewaySyncResult.ConflictUnknown token' -> return Storage.SyncResult.Conflict (__.LoadFromToken fold initial state token' log) + | GatewaySyncResult.Written token' -> return Storage.SyncResult.Written (token', fold state (Seq.ofList events)) } module Caching = open System.Runtime.Caching @@ -664,7 +718,7 @@ module Caching = let! syncRes = inner.TrySync streamName log (token, state) (events,state') match syncRes with | Storage.SyncResult.Conflict resync -> return Storage.SyncResult.Conflict (interceptAsync resync streamName) - | Storage.SyncResult.Written (token', state') -> return Storage.SyncResult.Written (token', state') } + | Storage.SyncResult.Written (token', state') -> return Storage.SyncResult.Written (intercept streamName (token', state')) } let applyCacheUpdatesWithSlidingExpiration (cache: Cache) @@ -679,7 +733,7 @@ module Caching = type private Folder<'event, 'state>(category : Category<'event, 'state>, fold: 'state -> 'event seq -> 'state, initial: 'state, ?readCache) = let loadAlgorithm streamName initial log = let batched = category.Load fold initial streamName log - let cached token state = category.LoadFromToken fold state token log + let cached token state = category.LoadFromToken fold initial state token log match readCache with | None -> batched | Some (cache : Caching.Cache, prefix : string) -> @@ -690,7 +744,7 @@ type private Folder<'event, 'state>(category : Category<'event, 'state>, fold: ' member __.Load (streamName : string) (log : ILogger) : Async = loadAlgorithm streamName initial log member __.TrySync _streamName(* TODO remove from main interface *) (log : ILogger) (token, state) (events : 'event list, state': 'state) : Async> = async { - let! syncRes = category.TrySync fold log (token, state) (events,state') + let! syncRes = category.TrySync fold initial log (token, state) (events,state') match syncRes with | Storage.SyncResult.Conflict resync -> return Storage.SyncResult.Conflict resync | Storage.SyncResult.Written (token',state') -> return Storage.SyncResult.Written (token',state') } @@ -701,9 +755,9 @@ type CachingStrategy = /// Prefix is used to distinguish multiple folds per stream | SlidingWindowPrefixed of Caching.Cache * window: TimeSpan * prefix: string -type EqxStreamBuilder<'event, 'state>(gateway : EqxGateway, codec, fold, initial, ?compaction, ?caching) = +type EqxStreamBuilder<'event, 'state>(gateway : EqxGateway, codec, fold, initial, ?access, ?caching) = member __.Create (databaseId, collectionId, streamName) : Equinox.IStream<'event, 'state> = - let category = Category<'event, 'state>(Collection(gateway, databaseId, collectionId), codec, ?access = compaction) + let category = Category<'event, 'state>(Collection(gateway, databaseId, collectionId), codec, ?access = access) let readCacheOption = match caching with @@ -744,25 +798,31 @@ module Initialization = return coll.Resource.Id } let createProc (client: IDocumentClient) (collectionUri: Uri) = async { - let f = """function multidocInsert(docs,expectedVersion,index) { + let f = """function multidocInsert(docs, expectedVersion, etag, index) { var response = getContext().getResponse(); var collection = getContext().getCollection(); var collectionLink = collection.getSelfLink(); if (!docs) throw new Error("docs argument is missing."); - for (var i=0; i Async.AwaitTaskCorrect |> Async.Ignore } @@ -840,7 +900,7 @@ type EqxConnector match tags with None -> () | Some tags -> for key, value in tags do yield sprintf "%s=%s" key value } let sanitizedName = name.Replace('\'','_').Replace(':','_') // sic; Align with logging for ES Adapter let client = new Client.DocumentClient(uri, key, connPolicy, Nullable(defaultArg defaultConsistencyLevel ConsistencyLevel.Session)) - log.Information("Eqx connecting to Cosmos with Connection Name {connectionName}", sanitizedName) + log.ForContext("Uri", uri).Information("Eqx connecting to Cosmos with Connection Name {connectionName}", sanitizedName) do! client.OpenAsync() |> Async.AwaitTaskCorrect return client :> IDocumentClient } diff --git a/tests/Equinox.Cosmos.Integration/CosmosFixturesInfrastructure.fs b/tests/Equinox.Cosmos.Integration/CosmosFixturesInfrastructure.fs index 48dd68c50..1c6be32c3 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosFixturesInfrastructure.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosFixturesInfrastructure.fs @@ -49,7 +49,7 @@ module SerilogHelpers = | (:? ScalarValue as x) -> Some x.Value | _ -> None [] - type EqxAct = Append | AppendConflict | SliceForward | SliceBackward | BatchForward | BatchBackward | Indexed + type EqxAct = Append | AppendConflict | SliceForward | SliceBackward | BatchForward | BatchBackward | Indexed | IndexedNotFound | IndexedCached let (|EqxAction|) (evt : Equinox.Cosmos.Log.Event) = match evt with | Equinox.Cosmos.Log.WriteSuccess _ -> EqxAct.Append @@ -59,6 +59,8 @@ module SerilogHelpers = | Equinox.Cosmos.Log.Batch (Equinox.Cosmos.Direction.Forward,_,_) -> EqxAct.BatchForward | Equinox.Cosmos.Log.Batch (Equinox.Cosmos.Direction.Backward,_,_) -> EqxAct.BatchBackward | Equinox.Cosmos.Log.Index _ -> EqxAct.Indexed + | Equinox.Cosmos.Log.IndexNotFound _ -> EqxAct.IndexedNotFound + | Equinox.Cosmos.Log.IndexCached _ -> EqxAct.IndexedCached let (|EqxEvent|_|) (logEvent : LogEvent) : Equinox.Cosmos.Log.Event option = logEvent.Properties.Values |> Seq.tryPick (function | SerilogScalar (:? Equinox.Cosmos.Log.Event as e) -> Some e diff --git a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs index ce582bb79..c8d5d38d7 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs @@ -4,6 +4,7 @@ open Equinox.Cosmos.Integration.Infrastructure open Equinox.Cosmos open Swensen.Unquote open System.Threading +open Serilog open System let serializationSettings = Newtonsoft.Json.Converters.FSharp.Settings.CreateCorrect() @@ -28,11 +29,15 @@ module Cart = let sliding20m = CachingStrategy.SlidingWindow (cache, TimeSpan.FromMinutes 20.) let resolveStream (StreamArgs args) = EqxStreamBuilder(gateway, codec, fold, initial, caching = sliding20m).Create(args) Backend.Cart.Service(log, resolveStream) - let createServiceWithCachingIndexed connection batchSize log cache = + let createServiceIndexed connection batchSize log = let gateway = createEqxGateway connection batchSize - let _sliding20m = CachingStrategy.SlidingWindow (cache, TimeSpan.FromMinutes 20.) // TODO let resolveStream (StreamArgs args) = EqxStreamBuilder(gateway, codec, fold, initial, AccessStrategy.IndexedSearch index).Create(args) Backend.Cart.Service(log, resolveStream) + let createServiceWithCachingIndexed connection batchSize log cache = + let gateway = createEqxGateway connection batchSize + let sliding20m = CachingStrategy.SlidingWindow (cache, TimeSpan.FromMinutes 20.) + let resolveStream (StreamArgs args) = EqxStreamBuilder(gateway, codec, fold, initial, AccessStrategy.IndexedSearch index, caching=sliding20m).Create(args) + Backend.Cart.Service(log, resolveStream) let createServiceWithCompactionAndCaching connection batchSize log cache = let gateway = createEqxGateway connection batchSize let sliding20m = CachingStrategy.SlidingWindow (cache, TimeSpan.FromMinutes 20.) @@ -70,6 +75,7 @@ type Tests(testOutputHelper) = let capture = LogCaptureBuffer() let logger = Serilog.LoggerConfiguration() + .WriteTo.Seq("http://localhost:5341") .WriteTo.Sink(testOutput) .WriteTo.Sink(capture) .CreateLogger() @@ -270,11 +276,10 @@ type Tests(testOutputHelper) = test <@ singleBatchForward = capture.ExternalCalls @> } - let singleIndexed = [EqxAct.Indexed] - let indexedReadAndAppend = singleIndexed @ [EqxAct.Append] + let primeIndex = [EqxAct.IndexedNotFound; EqxAct.SliceBackward; EqxAct.BatchBackward] [] - let ``Can roundtrip against Cosmos, correctly using the index to avoid redundant reads`` context skuId cartId = Async.RunSynchronously <| async { + let ``Can roundtrip against Cosmos, correctly using the index and cache to avoid redundant reads`` context skuId cartId = Async.RunSynchronously <| async { let log, capture = createLoggerWithCapture () let! conn = connectToSpecifiedCosmosOrSimulator log let batchSize = 10 @@ -286,18 +291,44 @@ type Tests(testOutputHelper) = do! addAndThenRemoveItemsManyTimes context cartId skuId service1 5 let! _ = service2.Read cartId + // ... should see a single read (with a) we are writes are cached + test <@ primeIndex @ [EqxAct.Append; EqxAct.IndexedCached] = capture.ExternalCalls @> + + // Add two more - the roundtrip should only incur a single read + capture.Clear() + do! addAndThenRemoveItemsManyTimes context cartId skuId service1 1 + test <@ [EqxAct.IndexedCached; EqxAct.Append] = capture.ExternalCalls @> + + // While we now have 12 events, we should be able to read them with a single call + capture.Clear() + let! _ = service2.Read cartId + test <@ [EqxAct.IndexedCached] = capture.ExternalCalls @> + } + + [] + let ``Can roundtrip against Cosmos, correctly using the index to avoid redundant reads`` context skuId cartId = Async.RunSynchronously <| async { + let log, capture = createLoggerWithCapture () + let! conn = connectToSpecifiedCosmosOrSimulator log + let batchSize = 10 + let createServiceIndexed () = Cart.createServiceIndexed conn batchSize log + let service1, service2 = createServiceIndexed (), createServiceIndexed () + + // Trigger 10 events, then reload + do! addAndThenRemoveItemsManyTimes context cartId skuId service1 5 + let! _ = service2.Read cartId + // ... should see a single read as we are writes are cached - test <@ indexedReadAndAppend @ singleIndexed = capture.ExternalCalls @> + test <@ primeIndex @ [EqxAct.Append; EqxAct.Indexed] = capture.ExternalCalls @> // Add two more - the roundtrip should only incur a single read capture.Clear() do! addAndThenRemoveItemsManyTimes context cartId skuId service1 1 - test <@ indexedReadAndAppend = capture.ExternalCalls @> + test <@ [EqxAct.Indexed; EqxAct.Append] = capture.ExternalCalls @> // While we now have 12 events, we should be able to read them with a single call capture.Clear() let! _ = service2.Read cartId - test <@ singleIndexed = capture.ExternalCalls @> + test <@ [EqxAct.Indexed] = capture.ExternalCalls @> } [] From 22d33620cc0e307ea1dec96520841e477c679a91 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 13 Nov 2018 16:43:03 +0000 Subject: [PATCH 36/66] Even more failure --- src/Equinox.Cosmos/Cosmos.fs | 2 +- tests/Equinox.Cosmos.Integration/CosmosIntegration.fs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Equinox.Cosmos/Cosmos.fs b/src/Equinox.Cosmos/Cosmos.fs index 46b1021e5..bdee6d557 100644 --- a/src/Equinox.Cosmos/Cosmos.fs +++ b/src/Equinox.Cosmos/Cosmos.fs @@ -812,7 +812,7 @@ module Initialization = if (-1 == expectedVersion) { collection.createDocument(collectionLink, index, { disableAutomaticIdGeneration : true}, callback); } else { - collection.replaceDocument(collection.getAltLink() + "/docs/" + index.id, index, { etag : etag }, callback); + collection.replaceDocument(collection.getAltLink() + "/docs/" + index.id, index, callback); } // can also contain { conflicts: [{t, d}] } representing conflicting events since expectedVersion // null/missing signifies events have been written, with no conflict diff --git a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs index c8d5d38d7..7d804f08d 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs @@ -292,7 +292,8 @@ type Tests(testOutputHelper) = let! _ = service2.Read cartId // ... should see a single read (with a) we are writes are cached - test <@ primeIndex @ [EqxAct.Append; EqxAct.IndexedCached] = capture.ExternalCalls @> + let! _ = service2.Read cartId + test <@ primeIndex @ [EqxAct.Append; EqxAct.IndexedCached; EqxAct.IndexedCached] = capture.ExternalCalls @> // Add two more - the roundtrip should only incur a single read capture.Clear() From f3b0a52cb762479ae8d4b50583729cbefa9ad38e Mon Sep 17 00:00:00 2001 From: Ming Luo Date: Tue, 13 Nov 2018 16:43:26 -0500 Subject: [PATCH 37/66] etag change return the etag of the created/replaced snapshot document, so we can have cache feature to insure the point reading snapshot only takes 1 RU. --- src/StoredProcedure.js | 44 ++++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/src/StoredProcedure.js b/src/StoredProcedure.js index ae86473b9..3574724be 100644 --- a/src/StoredProcedure.js +++ b/src/StoredProcedure.js @@ -1,12 +1,10 @@ - -function write (partitionkey, events, expectedVersion, pendingEvents, projections) { +function write2 (partitionkey, events, expectedVersion, pendingEvents, projections) { if (events === undefined || events==null) events = []; if (expectedVersion === undefined) expectedVersion = -2; if (pendingEvents === undefined) pendingEvents = null; if (projections === undefined || projections==null) projections = {}; - var response = getContext().getResponse(); var collection = getContext().getCollection(); var collectionLink = collection.getSelfLink(); @@ -15,11 +13,11 @@ function write (partitionkey, events, expectedVersion, pendingEvents, projection // Recursively queries for a document by id w/ support for continuation tokens. // Calls tryUpdate(document) as soon as the query returns a document. function tryQueryAndUpdate(continuation) { - var query = {query: "select * from root r where r.id = @id and r.p = @p", parameters: [{name: "@id", value: "-1"},{name: "@p", value: partitionkey}]}; + var query = {query: "select * from root r where r.id = @id and r.k = @k", parameters: [{name: "@id", value: "-1"},{name: "@p", value: partitionkey}]}; var requestOptions = {continuation: continuation}; var isAccepted = collection.queryDocuments(collectionLink, query, requestOptions, function (err, documents, responseOptions) { - if (err) throw err; + if (err) throw new Error("Error" + err.message); if (documents.length > 0) { // If the document is found, update it. @@ -46,10 +44,13 @@ function write (partitionkey, events, expectedVersion, pendingEvents, projection function insertEvents() { for (i=0; i Date: Wed, 14 Nov 2018 04:06:50 +0000 Subject: [PATCH 38/66] Complete handling for caching of index reads --- samples/Store/Integration/LogIntegration.fs | 4 +- src/Equinox.Cosmos/Cosmos.fs | 68 +++++++++---------- .../CosmosFixturesInfrastructure.fs | 2 +- .../CosmosIntegration.fs | 14 ++-- 4 files changed, 44 insertions(+), 44 deletions(-) diff --git a/samples/Store/Integration/LogIntegration.fs b/samples/Store/Integration/LogIntegration.fs index 15b6e4537..06a1e10a0 100644 --- a/samples/Store/Integration/LogIntegration.fs +++ b/samples/Store/Integration/LogIntegration.fs @@ -37,8 +37,8 @@ module EquinoxCosmosInterop = | Log.Batch (Direction.Forward,c,m) -> "EqxLoadF", m, Some c, m.ru | Log.Batch (Direction.Backward,c,m) -> "EqxLoadB", m, Some c, m.ru | Log.Index m -> "EqxLoadI", m, None, m.ru - | Log.IndexNotFound m -> "EqxLoadIN", m, None, m.ru - | Log.IndexCached m -> "EqxLoadIC", m, None, m.ru + | Log.IndexNotFound m -> "EqxLoadI404", m, None, m.ru + | Log.IndexNotModified m -> "EqxLoadI302", m, None, m.ru { action = action; stream = metric.stream; bytes = metric.bytes; count = metric.count; batches = batches interval = StopwatchInterval(metric.interval.StartTicks,metric.interval.EndTicks); ru = ru } diff --git a/src/Equinox.Cosmos/Cosmos.fs b/src/Equinox.Cosmos/Cosmos.fs index bdee6d557..4a38f1ec5 100644 --- a/src/Equinox.Cosmos/Cosmos.fs +++ b/src/Equinox.Cosmos/Cosmos.fs @@ -13,18 +13,6 @@ open System [] module DocDbExtensions = - type Client.RequestOptions with - /// Simplified ETag precondition builder - member options.ETag - with get () = - match options.AccessCondition with - | null -> null - | ac -> ac.Condition - - and set etag = - if String.IsNullOrEmpty etag then () else - options.AccessCondition <- Client.AccessCondition(Type = Client.AccessConditionType.IfMatch, Condition = etag) - /// Extracts the innermost exception from a nested hierarchy of Aggregate Exceptions let (|AggregateException|) (exn : exn) = let rec aux (e : exn) = @@ -34,31 +22,35 @@ module DocDbExtensions = | _ -> e aux exn - /// DocumentDB Error HttpStatusCode extractor let (|DocDbException|_|) (e : exn) = match e with | AggregateException (:? DocumentClientException as dce) -> Some dce | _ -> None /// Map Nullable to Option - let (|HasValue|_|) (x:Nullable<_>) = if x.HasValue then Some x.Value else None + let (|HasValue|Null|) (x:Nullable<_>) = + if x.HasValue then HasValue x.Value + else Null /// DocumentDB Error HttpStatusCode extractor let (|DocDbStatusCode|_|) (e : DocumentClientException) = match e.StatusCode with | HasValue x -> Some x - | _ -> None + | Null -> None - type ReadResult<'T> = Found of 'T | NotFound | PreconditionFailed + type ReadResult<'T> = Found of 'T | NotFound | NotModified type DocDbCollection(client : IDocumentClient, collectionUri) = member __.TryReadDocument(documentId : string, ?options : Client.RequestOptions): Async> = async { let! ct = Async.CancellationToken let options = defaultArg options null let docLink = sprintf "%O/docs/%s" collectionUri documentId try let! document = async { return! client.ReadDocumentAsync<'T>(docLink, options = options, cancellationToken = ct) |> Async.AwaitTaskCorrect } - return document.RequestCharge, Found document.Document + if document.StatusCode = System.Net.HttpStatusCode.NotModified then return document.RequestCharge, NotModified + // NB `.Document` will NRE if a IfNoneModified precondition triggers a NotModified result + else return document.RequestCharge, Found document.Document with | DocDbException (DocDbStatusCode System.Net.HttpStatusCode.NotFound as e) -> return e.RequestCharge, NotFound - | DocDbException (DocDbStatusCode System.Net.HttpStatusCode.PreconditionFailed as e) -> return e.RequestCharge, PreconditionFailed } + // NB while the docs suggest you may see a 412, the NotModified in the body of the try/with is actually what happens + | DocDbException (DocDbStatusCode System.Net.HttpStatusCode.PreconditionFailed as e) -> return e.RequestCharge, NotModified } module Store = [] @@ -144,6 +136,10 @@ module Store = { p: string // "{streamName}" id: string // "{-1}" + /// When we read, we need to capture the value so we can retain it for caching purposes; when we write, there's no point sending it as it would not be honored + [] + _etag: string + //w: int64 // 100: window size /// last index/i value m: int64 // {index} @@ -167,7 +163,7 @@ module Store = } static member IdConstant = "-1" static member Create (pos: Position) eventCount (eds: EventData[]) : IndexEvent = - { p = pos.streamName; id = IndexEvent.IdConstant; m = pos.IndexRel eventCount + { p = pos.streamName; id = IndexEvent.IdConstant; m = pos.IndexRel eventCount; _etag = null c = [| for ed in eds -> { t = ed.eventType; d = ed.data; m = ed.metadata } |] } and IndexProjection = { /// The Event Type, used to drive deserialization @@ -217,7 +213,7 @@ module Log = /// Individual read request for the Index, not found | IndexNotFound of Measurement /// Index read with Single RU Request Charge due to correct use of etag in cache - | IndexCached of Measurement + | IndexNotModified of Measurement /// Summarizes a set of Slices read together | Batch of Direction * slices: int * Measurement let prop name value (log : ILogger) = log.ForContext(name, value) @@ -320,27 +316,27 @@ module private Read = let! t, (ru, res : ReadResult) = getIndex pos |> Stopwatch.Time let log count bytes (f : Log.Measurement -> _) = log |> Log.event (f { stream = pos.streamName; interval = t; bytes = bytes; count = count; ru = ru }) match res with - | ReadResult.PreconditionFailed -> - (log 0 0 Log.IndexCached).Information("Eqx {action:l} {ms}ms rc={ru}", "IndexCached", (let e = t.Elapsed in e.TotalMilliseconds), ru) + | ReadResult.NotModified -> + (log 0 0 Log.IndexNotModified).Information("Eqx {action:l} {res} {ms}ms rc={ru}", "Index", 302, (let e = t.Elapsed in e.TotalMilliseconds), ru) | ReadResult.NotFound -> - (log 0 0 Log.IndexNotFound).Information("Eqx {action:l} {ms}ms rc={ru}", "IndexNotFound", (let e = t.Elapsed in e.TotalMilliseconds), ru) + (log 0 0 Log.IndexNotFound).Information("Eqx {action:l} {res} {ms}ms rc={ru}", "Index", 404, (let e = t.Elapsed in e.TotalMilliseconds), ru) | ReadResult.Found doc -> let log = let (|EventLen|) (x : Store.IndexProjection) = match x.d, x.m with Log.BlobLen bytes, Log.BlobLen metaBytes -> bytes + metaBytes let bytes, count = doc.c |> Array.sumBy (|EventLen|), doc.c.Length log bytes count Log.Index - let log = if (not << log.IsEnabled) Events.LogEventLevel.Debug then log else log |> Log.propProjectionEvents "Json" doc.c - log.Information("Eqx {action:l} {ms}ms rc={ru}", "Index", (let e = t.Elapsed in e.TotalMilliseconds), ru) + let log = if (not << log.IsEnabled) Events.LogEventLevel.Debug then log else log |> Log.propProjectionEvents "Json" doc.c |> Log.prop "etag" doc._etag + log.Information("Eqx {action:l} {res} {ms}ms rc={ru}", "Index", 200, (let e = t.Elapsed in e.TotalMilliseconds), ru) return ru, res } - type [] IndexResult = Unchanged | NotFound | Found of Store.Position * Store.IndexProjection[] + type [] IndexResult = NotModified | NotFound | Found of Store.Position * Store.IndexProjection[] /// `pos` being Some implies that the caller holds a cached value and hence is ready to deal with IndexResult.UnChanged let loadIndex (log : ILogger) retryPolicy client (pos : Store.Position): Async = async { let getIndex = getIndex client let! _rc, res = Log.withLoggedRetries retryPolicy "readAttempt" (loggedGetIndex getIndex pos) log match res with - | ReadResult.PreconditionFailed -> return IndexResult.Unchanged + | ReadResult.NotModified -> return IndexResult.NotModified | ReadResult.NotFound -> return IndexResult.NotFound - | ReadResult.Found index -> return IndexResult.Found ({ pos with index = Some index.m }, index.c) } + | ReadResult.Found index -> return IndexResult.Found ({ pos with index = Some index.m; etag=if index._etag=null then None else Some index._etag }, index.c) } let private getQuery (client : IDocumentClient) (pos:Store.Position) (direction: Direction) batchSize = let querySpec = @@ -505,9 +501,11 @@ module Token = let ofPreviousStreamVersionAndCompactionEventDataIndex prevStreamVersion compactionEventDataIndex eventsLength batchSize streamVersion' : Storage.StreamToken = ofCompactionEventNumber (Some (prevStreamVersion + 1L + int64 compactionEventDataIndex)) eventsLength batchSize streamVersion' let private unpackEqxStreamVersion (x : Storage.StreamToken) = let x : Token = unbox x.value in x.pos.Index + let private unpackEqxETag (x : Storage.StreamToken) = let x : Token = unbox x.value in x.pos.etag let supersedes current x = let currentVersion, newVersion = unpackEqxStreamVersion current, unpackEqxStreamVersion x - newVersion > currentVersion + let currentETag, newETag = unpackEqxETag current, unpackEqxETag x + newVersion > currentVersion || currentETag <> newETag type EqxConnection(client: IDocumentClient, ?readRetryPolicy (*: (int -> Async<'T>) -> Async<'T>*), ?writeRetryPolicy) = member __.Client = client @@ -535,9 +533,9 @@ type EqxGateway(conn : EqxConnection, batching : EqxBatchingPolicy) = let (|IEventDataArray|) events = [| for e in events -> e :> Store.IEventData |] member private __.InterpretIndexOrFallback log isCompactionEventType pos res: Async = async { match res with + | Read.IndexResult.NotModified -> return invalidOp "Not handled" | Read.IndexResult.Found (pos, projectionsAndEvents) when projectionsAndEvents |> Array.exists (fun x -> isCompactionEventType x.t) -> return Token.ofNonCompacting pos, projectionsAndEvents |> Seq.cast |> Array.ofSeq - | Read.IndexResult.Unchanged -> return invalidOp "Not handled" | _ -> let! streamToken, events = __.LoadBackwardsStoppingAtCompactionEvent log isCompactionEventType pos return streamToken, events |> Seq.cast |> Array.ofSeq } @@ -573,7 +571,7 @@ type EqxGateway(conn : EqxConnection, batching : EqxBatchingPolicy) = else let! res = Read.loadIndex log None(* TODO conn.ReadRetryPolicy*) conn.Client pos match res with - | Read.IndexResult.Unchanged -> + | Read.IndexResult.NotModified -> return LoadFromTokenResult.Unchanged | _ -> let! loaded = __.InterpretIndexOrFallback log isCompactionEventType.Value pos res @@ -803,21 +801,19 @@ module Initialization = var collection = getContext().getCollection(); var collectionLink = collection.getSelfLink(); if (!docs) throw new Error("docs argument is missing."); - if(index) { - var assignedEtag = null; + if (index) { function callback(err, doc, options) { if (err) throw err; - assignedEtag = doc._etag } if (-1 == expectedVersion) { collection.createDocument(collectionLink, index, { disableAutomaticIdGeneration : true}, callback); } else { collection.replaceDocument(collection.getAltLink() + "/docs/" + index.id, index, callback); } + response.setBody({ etag: null, conflicts: null }); + } else { // can also contain { conflicts: [{t, d}] } representing conflicting events since expectedVersion // null/missing signifies events have been written, with no conflict - response.setBody({ etag: assignedEtag, conflicts: null }); - } else { response.setBody({ etag: null, conflicts: null }); } for (var i=0; i EqxAct.BatchBackward | Equinox.Cosmos.Log.Index _ -> EqxAct.Indexed | Equinox.Cosmos.Log.IndexNotFound _ -> EqxAct.IndexedNotFound - | Equinox.Cosmos.Log.IndexCached _ -> EqxAct.IndexedCached + | Equinox.Cosmos.Log.IndexNotModified _ -> EqxAct.IndexedCached let (|EqxEvent|_|) (logEvent : LogEvent) : Equinox.Cosmos.Log.Event option = logEvent.Properties.Values |> Seq.tryPick (function | SerilogScalar (:? Equinox.Cosmos.Log.Event as e) -> Some e diff --git a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs index 7d804f08d..88943ec0f 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs @@ -277,6 +277,8 @@ type Tests(testOutputHelper) = } let primeIndex = [EqxAct.IndexedNotFound; EqxAct.SliceBackward; EqxAct.BatchBackward] + // When the test gets re-run to simplify, the stream will typically already have values + let primeIndexRerun = [EqxAct.Indexed] [] let ``Can roundtrip against Cosmos, correctly using the index and cache to avoid redundant reads`` context skuId cartId = Async.RunSynchronously <| async { @@ -291,11 +293,11 @@ type Tests(testOutputHelper) = do! addAndThenRemoveItemsManyTimes context cartId skuId service1 5 let! _ = service2.Read cartId - // ... should see a single read (with a) we are writes are cached - let! _ = service2.Read cartId - test <@ primeIndex @ [EqxAct.Append; EqxAct.IndexedCached; EqxAct.IndexedCached] = capture.ExternalCalls @> + // ... should see a single Indexed read given writes are cached + test <@ primeIndex @ [EqxAct.Append; EqxAct.Indexed] = capture.ExternalCalls + || primeIndexRerun @ [EqxAct.Append; EqxAct.Indexed] = capture.ExternalCalls@> - // Add two more - the roundtrip should only incur a single read + // Add two more - the roundtrip should only incur a single read, which should be cached by virtue of being a second one in successono capture.Clear() do! addAndThenRemoveItemsManyTimes context cartId skuId service1 1 test <@ [EqxAct.IndexedCached; EqxAct.Append] = capture.ExternalCalls @> @@ -303,7 +305,9 @@ type Tests(testOutputHelper) = // While we now have 12 events, we should be able to read them with a single call capture.Clear() let! _ = service2.Read cartId - test <@ [EqxAct.IndexedCached] = capture.ExternalCalls @> + let! _ = service2.Read cartId + // First read is a re-read, second is cached + test <@ [EqxAct.Indexed;EqxAct.IndexedCached] = capture.ExternalCalls @> } [] From 0777bbd032d8cc8d9d0f918411985f3efd4530d7 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 14 Nov 2018 05:21:23 +0000 Subject: [PATCH 39/66] Correct caching of Writes --- src/Equinox.Cosmos/Cosmos.fs | 19 +++++++++++-------- .../CosmosIntegration.fs | 12 ++++++------ 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/src/Equinox.Cosmos/Cosmos.fs b/src/Equinox.Cosmos/Cosmos.fs index 4a38f1ec5..1d67f753a 100644 --- a/src/Equinox.Cosmos/Cosmos.fs +++ b/src/Equinox.Cosmos/Cosmos.fs @@ -251,13 +251,15 @@ type EqxSyncResult = | ConflictUnknown of Store.Position | Conflict of Store.Position * events: Store.IEventData[] +// NB don't nest in a private module, or serialization will fail miserably ;) +[] +type WriteResponse = { etag: string; conflicts: Store.IndexProjection[] } + module private Write = - [] - type WriteResponse = { etag: string; conflicts: Store.IndexProjection[] } - let [] sprocName = "AtomicMultiDocInsert" + let [] sprocName = "EquinoxIndexedWrite" let private writeEventsAsync (client: IDocumentClient) (pos: Store.Position) (events: Store.EventData seq,maybeIndexEvents): Async = async { - let sprocUri = sprintf "%O/sprocs/%s" pos.collectionUri sprocName + let sprocLink = sprintf "%O/sprocs/%s" pos.collectionUri sprocName let opts = Client.RequestOptions(PartitionKey=PartitionKey(pos.streamName)) let! ct = Async.CancellationToken let events = events |> Seq.mapi (fun i ed -> Store.Event.Create pos (i+1) ed |> JsonConvert.SerializeObject) |> Seq.toArray @@ -267,7 +269,7 @@ module private Write = | None | Some [||] -> Unchecked.defaultof<_> | Some eds -> Store.IndexEvent.Create pos (events.Length) eds try - let! (res : Client.StoredProcedureResponse) = client.ExecuteStoredProcedureAsync(sprocUri, opts, ct, box events, box pos.Index, box pos.etag, box index) |> Async.AwaitTaskCorrect + let! (res : Client.StoredProcedureResponse) = client.ExecuteStoredProcedureAsync(sprocLink, opts, ct, box events, box pos.Index, box pos.etag, box index) |> Async.AwaitTaskCorrect match res.RequestCharge, (match res.Response.etag with null -> None | x -> Some x), res.Response.conflicts with | rc,e,null -> return rc, EqxSyncResult.Written { pos with index = Some (pos.IndexRel events.Length); etag=e } | rc,e,[||] -> return rc, EqxSyncResult.ConflictUnknown { pos with etag=e } @@ -361,7 +363,7 @@ module private Read = let log = if (not << log.IsEnabled) Events.LogEventLevel.Debug then log else log |> Log.propResolvedEvents "Json" slice let index = match slice |> Array.tryHead with Some head -> head.id | None -> null (log |> Log.prop "startIndex" pos.Index |> Log.prop "bytes" bytes |> Log.event evt) - .Information("Eqx {action:l} {count} {ms}ms i={index} rc={ru}", "Read", count, (let e = t.Elapsed in e.TotalMilliseconds), index, ru) + .Information("Eqx {action:l} {count} {direction} {ms}ms i={index} rc={ru}", "Query", count, direction, (let e = t.Elapsed in e.TotalMilliseconds), index, ru) return slice, ru } let private readBatches (log : ILogger) (readSlice: IDocumentQuery -> ILogger -> Async) @@ -796,7 +798,7 @@ module Initialization = return coll.Resource.Id } let createProc (client: IDocumentClient) (collectionUri: Uri) = async { - let f = """function multidocInsert(docs, expectedVersion, etag, index) { + let f = """function indexedWrite(docs, expectedVersion, etag, index) { var response = getContext().getResponse(); var collection = getContext().getCollection(); var collectionLink = collection.getSelfLink(); @@ -804,14 +806,15 @@ module Initialization = if (index) { function callback(err, doc, options) { if (err) throw err; + response.setBody({ etag: doc._etag, conflicts: null }); } if (-1 == expectedVersion) { collection.createDocument(collectionLink, index, { disableAutomaticIdGeneration : true}, callback); } else { collection.replaceDocument(collection.getAltLink() + "/docs/" + index.id, index, callback); } - response.setBody({ etag: null, conflicts: null }); } else { + // call always expects a parseable json response with `etag` and `conflicts` // can also contain { conflicts: [{t, d}] } representing conflicting events since expectedVersion // null/missing signifies events have been written, with no conflict response.setBody({ etag: null, conflicts: null }); diff --git a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs index 88943ec0f..a7ef73ff2 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs @@ -278,7 +278,7 @@ type Tests(testOutputHelper) = let primeIndex = [EqxAct.IndexedNotFound; EqxAct.SliceBackward; EqxAct.BatchBackward] // When the test gets re-run to simplify, the stream will typically already have values - let primeIndexRerun = [EqxAct.Indexed] + let primeIndexRerun = [EqxAct.IndexedCached] [] let ``Can roundtrip against Cosmos, correctly using the index and cache to avoid redundant reads`` context skuId cartId = Async.RunSynchronously <| async { @@ -293,9 +293,9 @@ type Tests(testOutputHelper) = do! addAndThenRemoveItemsManyTimes context cartId skuId service1 5 let! _ = service2.Read cartId - // ... should see a single Indexed read given writes are cached - test <@ primeIndex @ [EqxAct.Append; EqxAct.Indexed] = capture.ExternalCalls - || primeIndexRerun @ [EqxAct.Append; EqxAct.Indexed] = capture.ExternalCalls@> + // ... should see a single Cached Indexed read given writes are cached and writer emits etag + test <@ primeIndex @ [EqxAct.Append; EqxAct.IndexedCached] = capture.ExternalCalls + || primeIndexRerun @ [EqxAct.Append; EqxAct.IndexedCached] = capture.ExternalCalls@> // Add two more - the roundtrip should only incur a single read, which should be cached by virtue of being a second one in successono capture.Clear() @@ -306,8 +306,8 @@ type Tests(testOutputHelper) = capture.Clear() let! _ = service2.Read cartId let! _ = service2.Read cartId - // First read is a re-read, second is cached - test <@ [EqxAct.Indexed;EqxAct.IndexedCached] = capture.ExternalCalls @> + // First is cached because writer emits etag, second remains cached + test <@ [EqxAct.IndexedCached; EqxAct.IndexedCached] = capture.ExternalCalls @> } [] From 868b1f06f13d00d953f4331a5958f950ebbb35e5 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 14 Nov 2018 07:05:46 +0000 Subject: [PATCH 40/66] Compress snapshots in Index --- src/Equinox.Cosmos/Cosmos.fs | 44 ++++++++++++++-- .../VerbatimUtf8JsonConverterTests.fs | 52 ++++++++++++++----- 2 files changed, 79 insertions(+), 17 deletions(-) diff --git a/src/Equinox.Cosmos/Cosmos.fs b/src/Equinox.Cosmos/Cosmos.fs index 1d67f753a..8e7700306 100644 --- a/src/Equinox.Cosmos/Cosmos.fs +++ b/src/Equinox.Cosmos/Cosmos.fs @@ -10,9 +10,8 @@ open Newtonsoft.Json.Linq open Serilog open System - [] -module DocDbExtensions = +module private DocDbExtensions = /// Extracts the innermost exception from a nested hierarchy of Aggregate Exceptions let (|AggregateException|) (exn : exn) = let rec aux (e : exn) = @@ -53,6 +52,8 @@ module DocDbExtensions = | DocDbException (DocDbStatusCode System.Net.HttpStatusCode.PreconditionFailed as e) -> return e.RequestCharge, NotModified } module Store = + open System.IO + open System.IO.Compression [] type Position = { collectionUri: Uri; streamName: string; index: int64 option; etag: string option } @@ -165,21 +166,54 @@ module Store = static member Create (pos: Position) eventCount (eds: EventData[]) : IndexEvent = { p = pos.streamName; id = IndexEvent.IdConstant; m = pos.IndexRel eventCount; _etag = null c = [| for ed in eds -> { t = ed.eventType; d = ed.data; m = ed.metadata } |] } - and IndexProjection = + and [] IndexProjection = { /// The Event Type, used to drive deserialization t: string // required /// Event body, as UTF-8 encoded json ready to be injected into the Json being rendered for DocDb - [)>] + [)>] d: byte[] // required /// Optional metadata (null, or same as d, not written if missing) - [); JsonProperty(Required=Required.Default, NullValueHandling=NullValueHandling.Ignore)>] + [); JsonProperty(Required=Required.Default, NullValueHandling=NullValueHandling.Ignore)>] m: byte[] } // optional interface IEventData with member __.EventType = __.t member __.DataUtf8 = __.d member __.MetaUtf8 = __.m + + /// Manages zipping of the UTF-8 json bytes to make the index record minimal from the perspective of the writer stored proc + /// Only applied to snapshots in the Index + and Base64ZipUtf8JsonConverter() = + inherit JsonConverter() + let pickle (input : byte[]) : string = + if input = null then null else + + use output = new MemoryStream() + use compressor = new DeflateStream(output, CompressionLevel.Optimal) + compressor.Write(input,0,input.Length) + compressor.Close() + Convert.ToBase64String(output.ToArray()) + let unpickle str : byte[] = + if str = null then null else + + let compressedBytes = Convert.FromBase64String str + use input = new MemoryStream(compressedBytes) + use decompressor = new DeflateStream(input, CompressionMode.Decompress) + use output = new MemoryStream() + decompressor.CopyTo(output) + decompressor.Close() + output.ToArray() + + override __.CanConvert(objectType) = + typeof.Equals(objectType) + override __.ReadJson(reader, _, _, serializer) = + //( if reader.TokenType = JsonToken.Null then null else + serializer.Deserialize(reader, typedefof) :?> string |> unpickle |> box + override __.WriteJson(writer, value, serializer) = + let pickled = value |> unbox |> pickle + serializer.Serialize(writer, pickled) + (* Pseudocode: function sync(p, expectedVersion, windowSize, events) { if (i == 0) then { diff --git a/tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs b/tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs index 76122ed69..903c9f1c0 100644 --- a/tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs +++ b/tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs @@ -1,33 +1,61 @@ module Equinox.Cosmos.Integration.VerbatimUtf8JsonConverterTests open Equinox.Cosmos +open FsCheck.Xunit open Newtonsoft.Json open Swensen.Unquote open System open Xunit -let inline serialize (x:'t) = - let serializer = new JsonSerializer() - use sw = new System.IO.StringWriter() - use w = new JsonTextWriter(sw) - serializer.Serialize(w,x) - sw.ToString() - type Embedded = { embed : string } type Union = | A of Embedded | B of Embedded interface TypeShape.UnionContract.IUnionContract +let mkUnionEncoder () = Equinox.UnionCodec.JsonUtf8.Create(JsonSerializerSettings()) + [] -let ``VerbatimUtf8JsonConverter serializes properly`` () = - let unionEncoder = Equinox.UnionCodec.JsonUtf8.Create<_>(JsonSerializerSettings()) - let encoded = unionEncoder.Encode(A { embed = "\"" }) +let ``VerbatimUtf8JsonConverter encodes correctly`` () = + let encoded = mkUnionEncoder().Encode(A { embed = "\"" }) let e : Store.Event = { p = "streamName"; id = string 0; i = 0L c = DateTimeOffset.MinValue t = encoded.caseName d = encoded.payload m = null } - let res = serialize e - test <@ res.Contains """"d":{"embed":"\""}""" @> \ No newline at end of file + let res = JsonConvert.SerializeObject(e) + test <@ res.Contains """"d":{"embed":"\""}""" @> + +type Base64ZipUtf8JsonConverterTests() = + let unionEncoder = mkUnionEncoder () + + [] + let ``serializes, achieving compression`` () = + let encoded = unionEncoder.Encode(A { embed = String('x',5000) }) + let e : Store.IndexProjection = + { t = encoded.caseName + d = encoded.payload + m = null } + let res = JsonConvert.SerializeObject e + test <@ res.Contains("\"d\":\"") && res.Length < 100 @> + + [] + let roundtrips value = + let hasNulls = + match value with + | A x | B x when obj.ReferenceEquals(null, x) -> true + | A { embed = x } | B { embed = x } -> obj.ReferenceEquals(null, x) + if hasNulls then () else + + let encoded = unionEncoder.Encode value + let e : Store.IndexProjection = + { t = encoded.caseName + d = encoded.payload + m = null } + let ser = JsonConvert.SerializeObject(e) + test <@ ser.Contains("\"d\":\"") @> + let des = JsonConvert.DeserializeObject(ser) + let d : Equinox.UnionCodec.EncodedUnion<_> = { caseName = des.t; payload=des.d } + let decoded = unionEncoder.Decode d + test <@ value = decoded @> \ No newline at end of file From d5b29eee287e027c6eac8215c930a79f2ec4ff57 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 23 Nov 2018 14:53:59 +0000 Subject: [PATCH 41/66] Cosmos core events API (#49), remove rolling snapshots * Reorganize, adding Batch structure * Rework stored procedure * Remove rolling snapshots * Add explicit non-indexed mode --- samples/Store/Domain/Cart.fs | 2 +- samples/Store/Domain/ContactPreferences.fs | 4 +- samples/Store/Integration/CartIntegration.fs | 18 +- .../ContactPreferencesIntegration.fs | 14 +- .../Store/Integration/FavoritesIntegration.fs | 6 +- samples/Store/Integration/LogIntegration.fs | 18 +- src/Equinox.Cosmos/Backoff.fs | 105 ++ src/Equinox.Cosmos/Cosmos.fs | 1610 ++++++++++------- src/Equinox.Cosmos/Equinox.Cosmos.fsproj | 1 + .../CosmosCoreIntegration.fs | 270 +++ .../CosmosFixtures.fs | 20 +- .../CosmosFixturesInfrastructure.fs | 71 +- .../CosmosIntegration.fs | 245 +-- .../Equinox.Cosmos.Integration.fsproj | 5 +- ...onverterTests.fs => JsonConverterTests.fs} | 43 +- 15 files changed, 1496 insertions(+), 936 deletions(-) create mode 100644 src/Equinox.Cosmos/Backoff.fs create mode 100644 tests/Equinox.Cosmos.Integration/CosmosCoreIntegration.fs rename tests/Equinox.Cosmos.Integration/{VerbatimUtf8JsonConverterTests.fs => JsonConverterTests.fs} (60%) diff --git a/samples/Store/Domain/Cart.fs b/samples/Store/Domain/Cart.fs index d53cad63f..9e665c876 100644 --- a/samples/Store/Domain/Cart.fs +++ b/samples/Store/Domain/Cart.fs @@ -29,7 +29,7 @@ module Folds = let toSnapshot (s: State) : Events.Compaction.State = { items = [| for i in s.items -> { skuId = i.skuId; quantity = i.quantity; returnsWaived = i.returnsWaived } |] } let ofCompacted (s: Events.Compaction.State) : State = - { items = [ for i in s.items -> { skuId = i.skuId; quantity = i.quantity; returnsWaived = i.returnsWaived } ] } + { items = if s.items = null then [] else [ for i in s.items -> { skuId = i.skuId; quantity = i.quantity; returnsWaived = i.returnsWaived } ] } let initial = { items = [] } let evolve (state : State) event = let updateItems f = { state with items = f state.items } diff --git a/samples/Store/Domain/ContactPreferences.fs b/samples/Store/Domain/ContactPreferences.fs index 45ce4acd0..88cf9ded7 100644 --- a/samples/Store/Domain/ContactPreferences.fs +++ b/samples/Store/Domain/ContactPreferences.fs @@ -7,9 +7,11 @@ module Events = type Preferences = { manyPromotions : bool; littlePromotions : bool; productReview : bool; quickSurveys : bool } type Value = { email : string; preferences : Preferences } + let [] EventTypeName = "contactPreferencesChanged" type Event = - | []Updated of Value + | []Updated of Value interface TypeShape.UnionContract.IUnionContract + let eventTypeNames = System.Collections.Generic.HashSet([EventTypeName]) module Folds = type State = Events.Preferences diff --git a/samples/Store/Integration/CartIntegration.fs b/samples/Store/Integration/CartIntegration.fs index 7bc68178a..faa4ea720 100644 --- a/samples/Store/Integration/CartIntegration.fs +++ b/samples/Store/Integration/CartIntegration.fs @@ -1,6 +1,6 @@ module Samples.Store.Integration.CartIntegration -open Equinox.Cosmos +open Equinox.Cosmos.Builder open Equinox.Cosmos.Integration open Equinox.EventStore open Equinox.MemoryStore @@ -23,10 +23,10 @@ let resolveGesStreamWithRollingSnapshots gateway = let resolveGesStreamWithoutCustomAccessStrategy gateway = GesResolver(gateway, codec, fold, initial).Resolve -let resolveEqxStreamWithCompactionEventType gateway (StreamArgs args) = - EqxStreamBuilder(gateway, codec, fold, initial, Equinox.Cosmos.AccessStrategy.RollingSnapshots compact).Create(args) -let resolveEqxStreamWithoutCompactionSemantics gateway (StreamArgs args) = - EqxStreamBuilder(gateway, codec, fold, initial).Create(args) +let resolveEqxStreamWithProjection gateway = + EqxStreamBuilder(gateway, codec, fold, initial, AccessStrategy.Projection snapshot).Create +let resolveEqxStreamWithoutCustomAccessStrategy gateway = + EqxStreamBuilder(gateway, codec, fold, initial).Create let addAndThenRemoveItemsManyTimesExceptTheLastOne context cartId skuId (service: Backend.Cart.Service) count = service.FlowAsync(cartId, fun _ctx execute -> @@ -72,13 +72,13 @@ type Tests(testOutputHelper) = } [] - let ``Can roundtrip against Cosmos, correctly folding the events without compaction semantics`` args = Async.RunSynchronously <| async { - let! service = arrange connectToSpecifiedCosmosOrSimulator createEqxGateway resolveEqxStreamWithoutCompactionSemantics + let ``Can roundtrip against Cosmos, correctly folding the events without custom access strategy`` args = Async.RunSynchronously <| async { + let! service = arrange connectToSpecifiedCosmosOrSimulator createEqxStore resolveEqxStreamWithoutCustomAccessStrategy do! act service args } [] - let ``Can roundtrip against Cosmos, correctly folding the events with compaction`` args = Async.RunSynchronously <| async { - let! service = arrange connectToSpecifiedCosmosOrSimulator createEqxGateway resolveEqxStreamWithCompactionEventType + let ``Can roundtrip against Cosmos, correctly folding the events with With Projection`` args = Async.RunSynchronously <| async { + let! service = arrange connectToSpecifiedCosmosOrSimulator createEqxStore resolveEqxStreamWithProjection do! act service args } \ No newline at end of file diff --git a/samples/Store/Integration/ContactPreferencesIntegration.fs b/samples/Store/Integration/ContactPreferencesIntegration.fs index e2714eae4..c20349bbc 100644 --- a/samples/Store/Integration/ContactPreferencesIntegration.fs +++ b/samples/Store/Integration/ContactPreferencesIntegration.fs @@ -1,6 +1,6 @@ module Samples.Store.Integration.ContactPreferencesIntegration -open Equinox.Cosmos +open Equinox.Cosmos.Builder open Equinox.Cosmos.Integration open Equinox.EventStore open Equinox.MemoryStore @@ -21,10 +21,10 @@ let resolveStreamGesWithOptimizedStorageSemantics gateway = let resolveStreamGesWithoutAccessStrategy gateway = GesResolver(gateway defaultBatchSize, codec, fold, initial).Resolve -let resolveStreamEqxWithCompactionSemantics gateway (StreamArgs args) = - EqxStreamBuilder(gateway 1, codec, fold, initial, Equinox.Cosmos.AccessStrategy.EventsAreState).Create(args) -let resolveStreamEqxWithoutCompactionSemantics gateway (StreamArgs args) = - EqxStreamBuilder(gateway defaultBatchSize, codec, fold, initial).Create(args) +let resolveStreamEqxWithCompactionSemantics gateway = + EqxStreamBuilder(gateway 1, codec, fold, initial, AccessStrategy.AnyKnownEventType Domain.ContactPreferences.Events.eventTypeNames).Create +let resolveStreamEqxWithoutCompactionSemantics gateway = + EqxStreamBuilder(gateway defaultBatchSize, codec, fold, initial).Create type Tests(testOutputHelper) = let testOutput = TestOutputAdapter testOutputHelper @@ -63,12 +63,12 @@ type Tests(testOutputHelper) = [] let ``Can roundtrip against Cosmos, correctly folding the events with normal semantics`` args = Async.RunSynchronously <| async { - let! service = arrange connectToSpecifiedCosmosOrSimulator createEqxGateway resolveStreamEqxWithoutCompactionSemantics + let! service = arrange connectToSpecifiedCosmosOrSimulator createEqxStore resolveStreamEqxWithoutCompactionSemantics do! act service args } [] let ``Can roundtrip against Cosmos, correctly folding the events with compaction semantics`` args = Async.RunSynchronously <| async { - let! service = arrange connectToSpecifiedCosmosOrSimulator createEqxGateway resolveStreamEqxWithCompactionSemantics + let! service = arrange connectToSpecifiedCosmosOrSimulator createEqxStore resolveStreamEqxWithCompactionSemantics do! act service args } \ No newline at end of file diff --git a/samples/Store/Integration/FavoritesIntegration.fs b/samples/Store/Integration/FavoritesIntegration.fs index b7c0fce7e..6bb20b977 100644 --- a/samples/Store/Integration/FavoritesIntegration.fs +++ b/samples/Store/Integration/FavoritesIntegration.fs @@ -1,6 +1,6 @@ module Samples.Store.Integration.FavoritesIntegration -open Equinox.Cosmos +open Equinox.Cosmos.Builder open Equinox.Cosmos.Integration open Equinox.EventStore open Equinox.MemoryStore @@ -22,7 +22,7 @@ let createServiceGes gateway log = Backend.Favorites.Service(log, resolveStream) let createServiceEqx gateway log = - let resolveStream (StreamArgs args) = EqxStreamBuilder(gateway, codec, fold, initial, Equinox.Cosmos.AccessStrategy.RollingSnapshots compact).Create(args) + let resolveStream = EqxStreamBuilder(gateway, codec, fold, initial, AccessStrategy.Projection compact).Create Backend.Favorites.Service(log, resolveStream) type Tests(testOutputHelper) = @@ -59,7 +59,7 @@ type Tests(testOutputHelper) = let ``Can roundtrip against Cosmos, correctly folding the events`` args = Async.RunSynchronously <| async { let log = createLog () let! conn = connectToSpecifiedCosmosOrSimulator log - let gateway = createEqxGateway conn defaultBatchSize + let gateway = createEqxStore conn defaultBatchSize let service = createServiceEqx gateway log do! act service args } \ No newline at end of file diff --git a/samples/Store/Integration/LogIntegration.fs b/samples/Store/Integration/LogIntegration.fs index 06a1e10a0..832172573 100644 --- a/samples/Store/Integration/LogIntegration.fs +++ b/samples/Store/Integration/LogIntegration.fs @@ -31,7 +31,8 @@ module EquinoxCosmosInterop = let action, metric, batches, ru = match evt with | Log.WriteSuccess m -> "EqxAppendToStreamAsync", m, None, m.ru - | Log.WriteConflict m -> "EqxAppendToStreamAsync", m, None, m.ru + | Log.WriteConflict m -> "EqxAppendToStreamConflictAsync", m, None, m.ru + | Log.WriteResync m -> "EqxAppendToStreamResyncAsync", m, None, m.ru | Log.Slice (Direction.Forward,m) -> "EqxReadStreamEventsForwardAsync", m, None, m.ru | Log.Slice (Direction.Backward,m) -> "EqxReadStreamEventsBackwardAsync", m, None, m.ru | Log.Batch (Direction.Forward,c,m) -> "EqxLoadF", m, Some c, m.ru @@ -117,13 +118,14 @@ type Tests() = } [] - let ``Can roundtrip against Cosmos, hooking, extracting and substituting metrics in the logging information`` context cartId skuId = Async.RunSynchronously <| async { - let buffer = ResizeArray() + let ``Can roundtrip against Cosmos, hooking, extracting and substituting metrics in the logging information`` context skuId = Async.RunSynchronously <| async { let batchSize = defaultBatchSize - let (log,capture) = createLoggerWithMetricsExtraction buffer.Add + let buffer = ConcurrentQueue() + let log = createLoggerWithMetricsExtraction buffer.Enqueue let! conn = connectToSpecifiedCosmosOrSimulator log - let gateway = createEqxGateway conn batchSize - let service = Backend.Cart.Service(log, CartIntegration.resolveEqxStreamWithCompactionEventType gateway) - let itemCount, cartId = batchSize / 2 + 1, cartId () - do! act buffer capture service itemCount context cartId skuId "ReadStreamEventsBackwardAsync-Duration" + let gateway = createEqxStore conn batchSize + let service = Backend.Cart.Service(log, CartIntegration.resolveEqxStreamWithProjection gateway) + let itemCount = batchSize / 2 + 1 + let cartId = Guid.NewGuid() |> CartId + do! act buffer service itemCount context cartId skuId "Eqx Index " // one is a 404, one is a 200 } \ No newline at end of file diff --git a/src/Equinox.Cosmos/Backoff.fs b/src/Equinox.Cosmos/Backoff.fs new file mode 100644 index 000000000..c3d5d1a88 --- /dev/null +++ b/src/Equinox.Cosmos/Backoff.fs @@ -0,0 +1,105 @@ +namespace Equinox.Cosmos + +// NB this is a copy of the one in Backend - there is also one in Equinox/Infrastrcture.fs which this will be merged into + +open System + +/// Given a value, creates a function with one ignored argument which returns the value. + +/// A backoff strategy. +/// Accepts the attempt number and returns an interval in milliseconds to wait. +/// If None then backoff should stop. +type Backoff = int -> int option + +/// Operations on back off strategies represented as functions (int -> int option) +/// which take an attempt number and produce an interval. +module Backoff = + + let inline konst x _ = x + let private checkOverflow x = + if x = System.Int32.MinValue then 2000000000 + else x + + /// Stops immediately. + let never : Backoff = konst None + + /// Always returns a fixed interval. + let linear i : Backoff = konst (Some i) + + /// Modifies the interval. + let bind (f:int -> int option) (b:Backoff) = + fun i -> + match b i with + | Some x -> f x + | None -> None + + /// Modifies the interval. + let map (f:int -> int) (b:Backoff) : Backoff = + fun i -> + match b i with + | Some x -> f x |> checkOverflow |> Some + | None -> None + + /// Bounds the interval. + let bound mx = map (min mx) + + /// Creates a back-off strategy which increases the interval exponentially. + let exp (initialIntervalMs:int) (multiplier:float) : Backoff = + fun i -> (float initialIntervalMs) * (pown multiplier i) |> int |> checkOverflow |> Some + + /// Randomizes the output produced by a back-off strategy: + /// randomizedInterval = retryInterval * (random in range [1 - randomizationFactor, 1 + randomizationFactor]) + let rand (randomizationFactor:float) = + let rand = new System.Random() + let maxRand,minRand = (1.0 + randomizationFactor), (1.0 - randomizationFactor) + map (fun x -> (float x) * (rand.NextDouble() * (maxRand - minRand) + minRand) |> int) + + /// Uses a fibonacci sequence to genereate timeout intervals starting from the specified initial interval. + let fib (initialIntervalMs:int) : Backoff = + let rec fib n = + if n < 2 then initialIntervalMs + else fib (n - 1) + fib (n - 2) + fib >> checkOverflow >> Some + + /// Creates a stateful back-off strategy which keeps track of the number of attempts, + /// and a reset function which resets attempts to zero. + let keepCount (b:Backoff) : (unit -> int option) * (unit -> unit) = + let i = ref -1 + (fun () -> System.Threading.Interlocked.Increment i |> b), + (fun () -> i := -1) + + /// Bounds a backoff strategy to a specified maximum number of attempts. + let maxAttempts (max:int) (b:Backoff) : Backoff = + fun n -> if n > max then None else b n + + + // ------------------------------------------------------------------------------------------------------------------------ + // defaults + + /// 500ms + let [] DefaultInitialIntervalMs = 500 + + /// 60000ms + let [] DefaultMaxIntervalMs = 60000 + + /// 0.5 + let [] DefaultRandomizationFactor = 0.5 + + /// 1.5 + let [] DefaultMultiplier = 1.5 + + /// The default exponential and randomized back-off strategy with a provided initial interval. + /// DefaultMaxIntervalMs = 60,000 + /// DefaultRandomizationFactor = 0.5 + /// DefaultMultiplier = 1.5 + let DefaultExponentialBoundedRandomizedOf initialInternal = + exp initialInternal DefaultMultiplier + |> rand DefaultRandomizationFactor + |> bound DefaultMaxIntervalMs + + /// The default exponential and randomized back-off strategy. + /// DefaultInitialIntervalMs = 500 + /// DefaultMaxIntervalMs = 60,000 + /// DefaultRandomizationFactor = 0.5 + /// DefaultMultiplier = 1.5 + let DefaultExponentialBoundedRandomized = DefaultExponentialBoundedRandomizedOf DefaultInitialIntervalMs \ No newline at end of file diff --git a/src/Equinox.Cosmos/Cosmos.fs b/src/Equinox.Cosmos/Cosmos.fs index 8e7700306..ae5929741 100644 --- a/src/Equinox.Cosmos/Cosmos.fs +++ b/src/Equinox.Cosmos/Cosmos.fs @@ -1,17 +1,310 @@ -namespace Equinox.Cosmos +namespace Equinox.Cosmos.Internal.Json + +open Newtonsoft.Json.Linq +open Newtonsoft.Json + +/// Manages injecting prepared json into the data being submitted to DocDb as-is, on the basis we can trust it to be valid json as DocDb will need it to be +type VerbatimUtf8JsonConverter() = + inherit JsonConverter() + + override __.ReadJson(reader, _, _, _) = + let token = JToken.Load(reader) + if token.Type = JTokenType.Object then token.ToString() |> System.Text.Encoding.UTF8.GetBytes |> box + else Array.empty |> box + + override __.CanConvert(objectType) = + typeof.Equals(objectType) + + override __.WriteJson(writer, value, serializer) = + let array = value :?> byte[] + if array = null || Array.length array = 0 then serializer.Serialize(writer, null) + else writer.WriteRawValue(System.Text.Encoding.UTF8.GetString(array)) + +open System.IO +open System.IO.Compression + +/// Manages zipping of the UTF-8 json bytes to make the index record minimal from the perspective of the writer stored proc +/// Only applied to snapshots in the Index +type Base64ZipUtf8JsonConverter() = + inherit JsonConverter() + let pickle (input : byte[]) : string = + if input = null then null else + + use output = new MemoryStream() + use compressor = new DeflateStream(output, CompressionLevel.Optimal) + compressor.Write(input,0,input.Length) + compressor.Close() + System.Convert.ToBase64String(output.ToArray()) + let unpickle str : byte[] = + if str = null then null else + + let compressedBytes = System.Convert.FromBase64String str + use input = new MemoryStream(compressedBytes) + use decompressor = new DeflateStream(input, CompressionMode.Decompress) + use output = new MemoryStream() + decompressor.CopyTo(output) + output.ToArray() + + override __.CanConvert(objectType) = + typeof.Equals(objectType) + override __.ReadJson(reader, _, _, serializer) = + //( if reader.TokenType = JsonToken.Null then null else + serializer.Deserialize(reader, typedefof) :?> string |> unpickle |> box + override __.WriteJson(writer, value, serializer) = + let pickled = value |> unbox |> pickle + serializer.Serialize(writer, pickled) + +namespace Equinox.Cosmos.Events + +/// Common form for either a raw Event or a Projection +type IEvent = + /// The Event Type, used to drive deserialization + abstract member EventType : string + /// Event body, as UTF-8 encoded json ready to be injected into the Json being rendered for DocDb + abstract member Data : byte[] + /// Optional metadata (null, or same as d, not written if missing) + abstract member Meta : byte[] + +/// Represents an Event or Projection and its relative position in the event sequence +type IOrderedEvent = + inherit IEvent + /// The index into the event sequence of this event + abstract member Index : int64 + /// Indicates whether this is a primary event or a projection based on the events <= up to `Index` + abstract member IsProjection: bool + +/// Position and Etag to which an operation is relative +type [] Position = { index: int64; etag: string option } with + /// If we have strong reason to suspect a stream is empty, we won't have an etag (and Writer Stored Procedure special cases this) + static member internal FromKnownEmpty = Position.FromI 0L + /// NB very inefficient compared to FromDocument or using one already returned to you + static member internal FromI(i: int64) = { index = i; etag = None } + /// Just Do It mode + static member internal FromAppendAtEnd = Position.FromI -1L // sic - needs to yield -1 + /// NB very inefficient compared to FromDocument or using one already returned to you + static member internal FromMaxIndex(xs: IOrderedEvent[]) = + if Array.isEmpty xs then Position.FromKnownEmpty + else Position.FromI (1L + Seq.max (seq { for x in xs -> x.Index })) + +namespace Equinox.Cosmos.Store + +open Equinox.Cosmos.Events +open Newtonsoft.Json + +/// A 'normal' (frozen, not Pending) Batch of Events, without any Projections +type [] + Event = + { /// DocDb-mandated Partition Key, must be maintained within the document + /// Not actually required if running in single partition mode, but for simplicity, we always write it + p: string // "{streamName}" + + /// DocDb-mandated unique row key; needs to be unique within any partition it is maintained; must be string + /// At the present time, one can't perform an ORDER BY on this field, hence we also have i shadowing it + /// NB WipBatch uses a well known value here while it's actively 'open' + id: string // "{index}" + + /// When we read, we need to capture the value so we can retain it for caching purposes + /// NB this is not relevant to fill in when we pass it to the writing stored procedure + /// as it will do: 1. read 2. merge 3. write merged version contingent on the _etag not having changed + [] + _etag: string + + /// Same as `id`; necessitated by fact that it's not presently possible to do an ORDER BY on the row key + i: int64 // {index} + + /// Creation date (as opposed to system-defined _lastUpdated which is touched by triggers, replication etc.) + c: System.DateTimeOffset // ISO 8601 + + /// The Event Type, used to drive deserialization + t: string // required + + /// Event body, as UTF-8 encoded json ready to be injected into the Json being rendered for DocDb + [)>] + d: byte[] // required + + /// Optional metadata, as UTF-8 encoded json, ready to emit directly (null, not written if missing) + [)>] + [] + m: byte[] } // optional + /// Unless running in single partion mode (which would restrict us to 10GB per collection) + /// we need to nominate a partition key that will be in every document + static member PartitionKeyField = "p" + /// As one cannot sort by the implicit `id` field, we have an indexed `i` field for sort and range query use + static member IndexedFields = [Event.PartitionKeyField; "i"] + /// If we encounter a -1 doc, we're interested in its etag so we can re-read for one RU + member x.TryToPosition() = + if x.id <> WipBatch.WellKnownDocumentId then None + else Some { index = (let ``x.e.LongLength`` = 1L in x.i+``x.e.LongLength``); etag = match x._etag with null -> None | x -> Some x } + +/// The Special 'Pending' Batch Format +/// NB this Type does double duty as +/// a) transport for when we read it +/// b) a way of encoding a batch that the stored procedure will write in to the actual document +/// The stored representation has the following differences vs a 'normal' (frozen/completed) Batch +/// a) `id` and `i` = `-1` as WIP document currently always is +/// b) events are retained as in an `e` array, not top level fields +/// c) contains projections (`c`) +and [] + WipBatch = + { /// Partition key, as per Batch + p: string // "{streamName}" + /// Document Id within partition, as per Batch + id: string // "{-1}" - Well known IdConstant used while this remains the pending batch + + /// When we read, we need to capture the value so we can retain it for caching purposes + /// NB this is not relevant to fill in when we pass it to the writing stored procedure + /// as it will do: 1. read 2. merge 3. write merged version contingent on the _etag not having changed + [] + _etag: string + + /// base 'i' value for the Events held herein + _i: int64 + + /// Events + e: BatchEvent[] + + /// Projections + c: Projection[] } + /// arguably this should be a high nember to reflect fact it is the freshest ? + static member WellKnownDocumentId = "-1" + /// Create Position from [Wip]Batch record context (facilitating 1 RU reads) + member x.ToPosition() = { index = x._i+x.e.LongLength; etag = match x._etag with null -> None | x -> Some x } +/// A single event from the array held in a batch +and [] + BatchEvent = + { /// Creation date (as opposed to system-defined _lastUpdated which is touched by triggers, replication etc.) + c: System.DateTimeOffset // ISO 8601 + + /// The Event Type, used to drive deserialization + t: string // required + + /// Event body, as UTF-8 encoded json ready to be injected into the Json being rendered for DocDb + [)>] + d: byte[] // required + + /// Optional metadata, as UTF-8 encoded json, ready to emit directly (null, not written if missing) + [)>] + [] + m: byte[] } // optional +/// Projection based on the state at a given point in time `i` +and Projection = + { /// Base: Max index rolled into this projection + i: int64 + + ///// Indicates whether this is actually an event being retained to support a lagging projection + //x: bool + + /// The Event Type of this compaction/snapshot, used to drive deserialization + t: string // required + + /// Event body - Json -> UTF-8 -> Deflate -> Base64 + [)>] + d: byte[] // required + + /// Optional metadata, same encoding as `d` (can be null; not written if missing) + [)>] + [] + m: byte[] } // optional + +type Enum() = + static member Events (b:WipBatch) = + b.e |> Seq.mapi (fun offset x -> + { new IOrderedEvent with + member __.Index = b._i + int64 offset + member __.IsProjection = false + member __.EventType = x.t + member __.Data = x.d + member __.Meta = x.m }) + static member Events (i: int64, e:BatchEvent[]) = + e |> Seq.mapi (fun offset x -> + { new IOrderedEvent with + member __.Index = i + int64 offset + member __.IsProjection = false + member __.EventType = x.t + member __.Data = x.d + member __.Meta = x.m }) + static member Event (x:Event) = + Seq.singleton + { new IOrderedEvent with + member __.Index = x.i + member __.IsProjection = false + member __.EventType = x.t + member __.Data = x.d + member __.Meta = x.m } + static member Projections (xs: Projection[]) = seq { + for x in xs -> { new IOrderedEvent with + member __.Index = x.i + member __.IsProjection = true + member __.EventType = x.t + member __.Data = x.d + member __.Meta = x.m } } + static member EventsAndProjections (x:WipBatch): IOrderedEvent seq = + Enum.Projections x.c + +/// Reference to Collection and name that will be used as the location for the stream +type [] CollectionStream = { collectionUri: System.Uri; name: string } with + static member Create(collectionUri, name) = { collectionUri = collectionUri; name = name } + +namespace Equinox.Cosmos open Equinox +open Equinox.Cosmos.Events +open Equinox.Cosmos.Store open Equinox.Store open FSharp.Control open Microsoft.Azure.Documents -open Microsoft.Azure.Documents.Linq -open Newtonsoft.Json -open Newtonsoft.Json.Linq open Serilog open System +[] +type Direction = Forward | Backward with + override this.ToString() = match this with Forward -> "Forward" | Backward -> "Backward" + +module Log = + [] + type Measurement = { stream: string; interval: StopwatchInterval; bytes: int; count: int; ru: float } + [] + type Event = + | WriteSuccess of Measurement + | WriteResync of Measurement + | WriteConflict of Measurement + /// Individual read request in a Batch + | Slice of Direction * Measurement + /// Individual read request for the Index + | Index of Measurement + /// Individual read request for the Index, not found + | IndexNotFound of Measurement + /// Index read with Single RU Request Charge due to correct use of etag in cache + | IndexNotModified of Measurement + /// Summarizes a set of Slices read together + | Batch of Direction * slices: int * Measurement + let prop name value (log : ILogger) = log.ForContext(name, value) + let propData name (events: #IEvent seq) (log : ILogger) = + let items = seq { for e in events do yield sprintf "{\"%s\": %s}" e.EventType (System.Text.Encoding.UTF8.GetString e.Data) } + log.ForContext(name, sprintf "[%s]" (String.concat ",\n\r" items)) + let propEvents = propData "events" + let propDataProjections = Enum.Projections >> propData "projections" + + let withLoggedRetries<'t> retryPolicy (contextLabel : string) (f : ILogger -> Async<'t>) log: Async<'t> = + match retryPolicy with + | None -> f log + | Some retryPolicy -> + let withLoggingContextWrapping count = + let log = if count = 1 then log else log |> prop contextLabel count + f log + retryPolicy withLoggingContextWrapping + /// Attach a property to the log context to hold the metrics + // Sidestep Log.ForContext converting to a string; see https://github.com/serilog/serilog/issues/1124 + open Serilog.Events + let event (value : Event) (log : ILogger) = + let enrich (e : LogEvent) = e.AddPropertyIfAbsent(LogEventProperty("cosmosEvt", ScalarValue(value))) + log.ForContext({ new Serilog.Core.ILogEventEnricher with member __.Enrich(evt,_) = enrich evt }) + let (|BlobLen|) = function null -> 0 | (x : byte[]) -> x.Length + let (|EventLen|) (x: #IEvent) = let (BlobLen bytes), (BlobLen metaBytes) = x.Data, x.Meta in bytes+metaBytes + let (|BatchLen|) = Seq.sumBy (|EventLen|) + [] -module private DocDbExtensions = +module private DocDb = /// Extracts the innermost exception from a nested hierarchy of Aggregate Exceptions let (|AggregateException|) (exn : exn) = let rec aux (e : exn) = @@ -51,306 +344,207 @@ module private DocDbExtensions = // NB while the docs suggest you may see a 412, the NotModified in the body of the try/with is actually what happens | DocDbException (DocDbStatusCode System.Net.HttpStatusCode.PreconditionFailed as e) -> return e.RequestCharge, NotModified } -module Store = - open System.IO - open System.IO.Compression - [] - type Position = - { collectionUri: Uri; streamName: string; index: int64 option; etag: string option } - member __.Index : int64 = defaultArg __.index -1L - member __.IndexRel (offset: int) : int64 = __.index |> function - | Some index -> index+int64 offset - | None -> failwithf "Cannot IndexRel %A" __ - - type EventData = { eventType: string; data: byte[]; metadata: byte[] } - type IEventData = - /// The Event Type, used to drive deserialization - abstract member EventType : string - /// Event body, as UTF-8 encoded json ready to be injected into the Json being rendered for DocDb - abstract member DataUtf8 : byte[] - /// Optional metadata (null, or same as d, not written if missing) - abstract member MetaUtf8 : byte[] - - [] - type Event = - { (* DocDb-mandated essential elements *) - - // DocDb-mandated Partition Key, must be maintained within the document - // Not actually required if running in single partition mode, but for simplicity, we always write it - p: string // "{streamName}" - - // DocDb-mandated unique row key; needs to be unique within any partition it is maintained; must be a string - // At the present time, one can't perform an ORDER BY on this field, hence we also have i, which is identical - id: string // "{index}" - - // Same as `id`; necessitated by fact that it's not presently possible to do an ORDER BY on the row key - i: int64 // {index} - - (* Event payload elements *) - - /// Creation date (as opposed to sytem-defined _lastUpdated which is rewritten by triggers adnd/or replication) - c: DateTimeOffset // ISO 8601 - - /// The Event Type, used to drive deserialization - t: string // required - - /// Event body, as UTF-8 encoded json ready to be injected into the Json being rendered for DocDb - [)>] - d: byte[] // required - - /// Optional metadata (null, or same as d, not written if missing) - [); JsonProperty(Required=Required.Default, NullValueHandling=NullValueHandling.Ignore)>] - m: byte[] } // optional - /// Unless running in single partion mode (which would restrict us to 10GB per collection) - /// we need to nominate a partition key that will be in every document - static member PartitionKeyField = "p" - /// As one cannot sort by the implicit `id` field, we have an indexed `i` field which we use for sort and range query purporses - static member IndexedFields = [Event.PartitionKeyField; "i"] - static member Create (pos: Position) offset (ed: EventData) : Event = - { p = pos.streamName; id = string (pos.IndexRel offset); i = pos.IndexRel offset - c = DateTimeOffset.UtcNow - t = ed.eventType; d = ed.data; m = ed.metadata } - interface IEventData with - member __.EventType = __.t - member __.DataUtf8 = __.d - member __.MetaUtf8 = __.m - - /// Manages injecting prepared json into the data being submitted to DocDb as-is, on the basis we can trust it to be valid json as DocDb will need it to be - and VerbatimUtf8JsonConverter() = - inherit JsonConverter() - - override __.ReadJson(reader, _, _, _) = - let token = JToken.Load(reader) - if token.Type = JTokenType.Object then token.ToString() |> System.Text.Encoding.UTF8.GetBytes |> box - else Array.empty |> box - - override __.CanConvert(objectType) = - typeof.Equals(objectType) - - override __.WriteJson(writer, value, serializer) = - let array = value :?> byte[] - if array = null || Array.length array = 0 then serializer.Serialize(writer, null) - else writer.WriteRawValue(System.Text.Encoding.UTF8.GetString(array)) - - [] - type IndexEvent = - { p: string // "{streamName}" - id: string // "{-1}" - - /// When we read, we need to capture the value so we can retain it for caching purposes; when we write, there's no point sending it as it would not be honored - [] - _etag: string - - //w: int64 // 100: window size - /// last index/i value - m: int64 // {index} - - /// Compacted projections based on version identified by `m` - c: IndexProjection[] - - (*// Potential schema to manage Pending Events together with compaction events based on each one - // This scheme is more complete than the simple `c` encoding, which relies on every writer being able to write all salient snapshots - // For instance, in the case of blue/green deploys, older versions need to be able to coexist without destroying the perf for eachother - "x": [ - { "i":0, - "c":"ISO 8601" - "e":[ - [{"t":"added","d":"..."},{"t":"compacted/1","d":"..."}], - [{"t":"removed","d":"..."}], - ] - } - ] *) - //x: JObject[][] - } - static member IdConstant = "-1" - static member Create (pos: Position) eventCount (eds: EventData[]) : IndexEvent = - { p = pos.streamName; id = IndexEvent.IdConstant; m = pos.IndexRel eventCount; _etag = null - c = [| for ed in eds -> { t = ed.eventType; d = ed.data; m = ed.metadata } |] } - and [] IndexProjection = - { /// The Event Type, used to drive deserialization - t: string // required - - /// Event body, as UTF-8 encoded json ready to be injected into the Json being rendered for DocDb - [)>] - d: byte[] // required - - /// Optional metadata (null, or same as d, not written if missing) - [); JsonProperty(Required=Required.Default, NullValueHandling=NullValueHandling.Ignore)>] - m: byte[] } // optional - interface IEventData with - member __.EventType = __.t - member __.DataUtf8 = __.d - member __.MetaUtf8 = __.m - - /// Manages zipping of the UTF-8 json bytes to make the index record minimal from the perspective of the writer stored proc - /// Only applied to snapshots in the Index - and Base64ZipUtf8JsonConverter() = - inherit JsonConverter() - let pickle (input : byte[]) : string = - if input = null then null else - - use output = new MemoryStream() - use compressor = new DeflateStream(output, CompressionLevel.Optimal) - compressor.Write(input,0,input.Length) - compressor.Close() - Convert.ToBase64String(output.ToArray()) - let unpickle str : byte[] = - if str = null then null else - - let compressedBytes = Convert.FromBase64String str - use input = new MemoryStream(compressedBytes) - use decompressor = new DeflateStream(input, CompressionMode.Decompress) - use output = new MemoryStream() - decompressor.CopyTo(output) - decompressor.Close() - output.ToArray() - - override __.CanConvert(objectType) = - typeof.Equals(objectType) - override __.ReadJson(reader, _, _, serializer) = - //( if reader.TokenType = JsonToken.Null then null else - serializer.Deserialize(reader, typedefof) :?> string |> unpickle |> box - override __.WriteJson(writer, value, serializer) = - let pickled = value |> unbox |> pickle - serializer.Serialize(writer, pickled) - - (* Pseudocode: - function sync(p, expectedVersion, windowSize, events) { - if (i == 0) then { - coll.insert(p,0,{ p:p, id:-1, w:windowSize, m:flatLen(events)}) +module Sync = + // NB don't nest in a private module, or serialization will fail miserably ;) + [] + type SyncResponse = { etag: string; nextI: int64; conflicts: BatchEvent[] } + let [] sprocName = "EquinoxSync-SingleEvents-021" // NB need to renumber for any breaking change + let [] sprocBody = """ + +// Manages the merging of the supplied Request Batch, fulfilling one of the following end-states +// 1 Verify no current WIP batch, the incoming `req` becomes the WIP batch (the caller is entrusted to provide a valid and complete set of inputs, or it's GIGO) +// 2 Current WIP batch has space to accommodate the incoming projections (req.c) and events (req.e) - merge them in, replacing any superseded projections +// 3. Current WIP batch would become too large - remove WIP state from active document by replacing the well known id with a correct one; proceed as per 1 +function sync(req, expectedVersion) { + if (!req) throw new Error("Missing req argument"); + const collection = getContext().getCollection(); + const collectionLink = collection.getSelfLink(); + const response = getContext().getResponse(); + + // Locate the WIP (-1) batch (which may not exist) + const wipDocId = collection.getAltLink() + "/docs/" + req.id; + const isAccepted = collection.readDocument(wipDocId, {}, function (err, current) { + // Verify we dont have a conflicting write + if (expectedVersion === -1) { + executeUpsert(current); + } else if (!current && expectedVersion !== 0) { + // If there is no WIP page, the writer has no possible reason for writing at an index other than zero + response.setBody({ etag: null, nextI: 0, conflicts: [] }); + } else if (current && expectedVersion !== current._i + current.e.length) { + // Where possible, we extract conflicting events from e and/or c in order to avoid another read cycle + // yielding [] triggers the client to go loading the events itself + const conflicts = expectedVersion < current._i ? [] : current.e.slice(expectedVersion - current._i); + const nextI = current._i + current.e.length; + response.setBody({ etag: current._etag, nextI: nextI, conflicts: conflicts }); } else { - const i = doc.find(p=p && id=-1) - if(i.m <> expectedVersion) then emit from expectedVersion else - i.x.append(events) - for (var (i, c, e: [ {e1}, ...]) in events) { - coll.insert({p:p, id:i, i:i, c:c, e:e1) - } - // trim i.x to w total items in i.[e] - coll.update(p,id,i) + executeUpsert(current); } - } *) -[] -type Direction = Forward | Backward with - override this.ToString() = match this with Forward -> "Forward" | Backward -> "Backward" + }); + if (!isAccepted) throw new Error("readDocument not Accepted"); -module Log = - [] - type Measurement = { stream: string; interval: StopwatchInterval; bytes: int; count: int; ru: float } - [] - type Event = - | WriteSuccess of Measurement - | WriteConflict of Measurement - /// Individual read request in a Batch - | Slice of Direction * Measurement - /// Individual read request for the Index - | Index of Measurement - /// Individual read request for the Index, not found - | IndexNotFound of Measurement - /// Index read with Single RU Request Charge due to correct use of etag in cache - | IndexNotModified of Measurement - /// Summarizes a set of Slices read together - | Batch of Direction * slices: int * Measurement - let prop name value (log : ILogger) = log.ForContext(name, value) - let propEvents name (kvps : System.Collections.Generic.KeyValuePair seq) (log : ILogger) = - let items = seq { for kv in kvps do yield sprintf "{\"%s\": %s}" kv.Key kv.Value } - log.ForContext(name, sprintf "[%s]" (String.concat ",\n\r" items)) - let propEventData name (events : Store.EventData[]) (log : ILogger) = - log |> propEvents name (seq { for x in events -> Collections.Generic.KeyValuePair<_,_>(x.eventType, System.Text.Encoding.UTF8.GetString x.data)}) - let propResolvedEvents name (events : Store.Event[]) (log : ILogger) = - log |> propEvents name (seq { for x in events -> Collections.Generic.KeyValuePair<_,_>(x.t, System.Text.Encoding.UTF8.GetString x.d)}) - let propIEventDatas name (events : Store.IEventData[]) (log : ILogger) = - log |> propEvents name (seq { for x in events -> Collections.Generic.KeyValuePair<_,_>(x.EventType, System.Text.Encoding.UTF8.GetString x.DataUtf8)}) - let propProjectionEvents name (events : Store.IndexProjection[]) (log : ILogger) = - log |> propEvents name (seq { for x in events -> Collections.Generic.KeyValuePair<_,_>(x.t, System.Text.Encoding.UTF8.GetString x.d)}) - - open Serilog.Events - /// Attach a property to the log context to hold the metrics - // Sidestep Log.ForContext converting to a string; see https://github.com/serilog/serilog/issues/1124 - let event (value : Event) (log : ILogger) = - let enrich (e : LogEvent) = e.AddPropertyIfAbsent(LogEventProperty("cosmosEvt", ScalarValue(value))) - log.ForContext({ new Serilog.Core.ILogEventEnricher with member __.Enrich(evt,_) = enrich evt }) - let withLoggedRetries<'t> retryPolicy (contextLabel : string) (f : ILogger -> Async<'t>) log: Async<'t> = - match retryPolicy with - | None -> f log - | Some retryPolicy -> - let withLoggingContextWrapping count = - let log = if count = 1 then log else log |> prop contextLabel count - f log - retryPolicy withLoggingContextWrapping - let (|BlobLen|) = function null -> 0 | (x : byte[]) -> x.Length - -[] -type EqxSyncResult = - | Written of Store.Position - | ConflictUnknown of Store.Position - | Conflict of Store.Position * events: Store.IEventData[] - -// NB don't nest in a private module, or serialization will fail miserably ;) -[] -type WriteResponse = { etag: string; conflicts: Store.IndexProjection[] } - -module private Write = - let [] sprocName = "EquinoxIndexedWrite" + function executeUpsert(current) { + function callback(err, doc) { + if (err) throw err; + response.setBody({ etag: doc._etag, nextI: doc._i + doc.e.length, conflicts: null }); + } + // If we have hit a sensible limit for a slice in the WIP document, trim the events + if (current && current.e.length + req.e.length > 10) { + current._i = current._i + current.e.length; + current.e = req.e; + current.c = req.c; + + // as we've mutated the document in a manner that can conflict with other writers, out write needs to be contingent on no competing updates having taken place + finalize(current); + const isAccepted = collection.replaceDocument(current._self, current, { etag: current._etag }, callback); + if (!isAccepted) throw new Error("Unable to restart WIP batch."); + } else if (current) { + // Append the new events into the current batch + Array.prototype.push.apply(current.e, req.e); + // Replace all the projections + current.c = req.c; + // TODO: should remove only projections being superseded + + // as we've mutated the document in a manner that can conflict with other writers, out write needs to be contingent on no competing updates having taken place + finalize(current); + const isAccepted = collection.replaceDocument(current._self, current, { etag: current._etag }, callback); + if (!isAccepted) throw new Error("Unable to replace WIP batch."); + } else { + current = req; + current._i = 0; + // concurrency control is by virtue of fact that any conflicting writer will encounter a primary key violation (which will result in a retry) + finalize(current); + const isAccepted = collection.createDocument(collectionLink, current, { disableAutomaticIdGeneration: true }, callback); + if (!isAccepted) throw new Error("Unable to create WIP batch."); + } + for (i = 0; i < req.e.length; i++) { + const e = req.e[i]; + const eventI = current._i + current.e.length - req.e.length + i; + const doc = { + p: req.p, + id: eventI.toString(), + i: eventI, + c: e.c, + t: e.t, + d: e.d, + m: e.m + }; + const isAccepted = collection.createDocument(collectionLink, doc, function (err) { + if (err) throw err; + }); + if (!isAccepted) throw new Error("Unable to add event " + doc.i); + } + } - let private writeEventsAsync (client: IDocumentClient) (pos: Store.Position) (events: Store.EventData seq,maybeIndexEvents): Async = async { - let sprocLink = sprintf "%O/sprocs/%s" pos.collectionUri sprocName - let opts = Client.RequestOptions(PartitionKey=PartitionKey(pos.streamName)) + function finalize(current) { + current.i = -1; + current.id = current.i.toString(); + } +}""" + + [] + type Result = + | Written of Position + | Conflict of Position * events: IOrderedEvent[] + | ConflictUnknown of Position + + let private run (client: IDocumentClient) (stream: CollectionStream) (expectedVersion: int64 option, req: WipBatch) + : Async = async { + let sprocLink = sprintf "%O/sprocs/%s" stream.collectionUri sprocName + let opts = Client.RequestOptions(PartitionKey=PartitionKey(stream.name)) let! ct = Async.CancellationToken - let events = events |> Seq.mapi (fun i ed -> Store.Event.Create pos (i+1) ed |> JsonConvert.SerializeObject) |> Seq.toArray - if events.Length = 0 then invalidArg "eventsData" "must be non-empty" - let index : Store.IndexEvent = - match maybeIndexEvents with - | None | Some [||] -> Unchecked.defaultof<_> - | Some eds -> Store.IndexEvent.Create pos (events.Length) eds - try - let! (res : Client.StoredProcedureResponse) = client.ExecuteStoredProcedureAsync(sprocLink, opts, ct, box events, box pos.Index, box pos.etag, box index) |> Async.AwaitTaskCorrect - match res.RequestCharge, (match res.Response.etag with null -> None | x -> Some x), res.Response.conflicts with - | rc,e,null -> return rc, EqxSyncResult.Written { pos with index = Some (pos.IndexRel events.Length); etag=e } - | rc,e,[||] -> return rc, EqxSyncResult.ConflictUnknown { pos with etag=e } - | rc,e, xs -> return rc, EqxSyncResult.Conflict ({ pos with index = Some (pos.IndexRel xs.Length); etag=e }, Array.map (fun x -> x :> _) xs) - with DocDbException ex when ex.Message.Contains "already" -> // TODO this does not work for the SP - return ex.RequestCharge, EqxSyncResult.ConflictUnknown { pos with etag=None } } - - let bytes events = - let eventDataLen ({ data = Log.BlobLen bytes; metadata = Log.BlobLen metaBytes } : Store.EventData) = bytes + metaBytes - events |> Array.sumBy eventDataLen - - let private writeEventsLogged client (pos : Store.Position) (events : Store.EventData[], maybeIndexEvents) (log : ILogger): Async = async { - let log = if (not << log.IsEnabled) Events.LogEventLevel.Debug then log else log |> Log.propEventData "Json" events - let bytes, count = bytes events, events.Length + let ev = match expectedVersion with Some ev -> Position.FromI ev | None -> Position.FromAppendAtEnd + let! (res : Client.StoredProcedureResponse) = + client.ExecuteStoredProcedureAsync(sprocLink, opts, ct, box req, box ev.index) |> Async.AwaitTaskCorrect + + let newPos = { index = res.Response.nextI; etag = Option.ofObj res.Response.etag } + return res.RequestCharge, res.Response.conflicts |> function + | null -> Result.Written newPos + | [||] when newPos.index = 0L -> Result.Conflict (newPos, Array.empty) + | [||] -> Result.ConflictUnknown newPos + | xs -> Result.Conflict (newPos, Enum.Events (ev.index, xs) |> Array.ofSeq) } + + let private logged client (stream: CollectionStream) (expectedVersion, req: WipBatch) (log : ILogger) + : Async = async { + let verbose = log.IsEnabled Events.LogEventLevel.Debug + let log = if verbose then log |> Log.propEvents (Enum.Events req) |> Log.propDataProjections req.c else log + let (Log.BatchLen bytes), count = Enum.Events req, req.e.Length let log = log |> Log.prop "bytes" bytes - let writeLog = log |> Log.prop "stream" pos.streamName |> Log.prop "expectedVersion" pos.Index |> Log.prop "count" count - let! t, (ru,result) = writeEventsAsync client pos (events,maybeIndexEvents) |> Stopwatch.Time + let writeLog = + log |> Log.prop "stream" stream.name |> Log.prop "expectedVersion" expectedVersion + |> Log.prop "count" req.e.Length |> Log.prop "pcount" req.c.Length + let! t, (ru,result) = run client stream (expectedVersion, req) |> Stopwatch.Time let resultLog = - let mkMetric ru : Log.Measurement = { stream = pos.streamName; interval = t; bytes = bytes; count = count; ru = ru } - let logConflict () = writeLog.Information("Eqx TrySync WrongExpectedVersion writing {EventTypes}", [| for x in events -> x.eventType |]) + let mkMetric ru : Log.Measurement = { stream = stream.name; interval = t; bytes = bytes; count = count; ru = ru } + let logConflict () = writeLog.Information("Eqx TrySync Conflict writing {eventTypes}", [| for x in req.e -> x.t |]) match result with - | EqxSyncResult.Written pos -> + | Result.Written pos -> log |> Log.event (Log.WriteSuccess (mkMetric ru)) |> Log.prop "nextExpectedVersion" pos - | EqxSyncResult.ConflictUnknown pos -> + | Result.ConflictUnknown pos -> logConflict () log |> Log.event (Log.WriteConflict (mkMetric ru)) |> Log.prop "nextExpectedVersion" pos |> Log.prop "conflict" true - | EqxSyncResult.Conflict (pos, xs) -> + | Result.Conflict (pos, xs) -> logConflict () - let log = if (not << log.IsEnabled) Events.LogEventLevel.Debug then log else log |> Log.prop "nextExpectedVersion" pos |> Log.propIEventDatas "conflictJson" xs - log |> Log.event (Log.WriteConflict (mkMetric ru)) |> Log.prop "conflict" true - resultLog.Information("Eqx {action:l} {count} {ms}ms rc={ru}", "Write", events.Length, (let e = t.Elapsed in e.TotalMilliseconds), ru) + let log = if verbose then log |> Log.prop "nextExpectedVersion" pos |> Log.propData "conflicts" xs else log + log |> Log.event (Log.WriteResync(mkMetric ru)) |> Log.prop "conflict" true + resultLog.Information("Eqx {action:l} {count}+{pcount} {ms}ms rc={ru}", "Write", req.e.Length, req.c.Length, (let e = t.Elapsed in e.TotalMilliseconds), ru) return result } - let writeEvents (log : ILogger) retryPolicy client pk (events : Store.EventData[],maybeIndexEvents): Async = - let call = writeEventsLogged client pk (events,maybeIndexEvents) + let batch (log : ILogger) retryPolicy client pk batch: Async = + let call = logged client pk batch Log.withLoggedRetries retryPolicy "writeAttempt" call log - -module private Read = - let private getIndex (client : IDocumentClient) (pos:Store.Position) = - let coll = DocDbCollection(client, pos.collectionUri) - let ac = match pos.etag with None -> null | Some etag-> Client.AccessCondition(Type=Client.AccessConditionType.IfNoneMatch, Condition=etag) - let ro = Client.RequestOptions(PartitionKey=PartitionKey(pos.streamName), AccessCondition = ac) - coll.TryReadDocument(Store.IndexEvent.IdConstant, ro) - let private loggedGetIndex (getIndex : Store.Position -> Async<_>) (pos:Store.Position) (log: ILogger) = async { - let log = log |> Log.prop "stream" pos.streamName - let! t, (ru, res : ReadResult) = getIndex pos |> Stopwatch.Time - let log count bytes (f : Log.Measurement -> _) = log |> Log.event (f { stream = pos.streamName; interval = t; bytes = bytes; count = count; ru = ru }) + let mkBatch (stream: Store.CollectionStream) (events: IEvent[]) projections: WipBatch = + { p = stream.name; id = Store.WipBatch.WellKnownDocumentId; _i = -1L(*Server-managed*); _etag = null + e = [| for e in events -> { c = DateTimeOffset.UtcNow; t = e.EventType; d = e.Data; m = e.Meta } |] + c = Array.ofSeq projections } + let mkProjections baseIndex (projectionEvents: IEvent seq) : Store.Projection seq = + projectionEvents |> Seq.mapi (fun offset x -> { i = baseIndex + int64 offset; t = x.EventType; d = x.Data; m = x.Meta } : Store.Projection) + + module Initialization = + open System.Collections.ObjectModel + let createDatabase (client:IDocumentClient) dbName = async { + let opts = Client.RequestOptions(ConsistencyLevel = Nullable ConsistencyLevel.Session) + let! db = client.CreateDatabaseIfNotExistsAsync(Database(Id=dbName), options = opts) |> Async.AwaitTaskCorrect + return db.Resource.Id } + + let createCollection (client: IDocumentClient) (dbUri: Uri) collName ru = async { + let pkd = PartitionKeyDefinition() + pkd.Paths.Add(sprintf "/%s" Store.Event.PartitionKeyField) + let colld = DocumentCollection(Id = collName, PartitionKey = pkd) + + colld.IndexingPolicy.IndexingMode <- IndexingMode.Consistent + colld.IndexingPolicy.Automatic <- true + // Can either do a blacklist or a whitelist + // Given how long and variable the blacklist would be, we whitelist instead + colld.IndexingPolicy.ExcludedPaths <- Collection [|ExcludedPath(Path="/*")|] + // NB its critical to index the nominated PartitionKey field defined above or there will be runtime errors + colld.IndexingPolicy.IncludedPaths <- Collection [| for k in Store.Event.IndexedFields -> IncludedPath(Path=sprintf "/%s/?" k) |] + let! coll = client.CreateDocumentCollectionIfNotExistsAsync(dbUri, colld, Client.RequestOptions(OfferThroughput=Nullable ru)) |> Async.AwaitTaskCorrect + return coll.Resource.Id } + + let createProc (log: ILogger) (client: IDocumentClient) (collectionUri: Uri) = async { + let def = new StoredProcedure(Id = sprocName, Body = sprocBody) + log.Information("Creating stored procedure {sprocId}", def.Id) + // TODO ifnotexist semantics + return! client.CreateStoredProcedureAsync(collectionUri, def) |> Async.AwaitTaskCorrect |> Async.Ignore } + + let initialize log (client : IDocumentClient) dbName collName ru = async { + let! dbId = createDatabase client dbName + let dbUri = Client.UriFactory.CreateDatabaseUri dbId + let! collId = createCollection client dbUri collName ru + let collUri = Client.UriFactory.CreateDocumentCollectionUri (dbName, collId) + //let! _aux = createAux client dbUri collName auxRu + return! createProc log client collUri } + +module private Index = + let private get (client: IDocumentClient) (stream: CollectionStream, maybePos: Position option) = + let coll = DocDbCollection(client, stream.collectionUri) + let ac = match maybePos with Some { etag=Some etag } -> Client.AccessCondition(Type=Client.AccessConditionType.IfNoneMatch, Condition=etag) | _ -> null + let ro = Client.RequestOptions(PartitionKey=PartitionKey(stream.name), AccessCondition = ac) + coll.TryReadDocument(WipBatch.WellKnownDocumentId, ro) + let private loggedGet (get : CollectionStream * Position option -> Async<_>) (stream: CollectionStream, maybePos: Position option) (log: ILogger) = async { + let log = log |> Log.prop "stream" stream.name + let! t, (ru, res : ReadResult) = get (stream,maybePos) |> Stopwatch.Time + let log count bytes (f : Log.Measurement -> _) = log |> Log.event (f { stream = stream.name; interval = t; bytes = bytes; count = count; ru = ru }) match res with | ReadResult.NotModified -> (log 0 0 Log.IndexNotModified).Information("Eqx {action:l} {res} {ms}ms rc={ru}", "Index", 302, (let e = t.Elapsed in e.TotalMilliseconds), ru) @@ -358,353 +552,269 @@ module private Read = (log 0 0 Log.IndexNotFound).Information("Eqx {action:l} {res} {ms}ms rc={ru}", "Index", 404, (let e = t.Elapsed in e.TotalMilliseconds), ru) | ReadResult.Found doc -> let log = - let (|EventLen|) (x : Store.IndexProjection) = match x.d, x.m with Log.BlobLen bytes, Log.BlobLen metaBytes -> bytes + metaBytes - let bytes, count = doc.c |> Array.sumBy (|EventLen|), doc.c.Length + let (Log.BatchLen bytes), count = Enum.Projections doc.c, doc.c.Length log bytes count Log.Index - let log = if (not << log.IsEnabled) Events.LogEventLevel.Debug then log else log |> Log.propProjectionEvents "Json" doc.c |> Log.prop "etag" doc._etag + let log = if (not << log.IsEnabled) Events.LogEventLevel.Debug then log else log |> Log.propDataProjections doc.c |> Log.prop "etag" doc._etag log.Information("Eqx {action:l} {res} {ms}ms rc={ru}", "Index", 200, (let e = t.Elapsed in e.TotalMilliseconds), ru) return ru, res } - type [] IndexResult = NotModified | NotFound | Found of Store.Position * Store.IndexProjection[] + type [] Result = NotModified | NotFound | Found of Position * IOrderedEvent[] /// `pos` being Some implies that the caller holds a cached value and hence is ready to deal with IndexResult.UnChanged - let loadIndex (log : ILogger) retryPolicy client (pos : Store.Position): Async = async { - let getIndex = getIndex client - let! _rc, res = Log.withLoggedRetries retryPolicy "readAttempt" (loggedGetIndex getIndex pos) log + let tryLoad (log : ILogger) retryPolicy client (stream: CollectionStream) (maybePos: Position option): Async = async { + let get = get client + let! _rc, res = Log.withLoggedRetries retryPolicy "readAttempt" (loggedGet get (stream,maybePos)) log match res with - | ReadResult.NotModified -> return IndexResult.NotModified - | ReadResult.NotFound -> return IndexResult.NotFound - | ReadResult.Found index -> return IndexResult.Found ({ pos with index = Some index.m; etag=if index._etag=null then None else Some index._etag }, index.c) } + | ReadResult.NotModified -> return Result.NotModified + | ReadResult.NotFound -> return Result.NotFound + | ReadResult.Found doc -> return Result.Found (doc.ToPosition(), Enum.EventsAndProjections doc |> Array.ofSeq) } - let private getQuery (client : IDocumentClient) (pos:Store.Position) (direction: Direction) batchSize = + module private Query = + open Microsoft.Azure.Documents.Linq + let private mkQuery (client : IDocumentClient) maxItems (stream: CollectionStream) (direction: Direction) (startPos: Position option) = let querySpec = - match pos.index with - | None -> SqlQuerySpec(if direction = Direction.Forward then "SELECT * FROM c ORDER BY c.i ASC" else "SELECT * FROM c ORDER BY c.i DESC") - | Some index -> + match startPos with + | None -> SqlQuerySpec("SELECT * FROM c WHERE c.i!=-1 ORDER BY c.i " + if direction = Direction.Forward then "ASC" else "DESC") + | Some p -> let f = if direction = Direction.Forward then "c.i >= @id ORDER BY c.i ASC" else "c.i < @id ORDER BY c.i DESC" - SqlQuerySpec( "SELECT * FROM c WHERE " + f, SqlParameterCollection (Seq.singleton (SqlParameter("@id", index)))) - let feedOptions = new Client.FeedOptions(PartitionKey=PartitionKey(pos.streamName), MaxItemCount=Nullable batchSize) - client.CreateDocumentQuery(pos.collectionUri, querySpec, feedOptions).AsDocumentQuery() - - let (|EventLen|) (x : Store.Event) = match x.d, x.m with Log.BlobLen bytes, Log.BlobLen metaBytes -> bytes + metaBytes - let bytes events = events |> Array.sumBy (|EventLen|) + SqlQuerySpec("SELECT * FROM c WHERE c.i != -1 AND " + f, SqlParameterCollection [SqlParameter("@id", p.index)]) + let feedOptions = new Client.FeedOptions(PartitionKey=PartitionKey(stream.name), MaxItemCount=Nullable maxItems) + client.CreateDocumentQuery(stream.collectionUri, querySpec, feedOptions).AsDocumentQuery() - let private loggedQueryExecution (pos:Store.Position) direction (query: IDocumentQuery) (log: ILogger): Async = async { + // Unrolls the Batches in a response - note when reading backawards, the events are emitted in reverse order of index + let private handleSlice direction (stream: CollectionStream) (startPos: Position option) (query: IDocumentQuery) (log: ILogger) + : Async = async { let! ct = Async.CancellationToken - let! t, (res : Client.FeedResponse) = query.ExecuteNextAsync(ct) |> Async.AwaitTaskCorrect |> Stopwatch.Time - let slice, ru = Array.ofSeq res, res.RequestCharge - let bytes, count = bytes slice, slice.Length - let reqMetric : Log.Measurement = { stream = pos.streamName; interval = t; bytes = bytes; count = count; ru = ru } - let evt = Log.Slice (direction, reqMetric) - let log = if (not << log.IsEnabled) Events.LogEventLevel.Debug then log else log |> Log.propResolvedEvents "Json" slice - let index = match slice |> Array.tryHead with Some head -> head.id | None -> null - (log |> Log.prop "startIndex" pos.Index |> Log.prop "bytes" bytes |> Log.event evt) - .Information("Eqx {action:l} {count} {direction} {ms}ms i={index} rc={ru}", "Query", count, direction, (let e = t.Elapsed in e.TotalMilliseconds), index, ru) - return slice, ru } - - let private readBatches (log : ILogger) (readSlice: IDocumentQuery -> ILogger -> Async) + let! t, (res : Client.FeedResponse) = query.ExecuteNextAsync(ct) |> Async.AwaitTaskCorrect |> Stopwatch.Time + let batches, ru = Array.ofSeq res, res.RequestCharge + let events = batches |> Seq.collect Enum.Event |> Array.ofSeq + let (Log.BatchLen bytes), count = events, events.Length + let reqMetric : Log.Measurement = { stream = stream.name; interval = t; bytes = bytes; count = count; ru = ru } + // TODO investigate whether there is a way to avoid the potential cost (or whether there is significance to it) of these null responses + let log = if batches.Length = 0 && count = 0 && ru = 0. then log else let evt = Log.Slice (direction, reqMetric) in log |> Log.event evt + let log = if (not << log.IsEnabled) Events.LogEventLevel.Debug then log else log |> Log.propEvents events + let index = if count = 0 then Nullable () else Nullable <| Seq.min (seq { for x in batches -> x.i }) + (log |> Log.prop "startIndex" (match startPos with Some { index = i } -> Nullable i | _ -> Nullable()) |> Log.prop "bytes" bytes) + .Information("Eqx {action:l} {count}/{batches} {direction} {ms}ms i={index} rc={ru}", + "Query", count, batches.Length, direction, (let e = t.Elapsed in e.TotalMilliseconds), index, ru) + let maybePosition = batches |> Array.tryPick (fun x -> x.TryToPosition()) + return events, maybePosition, ru } + + let private runQuery (log : ILogger) (readSlice: IDocumentQuery -> ILogger -> Async) (maxPermittedBatchReads: int option) - (query: IDocumentQuery) - : AsyncSeq = - let rec loop batchCount : AsyncSeq = asyncSeq { + (query: IDocumentQuery) + : AsyncSeq = + let rec loop batchCount : AsyncSeq = asyncSeq { match maxPermittedBatchReads with | Some mpbr when batchCount >= mpbr -> log.Information "batch Limit exceeded"; invalidOp "batch Limit exceeded" | _ -> () let batchLog = log |> Log.prop "batchIndex" batchCount - let! slice = readSlice query batchLog + let! (slice : IOrderedEvent[] * Position option * float) = readSlice query batchLog yield slice if query.HasMoreResults then yield! loop (batchCount + 1) } loop 0 - let logBatchRead direction streamName interval events batchSize version (ru: float) (log : ILogger) = - let bytes, count = bytes events, events.Length + let private logBatchRead direction batchSize streamName interval (responsesCount, events : IOrderedEvent []) nextI (ru: float) (log : ILogger) = + let (Log.BatchLen bytes), count = events, events.Length let reqMetric : Log.Measurement = { stream = streamName; interval = interval; bytes = bytes; count = count; ru = ru } - let batches = (events.Length - 1)/batchSize + 1 let action = match direction with Direction.Forward -> "LoadF" | Direction.Backward -> "LoadB" - let evt = Log.Event.Batch (direction, batches, reqMetric) - (log |> Log.prop "bytes" bytes |> Log.event evt).Information( - "Eqx {action:l} stream={stream} {count}/{batches} {ms}ms i={index} rc={ru}", - action, streamName, count, batches, (let e = interval.Elapsed in e.TotalMilliseconds), version, ru) - - let private lastEventIndex (xs:Store.Event seq) : int64 = - match xs |> Seq.tryLast with - | None -> -1L - | Some last -> int64 last.id - - let loadForwardsFrom (log : ILogger) retryPolicy client batchSize maxPermittedBatchReads (pos): Async = async { - let mutable ru = 0.0 - let mergeBatches (batches: AsyncSeq) = async { - let! (events : Store.Event[]) = - batches - |> AsyncSeq.map (fun (events, r) -> ru <- ru + r; events) - |> AsyncSeq.concatSeq - |> AsyncSeq.toArrayAsync - return events, ru } - use query = getQuery client pos Direction.Forward batchSize - let call q = loggedQueryExecution pos Direction.Forward q - let retryingLoggingReadSlice q = Log.withLoggedRetries retryPolicy "readAttempt" (call q) - let direction = Direction.Forward - let log = log |> Log.prop "batchSize" batchSize |> Log.prop "direction" direction |> Log.prop "stream" pos.streamName - let batches : AsyncSeq = readBatches log retryingLoggingReadSlice maxPermittedBatchReads query - let! t, (events, ru) = mergeBatches batches |> Stopwatch.Time - query.Dispose() - let version = lastEventIndex events - log |> logBatchRead direction pos.streamName t events batchSize version ru - return { pos with index = Some version }, events } - - let partitionPayloadFrom firstUsedEventNumber : Store.Event[] -> int * int = - let acc (tu,tr) ((EventLen bytes) as y) = if y.id < firstUsedEventNumber then tu, tr + bytes else tu + bytes, tr - Array.fold acc (0,0) - let loadBackwardsUntilCompactionOrStart (log : ILogger) retryPolicy client batchSize maxPermittedBatchReads isCompactionEvent (pos : Store.Position) - : Async = async { - let mergeFromCompactionPointOrStartFromBackwardsStream (log : ILogger) (batchesBackward : AsyncSeq) - : Async = async { - let lastBatch = ref None + // TODO investigate whether there is a way to avoid the potential cost (or whether there is significance to it) of these null responses + let log = if count = 0 && ru = 0. then log else let evt = Log.Event.Batch (direction, responsesCount, reqMetric) in log |> Log.event evt + (log |> Log.prop "bytes" bytes |> Log.prop "batchSize" batchSize).Information( + "Eqx {action:l} {stream} v{nextI} {count}/{responses} {ms}ms rc={ru}", + action, streamName, nextI, count, responsesCount, (let e = interval.Elapsed in e.TotalMilliseconds), ru) + + let private inferPosition maybeIndexDocument (events: IOrderedEvent[]): Position = match maybeIndexDocument with Some p -> p | None -> Position.FromMaxIndex events + + let private calculateUsedVersusDroppedPayload stopIndex (xs: IOrderedEvent[]) : int * int = + let mutable used, dropped = 0, 0 + let mutable found = false + for x in xs do + let (Log.EventLen bytes) = x + if found then dropped <- dropped + bytes + else used <- used + bytes + if x.Index = stopIndex then found <- true + used, dropped + + let walk (log : ILogger) client retryPolicy maxItems maxRequests direction (stream: CollectionStream) startPos predicate + : Async = async { + let responseCount = ref 0 + let mergeBatches (log : ILogger) (batchesBackward : AsyncSeq) + : Async = async { + let mutable lastResponse = None + let mutable maybeIndexDocument = None let mutable ru = 0.0 - let! tempBackward = + let! events = batchesBackward - |> AsyncSeq.map (fun (events, r) -> lastBatch := Some events; ru <- ru + r; events) + |> AsyncSeq.map (fun (events, maybePos, r) -> + if maybeIndexDocument = None then maybeIndexDocument <- maybePos + lastResponse <- Some events; ru <- ru + r + incr responseCount + events) |> AsyncSeq.concatSeq |> AsyncSeq.takeWhileInclusive (fun x -> - if not (isCompactionEvent x) then true // continue the search + if not (predicate x) then true // continue the search else - match !lastBatch with - | None -> log.Information("Eqx Stop stream={stream} at={eventNumber}", pos.streamName, x.id) + match lastResponse with + | None -> log.Information("Eqx Stop stream={stream} at={index}", stream.name, x.Index) | Some batch -> - let used, residual = batch |> partitionPayloadFrom x.id - log.Information("Eqx Stop stream={stream} at={eventNumber} used={used} residual={residual}", pos.streamName, x.id, used, residual) + let used, residual = batch |> calculateUsedVersusDroppedPayload x.Index + log.Information("Eqx Stop stream={stream} at={index} used={used} residual={residual}", stream.name, x.Index, used, residual) false) |> AsyncSeq.toArrayAsync - let eventsForward = Array.Reverse(tempBackward); tempBackward // sic - relatively cheap, in-place reverse of something we own - return eventsForward, ru } - use query = getQuery client pos Direction.Backward batchSize - let call q = loggedQueryExecution pos Direction.Backward q - let retryingLoggingReadSlice q = Log.withLoggedRetries retryPolicy "readAttempt" (call q) - let log = log |> Log.prop "batchSize" batchSize |> Log.prop "stream" pos.streamName - let direction = Direction.Backward + return events, maybeIndexDocument, ru } + use query = mkQuery client maxItems stream direction startPos + let pullSlice = handleSlice direction stream startPos + let retryingLoggingReadSlice query = Log.withLoggedRetries retryPolicy "readAttempt" (pullSlice query) + let log = log |> Log.prop "batchSize" maxItems |> Log.prop "stream" stream.name let readlog = log |> Log.prop "direction" direction - let batchesBackward : AsyncSeq = readBatches readlog retryingLoggingReadSlice maxPermittedBatchReads query - let! t, (events, ru) = mergeFromCompactionPointOrStartFromBackwardsStream log batchesBackward |> Stopwatch.Time + let batches : AsyncSeq = runQuery readlog retryingLoggingReadSlice maxRequests query + let! t, (events, maybeIndexDocument, ru) = mergeBatches log batches |> Stopwatch.Time query.Dispose() - let version = lastEventIndex events - log |> logBatchRead direction pos.streamName t events batchSize version ru - return { pos with index = Some version } , events } + let pos = inferPosition maybeIndexDocument events -module UnionEncoderAdapters = - let private encodedEventOfStoredEvent (x : Store.Event) : UnionCodec.EncodedUnion = - { caseName = x.t; payload = x.d } - let private encodedEventOfStoredEventI (x : Store.IEventData) : UnionCodec.EncodedUnion = - { caseName = x.EventType; payload = x.DataUtf8 } - let private eventDataOfEncodedEvent (x : UnionCodec.EncodedUnion) : Store.EventData = - { eventType = x.caseName; data = x.payload; metadata = null } - let encodeEvents (codec : UnionCodec.IUnionEncoder<'event, byte[]>) (xs : 'event seq) : Store.EventData[] = - xs |> Seq.map (codec.Encode >> eventDataOfEncodedEvent) |> Seq.toArray - let decodeKnownEventsI (codec : UnionCodec.IUnionEncoder<'event, byte[]>) (xs : Store.IEventData seq) : 'event seq = - xs |> Seq.map encodedEventOfStoredEventI |> Seq.choose codec.TryDecode - let decodeKnownEvents (codec : UnionCodec.IUnionEncoder<'event, byte[]>) (xs : Store.Event seq) : 'event seq = - xs |> Seq.map encodedEventOfStoredEvent |> Seq.choose codec.TryDecode - -type []Token = { pos: Store.Position; compactionEventNumber: int64 option } + log |> logBatchRead direction maxItems stream.name t (!responseCount,events) pos.index ru + return pos, events } +module UnionEncoderAdapters = + let encodeEvent (codec : UnionCodec.IUnionEncoder<'event, byte[]>) (x : 'event) : IEvent = + let e = codec.Encode x + { new IEvent with + member __.EventType = e.caseName + member __.Data = e.payload + member __.Meta = null } + let decodeKnownEvents (codec : UnionCodec.IUnionEncoder<'event, byte[]>): IOrderedEvent seq -> 'event seq = + Seq.choose (fun x -> codec.TryDecode { caseName = x.EventType; payload = x.Data }) + +type [] Token = { stream: CollectionStream; pos: Position } module Token = - let private create compactionEventNumber batchCapacityLimit pos : Storage.StreamToken = - { value = box { pos = pos; compactionEventNumber = compactionEventNumber }; batchCapacityLimit = batchCapacityLimit } - /// No batching / compaction; we only need to retain the StreamVersion - let ofNonCompacting (pos : Store.Position) : Storage.StreamToken = - create None None pos - // headroom before compaction is necessary given the stated knowledge of the last (if known) `compactionEventNumberOption` - let private batchCapacityLimit compactedEventNumberOption unstoredEventsPending (batchSize : int) (streamVersion : int64) : int = - match compactedEventNumberOption with - | Some (compactionEventNumber : int64) -> (batchSize - unstoredEventsPending) - int (streamVersion - compactionEventNumber + 1L) |> max 0 - | None -> (batchSize - unstoredEventsPending) - (int streamVersion + 1) - 1 |> max 0 - let (*private*) ofCompactionEventNumber compactedEventNumberOption unstoredEventsPending batchSize (pos : Store.Position) : Storage.StreamToken = - let batchCapacityLimit = batchCapacityLimit compactedEventNumberOption unstoredEventsPending batchSize pos.Index - create compactedEventNumberOption (Some batchCapacityLimit) pos - /// Assume we have not seen any compaction events; use the batchSize and version to infer headroom - let ofUncompactedVersion batchSize pos : Storage.StreamToken = - ofCompactionEventNumber None 0 batchSize pos - /// Use previousToken plus the data we are adding and the position we are adding it to infer a headroom - let ofPreviousTokenAndEventsLength (previousToken : Storage.StreamToken) eventsLength batchSize pos : Storage.StreamToken = - let compactedEventNumber = (unbox previousToken.value).compactionEventNumber - ofCompactionEventNumber compactedEventNumber eventsLength batchSize pos - let ofPreviousTokenWithUpdatedPosition (previousToken : Storage.StreamToken) batchSize pos : Storage.StreamToken = - let compactedEventNumber = (unbox previousToken.value).compactionEventNumber - ofCompactionEventNumber compactedEventNumber 0 batchSize pos - /// Use an event just read from the stream to infer headroom - let ofCompactionResolvedEventAndVersion (compactionEvent: Store.Event) batchSize pos : Storage.StreamToken = - ofCompactionEventNumber (Some (int64 compactionEvent.id)) 0 batchSize pos - /// Use an event we are about to write to the stream to infer headroom - let ofPreviousStreamVersionAndCompactionEventDataIndex prevStreamVersion compactionEventDataIndex eventsLength batchSize streamVersion' : Storage.StreamToken = - ofCompactionEventNumber (Some (prevStreamVersion + 1L + int64 compactionEventDataIndex)) eventsLength batchSize streamVersion' - let private unpackEqxStreamVersion (x : Storage.StreamToken) = let x : Token = unbox x.value in x.pos.Index - let private unpackEqxETag (x : Storage.StreamToken) = let x : Token = unbox x.value in x.pos.etag - let supersedes current x = - let currentVersion, newVersion = unpackEqxStreamVersion current, unpackEqxStreamVersion x - let currentETag, newETag = unpackEqxETag current, unpackEqxETag x + let create stream pos : Storage.StreamToken = { value = box { stream = stream; pos = pos } } + let (|Unpack|) (token: Storage.StreamToken) : CollectionStream*Position = let t = unbox token.value in t.stream,t.pos + let supersedes (Unpack (_,currentPos)) (Unpack (_,xPos)) = + let currentVersion, newVersion = currentPos.index, xPos.index + let currentETag, newETag = currentPos.etag, xPos.etag newVersion > currentVersion || currentETag <> newETag +namespace Equinox.Cosmos.Builder + +open Equinox +open Equinox.Cosmos.Events // NB needs to be shadow by Equinox.Cosmos +open Equinox.Cosmos +open Equinox.Store.Infrastructure +open FSharp.Control +open Microsoft.Azure.Documents +open Serilog +open System +open System.Collections.Generic + +[] +module Internal = + [] + type InternalSyncResult = Written of Storage.StreamToken | ConflictUnknown of Storage.StreamToken | Conflict of Storage.StreamToken * IOrderedEvent[] + + [] + type LoadFromTokenResult = Unchanged | Found of Storage.StreamToken * IOrderedEvent[] + +/// Defines the policies in force for retrying with regard to transient failures calling CosmosDb (as opposed to application level concurrency conflicts) type EqxConnection(client: IDocumentClient, ?readRetryPolicy (*: (int -> Async<'T>) -> Async<'T>*), ?writeRetryPolicy) = member __.Client = client member __.ReadRetryPolicy = readRetryPolicy member __.WriteRetryPolicy = writeRetryPolicy - member __.Close = (client :?> Client.DocumentClient).Dispose() - -type EqxBatchingPolicy(getMaxBatchSize : unit -> int, ?batchCountLimit) = - new (maxBatchSize) = EqxBatchingPolicy(fun () -> maxBatchSize) - member __.BatchSize = getMaxBatchSize() - member __.MaxBatches = batchCountLimit - -[] -type GatewaySyncResult = Written of Storage.StreamToken | ConflictUnknown of Storage.StreamToken | Conflict of Storage.StreamToken * Store.IEventData[] - -[] -type LoadFromTokenResult = Unchanged | Found of Storage.StreamToken * Store.IEventData[] + //member __.Close = (client :?> Client.DocumentClient).Dispose() + +/// Defines the policies in force regarding how to constrain query responses +type EqxBatchingPolicy + ( // Max items to request in query response. Defaults to 10. + ?defaultMaxItems : int, + // Dynamic version of `defaultMaxItems`, allowing one to react to dynamic configuration changes. Default to using `defaultMaxItems` + ?getDefaultMaxItems : unit -> int, + /// Maximum number of trips to permit when slicing the work into multiple responses based on `MaxSlices`. Default: unlimited. + ?maxRequests) = + let getdefaultMaxItems = defaultArg getDefaultMaxItems (fun () -> defaultArg defaultMaxItems 10) + /// Limit for Maximum number of `Batch` records in a single query batch response + member __.MaxItems = getdefaultMaxItems () + /// Maximum number of trips to permit when slicing the work into multiple responses based on `MaxSlices` + member __.MaxRequests = maxRequests type EqxGateway(conn : EqxConnection, batching : EqxBatchingPolicy) = - let isResolvedEventEventType predicate (x:Store.Event) = predicate x.t - let tryIsResolvedEventEventType predicateOption = predicateOption |> Option.map isResolvedEventEventType - //let isResolvedEventDataEventType predicate (x:Store.Event) = predicate x.t - //let tryIsEventDataEventType predicateOption = predicateOption |> Option.map isResolvedEventDataEventType - let (|Pos|) (token: Storage.StreamToken) : Store.Position = (unbox token.value).pos - let (|IEventDataArray|) events = [| for e in events -> e :> Store.IEventData |] - member private __.InterpretIndexOrFallback log isCompactionEventType pos res: Async = async { + let eventTypesPredicate resolved = + let acc = HashSet() + fun (x: IOrderedEvent) -> + acc.Add x.EventType |> ignore + resolved acc + let (|Satisfies|_|) predicate (xs:IOrderedEvent[]) = + match Array.tryFindIndexBack predicate xs with + | None -> None + | Some index -> Array.sub xs index (xs.Length - index) |> Some + let loadBackwardsStopping log predicate stream: Async = async { + let! pos, events = Query.walk log conn.Client conn.ReadRetryPolicy batching.MaxItems batching.MaxRequests Direction.Backward stream None predicate + Array.Reverse events + return Token.create stream pos, events } + member __.LoadBackwardsStopping log predicate stream: Async = + let predicate = eventTypesPredicate predicate + loadBackwardsStopping log predicate stream + member __.Read log batchingOverride stream direction startPos predicate: Async = async { + let batching = defaultArg batchingOverride batching + let! pos, events = Query.walk log conn.Client conn.ReadRetryPolicy batching.MaxItems batching.MaxRequests direction stream startPos predicate + return Token.create stream pos, events } + member __.LoadFromProjectionsOrRollingSnapshots log predicate (stream,maybePos): Async = async { + let! res = Index.tryLoad log None(* TODO conn.ReadRetryPolicy*) conn.Client stream maybePos + let predicate = eventTypesPredicate predicate + match res with + | Index.Result.NotFound -> return Token.create stream Position.FromKnownEmpty, Array.empty + | Index.Result.NotModified -> return invalidOp "Not handled" + | Index.Result.Found (pos, Satisfies predicate enoughEvents) -> return Token.create stream pos, enoughEvents + | _ -> return! loadBackwardsStopping log predicate stream } + member __.GetPosition(log, stream, ?pos): Async = async { + let! res = Index.tryLoad log None(* TODO conn.ReadRetryPolicy*) conn.Client stream pos match res with - | Read.IndexResult.NotModified -> return invalidOp "Not handled" - | Read.IndexResult.Found (pos, projectionsAndEvents) when projectionsAndEvents |> Array.exists (fun x -> isCompactionEventType x.t) -> - return Token.ofNonCompacting pos, projectionsAndEvents |> Seq.cast |> Array.ofSeq + | Index.Result.NotFound -> return Token.create stream Position.FromKnownEmpty + | Index.Result.NotModified -> return Token.create stream pos.Value + | Index.Result.Found (pos, _projectionsAndEvents) -> return Token.create stream pos } + member __.LoadFromToken log (stream,pos) predicate: Async = async { + let predicate = eventTypesPredicate predicate + let! res = Index.tryLoad log None(* TODO conn.ReadRetryPolicy*) conn.Client stream (Some pos) + match res with + | Index.Result.NotFound -> return LoadFromTokenResult.Found (Token.create stream Position.FromKnownEmpty,Array.empty) + | Index.Result.NotModified -> return LoadFromTokenResult.Unchanged + | Index.Result.Found (pos, Satisfies predicate enoughEvents) -> return LoadFromTokenResult.Found (Token.create stream pos, enoughEvents) | _ -> - let! streamToken, events = __.LoadBackwardsStoppingAtCompactionEvent log isCompactionEventType pos - return streamToken, events |> Seq.cast |> Array.ofSeq } - member __.LoadBatched log isCompactionEventType (pos : Store.Position): Async = async { - let! pos, events = Read.loadForwardsFrom log conn.ReadRetryPolicy conn.Client batching.BatchSize batching.MaxBatches pos - match tryIsResolvedEventEventType isCompactionEventType with - | None -> return Token.ofNonCompacting pos, events - | Some isCompactionEvent -> - match events |> Array.tryFindBack isCompactionEvent with - | None -> return Token.ofUncompactedVersion batching.BatchSize pos, events - | Some resolvedEvent -> return Token.ofCompactionResolvedEventAndVersion resolvedEvent batching.BatchSize pos, events } - member __.LoadBackwardsStoppingAtCompactionEvent log isCompactionEventType pos: Async = async { - let isCompactionEvent = isResolvedEventEventType isCompactionEventType - let! pos, events = - Read.loadBackwardsUntilCompactionOrStart log conn.ReadRetryPolicy conn.Client batching.BatchSize batching.MaxBatches isCompactionEvent pos - match Array.tryHead events |> Option.filter isCompactionEvent with - | None -> return Token.ofUncompactedVersion batching.BatchSize pos, events - | Some resolvedEvent -> return Token.ofCompactionResolvedEventAndVersion resolvedEvent batching.BatchSize pos, events } - member __.IndexedOrBatched log isCompactionEventType pos: Async = async { - let! res = Read.loadIndex log None(* TODO conn.ReadRetryPolicy*) conn.Client pos - return! __.InterpretIndexOrFallback log isCompactionEventType pos res } - member __.LoadFromToken log (Pos pos as token) isCompactionEventType tryIndex: Async = async { - let ok r = LoadFromTokenResult.Found r - if not tryIndex then - let! pos, ((IEventDataArray xs) as events) = Read.loadForwardsFrom log conn.ReadRetryPolicy conn.Client batching.BatchSize batching.MaxBatches pos - let ok t = ok (t,xs) - match tryIsResolvedEventEventType isCompactionEventType with - | None -> return ok (Token.ofNonCompacting pos) - | Some isCompactionEvent -> - match events |> Array.tryFindBack isCompactionEvent with - | None -> return ok (Token.ofPreviousTokenAndEventsLength token events.Length batching.BatchSize pos) - | Some resolvedEvent -> return ok (Token.ofCompactionResolvedEventAndVersion resolvedEvent batching.BatchSize pos) - else - let! res = Read.loadIndex log None(* TODO conn.ReadRetryPolicy*) conn.Client pos - match res with - | Read.IndexResult.NotModified -> - return LoadFromTokenResult.Unchanged - | _ -> - let! loaded = __.InterpretIndexOrFallback log isCompactionEventType.Value pos res - return ok loaded } - member __.TrySync log (Pos pos as token) (encodedEvents: Store.EventData[],maybeIndexEvents) isCompactionEventType: Async = async { - let! wr = Write.writeEvents log conn.WriteRetryPolicy conn.Client pos (encodedEvents,maybeIndexEvents) + let! res = __.Read log None stream Direction.Forward (Some pos) (fun _ -> false) + return LoadFromTokenResult.Found res } + member __.Sync log stream (expectedVersion, batch: Store.WipBatch): Async = async { + let! wr = Sync.batch log conn.WriteRetryPolicy conn.Client stream (expectedVersion,batch) match wr with - | EqxSyncResult.Conflict (pos',events) -> - return GatewaySyncResult.Conflict (Token.ofPreviousTokenAndEventsLength token events.Length batching.BatchSize pos',events) - | EqxSyncResult.ConflictUnknown pos' -> - return GatewaySyncResult.ConflictUnknown (Token.ofPreviousTokenWithUpdatedPosition token batching.BatchSize pos') - | EqxSyncResult.Written pos' -> - - let token = - match isCompactionEventType with - | None -> Token.ofNonCompacting pos' - | Some isCompactionEvent -> - let isEventDataEventType predicate (x:Store.EventData) = predicate x.eventType - match encodedEvents |> Array.tryFindIndexBack (isEventDataEventType isCompactionEvent) with - | None -> Token.ofPreviousTokenAndEventsLength token encodedEvents.Length batching.BatchSize pos' - | Some compactionEventIndex -> - Token.ofPreviousStreamVersionAndCompactionEventDataIndex pos.Index compactionEventIndex encodedEvents.Length batching.BatchSize pos' - return GatewaySyncResult.Written token } - -type private Collection(gateway : EqxGateway, databaseId, collectionId) = - member __.Gateway = gateway - member __.CollectionUri = Client.UriFactory.CreateDocumentCollectionUri(databaseId, collectionId) - -[] -type SearchStrategy<'event> = - | EventType of string - | Predicate of ('event -> bool) - -[] -type AccessStrategy<'event,'state> = - | EventsAreState - | //[] - RollingSnapshots of eventType: string * compact: ('state -> 'event) - | IndexedSearch of (string -> bool) * index: ('state -> 'event seq) - -type private CompactionContext(eventsLen : int, capacityBeforeCompaction : int) = - /// Determines whether writing a Compaction event is warranted (based on the existing state and the current `Accumulated` changes) - member __.IsCompactionDue = eventsLen > capacityBeforeCompaction - -type private Category<'event, 'state>(coll : Collection, codec : UnionCodec.IUnionEncoder<'event, byte[]>, ?access : AccessStrategy<'event,'state>) = - let (|Pos|) streamName : Store.Position = { collectionUri = coll.CollectionUri; streamName = streamName; index = None; etag = None } - let compactionPredicate = - match access with - | None -> None - | Some (AccessStrategy.IndexedSearch (predicate,_)) -> Some predicate - | Some AccessStrategy.EventsAreState -> Some (fun _ -> true) - | Some (AccessStrategy.RollingSnapshots (et,_)) -> Some ((=) et) - let load (fold: 'state -> 'event seq -> 'state) initial loadF = async { - let! token, events = loadF - return token, fold initial (UnionEncoderAdapters.decodeKnownEvents codec events) } - let foldI (fold: 'state -> 'event seq -> 'state) initial token events = - token, fold initial (UnionEncoderAdapters.decodeKnownEventsI codec events) - let loadI (fold: 'state -> 'event seq -> 'state) initial loadF = async { - let! token, events = loadF - return foldI fold initial token events } - let loadAlgorithm fold (pos : Store.Position) initial log = - let batched = load fold initial (coll.Gateway.LoadBatched log None pos) - let compacted predicate = load fold initial (coll.Gateway.LoadBackwardsStoppingAtCompactionEvent log predicate pos) - let indexed predicate = loadI fold initial (coll.Gateway.IndexedOrBatched log predicate pos) - match access with - | Some (AccessStrategy.IndexedSearch (predicate,_)) -> indexed predicate - | None -> batched - | Some AccessStrategy.EventsAreState -> compacted (fun _ -> true) - | Some (AccessStrategy.RollingSnapshots (et,_)) -> compacted ((=) et) - member __.Load (fold: 'state -> 'event seq -> 'state) (initial: 'state) (Pos pos) (log : ILogger) : Async = - loadAlgorithm fold pos initial log - member __.LoadFromToken (fold: 'state -> 'event seq -> 'state) (initial: 'state) (state: 'state) token (log : ILogger) - : Async = async { - let indexed = match access with Some (AccessStrategy.IndexedSearch _) -> true | _ -> false - let! res = coll.Gateway.LoadFromToken log token compactionPredicate indexed + | Sync.Result.Conflict (pos',events) -> return InternalSyncResult.Conflict (Token.create stream pos',events) + | Sync.Result.ConflictUnknown pos' -> return InternalSyncResult.ConflictUnknown (Token.create stream pos') + | Sync.Result.Written pos' -> return InternalSyncResult.Written (Token.create stream pos') } + +type private Category<'event, 'state>(gateway : EqxGateway, codec : UnionCodec.IUnionEncoder<'event, byte[]>) = + let respond (fold: 'state -> 'event seq -> 'state) initial events : 'state = + fold initial (UnionEncoderAdapters.decodeKnownEvents codec events) + member __.Load includeProjections collectionStream fold initial predicate (log : ILogger): Async = async { + let! token, events = + if not includeProjections then gateway.LoadBackwardsStopping log predicate collectionStream + else gateway.LoadFromProjectionsOrRollingSnapshots log predicate (collectionStream,None) + return token, respond fold initial events } + member __.LoadFromToken (Token.Unpack streamPos, state: 'state as current) fold predicate (log : ILogger): Async = async { + let! res = gateway.LoadFromToken log streamPos predicate match res with - | LoadFromTokenResult.Unchanged -> return token, state - | LoadFromTokenResult.Found (token,events ) -> return foldI fold initial token events } - member __.TrySync (fold: 'state -> 'event seq -> 'state) initial (log : ILogger) - (token : Storage.StreamToken, state : 'state) - (events : 'event list, state' : 'state) : Async> = async { - let events, index = - match access with - | None | Some AccessStrategy.EventsAreState -> - events, None - | Some (AccessStrategy.RollingSnapshots (_,f)) -> - let cc = CompactionContext(List.length events, token.batchCapacityLimit.Value) - (if cc.IsCompactionDue then events @ [f state'] else events), None - | Some (AccessStrategy.IndexedSearch (_,index)) -> - events, Some (index state') - let encodedEvents : Store.EventData[] = UnionEncoderAdapters.encodeEvents codec (Seq.ofList events) - let maybeIndexEvents : Store.EventData[] option = index |> Option.map (UnionEncoderAdapters.encodeEvents codec) - let! syncRes = coll.Gateway.TrySync log token (encodedEvents,maybeIndexEvents) compactionPredicate - match syncRes with - | GatewaySyncResult.Conflict (token',events) -> return Storage.SyncResult.Conflict (async { return foldI fold initial token' events }) - | GatewaySyncResult.ConflictUnknown token' -> return Storage.SyncResult.Conflict (__.LoadFromToken fold initial state token' log) - | GatewaySyncResult.Written token' -> return Storage.SyncResult.Written (token', fold state (Seq.ofList events)) } + | LoadFromTokenResult.Unchanged -> return current + | LoadFromTokenResult.Found (token',events) -> return token', respond fold state events } + member __.Sync (Token.Unpack (stream,pos), state as current) (project: 'state -> 'event seq -> 'event seq) + (expectedVersion : int64 option, events, state') + fold predicate log + : Async> = async { + let encode = UnionEncoderAdapters.encodeEvent codec + let eventsEncoded, projectionsEncoded = Seq.map encode events |> Array.ofSeq, Seq.map encode (project state' events) + let baseIndex = pos.index + int64 (List.length events) + let projections = Sync.mkProjections baseIndex projectionsEncoded + let batch = Sync.mkBatch stream eventsEncoded projections + let! res = gateway.Sync log stream (expectedVersion,batch) + match res with + | InternalSyncResult.Conflict (token',events') -> return Storage.SyncResult.Conflict (async { return token', respond fold state events' }) + | InternalSyncResult.ConflictUnknown _token' -> return Storage.SyncResult.Conflict (__.LoadFromToken current fold predicate log) + | InternalSyncResult.Written token' -> return Storage.SyncResult.Written (token', state') } module Caching = open System.Runtime.Caching @@ -748,11 +858,12 @@ module Caching = interface ICategory<'event, 'state> with member __.Load (streamName : string) (log : ILogger) : Async = interceptAsync (inner.Load streamName log) streamName - member __.TrySync streamName (log : ILogger) (token, state) (events : 'event list, state' : 'state) : Async> = async { - let! syncRes = inner.TrySync streamName log (token, state) (events,state') + member __.TrySync (log : ILogger) (Token.Unpack (stream,_) as streamToken,state) (events : 'event list, state': 'state) + : Async> = async { + let! syncRes = inner.TrySync log (streamToken, state) (events,state') match syncRes with - | Storage.SyncResult.Conflict resync -> return Storage.SyncResult.Conflict (interceptAsync resync streamName) - | Storage.SyncResult.Written (token', state') -> return Storage.SyncResult.Written (intercept streamName (token', state')) } + | Storage.SyncResult.Conflict resync -> return Storage.SyncResult.Conflict (interceptAsync resync stream.name) + | Storage.SyncResult.Written (token', state') ->return Storage.SyncResult.Written (intercept stream.name (token', state')) } let applyCacheUpdatesWithSlidingExpiration (cache: Cache) @@ -764,111 +875,91 @@ module Caching = let addOrUpdateSlidingExpirationCacheEntry streamName = CacheEntry >> cache.UpdateIfNewer policy (prefix + streamName) CategoryTee<'event,'state>(category, addOrUpdateSlidingExpirationCacheEntry) :> _ -type private Folder<'event, 'state>(category : Category<'event, 'state>, fold: 'state -> 'event seq -> 'state, initial: 'state, ?readCache) = - let loadAlgorithm streamName initial log = - let batched = category.Load fold initial streamName log - let cached token state = category.LoadFromToken fold initial state token log - match readCache with - | None -> batched - | Some (cache : Caching.Cache, prefix : string) -> - match cache.TryGet(prefix + streamName) with - | None -> batched - | Some (token, state) -> cached token state +type private Folder<'event, 'state> + ( category : Category<'event, 'state>, fold: 'state -> 'event seq -> 'state, initial: 'state, + predicate : HashSet -> bool, + mkCollectionStream : string -> Store.CollectionStream, + // Whether or not a projection function is supplied controls whether reads consult the index or not + ?project: ('state -> 'event seq -> 'event seq), + ?readCache) = interface ICategory<'event, 'state> with - member __.Load (streamName : string) (log : ILogger) : Async = - loadAlgorithm streamName initial log - member __.TrySync _streamName(* TODO remove from main interface *) (log : ILogger) (token, state) (events : 'event list, state': 'state) : Async> = async { - let! syncRes = category.TrySync fold initial log (token, state) (events,state') + member __.Load streamName (log : ILogger): Async = + let collStream = mkCollectionStream streamName + let batched = category.Load (Option.isSome project) collStream fold initial predicate log + let cached tokenAndState = category.LoadFromToken tokenAndState fold predicate log + match readCache with + | None -> batched + | Some (cache : Caching.Cache, prefix : string) -> + match cache.TryGet(prefix + streamName) with + | None -> batched + | Some tokenAndState -> cached tokenAndState + member __.TrySync (log : ILogger) (Token.Unpack (_stream,pos) as streamToken,state) (events : 'event list, state': 'state) + : Async> = async { + let! syncRes = category.Sync (streamToken,state) (defaultArg project (fun _ _ -> Seq.empty)) (Some pos.index, events, state') fold predicate log match syncRes with - | Storage.SyncResult.Conflict resync -> return Storage.SyncResult.Conflict resync - | Storage.SyncResult.Written (token',state') -> return Storage.SyncResult.Written (token',state') } + | Storage.SyncResult.Conflict resync -> return Storage.SyncResult.Conflict resync + | Storage.SyncResult.Written (token',state') -> return Storage.SyncResult.Written (token',state') } + +/// Defines a process for mapping from a Stream Name to the appropriate storage area, allowing control over segregation / co-locating of data +type EqxCollections(selectDatabaseAndCollection : string -> string*string) = + new (databaseId, collectionId) = EqxCollections(fun _streamName -> databaseId, collectionId) + member __.CollectionForStream streamName = + let databaseId, collectionId = selectDatabaseAndCollection streamName + Store.CollectionStream.Create(Client.UriFactory.CreateDocumentCollectionUri(databaseId, collectionId), streamName) + +/// Pairs a Gateway, defining the retry policies for CosmosDb with an EqxCollections to +type EqxStore(gateway: EqxGateway, collections: EqxCollections) = + member __.Gateway = gateway + member __.Collections = collections [] type CachingStrategy = + /// Retain a single set of State, together with the associated etags + /// NB while a strategy like EventStore.Caching.SlidingWindowPrefixed is obviously easy to implement, the recommended approach is to + /// track all relevant data in the state, and/or have the `project` function ensure all relevant events get indexed quickly | SlidingWindow of Caching.Cache * window: TimeSpan - /// Prefix is used to distinguish multiple folds per stream - | SlidingWindowPrefixed of Caching.Cache * window: TimeSpan * prefix: string - -type EqxStreamBuilder<'event, 'state>(gateway : EqxGateway, codec, fold, initial, ?access, ?caching) = - member __.Create (databaseId, collectionId, streamName) : Equinox.IStream<'event, 'state> = - let category = Category<'event, 'state>(Collection(gateway, databaseId, collectionId), codec, ?access = access) +[] +type AccessStrategy<'event,'state> = + /// Require a configurable Set of Event Types to have been accumulated from a) projections + b) searching backward in the event stream + /// until `resolved` deems it so; fold foward based on those + /// When saving, `project` the 'state to seed the set of events that `resolved` will see first + | Projections of resolved: (ISet -> bool) * project: ('state -> 'event seq) + /// Simplified version of projection that only has a single Projection Event Type + /// Provides equivalent performance to Projections, just simplified function signatures + | Projection of eventType: string * ('state -> 'event) + /// Simplified version + | AnyKnownEventType of eventTypes: ISet + +type EqxStreamBuilder<'event, 'state>(store : EqxStore, codec, fold, initial, ?access, ?caching) = + member __.Create streamName : Equinox.IStream<'event, 'state> = let readCacheOption = match caching with | None -> None | Some (CachingStrategy.SlidingWindow(cache, _)) -> Some(cache, null) - | Some (CachingStrategy.SlidingWindowPrefixed(cache, _, prefix)) -> Some(cache, prefix) - let folder = Folder<'event, 'state>(category, fold, initial, ?readCache = readCacheOption) + let predicate, projectOption = + match access with + | None -> (fun _ -> false), None + | Some (AccessStrategy.Projections (predicate,project)) -> + predicate, + Some (fun state _events -> project state) + | Some (AccessStrategy.Projection (et,compact)) -> + (fun (ets: HashSet) -> ets.Contains et), + Some (fun state _events -> seq [compact state]) + | Some (AccessStrategy.AnyKnownEventType knownEventTypes) -> + (fun (ets: HashSet) -> knownEventTypes.Overlaps ets), + Some (fun _ events -> Seq.last events |> Seq.singleton) + let category = Category<'event, 'state>(store.Gateway, codec) + let folder = Folder<'event, 'state>(category, fold, initial, predicate, store.Collections.CollectionForStream, ?project=projectOption, ?readCache = readCacheOption) let category : ICategory<_,_> = match caching with | None -> folder :> _ | Some (CachingStrategy.SlidingWindow(cache, window)) -> Caching.applyCacheUpdatesWithSlidingExpiration cache null window folder - | Some (CachingStrategy.SlidingWindowPrefixed(cache, window, prefix)) -> - Caching.applyCacheUpdatesWithSlidingExpiration cache prefix window folder Equinox.Stream.create category streamName -module Initialization = - let createDatabase (client:IDocumentClient) dbName = async { - let opts = Client.RequestOptions(ConsistencyLevel = Nullable ConsistencyLevel.Session) - let! db = client.CreateDatabaseIfNotExistsAsync(Database(Id=dbName), options = opts) |> Async.AwaitTaskCorrect - return db.Resource.Id } - - let createCollection (client: IDocumentClient) (dbUri: Uri) collName ru = async { - let pkd = PartitionKeyDefinition() - pkd.Paths.Add(sprintf "/%s" Store.Event.PartitionKeyField) - let colld = DocumentCollection(Id = collName, PartitionKey = pkd) - - colld.IndexingPolicy.IndexingMode <- IndexingMode.Consistent - colld.IndexingPolicy.Automatic <- true - // Can either do a blacklist or a whitelist - // Given how long and variable the blacklist would be, we whitelist instead - colld.IndexingPolicy.ExcludedPaths <- System.Collections.ObjectModel.Collection [|ExcludedPath(Path="/*")|] - // NB its critical to index the nominated PartitionKey field defined above or there will be runtime errors - colld.IndexingPolicy.IncludedPaths <- System.Collections.ObjectModel.Collection [| for k in Store.Event.IndexedFields -> IncludedPath(Path=sprintf "/%s/?" k) |] - let! coll = client.CreateDocumentCollectionIfNotExistsAsync(dbUri, colld, Client.RequestOptions(OfferThroughput=Nullable ru)) |> Async.AwaitTaskCorrect - return coll.Resource.Id } - - let createProc (client: IDocumentClient) (collectionUri: Uri) = async { - let f = """function indexedWrite(docs, expectedVersion, etag, index) { - var response = getContext().getResponse(); - var collection = getContext().getCollection(); - var collectionLink = collection.getSelfLink(); - if (!docs) throw new Error("docs argument is missing."); - if (index) { - function callback(err, doc, options) { - if (err) throw err; - response.setBody({ etag: doc._etag, conflicts: null }); - } - if (-1 == expectedVersion) { - collection.createDocument(collectionLink, index, { disableAutomaticIdGeneration : true}, callback); - } else { - collection.replaceDocument(collection.getAltLink() + "/docs/" + index.id, index, callback); - } - } else { - // call always expects a parseable json response with `etag` and `conflicts` - // can also contain { conflicts: [{t, d}] } representing conflicting events since expectedVersion - // null/missing signifies events have been written, with no conflict - response.setBody({ etag: null, conflicts: null }); - } - for (var i=0; i Async.AwaitTaskCorrect |> Async.Ignore } - - let initialize (client : IDocumentClient) dbName collName ru = async { - let! dbId = createDatabase client dbName - let dbUri = Client.UriFactory.CreateDatabaseUri dbId - let! collId = createCollection client dbUri collName ru - let collUri = Client.UriFactory.CreateDocumentCollectionUri (dbName, collId) - //let! _aux = createAux client dbUri collName auxRu - return! createProc client collUri - } - [] type Discovery = | UriAndKey of databaseUri:Uri * key:string @@ -942,4 +1033,165 @@ type EqxConnector /// Yields a DocDbConnection configured per the specified strategy member __.Connect(name, discovery : Discovery) : Async = async { let! conn = connect(name, discovery) - return EqxConnection(conn, ?readRetryPolicy=readRetryPolicy, ?writeRetryPolicy=writeRetryPolicy) } \ No newline at end of file + return EqxConnection(conn, ?readRetryPolicy=readRetryPolicy, ?writeRetryPolicy=writeRetryPolicy) } + +namespace Equinox.Cosmos.Core + +open Equinox.Cosmos +open Equinox.Cosmos.Builder +open Equinox.Cosmos.Events +open FSharp.Control +open Equinox + +/// Outcome of appending events, specifying the new and/or conflicting events, together with the updated Target write position +[] +type AppendResult<'t> = + | Ok of pos: 't + | Conflict of index: 't * conflictingEvents: IOrderedEvent[] + | ConflictUnknown of index: 't + +/// Encapsulates the core facilites Equinox.Cosmos offers for operating directly on Events in Streams. +type EqxContext + ( /// Connection to CosmosDb with DocumentDb Transient Read and Write Retry policies + conn : EqxConnection, + /// Database + Collection selector + collections: EqxCollections, + /// Logger to write to - see https://github.com/serilog/serilog/wiki/Provided-Sinks for how to wire to your logger + logger : Serilog.ILogger, + /// Optional maximum number of Store.Batch records to retrieve as a set (how many Events are placed therein is controlled by maxEventsPerSlice). + /// Defaults to 10 + ?defaultMaxItems, + /// Alternate way of specifying defaultMaxItems which facilitates reading it from a cached dynamic configuration + ?getDefaultMaxItems) = + let getDefaultMaxItems = match getDefaultMaxItems with Some f -> f | None -> fun () -> defaultArg defaultMaxItems 10 + let batching = EqxBatchingPolicy(getDefaultMaxItems=getDefaultMaxItems) + let gateway = EqxGateway(conn, batching) + + let maxCountPredicate count = + let acc = ref (max (count-1) 0) + fun _ -> + if !acc = 0 then true else + decr acc + false + + let yieldPositionAndData res = async { + let! (Token.Unpack (_,pos')), data = res + return pos', data } + + member __.CreateStream(streamName) = collections.CollectionForStream streamName + + member internal __.GetInternal((stream, startPos), ?maxCount, ?direction) = async { + let direction = defaultArg direction Direction.Forward + if maxCount = Some 0 then + // Search semantics include the first hit so we need to special case this anyway + return Token.create stream (defaultArg startPos Position.FromKnownEmpty), Array.empty + else + let predicate = + match maxCount with + | Some limit -> maxCountPredicate limit + | None -> fun _ -> false + return! gateway.Read logger None stream direction startPos predicate } + + /// Establishes the current position of the stream in as effficient a manner as possible + /// (The ideal situation is that the preceding token is supplied as input in order to avail of 1RU low latency state checks) + member __.Sync(stream, ?position: Position) : Async = async { + //let indexed predicate = load fold initial (coll.Gateway.IndexedOrBatched log predicate (stream,None)) + let! (Token.Unpack (_,pos')) = gateway.GetPosition(logger, stream, ?pos=position) + return pos' } + + /// Reads in batches of `batchSize` from the specified `Position`, allowing the reader to efficiently walk away from a running query + /// ... NB as long as they Dispose! + member __.Walk(stream, batchSize, ?position, ?direction) : AsyncSeq = asyncSeq { + let! _pos,data = __.GetInternal((stream, position), batchSize, ?direction=direction) + // TODO add laziness + return AsyncSeq.ofSeq data } + + /// Reads all Events from a `Position` in a given `direction` + member __.Read(stream, ?position, ?maxCount, ?direction) : Async = + __.GetInternal((stream, position), ?maxCount=maxCount, ?direction=direction) |> yieldPositionAndData + + /// Appends the supplied batch of events, subject to a consistency check based on the `position` + /// Callers should implement appropriate idempotent handling, or use Equinox.Handler for that purpose + member __.Sync(stream, position, events: IEvent[]) : Async> = async { + let batch = Sync.mkBatch stream events Seq.empty + let! res = gateway.Sync logger stream (Some position.index,batch) + match res with + | Builder.Internal.InternalSyncResult.Written (Token.Unpack (_,pos)) -> return AppendResult.Ok pos + | Builder.Internal.InternalSyncResult.Conflict (Token.Unpack (_,pos),events) -> return AppendResult.Conflict (pos, events) + | Builder.Internal.InternalSyncResult.ConflictUnknown (Token.Unpack (_,pos)) -> return AppendResult.ConflictUnknown pos } + + /// Low level, non-idempotent call appending events to a stream without a concurrency control mechanism in play + /// NB Should be used sparingly; Equinox.Handler enables building equivalent equivalent idempotent handling with minimal code. + member __.NonIdempotentAppend(stream, events: IEvent[]) : Async = async { + let! res = __.Sync(stream, Position.FromAppendAtEnd, events) + match res with + | AppendResult.Ok token -> return token + | x -> return x |> sprintf "Conflict despite it being disabled %A" |> invalidOp } + +/// Api as defined in the Equinox Specification +/// Note the EqxContext APIs can yield better performance due to the fact that a Position tracks the etag of the Stream's WipBatch +module Events = + let private (|PositionIndex|) (x: Position) = x.index + let private stripSyncResult (f: Async>): Async> = async { + let! res = f + match res with + | AppendResult.Ok (PositionIndex index)-> return AppendResult.Ok index + | AppendResult.Conflict (PositionIndex index,events) -> return AppendResult.Conflict (index, events) + | AppendResult.ConflictUnknown (PositionIndex index) -> return AppendResult.ConflictUnknown index } + let private stripPosition (f: Async): Async = async { + let! (PositionIndex index) = f + return index } + let private dropPosition (f: Async): Async = async { + let! _,xs = f + return xs } + let (|MinPosition|) = function + | 0L -> None + | i -> Some (Position.FromI i) + let (|MaxPosition|) = function + | int64.MaxValue -> None + | i -> Some (Position.FromI (i + 1L)) + + /// Returns an aFromLastIndexs in the stream starting at the specified sequence number, + /// reading in batches of the specified size. + /// Returns an empty sequence if the stream is empty or if the sequence number is larger than the largest + /// sequence number in the stream. + let getAll (ctx: EqxContext) (streamName: string) (MinPosition index: int64) (batchSize: int): AsyncSeq = + ctx.Walk(ctx.CreateStream streamName, batchSize,?position=index) + + /// Returns an async array of events in the stream starting at the specified sequence number, + /// number of events to read is specified by batchSize + /// Returns an empty sequence if the stream is empty or if the sequence number is larger than the largest + /// sequence number in the stream. + let get (ctx: EqxContext) (streamName: string) (MinPosition index: int64) (maxCount: int): Async = + ctx.Read(ctx.CreateStream streamName, ?position=index, maxCount=maxCount) |> dropPosition + + /// Appends a batch of events to a stream at the specified expected sequence number. + /// If the specified expected sequence number does not match the stream, the events are not appended + /// and a failure is returned. + let append (ctx: EqxContext) (streamName: string) (index: int64) (events: IEvent[]): Async> = + ctx.Sync(ctx.CreateStream streamName, Position.FromI index, events) |> stripSyncResult + + /// Appends a batch of events to a stream at the the present Position without any conflict checks. + /// NB typically, it is recommended to ensure idempotency of operations by using the `append` and related API as + /// this facilitates ensuring consistency is maintained, and yields reduced latency and Request Charges impacts + /// (See equivalent APIs on `Context` that yield `Position` values) + let appendAtEnd (ctx: EqxContext) (streamName: string) (events: IEvent[]): Async = + ctx.NonIdempotentAppend(ctx.CreateStream streamName, events) |> stripPosition + + /// Returns an async sequence of events in the stream backwards starting from the specified sequence number, + /// reading in batches of the specified size. + /// Returns an empty sequence if the stream is empty or if the sequence number is smaller than the smallest + /// sequence number in the stream. + let getAllBackwards (ctx: EqxContext) (streamName: string) (MaxPosition index: int64) (maxCount: int): AsyncSeq = + ctx.Walk(ctx.CreateStream streamName, maxCount, ?position=index, direction=Direction.Backward) + + /// Returns an async array of events in the stream backwards starting from the specified sequence number, + /// number of events to read is specified by batchSize + /// Returns an empty sequence if the stream is empty or if the sequence number is smaller than the smallest + /// sequence number in the stream. + let getBackwards (ctx: EqxContext) (streamName: string) (MaxPosition index: int64) (maxCount: int): Async = + ctx.Read(ctx.CreateStream streamName, ?position=index, maxCount=maxCount, direction=Direction.Backward) |> dropPosition + + /// Obtains the `index` from the current write Position + let getNextIndex (ctx: EqxContext) (streamName: string) : Async = + ctx.Sync(ctx.CreateStream streamName) |> stripPosition \ No newline at end of file diff --git a/src/Equinox.Cosmos/Equinox.Cosmos.fsproj b/src/Equinox.Cosmos/Equinox.Cosmos.fsproj index c577971a0..97c20b955 100644 --- a/src/Equinox.Cosmos/Equinox.Cosmos.fsproj +++ b/src/Equinox.Cosmos/Equinox.Cosmos.fsproj @@ -11,6 +11,7 @@ + diff --git a/tests/Equinox.Cosmos.Integration/CosmosCoreIntegration.fs b/tests/Equinox.Cosmos.Integration/CosmosCoreIntegration.fs new file mode 100644 index 000000000..02b52c051 --- /dev/null +++ b/tests/Equinox.Cosmos.Integration/CosmosCoreIntegration.fs @@ -0,0 +1,270 @@ +module Equinox.Cosmos.Integration.CoreIntegration + +open Equinox.Cosmos.Integration.Infrastructure +open Equinox.Cosmos +open Equinox.Cosmos.Core +open FSharp.Control +open Newtonsoft.Json.Linq +open Swensen.Unquote +open Serilog +open System +open System.Text + +#nowarn "1182" // From hereon in, we may have some 'unused' privates (the tests) + +type EventData = { eventType: string; data: byte[] } with + interface Events.IEvent with + member __.EventType = __.eventType + member __.Data = __.data + member __.Meta = Encoding.UTF8.GetBytes("{\"m\":\"m\"}") + static member private Create(i, ?eventType, ?json) : Events.IEvent = + { eventType = sprintf "%s:%d" (defaultArg eventType "test_event") i + data = System.Text.Encoding.UTF8.GetBytes(defaultArg json "{\"d\":\"d\"}") } :> _ + static member Create(i, c) = Array.init c (fun x -> EventData.Create(x+i)) + +type Tests(testOutputHelper) = + inherit TestsWithLogCapture(testOutputHelper) + let log, capture = base.Log, base.Capture + + /// As we generate side-effects per run, we want each FSCheck-triggered invocation of the test run to work in its own stream + let testIterations = ref 0 + let (|TestStream|) (name: Guid) = + incr testIterations + sprintf "events-%O-%i" name !testIterations + let mkContextWithItemLimit conn defaultBatchSize = + EqxContext(conn,collections,log,?defaultMaxItems=defaultBatchSize) + let mkContext conn = mkContextWithItemLimit conn None + + let verifyRequestChargesMax rus = + let tripRequestCharges = [ for e, c in capture.RequestCharges -> sprintf "%A" e, c ] + test <@ float rus >= Seq.sum (Seq.map snd tripRequestCharges) @> + + [] + let append (TestStream streamName) = Async.RunSynchronously <| async { + let! conn = connectToSpecifiedCosmosOrSimulator log + let ctx = mkContext conn + + let index = 0L + let! res = Events.append ctx streamName index <| EventData.Create(0,1) + test <@ AppendResult.Ok 1L = res @> + test <@ [EqxAct.Append] = capture.ExternalCalls @> + verifyRequestChargesMax 14 // observed 12.03 // was 10 + // Clear the counters + capture.Clear() + + let! res = Events.append ctx streamName 1L <| EventData.Create(1,5) + test <@ AppendResult.Ok 6L = res @> + test <@ [EqxAct.Append] = capture.ExternalCalls @> + // We didnt request small batches or splitting so it's not dramatically more expensive to write N events + verifyRequestChargesMax 30 // observed 26.62 was 11 + } + + let blobEquals (x: byte[]) (y: byte[]) = System.Linq.Enumerable.SequenceEqual(x,y) + let stringOfUtf8 (x: byte[]) = Encoding.UTF8.GetString(x) + let xmlDiff (x: string) (y: string) = + match JsonDiffPatchDotNet.JsonDiffPatch().Diff(JToken.Parse x,JToken.Parse y) with + | null -> "" + | d -> string d + let verifyUtf8JsonEquals i x y = + let sx,sy = stringOfUtf8 x, stringOfUtf8 y + test <@ ignore i; blobEquals x y || "" = xmlDiff sx sy @> + + let add6EventsIn2Batches ctx streamName = async { + let index = 0L + let! res = Events.append ctx streamName index <| EventData.Create(0,1) + + test <@ AppendResult.Ok 1L = res @> + let! res = Events.append ctx streamName 1L <| EventData.Create(1,5) + test <@ AppendResult.Ok 6L = res @> + // Only start counting RUs from here + capture.Clear() + return EventData.Create(0,6) + } + + let verifyCorrectEventsEx direction baseIndex (expected: Events.IEvent []) (xs: Events.IOrderedEvent[]) = + let xs, baseIndex = + if direction = Direction.Forward then xs, baseIndex + else Array.rev xs, baseIndex - int64 (Array.length expected) + 1L + test <@ expected.Length = xs.Length @> + test <@ [for i in 0..expected.Length - 1 -> baseIndex + int64 i] = [for r in xs -> r.Index] @> + test <@ [for e in expected -> e.EventType] = [ for r in xs -> r.EventType ] @> + for i,x,y in Seq.mapi2 (fun i x y -> i,x,y) [for e in expected -> e.Data] [for r in xs -> r.Data] do + verifyUtf8JsonEquals i x y + let verifyCorrectEventsBackward = verifyCorrectEventsEx Direction.Backward + let verifyCorrectEvents = verifyCorrectEventsEx Direction.Forward + + [] + let ``appendAtEnd and getNextIndex`` (extras, TestStream streamName) = Async.RunSynchronously <| async { + let! conn = connectToSpecifiedCosmosOrSimulator log + let ctx = mkContextWithItemLimit conn (Some 1) + + // If a fail triggers a rerun, we need to dump the previous log entries captured + capture.Clear() + let! pos = Events.getNextIndex ctx streamName + test <@ [EqxAct.IndexNotFound] = capture.ExternalCalls @> + 0L =! pos + verifyRequestChargesMax 1 // for a 404 by definition + capture.Clear() + + let mutable pos = 0L + let ae = false // TODO fix bug + for appendBatchSize in [4; 5; 9] do + if ae then + let! res = Events.appendAtEnd ctx streamName <| EventData.Create (int pos,appendBatchSize) + pos <- pos + int64 appendBatchSize + //let! res = Events.append ctx streamName pos (Array.replicate appendBatchSize event) + test <@ [EqxAct.Append] = capture.ExternalCalls @> + pos =! res + else + let! res = Events.append ctx streamName pos <| EventData.Create (int pos,appendBatchSize) + pos <- pos + int64 appendBatchSize + //let! res = Events.append ctx streamName pos (Array.replicate appendBatchSize event) + test <@ [EqxAct.Append] = capture.ExternalCalls @> + AppendResult.Ok pos =! res + verifyRequestChargesMax 50 // was 20, observed 41.64 // 15.59 observed + capture.Clear() + + let! res = Events.appendAtEnd ctx streamName <| EventData.Create (int pos,42) + pos <- pos + 42L + pos =! res + test <@ [EqxAct.Append] = capture.ExternalCalls @> + verifyRequestChargesMax 180 // observed 167.32 // was 20 + capture.Clear() + + let! res = Events.getNextIndex ctx streamName + test <@ [EqxAct.Index] = capture.ExternalCalls @> + verifyRequestChargesMax 2 + capture.Clear() + pos =! res + + // Demonstrate benefit/mechanism for using the Position-based API to avail of the etag tracking + let stream = ctx.CreateStream streamName + + let max = 2000 // observed to time out server side // WAS 5000 + let extrasCount = match extras with x when x * 100 > max -> max | x when x < 1 -> 1 | x -> x*100 + let! _pos = ctx.NonIdempotentAppend(stream, EventData.Create (int pos,extrasCount)) + test <@ [EqxAct.Append] = capture.ExternalCalls @> + verifyRequestChargesMax 7000 // 6867.7 observed // was 300 // 278 observed + capture.Clear() + + let! pos = ctx.Sync(stream,?position=None) + test <@ [EqxAct.Index] = capture.ExternalCalls @> + verifyRequestChargesMax 50 // 41 observed // for a 200, you'll pay a lot (we omitted to include the position that NonIdempotentAppend yielded) + capture.Clear() + + let! _pos = ctx.Sync(stream,pos) + test <@ [EqxAct.IndexNotModified] = capture.ExternalCalls @> + verifyRequestChargesMax 1 // for a 302 by definition - when an etag IfNotMatch is honored, you only pay one RU + capture.Clear() + } + + [] + let ``append - fails on non-matching`` (TestStream streamName) = Async.RunSynchronously <| async { + let! conn = connectToSpecifiedCosmosOrSimulator log + let ctx = mkContext conn + + // Attempt to write, skipping Index 0 + let! res = Events.append ctx streamName 1L <| EventData.Create(0,1) + test <@ [EqxAct.Resync] = capture.ExternalCalls @> + // The response aligns with a normal conflict in that it passes the entire set of conflicting events () + test <@ AppendResult.Conflict (0L,[||]) = res @> + verifyRequestChargesMax 5 + capture.Clear() + + // Now write at the correct position + let expected = EventData.Create(1,1) + let! res = Events.append ctx streamName 0L expected + test <@ AppendResult.Ok 1L = res @> + test <@ [EqxAct.Append] = capture.ExternalCalls @> + verifyRequestChargesMax 12 // was 10, observed 10.57 + capture.Clear() + + // Try overwriting it (a competing consumer would see the same) + let! res = Events.append ctx streamName 0L <| EventData.Create(-42,2) + // This time we get passed the conflicting events - we pay a little for that, but that's unavoidable + match res with + | AppendResult.Conflict (1L, e) -> verifyCorrectEvents 0L expected e + | x -> x |> failwithf "Unexpected %A" + test <@ [EqxAct.Resync] = capture.ExternalCalls @> + verifyRequestChargesMax 5 // observed 4.21 // was 4 + capture.Clear() + } + + (* Forward *) + + [] + let get (TestStream streamName) = Async.RunSynchronously <| async { + let! conn = connectToSpecifiedCosmosOrSimulator log + let ctx = mkContextWithItemLimit conn (Some 3) + + // We're going to ignore the first, to prove we can + let! expected = add6EventsIn2Batches ctx streamName + let expected = Array.tail expected + + let! res = Events.get ctx streamName 1L 10 + + verifyCorrectEvents 1L expected res + + test <@ List.replicate 2 EqxAct.SliceForward @ [EqxAct.BatchForward] = capture.ExternalCalls @> + verifyRequestChargesMax 8 // observed 6.14 // was 3 + } + + [] + let ``get (in 2 batches)`` (TestStream streamName) = Async.RunSynchronously <| async { + let! conn = connectToSpecifiedCosmosOrSimulator log + let ctx = mkContextWithItemLimit conn (Some 2) + + let! expected = add6EventsIn2Batches ctx streamName + let expected = Array.tail expected |> Array.take 3 + + let! res = Events.get ctx streamName 1L 3 + + verifyCorrectEvents 1L expected res + + // 2 items atm + test <@ [EqxAct.SliceForward; EqxAct.SliceForward; EqxAct.BatchForward] = capture.ExternalCalls @> + verifyRequestChargesMax 7 // observed 6.14 // was 6 + } + + [] + let getAll (TestStream streamName) = Async.RunSynchronously <| async { + let! conn = connectToSpecifiedCosmosOrSimulator log + let ctx = mkContextWithItemLimit conn (Some 2) + + let! expected = add6EventsIn2Batches ctx streamName + + let! res = Events.get ctx streamName 1L 4 // Events.getAll >> AsyncSeq.concatSeq |> AsyncSeq.toArrayAsync + let expected = expected |> Array.tail |> Array.take 4 + + verifyCorrectEvents 1L expected res + + // TODO [implement and] prove laziness + test <@ List.replicate 2 EqxAct.SliceForward @ [EqxAct.BatchForward] = capture.ExternalCalls @> + verifyRequestChargesMax 10 // observed 8.99 // was 3 + } + + (* Backward *) + + [] + let getBackwards (TestStream streamName) = Async.RunSynchronously <| async { + let! conn = connectToSpecifiedCosmosOrSimulator log + let ctx = mkContextWithItemLimit conn (Some 2) + + let! expected = add6EventsIn2Batches ctx streamName + + // We want to skip reading the last + let expected = Array.take 5 expected + + let! res = Events.getBackwards ctx streamName 4L 5 + + verifyCorrectEventsBackward 4L expected res + + test <@ List.replicate 3 EqxAct.SliceBackward @ [EqxAct.BatchBackward] = capture.ExternalCalls @> + verifyRequestChargesMax 10 // observed 8.98 // was 3 + } + + // TODO AsyncSeq version + + // TODO 2 batches backward test + + // TODO mine other integration tests \ No newline at end of file diff --git a/tests/Equinox.Cosmos.Integration/CosmosFixtures.fs b/tests/Equinox.Cosmos.Integration/CosmosFixtures.fs index 7a3192c20..467c1bf57 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosFixtures.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosFixtures.fs @@ -1,9 +1,12 @@ [] module Equinox.Cosmos.Integration.CosmosFixtures -open Equinox.Cosmos +open Equinox.Cosmos.Builder open System +module Option = + let defaultValue def option = defaultArg option def + /// Standing up an Equinox instance is necessary to run for test purposes; either: /// - replace connection below with a connection string or Uri+Key for an initialized Equinox instance /// - Create a local Equinox via dotnet run cli/Equinox.cli -s $env:EQUINOX_COSMOS_CONNECTION -d test -c $env:EQUINOX_COSMOS_COLLECTION provision -ru 10000 @@ -11,6 +14,7 @@ let private connectToCosmos (log: Serilog.ILogger) name discovery = EqxConnector(log=log, requestTimeout=TimeSpan.FromSeconds 3., maxRetryAttemptsOnThrottledRequests=2, maxRetryWaitTimeInSeconds=60) .Connect(name, discovery) let private read env = Environment.GetEnvironmentVariable env |> Option.ofObj +let (|Default|) def name = (read name),def ||> defaultArg let connectToSpecifiedCosmosOrSimulator (log: Serilog.ILogger) = match read "EQUINOX_COSMOS_CONNECTION" with @@ -21,9 +25,13 @@ let connectToSpecifiedCosmosOrSimulator (log: Serilog.ILogger) = Discovery.FromConnectionString connectionString |> connectToCosmos log "EQUINOX_COSMOS_CONNECTION" -let (|StreamArgs|) streamName = - let databaseId, collectionId = defaultArg (read "EQUINOX_COSMOS_DATABASE") "equinox-test", defaultArg (read "EQUINOX_COSMOS_COLLECTION") "equinox-test" - databaseId, collectionId, streamName - let defaultBatchSize = 500 -let createEqxGateway connection batchSize = EqxGateway(connection, EqxBatchingPolicy(maxBatchSize = batchSize)) \ No newline at end of file + +let collections = + EqxCollections( + read "EQUINOX_COSMOS_DATABASE" |> Option.defaultValue "equinox-test", + read "EQUINOX_COSMOS_COLLECTION" |> Option.defaultValue "equinox-test") + +let createEqxStore connection batchSize = + let gateway = EqxGateway(connection, EqxBatchingPolicy(defaultMaxItems=batchSize)) + EqxStore(gateway, collections) \ No newline at end of file diff --git a/tests/Equinox.Cosmos.Integration/CosmosFixturesInfrastructure.fs b/tests/Equinox.Cosmos.Integration/CosmosFixturesInfrastructure.fs index 31fc6d2cb..1a88db4e2 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosFixturesInfrastructure.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosFixturesInfrastructure.fs @@ -3,7 +3,9 @@ module Equinox.Cosmos.Integration.Infrastructure open Domain open FsCheck +open Serilog open System +open Serilog.Core type FsCheckGenerators = static member SkuId = Arb.generate |> Gen.map SkuId |> Arb.fromGen @@ -37,30 +39,53 @@ type TestOutputAdapter(testOutput : Xunit.Abstractions.ITestOutputHelper) = [] module SerilogHelpers = - open Serilog open Serilog.Events let createLogger sink = LoggerConfiguration() .WriteTo.Sink(sink) + .WriteTo.Seq("http://localhost:5341") .CreateLogger() let (|SerilogScalar|_|) : Serilog.Events.LogEventPropertyValue -> obj option = function | (:? ScalarValue as x) -> Some x.Value | _ -> None + open Equinox.Cosmos [] - type EqxAct = Append | AppendConflict | SliceForward | SliceBackward | BatchForward | BatchBackward | Indexed | IndexedNotFound | IndexedCached + type EqxAct = Append | Resync | Conflict | SliceForward | SliceBackward | BatchForward | BatchBackward | Index | IndexNotFound | IndexNotModified | SliceWaste let (|EqxAction|) (evt : Equinox.Cosmos.Log.Event) = match evt with - | Equinox.Cosmos.Log.WriteSuccess _ -> EqxAct.Append - | Equinox.Cosmos.Log.WriteConflict _ -> EqxAct.AppendConflict - | Equinox.Cosmos.Log.Slice (Equinox.Cosmos.Direction.Forward,_) -> EqxAct.SliceForward - | Equinox.Cosmos.Log.Slice (Equinox.Cosmos.Direction.Backward,_) -> EqxAct.SliceBackward - | Equinox.Cosmos.Log.Batch (Equinox.Cosmos.Direction.Forward,_,_) -> EqxAct.BatchForward - | Equinox.Cosmos.Log.Batch (Equinox.Cosmos.Direction.Backward,_,_) -> EqxAct.BatchBackward - | Equinox.Cosmos.Log.Index _ -> EqxAct.Indexed - | Equinox.Cosmos.Log.IndexNotFound _ -> EqxAct.IndexedNotFound - | Equinox.Cosmos.Log.IndexNotModified _ -> EqxAct.IndexedCached + | Log.WriteSuccess _ -> EqxAct.Append + | Log.WriteResync _ -> EqxAct.Resync + | Log.WriteConflict _ -> EqxAct.Conflict + | Log.Slice (Direction.Forward,{count = 0}) -> EqxAct.SliceWaste // TODO remove, see comment where these are emitted + | Log.Slice (Direction.Forward,_) -> EqxAct.SliceForward + | Log.Slice (Direction.Backward,{count = 0}) -> EqxAct.SliceWaste // TODO remove, see comment where these are emitted + | Log.Slice (Direction.Backward,_) -> EqxAct.SliceBackward + | Log.Batch (Direction.Forward,_,_) -> EqxAct.BatchForward + | Log.Batch (Direction.Backward,_,_) -> EqxAct.BatchBackward + | Log.Index _ -> EqxAct.Index + | Log.IndexNotFound _ -> EqxAct.IndexNotFound + | Log.IndexNotModified _ -> EqxAct.IndexNotModified + let inline (|Stats|) ({ ru = ru }: Equinox.Cosmos.Log.Measurement) = ru + let (|CosmosReadRu|CosmosWriteRu|CosmosResyncRu|CosmosSliceRu|) (evt : Equinox.Cosmos.Log.Event) = + match evt with + | Log.Index (Stats s) + | Log.IndexNotFound (Stats s) + | Log.IndexNotModified (Stats s) + | Log.Batch (_,_, (Stats s)) -> CosmosReadRu s + | Log.WriteSuccess (Stats s) + | Log.WriteConflict (Stats s) -> CosmosWriteRu s + | Log.WriteResync (Stats s) -> CosmosResyncRu s + // slices are rolled up into batches so be sure not to double-count + | Log.Slice (_,Stats s) -> CosmosSliceRu s + /// Facilitates splitting between events with direct charges vs synthetic events Equinox generates to avoid double counting + let (|CosmosRequestCharge|EquinoxChargeRollup|) c = + match c with + | CosmosSliceRu _ -> + EquinoxChargeRollup + | CosmosReadRu rc | CosmosWriteRu rc | CosmosResyncRu rc as e -> + CosmosRequestCharge (e,rc) let (|EqxEvent|_|) (logEvent : LogEvent) : Equinox.Cosmos.Log.Event option = logEvent.Properties.Values |> Seq.tryPick (function | SerilogScalar (:? Equinox.Cosmos.Log.Event as e) -> Some e @@ -80,6 +105,26 @@ module SerilogHelpers = captured.Add logEvent interface Serilog.Core.ILogEventSink with member __.Emit logEvent = writeSerilogEvent logEvent member __.Clear () = captured.Clear() - member __.Entries = captured.ToArray() member __.ChooseCalls chooser = captured |> Seq.choose chooser |> List.ofSeq - member __.ExternalCalls = __.ChooseCalls (function EqxEvent (EqxAction act) -> Some act | _ -> None) \ No newline at end of file + member __.ExternalCalls = __.ChooseCalls (function EqxEvent (EqxAction act) (*when act <> EqxAct.Waste*) -> Some act | _ -> None) + member __.RequestCharges = __.ChooseCalls (function EqxEvent (CosmosRequestCharge e) -> Some e | _ -> None) + +type TestsWithLogCapture(testOutputHelper) = + let log, capture = TestsWithLogCapture.CreateLoggerWithCapture testOutputHelper + + /// NB the returned Logger must be Dispose()'d to guarantee all log output has been flushed upon completion of a test + static member CreateLoggerWithCapture testOutputHelper : Logger* LogCaptureBuffer = + let testOutput = TestOutputAdapter testOutputHelper + let capture = LogCaptureBuffer() + let logger = + Serilog.LoggerConfiguration() + .WriteTo.Seq("http://localhost:5341") + .WriteTo.Sink(testOutput) + .WriteTo.Sink(capture) + .CreateLogger() + logger, capture + + member __.Capture = capture + member __.Log = log + + interface IDisposable with member __.Dispose() = log.Dispose() \ No newline at end of file diff --git a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs index a7ef73ff2..a0019a3d9 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs @@ -1,10 +1,10 @@ module Equinox.Cosmos.Integration.CosmosIntegration +open Domain open Equinox.Cosmos.Integration.Infrastructure -open Equinox.Cosmos +open Equinox.Cosmos.Builder open Swensen.Unquote open System.Threading -open Serilog open System let serializationSettings = Newtonsoft.Json.Converters.FSharp.Settings.CreateCorrect() @@ -12,53 +12,38 @@ let genCodec<'Union when 'Union :> TypeShape.UnionContract.IUnionContract>() = Equinox.UnionCodec.JsonUtf8.Create<'Union>(serializationSettings) module Cart = - let fold, initial, compact, index = Domain.Cart.Folds.fold, Domain.Cart.Folds.initial, Domain.Cart.Folds.compact, Domain.Cart.Folds.index + let fold, initial, project = Domain.Cart.Folds.fold, Domain.Cart.Folds.initial, Domain.Cart.Folds.compact let codec = genCodec() let createServiceWithoutOptimization connection batchSize log = - let gateway = createEqxGateway connection batchSize - let resolveStream (StreamArgs args) = - EqxStreamBuilder(gateway, codec, fold, initial).Create(args) + let store = createEqxStore connection batchSize + let resolveStream = EqxStreamBuilder(store, codec, fold, initial).Create Backend.Cart.Service(log, resolveStream) - let createServiceWithCompaction connection batchSize log = - let gateway = createEqxGateway connection batchSize - let resolveStream (StreamArgs args) = - EqxStreamBuilder(gateway, codec, fold, initial, AccessStrategy.RollingSnapshots compact).Create(args) + let createServiceWithProjection connection batchSize log = + let store = createEqxStore connection batchSize + let resolveStream = EqxStreamBuilder(store, codec, fold, initial, AccessStrategy.Projection project).Create Backend.Cart.Service(log, resolveStream) - let createServiceWithCaching connection batchSize log cache = - let gateway = createEqxGateway connection batchSize + let createServiceWithProjectionAndCaching connection batchSize log cache = + let store = createEqxStore connection batchSize let sliding20m = CachingStrategy.SlidingWindow (cache, TimeSpan.FromMinutes 20.) - let resolveStream (StreamArgs args) = EqxStreamBuilder(gateway, codec, fold, initial, caching = sliding20m).Create(args) - Backend.Cart.Service(log, resolveStream) - let createServiceIndexed connection batchSize log = - let gateway = createEqxGateway connection batchSize - let resolveStream (StreamArgs args) = EqxStreamBuilder(gateway, codec, fold, initial, AccessStrategy.IndexedSearch index).Create(args) - Backend.Cart.Service(log, resolveStream) - let createServiceWithCachingIndexed connection batchSize log cache = - let gateway = createEqxGateway connection batchSize - let sliding20m = CachingStrategy.SlidingWindow (cache, TimeSpan.FromMinutes 20.) - let resolveStream (StreamArgs args) = EqxStreamBuilder(gateway, codec, fold, initial, AccessStrategy.IndexedSearch index, caching=sliding20m).Create(args) - Backend.Cart.Service(log, resolveStream) - let createServiceWithCompactionAndCaching connection batchSize log cache = - let gateway = createEqxGateway connection batchSize - let sliding20m = CachingStrategy.SlidingWindow (cache, TimeSpan.FromMinutes 20.) - let resolveStream (StreamArgs args) = EqxStreamBuilder(gateway, codec, fold, initial, AccessStrategy.RollingSnapshots compact, sliding20m).Create(args) + let resolveStream = EqxStreamBuilder(store, codec, fold, initial, AccessStrategy.Projection project, sliding20m).Create Backend.Cart.Service(log, resolveStream) module ContactPreferences = - let fold, initial = Domain.ContactPreferences.Folds.fold, Domain.ContactPreferences.Folds.initial + let fold, initial, eventTypes = Domain.ContactPreferences.Folds.fold, Domain.ContactPreferences.Folds.initial, Domain.ContactPreferences.Events.eventTypeNames let codec = genCodec() let createServiceWithoutOptimization createGateway defaultBatchSize log _ignoreWindowSize _ignoreCompactionPredicate = let gateway = createGateway defaultBatchSize - let resolveStream (StreamArgs args) = EqxStreamBuilder(gateway, codec, fold, initial).Create(args) + let resolveStream = EqxStreamBuilder(gateway, codec, fold, initial).Create Backend.ContactPreferences.Service(log, resolveStream) let createService createGateway log = - let resolveStream (StreamArgs args) = EqxStreamBuilder(createGateway 1, codec, fold, initial, AccessStrategy.EventsAreState).Create(args) + let resolveStream = EqxStreamBuilder(createGateway 1, codec, fold, initial, AccessStrategy.AnyKnownEventType eventTypes).Create Backend.ContactPreferences.Service(log, resolveStream) #nowarn "1182" // From hereon in, we may have some 'unused' privates (the tests) type Tests(testOutputHelper) = - let testOutput = TestOutputAdapter testOutputHelper + inherit TestsWithLogCapture(testOutputHelper) + let log,capture = base.Log, base.Capture let addAndThenRemoveItems exceptTheLastOne context cartId skuId (service: Backend.Cart.Service) count = service.FlowAsync(cartId, fun _ctx execute -> @@ -71,33 +56,19 @@ type Tests(testOutputHelper) = let addAndThenRemoveItemsManyTimesExceptTheLastOne context cartId skuId service count = addAndThenRemoveItems true context cartId skuId service count - let createLoggerWithCapture () = - let capture = LogCaptureBuffer() - let logger = - Serilog.LoggerConfiguration() - .WriteTo.Seq("http://localhost:5341") - .WriteTo.Sink(testOutput) - .WriteTo.Sink(capture) - .CreateLogger() - logger, capture - - let singleSliceForward = EqxAct.SliceForward - let singleBatchForward = [EqxAct.SliceForward; EqxAct.BatchForward] - let batchForwardAndAppend = singleBatchForward @ [EqxAct.Append] - [] - let ``Can roundtrip against Cosmos, correctly batching the reads [without any optimizations]`` context cartId skuId = Async.RunSynchronously <| async { - let log, capture = createLoggerWithCapture () + let ``Can roundtrip against Cosmos, correctly batching the reads [without using the Index for reads]`` context skuId = Async.RunSynchronously <| async { let! conn = connectToSpecifiedCosmosOrSimulator log let batchSize = 3 let service = Cart.createServiceWithoutOptimization conn batchSize log + capture.Clear() // for re-runs of the test + let cartId = Guid.NewGuid() |> CartId // The command processing should trigger only a single read and a single write call let addRemoveCount = 6 do! addAndThenRemoveItemsManyTimesExceptTheLastOne context cartId skuId service addRemoveCount - test <@ batchForwardAndAppend = capture.ExternalCalls @> - + test <@ [EqxAct.SliceWaste; EqxAct.BatchBackward; EqxAct.Append] = capture.ExternalCalls @> // Restart the counting capture.Clear() @@ -108,20 +79,24 @@ type Tests(testOutputHelper) = // Need to read 4 batches to read 11 events in batches of 3 let expectedBatches = ceil(float expectedEventCount/float batchSize) |> int - test <@ List.replicate (expectedBatches-1) singleSliceForward @ singleBatchForward = capture.ExternalCalls @> + test <@ List.replicate (expectedBatches-1) EqxAct.SliceBackward @ [EqxAct.SliceBackward; EqxAct.BatchBackward] = capture.ExternalCalls @> } [] - let ``Can roundtrip against Cosmos, managing sync conflicts by retrying [without any optimizations]`` ctx initialState = Async.RunSynchronously <| async { - let log1, capture1 = createLoggerWithCapture () + let ``Can roundtrip against Cosmos, managing sync conflicts by retrying`` withOptimizations ctx initialState = Async.RunSynchronously <| async { + let log1, capture1 = log, capture + capture1.Clear() let! conn = connectToSpecifiedCosmosOrSimulator log1 // Ensure batching is included at some point in the proceedings let batchSize = 3 - let context, cartId, (sku11, sku12, sku21, sku22) = ctx + let context, (sku11, sku12, sku21, sku22) = ctx + let cartId = Guid.NewGuid() |> CartId // establish base stream state - let service1 = Cart.createServiceWithoutOptimization conn batchSize log1 + let service1 = + if withOptimizations then Cart.createServiceWithProjection conn batchSize log1 + else Cart.createServiceWithProjection conn batchSize log1 let! maybeInitialSku = let (streamEmpty, skuId) = initialState async { @@ -153,8 +128,9 @@ type Tests(testOutputHelper) = do! act prepare service1 sku12 12 // Signal conflict generated do! s4 } - let log2, capture2 = createLoggerWithCapture () - let service2 = Cart.createServiceWithoutOptimization conn batchSize log2 + let log2, capture2 = TestsWithLogCapture.CreateLoggerWithCapture testOutputHelper + use _flush = log2 + let service2 = Cart.createServiceWithProjection conn batchSize log2 let t2 = async { // Signal we have state, wait for other to do same, engineer conflict let prepare = async { @@ -180,59 +156,22 @@ type Tests(testOutputHelper) = && has sku11 11 && has sku12 12 && has sku21 21 && has sku22 22 @> // Intended conflicts pertained - let hadConflict= function EqxEvent (EqxAction EqxAct.AppendConflict) -> Some () | _ -> None - test <@ [1; 1] = [for c in [capture1; capture2] -> c.ChooseCalls hadConflict |> List.length] @> + let conflict = function EqxAct.Conflict | EqxAct.Resync as x -> Some x | _ -> None + test <@ let c2 = List.choose conflict capture2.ExternalCalls + [EqxAct.Resync] = List.choose conflict capture1.ExternalCalls + && [EqxAct.Resync] = c2 @> } let singleBatchBackwards = [EqxAct.SliceBackward; EqxAct.BatchBackward] let batchBackwardsAndAppend = singleBatchBackwards @ [EqxAct.Append] [] - let ``Can roundtrip against Cosmos, correctly compacting to avoid redundant reads`` context skuId cartId = Async.RunSynchronously <| async { - let log, capture = createLoggerWithCapture () - let! conn = connectToSpecifiedCosmosOrSimulator log - let batchSize = 10 - let service = Cart.createServiceWithCompaction conn batchSize log - - // Trigger 10 events, then reload - do! addAndThenRemoveItemsManyTimes context cartId skuId service 5 - let! _ = service.Read cartId - - // ... should see a single read as we are inside the batch threshold - test <@ batchBackwardsAndAppend @ singleBatchBackwards = capture.ExternalCalls @> - - // Add two more, which should push it over the threshold and hence trigger inclusion of a snapshot event (but not incurr extra roundtrips) - capture.Clear() - do! addAndThenRemoveItemsManyTimes context cartId skuId service 1 - test <@ batchBackwardsAndAppend = capture.ExternalCalls @> - - // While we now have 13 events, we should be able to read them with a single call - capture.Clear() - let! _ = service.Read cartId - test <@ singleBatchBackwards = capture.ExternalCalls @> - - // Add 8 more; total of 21 should not trigger snapshotting as Event Number 12 (the 13th one) is a shapshot - capture.Clear() - do! addAndThenRemoveItemsManyTimes context cartId skuId service 4 - test <@ batchBackwardsAndAppend = capture.ExternalCalls @> - - // While we now have 21 events, we should be able to read them with a single call - capture.Clear() - let! _ = service.Read cartId - // ... and trigger a second snapshotting (inducing a single additional read + write) - do! addAndThenRemoveItemsManyTimes context cartId skuId service 1 - // and reload the 24 events with a single read - let! _ = service.Read cartId - test <@ singleBatchBackwards @ batchBackwardsAndAppend @ singleBatchBackwards = capture.ExternalCalls @> - } - - [] - let ``Can correctly read and update against Cosmos with EventsAreState Access Strategy`` id value = Async.RunSynchronously <| async { - let log, capture = createLoggerWithCapture () + let ``Can correctly read and update against Cosmos with EventsAreState Access Strategy`` value = Async.RunSynchronously <| async { let! conn = connectToSpecifiedCosmosOrSimulator log - let service = ContactPreferences.createService (createEqxGateway conn) log + let service = ContactPreferences.createService (createEqxStore conn) log - let (Domain.ContactPreferences.Id email) = id + let email = let g = System.Guid.NewGuid() in g.ToString "N" + //let (Domain.ContactPreferences.Id email) = id () // Feed some junk into the stream for i in 0..11 do let quickSurveysValue = i % 2 = 0 @@ -246,134 +185,62 @@ type Tests(testOutputHelper) = let! result = service.Read email test <@ value = result @> - test <@ batchBackwardsAndAppend @ singleBatchBackwards = capture.ExternalCalls @> + test <@ [EqxAct.Index; EqxAct.Append; EqxAct.Index] = capture.ExternalCalls @> } [] - let ``Can roundtrip against Cosmos, correctly caching to avoid redundant reads`` context skuId cartId = Async.RunSynchronously <| async { - let log, capture = createLoggerWithCapture () + let ``Can roundtrip against Cosmos, using Projection to avoid queries`` context skuId = Async.RunSynchronously <| async { let! conn = connectToSpecifiedCosmosOrSimulator log let batchSize = 10 - let cache = Caching.Cache("cart", sizeMb = 50) - let createServiceCached () = Cart.createServiceWithCaching conn batchSize log cache - let service1, service2 = createServiceCached (), createServiceCached () + let createServiceIndexed () = Cart.createServiceWithProjection conn batchSize log + let service1, service2 = createServiceIndexed (), createServiceIndexed () + capture.Clear() // Trigger 10 events, then reload + let cartId = Guid.NewGuid() |> CartId do! addAndThenRemoveItemsManyTimes context cartId skuId service1 5 let! _ = service2.Read cartId // ... should see a single read as we are writes are cached - test <@ batchForwardAndAppend @ singleBatchForward = capture.ExternalCalls @> + test <@ [EqxAct.IndexNotFound; EqxAct.Append; EqxAct.Index] = capture.ExternalCalls @> // Add two more - the roundtrip should only incur a single read capture.Clear() do! addAndThenRemoveItemsManyTimes context cartId skuId service1 1 - test <@ batchForwardAndAppend = capture.ExternalCalls @> + test <@ [EqxAct.Index; EqxAct.Append] = capture.ExternalCalls @> // While we now have 12 events, we should be able to read them with a single call capture.Clear() let! _ = service2.Read cartId - test <@ singleBatchForward = capture.ExternalCalls @> + test <@ [EqxAct.Index] = capture.ExternalCalls @> } - let primeIndex = [EqxAct.IndexedNotFound; EqxAct.SliceBackward; EqxAct.BatchBackward] - // When the test gets re-run to simplify, the stream will typically already have values - let primeIndexRerun = [EqxAct.IndexedCached] - [] - let ``Can roundtrip against Cosmos, correctly using the index and cache to avoid redundant reads`` context skuId cartId = Async.RunSynchronously <| async { - let log, capture = createLoggerWithCapture () + let ``Can roundtrip against Cosmos, correctly using Projection and Cache to avoid redundant reads`` context skuId = Async.RunSynchronously <| async { let! conn = connectToSpecifiedCosmosOrSimulator log let batchSize = 10 let cache = Caching.Cache("cart", sizeMb = 50) - let createServiceCached () = Cart.createServiceWithCachingIndexed conn batchSize log cache + let createServiceCached () = Cart.createServiceWithProjectionAndCaching conn batchSize log cache let service1, service2 = createServiceCached (), createServiceCached () + capture.Clear() // Trigger 10 events, then reload + let cartId = Guid.NewGuid() |> CartId do! addAndThenRemoveItemsManyTimes context cartId skuId service1 5 let! _ = service2.Read cartId // ... should see a single Cached Indexed read given writes are cached and writer emits etag - test <@ primeIndex @ [EqxAct.Append; EqxAct.IndexedCached] = capture.ExternalCalls - || primeIndexRerun @ [EqxAct.Append; EqxAct.IndexedCached] = capture.ExternalCalls@> + test <@ [EqxAct.IndexNotFound; EqxAct.Append; EqxAct.IndexNotModified] = capture.ExternalCalls @> // Add two more - the roundtrip should only incur a single read, which should be cached by virtue of being a second one in successono capture.Clear() do! addAndThenRemoveItemsManyTimes context cartId skuId service1 1 - test <@ [EqxAct.IndexedCached; EqxAct.Append] = capture.ExternalCalls @> + test <@ [EqxAct.IndexNotModified; EqxAct.Append] = capture.ExternalCalls @> // While we now have 12 events, we should be able to read them with a single call capture.Clear() let! _ = service2.Read cartId let! _ = service2.Read cartId // First is cached because writer emits etag, second remains cached - test <@ [EqxAct.IndexedCached; EqxAct.IndexedCached] = capture.ExternalCalls @> - } - - [] - let ``Can roundtrip against Cosmos, correctly using the index to avoid redundant reads`` context skuId cartId = Async.RunSynchronously <| async { - let log, capture = createLoggerWithCapture () - let! conn = connectToSpecifiedCosmosOrSimulator log - let batchSize = 10 - let createServiceIndexed () = Cart.createServiceIndexed conn batchSize log - let service1, service2 = createServiceIndexed (), createServiceIndexed () - - // Trigger 10 events, then reload - do! addAndThenRemoveItemsManyTimes context cartId skuId service1 5 - let! _ = service2.Read cartId - - // ... should see a single read as we are writes are cached - test <@ primeIndex @ [EqxAct.Append; EqxAct.Indexed] = capture.ExternalCalls @> - - // Add two more - the roundtrip should only incur a single read - capture.Clear() - do! addAndThenRemoveItemsManyTimes context cartId skuId service1 1 - test <@ [EqxAct.Indexed; EqxAct.Append] = capture.ExternalCalls @> - - // While we now have 12 events, we should be able to read them with a single call - capture.Clear() - let! _ = service2.Read cartId - test <@ [EqxAct.Indexed] = capture.ExternalCalls @> - } - - [] - let ``Can combine compaction with caching against Cosmos`` context skuId cartId = Async.RunSynchronously <| async { - let log, capture = createLoggerWithCapture () - let! conn = connectToSpecifiedCosmosOrSimulator log - let batchSize = 10 - let service1 = Cart.createServiceWithCompaction conn batchSize log - let cache = Caching.Cache("cart", sizeMb = 50) - let service2 = Cart.createServiceWithCompactionAndCaching conn batchSize log cache - - // Trigger 10 events, then reload - do! addAndThenRemoveItemsManyTimes context cartId skuId service1 5 - let! _ = service2.Read cartId - - // ... should see a single read as we are inside the batch threshold - test <@ batchBackwardsAndAppend @ singleBatchBackwards = capture.ExternalCalls @> - - // Add two more, which should push it over the threshold and hence trigger inclusion of a snapshot event (but not incurr extra roundtrips) - capture.Clear() - do! addAndThenRemoveItemsManyTimes context cartId skuId service1 1 - test <@ batchBackwardsAndAppend = capture.ExternalCalls @> - - // While we now have 13 events, we whould be able to read them backwards with a single call - capture.Clear() - let! _ = service1.Read cartId - test <@ singleBatchBackwards = capture.ExternalCalls @> - - // Add 8 more; total of 21 should not trigger snapshotting as Event Number 12 (the 13th one) is a shapshot - capture.Clear() - do! addAndThenRemoveItemsManyTimes context cartId skuId service1 4 - test <@ batchBackwardsAndAppend = capture.ExternalCalls @> - - // While we now have 21 events, we should be able to read them with a single call - capture.Clear() - let! _ = service1.Read cartId - // ... and trigger a second snapshotting (inducing a single additional read + write) - do! addAndThenRemoveItemsManyTimes context cartId skuId service1 1 - // and we _could_ reload the 24 events with a single read if reading backwards. However we are using the cache, which last saw it with 10 events, which necessitates two reads - let! _ = service2.Read cartId - let suboptimalExtraSlice = [singleSliceForward] - test <@ singleBatchBackwards @ batchBackwardsAndAppend @ suboptimalExtraSlice @ singleBatchForward = capture.ExternalCalls @> + test <@ [EqxAct.IndexNotModified; EqxAct.IndexNotModified] = capture.ExternalCalls @> } \ No newline at end of file diff --git a/tests/Equinox.Cosmos.Integration/Equinox.Cosmos.Integration.fsproj b/tests/Equinox.Cosmos.Integration/Equinox.Cosmos.Integration.fsproj index 62f071fb0..796526db0 100644 --- a/tests/Equinox.Cosmos.Integration/Equinox.Cosmos.Integration.fsproj +++ b/tests/Equinox.Cosmos.Integration/Equinox.Cosmos.Integration.fsproj @@ -11,7 +11,8 @@ - + + @@ -24,6 +25,8 @@ + + diff --git a/tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs b/tests/Equinox.Cosmos.Integration/JsonConverterTests.fs similarity index 60% rename from tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs rename to tests/Equinox.Cosmos.Integration/JsonConverterTests.fs index 903c9f1c0..06b795722 100644 --- a/tests/Equinox.Cosmos.Integration/VerbatimUtf8JsonConverterTests.fs +++ b/tests/Equinox.Cosmos.Integration/JsonConverterTests.fs @@ -1,4 +1,4 @@ -module Equinox.Cosmos.Integration.VerbatimUtf8JsonConverterTests +module Equinox.Cosmos.Integration.JsonConverterTests open Equinox.Cosmos open FsCheck.Xunit @@ -15,26 +15,30 @@ type Union = let mkUnionEncoder () = Equinox.UnionCodec.JsonUtf8.Create(JsonSerializerSettings()) -[] -let ``VerbatimUtf8JsonConverter encodes correctly`` () = - let encoded = mkUnionEncoder().Encode(A { embed = "\"" }) - let e : Store.Event = - { p = "streamName"; id = string 0; i = 0L - c = DateTimeOffset.MinValue - t = encoded.caseName - d = encoded.payload - m = null } - let res = JsonConvert.SerializeObject(e) - test <@ res.Contains """"d":{"embed":"\""}""" @> - -type Base64ZipUtf8JsonConverterTests() = +type VerbatimUtf8Tests() = + let unionEncoder = mkUnionEncoder () + + [] + let ``encodes correctly`` () = + let encoded = mkUnionEncoder().Encode(A { embed = "\"" }) + let e : Store.Event = + { p = "streamName"; id = string 0; i = 0L; _etag=null + c = DateTimeOffset.MinValue + t = encoded.caseName + d = encoded.payload + m = null } + let res = JsonConvert.SerializeObject(e) + test <@ res.Contains """"d":{"embed":"\""}""" @> + +type Base64ZipUtf8Tests() = let unionEncoder = mkUnionEncoder () [] let ``serializes, achieving compression`` () = let encoded = unionEncoder.Encode(A { embed = String('x',5000) }) - let e : Store.IndexProjection = - { t = encoded.caseName + let e : Store.Projection = + { i = 42L + t = encoded.caseName d = encoded.payload m = null } let res = JsonConvert.SerializeObject e @@ -49,13 +53,14 @@ type Base64ZipUtf8JsonConverterTests() = if hasNulls then () else let encoded = unionEncoder.Encode value - let e : Store.IndexProjection = - { t = encoded.caseName + let e : Store.Projection = + { i = 42L + t = encoded.caseName d = encoded.payload m = null } let ser = JsonConvert.SerializeObject(e) test <@ ser.Contains("\"d\":\"") @> - let des = JsonConvert.DeserializeObject(ser) + let des = JsonConvert.DeserializeObject(ser) let d : Equinox.UnionCodec.EncodedUnion<_> = { caseName = des.t; payload=des.d } let decoded = unionEncoder.Decode d test <@ value = decoded @> \ No newline at end of file From a7c4e604677a655b230b858afd569d9154d4c10e Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Mon, 26 Nov 2018 21:23:39 +0000 Subject: [PATCH 42/66] Extract json helpers to file --- src/Equinox.Cosmos/Backoff.fs | 105 ----------------------- src/Equinox.Cosmos/Cosmos.fs | 58 +------------ src/Equinox.Cosmos/CosmosInternalJson.fs | 55 ++++++++++++ src/Equinox.Cosmos/Equinox.Cosmos.fsproj | 2 +- 4 files changed, 57 insertions(+), 163 deletions(-) delete mode 100644 src/Equinox.Cosmos/Backoff.fs create mode 100644 src/Equinox.Cosmos/CosmosInternalJson.fs diff --git a/src/Equinox.Cosmos/Backoff.fs b/src/Equinox.Cosmos/Backoff.fs deleted file mode 100644 index c3d5d1a88..000000000 --- a/src/Equinox.Cosmos/Backoff.fs +++ /dev/null @@ -1,105 +0,0 @@ -namespace Equinox.Cosmos - -// NB this is a copy of the one in Backend - there is also one in Equinox/Infrastrcture.fs which this will be merged into - -open System - -/// Given a value, creates a function with one ignored argument which returns the value. - -/// A backoff strategy. -/// Accepts the attempt number and returns an interval in milliseconds to wait. -/// If None then backoff should stop. -type Backoff = int -> int option - -/// Operations on back off strategies represented as functions (int -> int option) -/// which take an attempt number and produce an interval. -module Backoff = - - let inline konst x _ = x - let private checkOverflow x = - if x = System.Int32.MinValue then 2000000000 - else x - - /// Stops immediately. - let never : Backoff = konst None - - /// Always returns a fixed interval. - let linear i : Backoff = konst (Some i) - - /// Modifies the interval. - let bind (f:int -> int option) (b:Backoff) = - fun i -> - match b i with - | Some x -> f x - | None -> None - - /// Modifies the interval. - let map (f:int -> int) (b:Backoff) : Backoff = - fun i -> - match b i with - | Some x -> f x |> checkOverflow |> Some - | None -> None - - /// Bounds the interval. - let bound mx = map (min mx) - - /// Creates a back-off strategy which increases the interval exponentially. - let exp (initialIntervalMs:int) (multiplier:float) : Backoff = - fun i -> (float initialIntervalMs) * (pown multiplier i) |> int |> checkOverflow |> Some - - /// Randomizes the output produced by a back-off strategy: - /// randomizedInterval = retryInterval * (random in range [1 - randomizationFactor, 1 + randomizationFactor]) - let rand (randomizationFactor:float) = - let rand = new System.Random() - let maxRand,minRand = (1.0 + randomizationFactor), (1.0 - randomizationFactor) - map (fun x -> (float x) * (rand.NextDouble() * (maxRand - minRand) + minRand) |> int) - - /// Uses a fibonacci sequence to genereate timeout intervals starting from the specified initial interval. - let fib (initialIntervalMs:int) : Backoff = - let rec fib n = - if n < 2 then initialIntervalMs - else fib (n - 1) + fib (n - 2) - fib >> checkOverflow >> Some - - /// Creates a stateful back-off strategy which keeps track of the number of attempts, - /// and a reset function which resets attempts to zero. - let keepCount (b:Backoff) : (unit -> int option) * (unit -> unit) = - let i = ref -1 - (fun () -> System.Threading.Interlocked.Increment i |> b), - (fun () -> i := -1) - - /// Bounds a backoff strategy to a specified maximum number of attempts. - let maxAttempts (max:int) (b:Backoff) : Backoff = - fun n -> if n > max then None else b n - - - // ------------------------------------------------------------------------------------------------------------------------ - // defaults - - /// 500ms - let [] DefaultInitialIntervalMs = 500 - - /// 60000ms - let [] DefaultMaxIntervalMs = 60000 - - /// 0.5 - let [] DefaultRandomizationFactor = 0.5 - - /// 1.5 - let [] DefaultMultiplier = 1.5 - - /// The default exponential and randomized back-off strategy with a provided initial interval. - /// DefaultMaxIntervalMs = 60,000 - /// DefaultRandomizationFactor = 0.5 - /// DefaultMultiplier = 1.5 - let DefaultExponentialBoundedRandomizedOf initialInternal = - exp initialInternal DefaultMultiplier - |> rand DefaultRandomizationFactor - |> bound DefaultMaxIntervalMs - - /// The default exponential and randomized back-off strategy. - /// DefaultInitialIntervalMs = 500 - /// DefaultMaxIntervalMs = 60,000 - /// DefaultRandomizationFactor = 0.5 - /// DefaultMultiplier = 1.5 - let DefaultExponentialBoundedRandomized = DefaultExponentialBoundedRandomizedOf DefaultInitialIntervalMs \ No newline at end of file diff --git a/src/Equinox.Cosmos/Cosmos.fs b/src/Equinox.Cosmos/Cosmos.fs index ae5929741..94f34531e 100644 --- a/src/Equinox.Cosmos/Cosmos.fs +++ b/src/Equinox.Cosmos/Cosmos.fs @@ -1,60 +1,4 @@ -namespace Equinox.Cosmos.Internal.Json - -open Newtonsoft.Json.Linq -open Newtonsoft.Json - -/// Manages injecting prepared json into the data being submitted to DocDb as-is, on the basis we can trust it to be valid json as DocDb will need it to be -type VerbatimUtf8JsonConverter() = - inherit JsonConverter() - - override __.ReadJson(reader, _, _, _) = - let token = JToken.Load(reader) - if token.Type = JTokenType.Object then token.ToString() |> System.Text.Encoding.UTF8.GetBytes |> box - else Array.empty |> box - - override __.CanConvert(objectType) = - typeof.Equals(objectType) - - override __.WriteJson(writer, value, serializer) = - let array = value :?> byte[] - if array = null || Array.length array = 0 then serializer.Serialize(writer, null) - else writer.WriteRawValue(System.Text.Encoding.UTF8.GetString(array)) - -open System.IO -open System.IO.Compression - -/// Manages zipping of the UTF-8 json bytes to make the index record minimal from the perspective of the writer stored proc -/// Only applied to snapshots in the Index -type Base64ZipUtf8JsonConverter() = - inherit JsonConverter() - let pickle (input : byte[]) : string = - if input = null then null else - - use output = new MemoryStream() - use compressor = new DeflateStream(output, CompressionLevel.Optimal) - compressor.Write(input,0,input.Length) - compressor.Close() - System.Convert.ToBase64String(output.ToArray()) - let unpickle str : byte[] = - if str = null then null else - - let compressedBytes = System.Convert.FromBase64String str - use input = new MemoryStream(compressedBytes) - use decompressor = new DeflateStream(input, CompressionMode.Decompress) - use output = new MemoryStream() - decompressor.CopyTo(output) - output.ToArray() - - override __.CanConvert(objectType) = - typeof.Equals(objectType) - override __.ReadJson(reader, _, _, serializer) = - //( if reader.TokenType = JsonToken.Null then null else - serializer.Deserialize(reader, typedefof) :?> string |> unpickle |> box - override __.WriteJson(writer, value, serializer) = - let pickled = value |> unbox |> pickle - serializer.Serialize(writer, pickled) - -namespace Equinox.Cosmos.Events +namespace Equinox.Cosmos.Events /// Common form for either a raw Event or a Projection type IEvent = diff --git a/src/Equinox.Cosmos/CosmosInternalJson.fs b/src/Equinox.Cosmos/CosmosInternalJson.fs new file mode 100644 index 000000000..be9d10fd6 --- /dev/null +++ b/src/Equinox.Cosmos/CosmosInternalJson.fs @@ -0,0 +1,55 @@ +namespace Equinox.Cosmos.Internal.Json + +open Newtonsoft.Json.Linq +open Newtonsoft.Json + +/// Manages injecting prepared json into the data being submitted to DocDb as-is, on the basis we can trust it to be valid json as DocDb will need it to be +type VerbatimUtf8JsonConverter() = + inherit JsonConverter() + + override __.ReadJson(reader, _, _, _) = + let token = JToken.Load(reader) + if token.Type = JTokenType.Object then token.ToString() |> System.Text.Encoding.UTF8.GetBytes |> box + else Array.empty |> box + + override __.CanConvert(objectType) = + typeof.Equals(objectType) + + override __.WriteJson(writer, value, serializer) = + let array = value :?> byte[] + if array = null || Array.length array = 0 then serializer.Serialize(writer, null) + else writer.WriteRawValue(System.Text.Encoding.UTF8.GetString(array)) + +open System.IO +open System.IO.Compression + +/// Manages zipping of the UTF-8 json bytes to make the index record minimal from the perspective of the writer stored proc +/// Only applied to snapshots in the Index +type Base64ZipUtf8JsonConverter() = + inherit JsonConverter() + let pickle (input : byte[]) : string = + if input = null then null else + + use output = new MemoryStream() + use compressor = new DeflateStream(output, CompressionLevel.Optimal) + compressor.Write(input,0,input.Length) + compressor.Close() + System.Convert.ToBase64String(output.ToArray()) + let unpickle str : byte[] = + if str = null then null else + + let compressedBytes = System.Convert.FromBase64String str + use input = new MemoryStream(compressedBytes) + use decompressor = new DeflateStream(input, CompressionMode.Decompress) + use output = new MemoryStream() + decompressor.CopyTo(output) + output.ToArray() + + override __.CanConvert(objectType) = + typeof.Equals(objectType) + override __.ReadJson(reader, _, _, serializer) = + //( if reader.TokenType = JsonToken.Null then null else + serializer.Deserialize(reader, typedefof) :?> string |> unpickle |> box + override __.WriteJson(writer, value, serializer) = + let pickled = value |> unbox |> pickle + serializer.Serialize(writer, pickled) diff --git a/src/Equinox.Cosmos/Equinox.Cosmos.fsproj b/src/Equinox.Cosmos/Equinox.Cosmos.fsproj index 97c20b955..c4fbd86eb 100644 --- a/src/Equinox.Cosmos/Equinox.Cosmos.fsproj +++ b/src/Equinox.Cosmos/Equinox.Cosmos.fsproj @@ -11,7 +11,7 @@ - + From 92cb45c10f629f3802eb2fef1f875c53822aac7e Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 23 Nov 2018 18:08:02 +0000 Subject: [PATCH 43/66] Align with ES cleanup --- samples/Store/Domain/ContactPreferences.fs | 4 +- samples/Store/Integration/CartIntegration.fs | 3 +- .../ContactPreferencesIntegration.fs | 10 +- .../Store/Integration/FavoritesIntegration.fs | 3 +- src/Equinox.Cosmos/Cosmos.fs | 140 +++++++++--------- .../CosmosCoreIntegration.fs | 6 +- .../CosmosFixturesInfrastructure.fs | 18 +-- .../CosmosIntegration.fs | 11 +- 8 files changed, 95 insertions(+), 100 deletions(-) diff --git a/samples/Store/Domain/ContactPreferences.fs b/samples/Store/Domain/ContactPreferences.fs index 88cf9ded7..45ce4acd0 100644 --- a/samples/Store/Domain/ContactPreferences.fs +++ b/samples/Store/Domain/ContactPreferences.fs @@ -7,11 +7,9 @@ module Events = type Preferences = { manyPromotions : bool; littlePromotions : bool; productReview : bool; quickSurveys : bool } type Value = { email : string; preferences : Preferences } - let [] EventTypeName = "contactPreferencesChanged" type Event = - | []Updated of Value + | []Updated of Value interface TypeShape.UnionContract.IUnionContract - let eventTypeNames = System.Collections.Generic.HashSet([EventTypeName]) module Folds = type State = Events.Preferences diff --git a/samples/Store/Integration/CartIntegration.fs b/samples/Store/Integration/CartIntegration.fs index faa4ea720..3b8e93405 100644 --- a/samples/Store/Integration/CartIntegration.fs +++ b/samples/Store/Integration/CartIntegration.fs @@ -23,8 +23,9 @@ let resolveGesStreamWithRollingSnapshots gateway = let resolveGesStreamWithoutCustomAccessStrategy gateway = GesResolver(gateway, codec, fold, initial).Resolve +let projection = "Compacted",snd snapshot let resolveEqxStreamWithProjection gateway = - EqxStreamBuilder(gateway, codec, fold, initial, AccessStrategy.Projection snapshot).Create + EqxStreamBuilder(gateway, codec, fold, initial, AccessStrategy.Projection projection).Create let resolveEqxStreamWithoutCustomAccessStrategy gateway = EqxStreamBuilder(gateway, codec, fold, initial).Create diff --git a/samples/Store/Integration/ContactPreferencesIntegration.fs b/samples/Store/Integration/ContactPreferencesIntegration.fs index c20349bbc..3b9b523de 100644 --- a/samples/Store/Integration/ContactPreferencesIntegration.fs +++ b/samples/Store/Integration/ContactPreferencesIntegration.fs @@ -21,9 +21,9 @@ let resolveStreamGesWithOptimizedStorageSemantics gateway = let resolveStreamGesWithoutAccessStrategy gateway = GesResolver(gateway defaultBatchSize, codec, fold, initial).Resolve -let resolveStreamEqxWithCompactionSemantics gateway = - EqxStreamBuilder(gateway 1, codec, fold, initial, AccessStrategy.AnyKnownEventType Domain.ContactPreferences.Events.eventTypeNames).Create -let resolveStreamEqxWithoutCompactionSemantics gateway = +let resolveStreamEqxWithKnownEventTypeSemantics gateway = + EqxStreamBuilder(gateway 1, codec, fold, initial, AccessStrategy.AnyKnownEventType (System.Collections.Generic.HashSet ["contactPreferencesChanged"])).Create +let resolveStreamEqxWithoutCustomAccessStrategy gateway = EqxStreamBuilder(gateway defaultBatchSize, codec, fold, initial).Create type Tests(testOutputHelper) = @@ -63,12 +63,12 @@ type Tests(testOutputHelper) = [] let ``Can roundtrip against Cosmos, correctly folding the events with normal semantics`` args = Async.RunSynchronously <| async { - let! service = arrange connectToSpecifiedCosmosOrSimulator createEqxStore resolveStreamEqxWithoutCompactionSemantics + let! service = arrange connectToSpecifiedCosmosOrSimulator createEqxStore resolveStreamEqxWithoutCustomAccessStrategy do! act service args } [] let ``Can roundtrip against Cosmos, correctly folding the events with compaction semantics`` args = Async.RunSynchronously <| async { - let! service = arrange connectToSpecifiedCosmosOrSimulator createEqxStore resolveStreamEqxWithCompactionSemantics + let! service = arrange connectToSpecifiedCosmosOrSimulator createEqxStore resolveStreamEqxWithKnownEventTypeSemantics do! act service args } \ No newline at end of file diff --git a/samples/Store/Integration/FavoritesIntegration.fs b/samples/Store/Integration/FavoritesIntegration.fs index 6bb20b977..5a0d20ef9 100644 --- a/samples/Store/Integration/FavoritesIntegration.fs +++ b/samples/Store/Integration/FavoritesIntegration.fs @@ -22,7 +22,8 @@ let createServiceGes gateway log = Backend.Favorites.Service(log, resolveStream) let createServiceEqx gateway log = - let resolveStream = EqxStreamBuilder(gateway, codec, fold, initial, AccessStrategy.Projection compact).Create + let projection = "Compacted",snd snapshot + let resolveStream = EqxStreamBuilder(gateway, codec, fold, initial, AccessStrategy.Projection projection).Create Backend.Favorites.Service(log, resolveStream) type Tests(testOutputHelper) = diff --git a/src/Equinox.Cosmos/Cosmos.fs b/src/Equinox.Cosmos/Cosmos.fs index 94f34531e..592db84d9 100644 --- a/src/Equinox.Cosmos/Cosmos.fs +++ b/src/Equinox.Cosmos/Cosmos.fs @@ -10,7 +10,7 @@ type IEvent = abstract member Meta : byte[] /// Represents an Event or Projection and its relative position in the event sequence -type IOrderedEvent = +type IIndexedEvent = inherit IEvent /// The index into the event sequence of this event abstract member Index : int64 @@ -26,7 +26,7 @@ type [] Position = { index: int64; etag: string option } with /// Just Do It mode static member internal FromAppendAtEnd = Position.FromI -1L // sic - needs to yield -1 /// NB very inefficient compared to FromDocument or using one already returned to you - static member internal FromMaxIndex(xs: IOrderedEvent[]) = + static member internal FromMaxIndex(xs: IIndexedEvent[]) = if Array.isEmpty xs then Position.FromKnownEmpty else Position.FromI (1L + Seq.max (seq { for x in xs -> x.Index })) @@ -153,7 +153,7 @@ and Projection = type Enum() = static member Events (b:WipBatch) = b.e |> Seq.mapi (fun offset x -> - { new IOrderedEvent with + { new IIndexedEvent with member __.Index = b._i + int64 offset member __.IsProjection = false member __.EventType = x.t @@ -161,7 +161,7 @@ type Enum() = member __.Meta = x.m }) static member Events (i: int64, e:BatchEvent[]) = e |> Seq.mapi (fun offset x -> - { new IOrderedEvent with + { new IIndexedEvent with member __.Index = i + int64 offset member __.IsProjection = false member __.EventType = x.t @@ -169,20 +169,20 @@ type Enum() = member __.Meta = x.m }) static member Event (x:Event) = Seq.singleton - { new IOrderedEvent with + { new IIndexedEvent with member __.Index = x.i member __.IsProjection = false member __.EventType = x.t member __.Data = x.d member __.Meta = x.m } static member Projections (xs: Projection[]) = seq { - for x in xs -> { new IOrderedEvent with + for x in xs -> { new IIndexedEvent with member __.Index = x.i member __.IsProjection = true member __.EventType = x.t member __.Data = x.d member __.Meta = x.m } } - static member EventsAndProjections (x:WipBatch): IOrderedEvent seq = + static member EventsAndProjections (x:WipBatch): IIndexedEvent seq = Enum.Projections x.c /// Reference to Collection and name that will be used as the location for the stream @@ -204,6 +204,8 @@ open System type Direction = Forward | Backward with override this.ToString() = match this with Forward -> "Forward" | Backward -> "Backward" +type IRetryPolicy = abstract member Execute: (int -> Async<'T>) -> Async<'T> + module Log = [] type Measurement = { stream: string; interval: StopwatchInterval; bytes: int; count: int; ru: float } @@ -229,14 +231,14 @@ module Log = let propEvents = propData "events" let propDataProjections = Enum.Projections >> propData "projections" - let withLoggedRetries<'t> retryPolicy (contextLabel : string) (f : ILogger -> Async<'t>) log: Async<'t> = + let withLoggedRetries<'t> (retryPolicy: IRetryPolicy option) (contextLabel : string) (f : ILogger -> Async<'t>) log: Async<'t> = match retryPolicy with | None -> f log | Some retryPolicy -> let withLoggingContextWrapping count = let log = if count = 1 then log else log |> prop contextLabel count f log - retryPolicy withLoggingContextWrapping + retryPolicy.Execute withLoggingContextWrapping /// Attach a property to the log context to hold the metrics // Sidestep Log.ForContext converting to a string; see https://github.com/serilog/serilog/issues/1124 open Serilog.Events @@ -388,7 +390,7 @@ function sync(req, expectedVersion) { [] type Result = | Written of Position - | Conflict of Position * events: IOrderedEvent[] + | Conflict of Position * events: IIndexedEvent[] | ConflictUnknown of Position let private run (client: IDocumentClient) (stream: CollectionStream) (expectedVersion: int64 option, req: WipBatch) @@ -419,7 +421,7 @@ function sync(req, expectedVersion) { let! t, (ru,result) = run client stream (expectedVersion, req) |> Stopwatch.Time let resultLog = let mkMetric ru : Log.Measurement = { stream = stream.name; interval = t; bytes = bytes; count = count; ru = ru } - let logConflict () = writeLog.Information("Eqx TrySync Conflict writing {eventTypes}", [| for x in req.e -> x.t |]) + let logConflict () = writeLog.Information("Eqx Sync Conflict writing {eventTypes}", [| for x in req.e -> x.t |]) match result with | Result.Written pos -> log |> Log.event (Log.WriteSuccess (mkMetric ru)) |> Log.prop "nextExpectedVersion" pos @@ -501,7 +503,7 @@ module private Index = let log = if (not << log.IsEnabled) Events.LogEventLevel.Debug then log else log |> Log.propDataProjections doc.c |> Log.prop "etag" doc._etag log.Information("Eqx {action:l} {res} {ms}ms rc={ru}", "Index", 200, (let e = t.Elapsed in e.TotalMilliseconds), ru) return ru, res } - type [] Result = NotModified | NotFound | Found of Position * IOrderedEvent[] + type [] Result = NotModified | NotFound | Found of Position * IIndexedEvent[] /// `pos` being Some implies that the caller holds a cached value and hence is ready to deal with IndexResult.UnChanged let tryLoad (log : ILogger) retryPolicy client (stream: CollectionStream) (maybePos: Position option): Async = async { let get = get client @@ -525,7 +527,7 @@ module private Index = // Unrolls the Batches in a response - note when reading backawards, the events are emitted in reverse order of index let private handleSlice direction (stream: CollectionStream) (startPos: Position option) (query: IDocumentQuery) (log: ILogger) - : Async = async { + : Async = async { let! ct = Async.CancellationToken let! t, (res : Client.FeedResponse) = query.ExecuteNextAsync(ct) |> Async.AwaitTaskCorrect |> Stopwatch.Time let batches, ru = Array.ofSeq res, res.RequestCharge @@ -542,23 +544,23 @@ module private Index = let maybePosition = batches |> Array.tryPick (fun x -> x.TryToPosition()) return events, maybePosition, ru } - let private runQuery (log : ILogger) (readSlice: IDocumentQuery -> ILogger -> Async) + let private runQuery (log : ILogger) (readSlice: IDocumentQuery -> ILogger -> Async) (maxPermittedBatchReads: int option) (query: IDocumentQuery) - : AsyncSeq = - let rec loop batchCount : AsyncSeq = asyncSeq { + : AsyncSeq = + let rec loop batchCount : AsyncSeq = asyncSeq { match maxPermittedBatchReads with | Some mpbr when batchCount >= mpbr -> log.Information "batch Limit exceeded"; invalidOp "batch Limit exceeded" | _ -> () let batchLog = log |> Log.prop "batchIndex" batchCount - let! (slice : IOrderedEvent[] * Position option * float) = readSlice query batchLog + let! (slice : IIndexedEvent[] * Position option * float) = readSlice query batchLog yield slice if query.HasMoreResults then yield! loop (batchCount + 1) } loop 0 - let private logBatchRead direction batchSize streamName interval (responsesCount, events : IOrderedEvent []) nextI (ru: float) (log : ILogger) = + let private logBatchRead direction batchSize streamName interval (responsesCount, events : IIndexedEvent []) nextI (ru: float) (log : ILogger) = let (Log.BatchLen bytes), count = events, events.Length let reqMetric : Log.Measurement = { stream = streamName; interval = interval; bytes = bytes; count = count; ru = ru } let action = match direction with Direction.Forward -> "LoadF" | Direction.Backward -> "LoadB" @@ -568,9 +570,7 @@ module private Index = "Eqx {action:l} {stream} v{nextI} {count}/{responses} {ms}ms rc={ru}", action, streamName, nextI, count, responsesCount, (let e = interval.Elapsed in e.TotalMilliseconds), ru) - let private inferPosition maybeIndexDocument (events: IOrderedEvent[]): Position = match maybeIndexDocument with Some p -> p | None -> Position.FromMaxIndex events - - let private calculateUsedVersusDroppedPayload stopIndex (xs: IOrderedEvent[]) : int * int = + let private calculateUsedVersusDroppedPayload stopIndex (xs: IIndexedEvent[]) : int * int = let mutable used, dropped = 0, 0 let mutable found = false for x in xs do @@ -581,10 +581,10 @@ module private Index = used, dropped let walk (log : ILogger) client retryPolicy maxItems maxRequests direction (stream: CollectionStream) startPos predicate - : Async = async { + : Async = async { let responseCount = ref 0 - let mergeBatches (log : ILogger) (batchesBackward : AsyncSeq) - : Async = async { + let mergeBatches (log : ILogger) (batchesBackward : AsyncSeq) + : Async = async { let mutable lastResponse = None let mutable maybeIndexDocument = None let mutable ru = 0.0 @@ -612,24 +612,14 @@ module private Index = let retryingLoggingReadSlice query = Log.withLoggedRetries retryPolicy "readAttempt" (pullSlice query) let log = log |> Log.prop "batchSize" maxItems |> Log.prop "stream" stream.name let readlog = log |> Log.prop "direction" direction - let batches : AsyncSeq = runQuery readlog retryingLoggingReadSlice maxRequests query + let batches : AsyncSeq = runQuery readlog retryingLoggingReadSlice maxRequests query let! t, (events, maybeIndexDocument, ru) = mergeBatches log batches |> Stopwatch.Time query.Dispose() - let pos = inferPosition maybeIndexDocument events + let pos = match maybeIndexDocument with Some p -> p | None -> Position.FromMaxIndex events log |> logBatchRead direction maxItems stream.name t (!responseCount,events) pos.index ru return pos, events } -module UnionEncoderAdapters = - let encodeEvent (codec : UnionCodec.IUnionEncoder<'event, byte[]>) (x : 'event) : IEvent = - let e = codec.Encode x - { new IEvent with - member __.EventType = e.caseName - member __.Data = e.payload - member __.Meta = null } - let decodeKnownEvents (codec : UnionCodec.IUnionEncoder<'event, byte[]>): IOrderedEvent seq -> 'event seq = - Seq.choose (fun x -> codec.TryDecode { caseName = x.EventType; payload = x.Data }) - type [] Token = { stream: CollectionStream; pos: Position } module Token = let create stream pos : Storage.StreamToken = { value = box { stream = stream; pos = pos } } @@ -654,17 +644,16 @@ open System.Collections.Generic [] module Internal = [] - type InternalSyncResult = Written of Storage.StreamToken | ConflictUnknown of Storage.StreamToken | Conflict of Storage.StreamToken * IOrderedEvent[] + type InternalSyncResult = Written of Storage.StreamToken | ConflictUnknown of Storage.StreamToken | Conflict of Storage.StreamToken * IIndexedEvent[] [] - type LoadFromTokenResult = Unchanged | Found of Storage.StreamToken * IOrderedEvent[] + type LoadFromTokenResult = Unchanged | Found of Storage.StreamToken * IIndexedEvent[] -/// Defines the policies in force for retrying with regard to transient failures calling CosmosDb (as opposed to application level concurrency conflicts) -type EqxConnection(client: IDocumentClient, ?readRetryPolicy (*: (int -> Async<'T>) -> Async<'T>*), ?writeRetryPolicy) = +/// Defines policies for retrying with respect to transient failures calling CosmosDb (as opposed to application level concurrency conflicts) +type EqxConnection(client: IDocumentClient, ?readRetryPolicy: IRetryPolicy, ?writeRetryPolicy: IRetryPolicy) = member __.Client = client member __.ReadRetryPolicy = readRetryPolicy member __.WriteRetryPolicy = writeRetryPolicy - //member __.Close = (client :?> Client.DocumentClient).Dispose() /// Defines the policies in force regarding how to constrain query responses type EqxBatchingPolicy @@ -682,26 +671,26 @@ type EqxBatchingPolicy type EqxGateway(conn : EqxConnection, batching : EqxBatchingPolicy) = let eventTypesPredicate resolved = - let acc = HashSet() - fun (x: IOrderedEvent) -> + let acc = System.Collections.Generic.HashSet() + fun (x: IIndexedEvent) -> acc.Add x.EventType |> ignore resolved acc - let (|Satisfies|_|) predicate (xs:IOrderedEvent[]) = + let (|Satisfies|_|) predicate (xs:IIndexedEvent[]) = match Array.tryFindIndexBack predicate xs with | None -> None | Some index -> Array.sub xs index (xs.Length - index) |> Some - let loadBackwardsStopping log predicate stream: Async = async { + let loadBackwardsStopping log predicate stream: Async = async { let! pos, events = Query.walk log conn.Client conn.ReadRetryPolicy batching.MaxItems batching.MaxRequests Direction.Backward stream None predicate Array.Reverse events return Token.create stream pos, events } - member __.LoadBackwardsStopping log predicate stream: Async = + member __.LoadBackwardsStopping log predicate stream: Async = let predicate = eventTypesPredicate predicate loadBackwardsStopping log predicate stream - member __.Read log batchingOverride stream direction startPos predicate: Async = async { + member __.Read log batchingOverride stream direction startPos predicate: Async = async { let batching = defaultArg batchingOverride batching let! pos, events = Query.walk log conn.Client conn.ReadRetryPolicy batching.MaxItems batching.MaxRequests direction stream startPos predicate return Token.create stream pos, events } - member __.LoadFromProjectionsOrRollingSnapshots log predicate (stream,maybePos): Async = async { + member __.LoadFromProjectionsOrRollingSnapshots log predicate (stream,maybePos): Async = async { let! res = Index.tryLoad log None(* TODO conn.ReadRetryPolicy*) conn.Client stream maybePos let predicate = eventTypesPredicate predicate match res with @@ -733,8 +722,10 @@ type EqxGateway(conn : EqxConnection, batching : EqxBatchingPolicy) = | Sync.Result.Written pos' -> return InternalSyncResult.Written (Token.create stream pos') } type private Category<'event, 'state>(gateway : EqxGateway, codec : UnionCodec.IUnionEncoder<'event, byte[]>) = + let tryDecode (x: #IEvent) = codec.TryDecode { caseName = x.EventType; payload = x.Data } + let (|TryDecodeFold|) (fold: 'state -> 'event seq -> 'state) initial (events: IIndexedEvent seq) : 'state = Seq.choose tryDecode events |> fold initial let respond (fold: 'state -> 'event seq -> 'state) initial events : 'state = - fold initial (UnionEncoderAdapters.decodeKnownEvents codec events) + fold initial (Seq.choose tryDecode events) member __.Load includeProjections collectionStream fold initial predicate (log : ILogger): Async = async { let! token, events = if not includeProjections then gateway.LoadBackwardsStopping log predicate collectionStream @@ -749,8 +740,14 @@ type private Category<'event, 'state>(gateway : EqxGateway, codec : UnionCodec.I (expectedVersion : int64 option, events, state') fold predicate log : Async> = async { - let encode = UnionEncoderAdapters.encodeEvent codec - let eventsEncoded, projectionsEncoded = Seq.map encode events |> Array.ofSeq, Seq.map encode (project state' events) + let encodeEvent (x : 'event) : IEvent = + let e = codec.Encode x + { new IEvent with + member __.EventType = e.caseName + member __.Data = e.payload + member __.Meta = null } + let state' = fold state (Seq.ofList events) + let eventsEncoded, projectionsEncoded = Seq.map encodeEvent events |> Array.ofSeq, Seq.map encodeEvent (project state' events) let baseIndex = pos.index + int64 (List.length events) let projections = Sync.mkProjections baseIndex projectionsEncoded let batch = Sync.mkBatch stream eventsEncoded projections @@ -802,9 +799,9 @@ module Caching = interface ICategory<'event, 'state> with member __.Load (streamName : string) (log : ILogger) : Async = interceptAsync (inner.Load streamName log) streamName - member __.TrySync (log : ILogger) (Token.Unpack (stream,_) as streamToken,state) (events : 'event list, state': 'state) + member __.TrySync (log : ILogger) (Token.Unpack (stream,_) as streamToken,state) (events : 'event list) : Async> = async { - let! syncRes = inner.TrySync log (streamToken, state) (events,state') + let! syncRes = inner.TrySync log (streamToken, state) events match syncRes with | Storage.SyncResult.Conflict resync -> return Storage.SyncResult.Conflict (interceptAsync resync stream.name) | Storage.SyncResult.Written (token', state') ->return Storage.SyncResult.Written (intercept stream.name (token', state')) } @@ -821,7 +818,7 @@ module Caching = type private Folder<'event, 'state> ( category : Category<'event, 'state>, fold: 'state -> 'event seq -> 'state, initial: 'state, - predicate : HashSet -> bool, + predicate : System.Collections.Generic.HashSet -> bool, mkCollectionStream : string -> Store.CollectionStream, // Whether or not a projection function is supplied controls whether reads consult the index or not ?project: ('state -> 'event seq -> 'event seq), @@ -837,9 +834,9 @@ type private Folder<'event, 'state> match cache.TryGet(prefix + streamName) with | None -> batched | Some tokenAndState -> cached tokenAndState - member __.TrySync (log : ILogger) (Token.Unpack (_stream,pos) as streamToken,state) (events : 'event list, state': 'state) + member __.TrySync (log : ILogger) (Token.Unpack (_stream,pos) as streamToken,state) (events : 'event list) : Async> = async { - let! syncRes = category.Sync (streamToken,state) (defaultArg project (fun _ _ -> Seq.empty)) (Some pos.index, events, state') fold predicate log + let! syncRes = category.Sync (streamToken,state) (defaultArg project (fun _ _ -> Seq.empty)) (Some pos.index, events, fold state events) fold predicate log match syncRes with | Storage.SyncResult.Conflict resync -> return Storage.SyncResult.Conflict resync | Storage.SyncResult.Written (token',state') -> return Storage.SyncResult.Written (token',state') } @@ -873,7 +870,7 @@ type AccessStrategy<'event,'state> = /// Provides equivalent performance to Projections, just simplified function signatures | Projection of eventType: string * ('state -> 'event) /// Simplified version - | AnyKnownEventType of eventTypes: ISet + | AnyKnownEventType of eventTypes: System.Collections.Generic.ISet type EqxStreamBuilder<'event, 'state>(store : EqxStore, codec, fold, initial, ?access, ?caching) = member __.Create streamName : Equinox.IStream<'event, 'state> = @@ -888,10 +885,10 @@ type EqxStreamBuilder<'event, 'state>(store : EqxStore, codec, fold, initial, ?a predicate, Some (fun state _events -> project state) | Some (AccessStrategy.Projection (et,compact)) -> - (fun (ets: HashSet) -> ets.Contains et), + (fun (ets: System.Collections.Generic.HashSet) -> ets.Contains et), Some (fun state _events -> seq [compact state]) | Some (AccessStrategy.AnyKnownEventType knownEventTypes) -> - (fun (ets: HashSet) -> knownEventTypes.Overlaps ets), + (fun (ets: System.Collections.Generic.HashSet) -> knownEventTypes.Overlaps ets), Some (fun _ events -> Seq.last events |> Seq.singleton) let category = Category<'event, 'state>(store.Gateway, codec) let folder = Folder<'event, 'state>(category, fold, initial, predicate, store.Collections.CollectionForStream, ?project=projectOption, ?readCache = readCacheOption) @@ -985,13 +982,12 @@ open Equinox.Cosmos open Equinox.Cosmos.Builder open Equinox.Cosmos.Events open FSharp.Control -open Equinox /// Outcome of appending events, specifying the new and/or conflicting events, together with the updated Target write position [] type AppendResult<'t> = | Ok of pos: 't - | Conflict of index: 't * conflictingEvents: IOrderedEvent[] + | Conflict of index: 't * conflictingEvents: IIndexedEvent[] | ConflictUnknown of index: 't /// Encapsulates the core facilites Equinox.Cosmos offers for operating directly on Events in Streams. @@ -1030,11 +1026,11 @@ type EqxContext // Search semantics include the first hit so we need to special case this anyway return Token.create stream (defaultArg startPos Position.FromKnownEmpty), Array.empty else - let predicate = + let isOrigin = match maxCount with | Some limit -> maxCountPredicate limit | None -> fun _ -> false - return! gateway.Read logger None stream direction startPos predicate } + return! gateway.Read logger None stream direction startPos isOrigin } /// Establishes the current position of the stream in as effficient a manner as possible /// (The ideal situation is that the preceding token is supplied as input in order to avail of 1RU low latency state checks) @@ -1045,13 +1041,13 @@ type EqxContext /// Reads in batches of `batchSize` from the specified `Position`, allowing the reader to efficiently walk away from a running query /// ... NB as long as they Dispose! - member __.Walk(stream, batchSize, ?position, ?direction) : AsyncSeq = asyncSeq { + member __.Walk(stream, batchSize, ?position, ?direction) : AsyncSeq = asyncSeq { let! _pos,data = __.GetInternal((stream, position), batchSize, ?direction=direction) // TODO add laziness return AsyncSeq.ofSeq data } /// Reads all Events from a `Position` in a given `direction` - member __.Read(stream, ?position, ?maxCount, ?direction) : Async = + member __.Read(stream, ?position, ?maxCount, ?direction) : Async = __.GetInternal((stream, position), ?maxCount=maxCount, ?direction=direction) |> yieldPositionAndData /// Appends the supplied batch of events, subject to a consistency check based on the `position` @@ -1085,7 +1081,7 @@ module Events = let private stripPosition (f: Async): Async = async { let! (PositionIndex index) = f return index } - let private dropPosition (f: Async): Async = async { + let private dropPosition (f: Async): Async = async { let! _,xs = f return xs } let (|MinPosition|) = function @@ -1095,18 +1091,18 @@ module Events = | int64.MaxValue -> None | i -> Some (Position.FromI (i + 1L)) - /// Returns an aFromLastIndexs in the stream starting at the specified sequence number, + /// Returns an async sequence of events in the stream starting at the specified sequence number, /// reading in batches of the specified size. /// Returns an empty sequence if the stream is empty or if the sequence number is larger than the largest /// sequence number in the stream. - let getAll (ctx: EqxContext) (streamName: string) (MinPosition index: int64) (batchSize: int): AsyncSeq = - ctx.Walk(ctx.CreateStream streamName, batchSize,?position=index) + let getAll (ctx: EqxContext) (streamName: string) (MinPosition index: int64) (batchSize: int): AsyncSeq = + ctx.Walk(ctx.CreateStream streamName, batchSize, ?position=index) /// Returns an async array of events in the stream starting at the specified sequence number, /// number of events to read is specified by batchSize /// Returns an empty sequence if the stream is empty or if the sequence number is larger than the largest /// sequence number in the stream. - let get (ctx: EqxContext) (streamName: string) (MinPosition index: int64) (maxCount: int): Async = + let get (ctx: EqxContext) (streamName: string) (MinPosition index: int64) (maxCount: int): Async = ctx.Read(ctx.CreateStream streamName, ?position=index, maxCount=maxCount) |> dropPosition /// Appends a batch of events to a stream at the specified expected sequence number. @@ -1126,14 +1122,14 @@ module Events = /// reading in batches of the specified size. /// Returns an empty sequence if the stream is empty or if the sequence number is smaller than the smallest /// sequence number in the stream. - let getAllBackwards (ctx: EqxContext) (streamName: string) (MaxPosition index: int64) (maxCount: int): AsyncSeq = + let getAllBackwards (ctx: EqxContext) (streamName: string) (MaxPosition index: int64) (maxCount: int): AsyncSeq = ctx.Walk(ctx.CreateStream streamName, maxCount, ?position=index, direction=Direction.Backward) /// Returns an async array of events in the stream backwards starting from the specified sequence number, /// number of events to read is specified by batchSize /// Returns an empty sequence if the stream is empty or if the sequence number is smaller than the smallest /// sequence number in the stream. - let getBackwards (ctx: EqxContext) (streamName: string) (MaxPosition index: int64) (maxCount: int): Async = + let getBackwards (ctx: EqxContext) (streamName: string) (MaxPosition index: int64) (maxCount: int): Async = ctx.Read(ctx.CreateStream streamName, ?position=index, maxCount=maxCount, direction=Direction.Backward) |> dropPosition /// Obtains the `index` from the current write Position diff --git a/tests/Equinox.Cosmos.Integration/CosmosCoreIntegration.fs b/tests/Equinox.Cosmos.Integration/CosmosCoreIntegration.fs index 02b52c051..8e0622993 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosCoreIntegration.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosCoreIntegration.fs @@ -12,7 +12,7 @@ open System.Text #nowarn "1182" // From hereon in, we may have some 'unused' privates (the tests) -type EventData = { eventType: string; data: byte[] } with +type EventData = { eventType:string; data: byte[] } with interface Events.IEvent with member __.EventType = __.eventType member __.Data = __.data @@ -56,7 +56,7 @@ type Tests(testOutputHelper) = test <@ AppendResult.Ok 6L = res @> test <@ [EqxAct.Append] = capture.ExternalCalls @> // We didnt request small batches or splitting so it's not dramatically more expensive to write N events - verifyRequestChargesMax 30 // observed 26.62 was 11 + verifyRequestChargesMax 29 // observed 28.61 // was 11 } let blobEquals (x: byte[]) (y: byte[]) = System.Linq.Enumerable.SequenceEqual(x,y) @@ -81,7 +81,7 @@ type Tests(testOutputHelper) = return EventData.Create(0,6) } - let verifyCorrectEventsEx direction baseIndex (expected: Events.IEvent []) (xs: Events.IOrderedEvent[]) = + let verifyCorrectEventsEx direction baseIndex (expected: Events.IEvent []) (xs: Events.IIndexedEvent[]) = let xs, baseIndex = if direction = Direction.Forward then xs, baseIndex else Array.rev xs, baseIndex - int64 (Array.length expected) + 1L diff --git a/tests/Equinox.Cosmos.Integration/CosmosFixturesInfrastructure.fs b/tests/Equinox.Cosmos.Integration/CosmosFixturesInfrastructure.fs index 1a88db4e2..152a4ade1 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosFixturesInfrastructure.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosFixturesInfrastructure.fs @@ -68,23 +68,21 @@ module SerilogHelpers = | Log.IndexNotFound _ -> EqxAct.IndexNotFound | Log.IndexNotModified _ -> EqxAct.IndexNotModified let inline (|Stats|) ({ ru = ru }: Equinox.Cosmos.Log.Measurement) = ru - let (|CosmosReadRu|CosmosWriteRu|CosmosResyncRu|CosmosSliceRu|) (evt : Equinox.Cosmos.Log.Event) = - match evt with + let (|CosmosReadRc|CosmosWriteRc|CosmosResyncRc|CosmosSliceRc|) = function | Log.Index (Stats s) | Log.IndexNotFound (Stats s) | Log.IndexNotModified (Stats s) - | Log.Batch (_,_, (Stats s)) -> CosmosReadRu s + | Log.Batch (_,_, (Stats s)) -> CosmosReadRc s | Log.WriteSuccess (Stats s) - | Log.WriteConflict (Stats s) -> CosmosWriteRu s - | Log.WriteResync (Stats s) -> CosmosResyncRu s + | Log.WriteConflict (Stats s) -> CosmosWriteRc s + | Log.WriteResync (Stats s) -> CosmosResyncRc s // slices are rolled up into batches so be sure not to double-count - | Log.Slice (_,Stats s) -> CosmosSliceRu s + | Log.Slice (_,Stats s) -> CosmosSliceRc s /// Facilitates splitting between events with direct charges vs synthetic events Equinox generates to avoid double counting - let (|CosmosRequestCharge|EquinoxChargeRollup|) c = - match c with - | CosmosSliceRu _ -> + let (|CosmosRequestCharge|EquinoxChargeRollup|) = function + | CosmosSliceRc _ -> EquinoxChargeRollup - | CosmosReadRu rc | CosmosWriteRu rc | CosmosResyncRu rc as e -> + | CosmosReadRc rc | CosmosWriteRc rc | CosmosResyncRc rc as e -> CosmosRequestCharge (e,rc) let (|EqxEvent|_|) (logEvent : LogEvent) : Equinox.Cosmos.Log.Event option = logEvent.Properties.Values |> Seq.tryPick (function diff --git a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs index a0019a3d9..1ccbbc205 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs @@ -12,31 +12,32 @@ let genCodec<'Union when 'Union :> TypeShape.UnionContract.IUnionContract>() = Equinox.UnionCodec.JsonUtf8.Create<'Union>(serializationSettings) module Cart = - let fold, initial, project = Domain.Cart.Folds.fold, Domain.Cart.Folds.initial, Domain.Cart.Folds.compact + let fold, initial, snapshot = Domain.Cart.Folds.fold, Domain.Cart.Folds.initial, Domain.Cart.Folds.snapshot let codec = genCodec() let createServiceWithoutOptimization connection batchSize log = let store = createEqxStore connection batchSize let resolveStream = EqxStreamBuilder(store, codec, fold, initial).Create Backend.Cart.Service(log, resolveStream) + let projection = "Compacted",snd snapshot let createServiceWithProjection connection batchSize log = let store = createEqxStore connection batchSize - let resolveStream = EqxStreamBuilder(store, codec, fold, initial, AccessStrategy.Projection project).Create + let resolveStream = EqxStreamBuilder(store, codec, fold, initial, AccessStrategy.Projection projection).Create Backend.Cart.Service(log, resolveStream) let createServiceWithProjectionAndCaching connection batchSize log cache = let store = createEqxStore connection batchSize let sliding20m = CachingStrategy.SlidingWindow (cache, TimeSpan.FromMinutes 20.) - let resolveStream = EqxStreamBuilder(store, codec, fold, initial, AccessStrategy.Projection project, sliding20m).Create + let resolveStream = EqxStreamBuilder(store, codec, fold, initial, AccessStrategy.Projection projection, sliding20m).Create Backend.Cart.Service(log, resolveStream) module ContactPreferences = - let fold, initial, eventTypes = Domain.ContactPreferences.Folds.fold, Domain.ContactPreferences.Folds.initial, Domain.ContactPreferences.Events.eventTypeNames + let fold, initial = Domain.ContactPreferences.Folds.fold, Domain.ContactPreferences.Folds.initial let codec = genCodec() let createServiceWithoutOptimization createGateway defaultBatchSize log _ignoreWindowSize _ignoreCompactionPredicate = let gateway = createGateway defaultBatchSize let resolveStream = EqxStreamBuilder(gateway, codec, fold, initial).Create Backend.ContactPreferences.Service(log, resolveStream) let createService createGateway log = - let resolveStream = EqxStreamBuilder(createGateway 1, codec, fold, initial, AccessStrategy.AnyKnownEventType eventTypes).Create + let resolveStream = EqxStreamBuilder(createGateway 1, codec, fold, initial, AccessStrategy.AnyKnownEventType (System.Collections.Generic.HashSet ["contactPreferencesChanged"])).Create Backend.ContactPreferences.Service(log, resolveStream) #nowarn "1182" // From hereon in, we may have some 'unused' privates (the tests) From 8dfcaa2858e957044213a1902ae536a07dd0b71e Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Mon, 26 Nov 2018 23:25:14 +0000 Subject: [PATCH 44/66] Clarify projections/unfolding (#53) --- samples/Store/Integration/CartIntegration.fs | 3 +- .../ContactPreferencesIntegration.fs | 2 +- .../Store/Integration/FavoritesIntegration.fs | 3 +- samples/Store/Integration/LogIntegration.fs | 26 +- src/Equinox.Cosmos/Cosmos.fs | 364 ++++++++---------- .../CosmosCoreIntegration.fs | 16 +- .../CosmosFixturesInfrastructure.fs | 53 +-- .../CosmosIntegration.fs | 26 +- .../JsonConverterTests.fs | 8 +- 9 files changed, 236 insertions(+), 265 deletions(-) diff --git a/samples/Store/Integration/CartIntegration.fs b/samples/Store/Integration/CartIntegration.fs index 3b8e93405..3001083f0 100644 --- a/samples/Store/Integration/CartIntegration.fs +++ b/samples/Store/Integration/CartIntegration.fs @@ -23,9 +23,8 @@ let resolveGesStreamWithRollingSnapshots gateway = let resolveGesStreamWithoutCustomAccessStrategy gateway = GesResolver(gateway, codec, fold, initial).Resolve -let projection = "Compacted",snd snapshot let resolveEqxStreamWithProjection gateway = - EqxStreamBuilder(gateway, codec, fold, initial, AccessStrategy.Projection projection).Create + EqxStreamBuilder(gateway, codec, fold, initial, AccessStrategy.Snapshot snapshot).Create let resolveEqxStreamWithoutCustomAccessStrategy gateway = EqxStreamBuilder(gateway, codec, fold, initial).Create diff --git a/samples/Store/Integration/ContactPreferencesIntegration.fs b/samples/Store/Integration/ContactPreferencesIntegration.fs index 3b9b523de..dae2aa140 100644 --- a/samples/Store/Integration/ContactPreferencesIntegration.fs +++ b/samples/Store/Integration/ContactPreferencesIntegration.fs @@ -22,7 +22,7 @@ let resolveStreamGesWithoutAccessStrategy gateway = GesResolver(gateway defaultBatchSize, codec, fold, initial).Resolve let resolveStreamEqxWithKnownEventTypeSemantics gateway = - EqxStreamBuilder(gateway 1, codec, fold, initial, AccessStrategy.AnyKnownEventType (System.Collections.Generic.HashSet ["contactPreferencesChanged"])).Create + EqxStreamBuilder(gateway 1, codec, fold, initial, AccessStrategy.AnyKnownEventType).Create let resolveStreamEqxWithoutCustomAccessStrategy gateway = EqxStreamBuilder(gateway defaultBatchSize, codec, fold, initial).Create diff --git a/samples/Store/Integration/FavoritesIntegration.fs b/samples/Store/Integration/FavoritesIntegration.fs index 5a0d20ef9..3bd07e6f5 100644 --- a/samples/Store/Integration/FavoritesIntegration.fs +++ b/samples/Store/Integration/FavoritesIntegration.fs @@ -22,8 +22,7 @@ let createServiceGes gateway log = Backend.Favorites.Service(log, resolveStream) let createServiceEqx gateway log = - let projection = "Compacted",snd snapshot - let resolveStream = EqxStreamBuilder(gateway, codec, fold, initial, AccessStrategy.Projection projection).Create + let resolveStream = EqxStreamBuilder(gateway, codec, fold, initial, AccessStrategy.Snapshot snapshot).Create Backend.Favorites.Service(log, resolveStream) type Tests(testOutputHelper) = diff --git a/samples/Store/Integration/LogIntegration.fs b/samples/Store/Integration/LogIntegration.fs index 832172573..57445d470 100644 --- a/samples/Store/Integration/LogIntegration.fs +++ b/samples/Store/Integration/LogIntegration.fs @@ -25,22 +25,22 @@ module EquinoxEsInterop = module EquinoxCosmosInterop = open Equinox.Cosmos [] - type FlatMetric = { action: string; stream: string; interval: StopwatchInterval; bytes: int; count: int; batches: int option; ru: float } with + type FlatMetric = { action: string; stream: string; interval: StopwatchInterval; bytes: int; count: int; responses: int option; ru: float } with override __.ToString() = sprintf "%s-Stream=%s %s-Elapsed=%O Ru=%O" __.action __.stream __.action __.interval.Elapsed __.ru let flatten (evt : Log.Event) : FlatMetric = let action, metric, batches, ru = match evt with - | Log.WriteSuccess m -> "EqxAppendToStreamAsync", m, None, m.ru - | Log.WriteConflict m -> "EqxAppendToStreamConflictAsync", m, None, m.ru - | Log.WriteResync m -> "EqxAppendToStreamResyncAsync", m, None, m.ru - | Log.Slice (Direction.Forward,m) -> "EqxReadStreamEventsForwardAsync", m, None, m.ru - | Log.Slice (Direction.Backward,m) -> "EqxReadStreamEventsBackwardAsync", m, None, m.ru - | Log.Batch (Direction.Forward,c,m) -> "EqxLoadF", m, Some c, m.ru - | Log.Batch (Direction.Backward,c,m) -> "EqxLoadB", m, Some c, m.ru - | Log.Index m -> "EqxLoadI", m, None, m.ru - | Log.IndexNotFound m -> "EqxLoadI404", m, None, m.ru - | Log.IndexNotModified m -> "EqxLoadI302", m, None, m.ru - { action = action; stream = metric.stream; bytes = metric.bytes; count = metric.count; batches = batches + | Log.Tip m -> "CosmosTip", m, None, m.ru + | Log.TipNotFound m -> "CosmosTip404", m, None, m.ru + | Log.TipNotModified m -> "CosmosTip302", m, None, m.ru + | Log.Query (Direction.Forward,c,m) -> "CosmosQueryF", m, Some c, m.ru + | Log.Query (Direction.Backward,c,m) -> "CosmosQueryB", m, Some c, m.ru + | Log.Response (Direction.Forward,m) -> "CosmosResponseF", m, None, m.ru + | Log.Response (Direction.Backward,m) -> "CosmosResponseB", m, None, m.ru + | Log.SyncSuccess m -> "CosmosSync200", m, None, m.ru + | Log.SyncConflict m -> "CosmosSync409", m, None, m.ru + | Log.SyncResync m -> "CosmosSyncResync", m, None, m.ru + { action = action; stream = metric.stream; bytes = metric.bytes; count = metric.count; responses = batches interval = StopwatchInterval(metric.interval.StartTicks,metric.interval.EndTicks); ru = ru } type SerilogMetricsExtractor(emit : string -> unit) = @@ -127,5 +127,5 @@ type Tests() = let service = Backend.Cart.Service(log, CartIntegration.resolveEqxStreamWithProjection gateway) let itemCount = batchSize / 2 + 1 let cartId = Guid.NewGuid() |> CartId - do! act buffer service itemCount context cartId skuId "Eqx Index " // one is a 404, one is a 200 + do! act buffer service itemCount context cartId skuId "EqxCosmos Tip " // one is a 404, one is a 200 } \ No newline at end of file diff --git a/src/Equinox.Cosmos/Cosmos.fs b/src/Equinox.Cosmos/Cosmos.fs index 592db84d9..fe02d679f 100644 --- a/src/Equinox.Cosmos/Cosmos.fs +++ b/src/Equinox.Cosmos/Cosmos.fs @@ -1,6 +1,6 @@ namespace Equinox.Cosmos.Events -/// Common form for either a raw Event or a Projection +/// Common form for either a Domain Event or an Unfolded Event type IEvent = /// The Event Type, used to drive deserialization abstract member EventType : string @@ -9,13 +9,13 @@ type IEvent = /// Optional metadata (null, or same as d, not written if missing) abstract member Meta : byte[] -/// Represents an Event or Projection and its relative position in the event sequence +/// Represents a Domain Event or Unfold, together with it's Index in the event sequence type IIndexedEvent = inherit IEvent /// The index into the event sequence of this event abstract member Index : int64 - /// Indicates whether this is a primary event or a projection based on the events <= up to `Index` - abstract member IsProjection: bool + /// Indicates whether this is a Domain Event or an Unfolded Event based on the state inferred from the events up to `Index` + abstract member IsUnfold: bool /// Position and Etag to which an operation is relative type [] Position = { index: int64; etag: string option } with @@ -35,7 +35,7 @@ namespace Equinox.Cosmos.Store open Equinox.Cosmos.Events open Newtonsoft.Json -/// A 'normal' (frozen, not Pending) Batch of Events, without any Projections +/// A 'normal' (frozen, not Tip) Batch of Events (without any Unfolds) type [] Event = { /// DocDb-mandated Partition Key, must be maintained within the document @@ -56,10 +56,10 @@ type [] /// Same as `id`; necessitated by fact that it's not presently possible to do an ORDER BY on the row key i: int64 // {index} - /// Creation date (as opposed to system-defined _lastUpdated which is touched by triggers, replication etc.) + /// Creation datetime (as opposed to system-defined _lastUpdated which is touched by triggers, replication etc.) c: System.DateTimeOffset // ISO 8601 - /// The Event Type, used to drive deserialization + /// The Case (Event Type), used to drive deserialization t: string // required /// Event body, as UTF-8 encoded json ready to be injected into the Json being rendered for DocDb @@ -77,19 +77,18 @@ type [] static member IndexedFields = [Event.PartitionKeyField; "i"] /// If we encounter a -1 doc, we're interested in its etag so we can re-read for one RU member x.TryToPosition() = - if x.id <> WipBatch.WellKnownDocumentId then None + if x.id <> Tip.WellKnownDocumentId then None else Some { index = (let ``x.e.LongLength`` = 1L in x.i+``x.e.LongLength``); etag = match x._etag with null -> None | x -> Some x } - /// The Special 'Pending' Batch Format /// NB this Type does double duty as /// a) transport for when we read it -/// b) a way of encoding a batch that the stored procedure will write in to the actual document +/// b) a way of encoding a batch that the stored procedure will write in to the actual document (`i` is -1 until Stored Proc computes it) /// The stored representation has the following differences vs a 'normal' (frozen/completed) Batch /// a) `id` and `i` = `-1` as WIP document currently always is /// b) events are retained as in an `e` array, not top level fields -/// c) contains projections (`c`) +/// c) contains unfolds (`c`) and [] - WipBatch = + Tip = { /// Partition key, as per Batch p: string // "{streamName}" /// Document Id within partition, as per Batch @@ -107,11 +106,11 @@ and [] /// Events e: BatchEvent[] - /// Projections - c: Projection[] } + /// Compaction/Snapshot/Projection events + c: Unfold[] } /// arguably this should be a high nember to reflect fact it is the freshest ? static member WellKnownDocumentId = "-1" - /// Create Position from [Wip]Batch record context (facilitating 1 RU reads) + /// Create Position from Tip record context (facilitating 1 RU reads) member x.ToPosition() = { index = x._i+x.e.LongLength; etag = match x._etag with null -> None | x -> Some x } /// A single event from the array held in a batch and [] @@ -130,15 +129,12 @@ and [] [)>] [] m: byte[] } // optional -/// Projection based on the state at a given point in time `i` -and Projection = - { /// Base: Max index rolled into this projection +/// Compaction/Snapshot/Projection Event based on the state at a given point in time `i` +and Unfold = + { /// Base: Stream Position (Version) of State from which this Unfold Event was generated i: int64 - ///// Indicates whether this is actually an event being retained to support a lagging projection - //x: bool - - /// The Event Type of this compaction/snapshot, used to drive deserialization + /// The Case (Event Type) of this compaction/snapshot, used to drive deserialization t: string // required /// Event body - Json -> UTF-8 -> Deflate -> Base64 @@ -151,39 +147,39 @@ and Projection = m: byte[] } // optional type Enum() = - static member Events (b:WipBatch) = + static member Events(b: Tip) = b.e |> Seq.mapi (fun offset x -> - { new IIndexedEvent with - member __.Index = b._i + int64 offset - member __.IsProjection = false - member __.EventType = x.t - member __.Data = x.d - member __.Meta = x.m }) - static member Events (i: int64, e:BatchEvent[]) = + { new IIndexedEvent with + member __.Index = b._i + int64 offset + member __.IsUnfold = false + member __.EventType = x.t + member __.Data = x.d + member __.Meta = x.m }) + static member Events(i: int64, e: BatchEvent[]) = e |> Seq.mapi (fun offset x -> { new IIndexedEvent with member __.Index = i + int64 offset - member __.IsProjection = false + member __.IsUnfold = false member __.EventType = x.t member __.Data = x.d member __.Meta = x.m }) - static member Event (x:Event) = + static member Event(x: Event) = Seq.singleton { new IIndexedEvent with member __.Index = x.i - member __.IsProjection = false + member __.IsUnfold = false member __.EventType = x.t member __.Data = x.d member __.Meta = x.m } - static member Projections (xs: Projection[]) = seq { + static member Unfolds(xs: Unfold[]) = seq { for x in xs -> { new IIndexedEvent with member __.Index = x.i - member __.IsProjection = true + member __.IsUnfold = true member __.EventType = x.t member __.Data = x.d member __.Meta = x.m } } - static member EventsAndProjections (x:WipBatch): IIndexedEvent seq = - Enum.Projections x.c + static member EventsAndUnfolds(x:Tip): IIndexedEvent seq = + Enum.Unfolds x.c /// Reference to Collection and name that will be used as the location for the stream type [] CollectionStream = { collectionUri: System.Uri; name: string } with @@ -211,25 +207,25 @@ module Log = type Measurement = { stream: string; interval: StopwatchInterval; bytes: int; count: int; ru: float } [] type Event = - | WriteSuccess of Measurement - | WriteResync of Measurement - | WriteConflict of Measurement + /// Individual read request for the Tip + | Tip of Measurement + /// Individual read request for the Tip, not found + | TipNotFound of Measurement + /// Tip read with Single RU Request Charge due to correct use of etag in cache + | TipNotModified of Measurement + /// Summarizes a set of Responses for a given Read request + | Query of Direction * responses: int * Measurement /// Individual read request in a Batch - | Slice of Direction * Measurement - /// Individual read request for the Index - | Index of Measurement - /// Individual read request for the Index, not found - | IndexNotFound of Measurement - /// Index read with Single RU Request Charge due to correct use of etag in cache - | IndexNotModified of Measurement - /// Summarizes a set of Slices read together - | Batch of Direction * slices: int * Measurement + | Response of Direction * Measurement + | SyncSuccess of Measurement + | SyncResync of Measurement + | SyncConflict of Measurement let prop name value (log : ILogger) = log.ForContext(name, value) let propData name (events: #IEvent seq) (log : ILogger) = let items = seq { for e in events do yield sprintf "{\"%s\": %s}" e.EventType (System.Text.Encoding.UTF8.GetString e.Data) } log.ForContext(name, sprintf "[%s]" (String.concat ",\n\r" items)) let propEvents = propData "events" - let propDataProjections = Enum.Projections >> propData "projections" + let propDataUnfolds = Enum.Unfolds >> propData "unfolds" let withLoggedRetries<'t> (retryPolicy: IRetryPolicy option) (contextLabel : string) (f : ILogger -> Async<'t>) log: Async<'t> = match retryPolicy with @@ -258,7 +254,6 @@ module private DocDb = | :? AggregateException as agg when agg.InnerExceptions.Count = 1 -> aux agg.InnerExceptions.[0] | _ -> e - aux exn /// DocumentDB Error HttpStatusCode extractor let (|DocDbException|_|) (e : exn) = @@ -393,7 +388,7 @@ function sync(req, expectedVersion) { | Conflict of Position * events: IIndexedEvent[] | ConflictUnknown of Position - let private run (client: IDocumentClient) (stream: CollectionStream) (expectedVersion: int64 option, req: WipBatch) + let private run (client: IDocumentClient) (stream: CollectionStream) (expectedVersion: int64 option, req: Tip) : Async = async { let sprocLink = sprintf "%O/sprocs/%s" stream.collectionUri sprocName let opts = Client.RequestOptions(PartitionKey=PartitionKey(stream.name)) @@ -409,41 +404,41 @@ function sync(req, expectedVersion) { | [||] -> Result.ConflictUnknown newPos | xs -> Result.Conflict (newPos, Enum.Events (ev.index, xs) |> Array.ofSeq) } - let private logged client (stream: CollectionStream) (expectedVersion, req: WipBatch) (log : ILogger) + let private logged client (stream: CollectionStream) (expectedVersion, req: Tip) (log : ILogger) : Async = async { let verbose = log.IsEnabled Events.LogEventLevel.Debug - let log = if verbose then log |> Log.propEvents (Enum.Events req) |> Log.propDataProjections req.c else log + let log = if verbose then log |> Log.propEvents (Enum.Events req) |> Log.propDataUnfolds req.c else log let (Log.BatchLen bytes), count = Enum.Events req, req.e.Length let log = log |> Log.prop "bytes" bytes let writeLog = log |> Log.prop "stream" stream.name |> Log.prop "expectedVersion" expectedVersion - |> Log.prop "count" req.e.Length |> Log.prop "pcount" req.c.Length - let! t, (ru,result) = run client stream (expectedVersion, req) |> Stopwatch.Time + |> Log.prop "count" req.e.Length |> Log.prop "ucount" req.c.Length + let! t, (ru,result) = run client stream (expectedVersion,req) |> Stopwatch.Time let resultLog = let mkMetric ru : Log.Measurement = { stream = stream.name; interval = t; bytes = bytes; count = count; ru = ru } - let logConflict () = writeLog.Information("Eqx Sync Conflict writing {eventTypes}", [| for x in req.e -> x.t |]) + let logConflict () = writeLog.Information("EqxCosmos Sync: Conflict writing {eventTypes}", [| for x in req.e -> x.t |]) match result with | Result.Written pos -> - log |> Log.event (Log.WriteSuccess (mkMetric ru)) |> Log.prop "nextExpectedVersion" pos + log |> Log.event (Log.SyncSuccess (mkMetric ru)) |> Log.prop "nextExpectedVersion" pos | Result.ConflictUnknown pos -> logConflict () - log |> Log.event (Log.WriteConflict (mkMetric ru)) |> Log.prop "nextExpectedVersion" pos |> Log.prop "conflict" true + log |> Log.event (Log.SyncConflict (mkMetric ru)) |> Log.prop "nextExpectedVersion" pos |> Log.prop "conflict" true | Result.Conflict (pos, xs) -> logConflict () let log = if verbose then log |> Log.prop "nextExpectedVersion" pos |> Log.propData "conflicts" xs else log - log |> Log.event (Log.WriteResync(mkMetric ru)) |> Log.prop "conflict" true - resultLog.Information("Eqx {action:l} {count}+{pcount} {ms}ms rc={ru}", "Write", req.e.Length, req.c.Length, (let e = t.Elapsed in e.TotalMilliseconds), ru) + log |> Log.event (Log.SyncResync(mkMetric ru)) |> Log.prop "conflict" true + resultLog.Information("EqxCosmos {action:l} {count}+{ucount} {ms}ms rc={ru}", "Sync", req.e.Length, req.c.Length, (let e = t.Elapsed in e.TotalMilliseconds), ru) return result } let batch (log : ILogger) retryPolicy client pk batch: Async = let call = logged client pk batch Log.withLoggedRetries retryPolicy "writeAttempt" call log - let mkBatch (stream: Store.CollectionStream) (events: IEvent[]) projections: WipBatch = - { p = stream.name; id = Store.WipBatch.WellKnownDocumentId; _i = -1L(*Server-managed*); _etag = null + let mkBatch (stream: Store.CollectionStream) (events: IEvent[]) unfolds: Tip = + { p = stream.name; id = Store.Tip.WellKnownDocumentId; _i = -1L(*Server-managed*); _etag = null e = [| for e in events -> { c = DateTimeOffset.UtcNow; t = e.EventType; d = e.Data; m = e.Meta } |] - c = Array.ofSeq projections } - let mkProjections baseIndex (projectionEvents: IEvent seq) : Store.Projection seq = - projectionEvents |> Seq.mapi (fun offset x -> { i = baseIndex + int64 offset; t = x.EventType; d = x.Data; m = x.Meta } : Store.Projection) + c = Array.ofSeq unfolds } + let mkUnfold baseIndex (unfolds: IEvent seq) : Store.Unfold seq = + unfolds |> Seq.mapi (fun offset x -> { i = baseIndex + int64 offset; t = x.EventType; d = x.Data; m = x.Meta } : Store.Unfold) module Initialization = open System.Collections.ObjectModel @@ -481,27 +476,27 @@ function sync(req, expectedVersion) { //let! _aux = createAux client dbUri collName auxRu return! createProc log client collUri } -module private Index = +module private Tip = let private get (client: IDocumentClient) (stream: CollectionStream, maybePos: Position option) = let coll = DocDbCollection(client, stream.collectionUri) let ac = match maybePos with Some { etag=Some etag } -> Client.AccessCondition(Type=Client.AccessConditionType.IfNoneMatch, Condition=etag) | _ -> null let ro = Client.RequestOptions(PartitionKey=PartitionKey(stream.name), AccessCondition = ac) - coll.TryReadDocument(WipBatch.WellKnownDocumentId, ro) + coll.TryReadDocument(Tip.WellKnownDocumentId, ro) let private loggedGet (get : CollectionStream * Position option -> Async<_>) (stream: CollectionStream, maybePos: Position option) (log: ILogger) = async { let log = log |> Log.prop "stream" stream.name - let! t, (ru, res : ReadResult) = get (stream,maybePos) |> Stopwatch.Time + let! t, (ru, res : ReadResult) = get (stream,maybePos) |> Stopwatch.Time let log count bytes (f : Log.Measurement -> _) = log |> Log.event (f { stream = stream.name; interval = t; bytes = bytes; count = count; ru = ru }) match res with | ReadResult.NotModified -> - (log 0 0 Log.IndexNotModified).Information("Eqx {action:l} {res} {ms}ms rc={ru}", "Index", 302, (let e = t.Elapsed in e.TotalMilliseconds), ru) + (log 0 0 Log.TipNotModified).Information("EqxCosmos {action:l} {res} {ms}ms rc={ru}", "Tip", 302, (let e = t.Elapsed in e.TotalMilliseconds), ru) | ReadResult.NotFound -> - (log 0 0 Log.IndexNotFound).Information("Eqx {action:l} {res} {ms}ms rc={ru}", "Index", 404, (let e = t.Elapsed in e.TotalMilliseconds), ru) + (log 0 0 Log.TipNotFound).Information("EqxCosmos {action:l} {res} {ms}ms rc={ru}", "Tip", 404, (let e = t.Elapsed in e.TotalMilliseconds), ru) | ReadResult.Found doc -> let log = - let (Log.BatchLen bytes), count = Enum.Projections doc.c, doc.c.Length - log bytes count Log.Index - let log = if (not << log.IsEnabled) Events.LogEventLevel.Debug then log else log |> Log.propDataProjections doc.c |> Log.prop "etag" doc._etag - log.Information("Eqx {action:l} {res} {ms}ms rc={ru}", "Index", 200, (let e = t.Elapsed in e.TotalMilliseconds), ru) + let (Log.BatchLen bytes), count = Enum.Unfolds doc.c, doc.c.Length + log bytes count Log.Tip + let log = if (not << log.IsEnabled) Events.LogEventLevel.Debug then log else log |> Log.propDataUnfolds doc.c |> Log.prop "etag" doc._etag + log.Information("EqxCosmos {action:l} {res} {ms}ms rc={ru}", "Tip", 200, (let e = t.Elapsed in e.TotalMilliseconds), ru) return ru, res } type [] Result = NotModified | NotFound | Found of Position * IIndexedEvent[] /// `pos` being Some implies that the caller holds a cached value and hence is ready to deal with IndexResult.UnChanged @@ -511,7 +506,7 @@ module private Index = match res with | ReadResult.NotModified -> return Result.NotModified | ReadResult.NotFound -> return Result.NotFound - | ReadResult.Found doc -> return Result.Found (doc.ToPosition(), Enum.EventsAndProjections doc |> Array.ofSeq) } + | ReadResult.Found doc -> return Result.Found (doc.ToPosition(), Enum.EventsAndUnfolds doc |> Array.ofSeq) } module private Query = open Microsoft.Azure.Documents.Linq @@ -525,8 +520,8 @@ module private Index = let feedOptions = new Client.FeedOptions(PartitionKey=PartitionKey(stream.name), MaxItemCount=Nullable maxItems) client.CreateDocumentQuery(stream.collectionUri, querySpec, feedOptions).AsDocumentQuery() - // Unrolls the Batches in a response - note when reading backawards, the events are emitted in reverse order of index - let private handleSlice direction (stream: CollectionStream) (startPos: Position option) (query: IDocumentQuery) (log: ILogger) + // Unrolls the Batches in a response - note when reading backwards, the events are emitted in reverse order of index + let private handleResponse direction (stream: CollectionStream) (startPos: Position option) (query: IDocumentQuery) (log: ILogger) : Async = async { let! ct = Async.CancellationToken let! t, (res : Client.FeedResponse) = query.ExecuteNextAsync(ct) |> Async.AwaitTaskCorrect |> Stopwatch.Time @@ -535,16 +530,16 @@ module private Index = let (Log.BatchLen bytes), count = events, events.Length let reqMetric : Log.Measurement = { stream = stream.name; interval = t; bytes = bytes; count = count; ru = ru } // TODO investigate whether there is a way to avoid the potential cost (or whether there is significance to it) of these null responses - let log = if batches.Length = 0 && count = 0 && ru = 0. then log else let evt = Log.Slice (direction, reqMetric) in log |> Log.event evt + let log = if batches.Length = 0 && count = 0 && ru = 0. then log else let evt = Log.Response (direction, reqMetric) in log |> Log.event evt let log = if (not << log.IsEnabled) Events.LogEventLevel.Debug then log else log |> Log.propEvents events let index = if count = 0 then Nullable () else Nullable <| Seq.min (seq { for x in batches -> x.i }) (log |> Log.prop "startIndex" (match startPos with Some { index = i } -> Nullable i | _ -> Nullable()) |> Log.prop "bytes" bytes) - .Information("Eqx {action:l} {count}/{batches} {direction} {ms}ms i={index} rc={ru}", - "Query", count, batches.Length, direction, (let e = t.Elapsed in e.TotalMilliseconds), index, ru) + .Information("EqxCosmos {action:l} {count}/{batches} {direction} {ms}ms i={index} rc={ru}", + "Response", count, batches.Length, direction, (let e = t.Elapsed in e.TotalMilliseconds), index, ru) let maybePosition = batches |> Array.tryPick (fun x -> x.TryToPosition()) return events, maybePosition, ru } - let private runQuery (log : ILogger) (readSlice: IDocumentQuery -> ILogger -> Async) + let private run (log : ILogger) (readSlice: IDocumentQuery -> ILogger -> Async) (maxPermittedBatchReads: int option) (query: IDocumentQuery) : AsyncSeq = @@ -560,14 +555,14 @@ module private Index = yield! loop (batchCount + 1) } loop 0 - let private logBatchRead direction batchSize streamName interval (responsesCount, events : IIndexedEvent []) nextI (ru: float) (log : ILogger) = + let private logQuery direction batchSize streamName interval (responsesCount, events : IIndexedEvent []) nextI (ru: float) (log : ILogger) = let (Log.BatchLen bytes), count = events, events.Length let reqMetric : Log.Measurement = { stream = streamName; interval = interval; bytes = bytes; count = count; ru = ru } - let action = match direction with Direction.Forward -> "LoadF" | Direction.Backward -> "LoadB" + let action = match direction with Direction.Forward -> "QueryF" | Direction.Backward -> "QueryB" // TODO investigate whether there is a way to avoid the potential cost (or whether there is significance to it) of these null responses - let log = if count = 0 && ru = 0. then log else let evt = Log.Event.Batch (direction, responsesCount, reqMetric) in log |> Log.event evt + let log = if count = 0 && ru = 0. then log else let evt = Log.Event.Query (direction, responsesCount, reqMetric) in log |> Log.event evt (log |> Log.prop "bytes" bytes |> Log.prop "batchSize" batchSize).Information( - "Eqx {action:l} {stream} v{nextI} {count}/{responses} {ms}ms rc={ru}", + "EqxCosmos {action:l} {stream} v{nextI} {count}/{responses} {ms}ms rc={ru}", action, streamName, nextI, count, responsesCount, (let e = interval.Elapsed in e.TotalMilliseconds), ru) let private calculateUsedVersusDroppedPayload stopIndex (xs: IIndexedEvent[]) : int * int = @@ -580,45 +575,45 @@ module private Index = if x.Index = stopIndex then found <- true used, dropped - let walk (log : ILogger) client retryPolicy maxItems maxRequests direction (stream: CollectionStream) startPos predicate - : Async = async { + let walk<'event> (log : ILogger) client retryPolicy maxItems maxRequests direction (stream: CollectionStream) startPos + (tryDecode : IIndexedEvent -> 'event option, isOrigin: 'event -> bool) + : Async = async { let responseCount = ref 0 - let mergeBatches (log : ILogger) (batchesBackward : AsyncSeq) - : Async = async { - let mutable lastResponse = None - let mutable maybeIndexDocument = None - let mutable ru = 0.0 + let mergeBatches (log : ILogger) (batchesBackward: AsyncSeq) = async { + let mutable lastResponse, mapbeTipPos, ru = None, None, 0. let! events = batchesBackward |> AsyncSeq.map (fun (events, maybePos, r) -> - if maybeIndexDocument = None then maybeIndexDocument <- maybePos + if mapbeTipPos = None then mapbeTipPos <- maybePos lastResponse <- Some events; ru <- ru + r incr responseCount - events) + events |> Array.map (fun x -> x, tryDecode x)) |> AsyncSeq.concatSeq - |> AsyncSeq.takeWhileInclusive (fun x -> - if not (predicate x) then true // continue the search - else + |> AsyncSeq.takeWhileInclusive (function + | x, Some e when isOrigin e -> match lastResponse with - | None -> log.Information("Eqx Stop stream={stream} at={index}", stream.name, x.Index) + | None -> log.Information("EqxCosmos Stop stream={stream} at={index} {case}", stream.name, x.Index, x.EventType) | Some batch -> let used, residual = batch |> calculateUsedVersusDroppedPayload x.Index - log.Information("Eqx Stop stream={stream} at={index} used={used} residual={residual}", stream.name, x.Index, used, residual) - false) + log.Information("EqxCosmos Stop stream={stream} at={index} {case} used={used} residual={residual}", + stream.name, x.Index, x.EventType, used, residual) + false + | _ -> true) (*continue the search*) |> AsyncSeq.toArrayAsync - return events, maybeIndexDocument, ru } + return events, mapbeTipPos, ru } use query = mkQuery client maxItems stream direction startPos - let pullSlice = handleSlice direction stream startPos + let pullSlice = handleResponse direction stream startPos let retryingLoggingReadSlice query = Log.withLoggedRetries retryPolicy "readAttempt" (pullSlice query) let log = log |> Log.prop "batchSize" maxItems |> Log.prop "stream" stream.name let readlog = log |> Log.prop "direction" direction - let batches : AsyncSeq = runQuery readlog retryingLoggingReadSlice maxRequests query - let! t, (events, maybeIndexDocument, ru) = mergeBatches log batches |> Stopwatch.Time + let batches : AsyncSeq = run readlog retryingLoggingReadSlice maxRequests query + let! t, (events, maybeTipPos, ru) = mergeBatches log batches |> Stopwatch.Time query.Dispose() - let pos = match maybeIndexDocument with Some p -> p | None -> Position.FromMaxIndex events + let raws, decoded = (Array.map fst events), (events |> Seq.choose snd |> Array.ofSeq) + let pos = match maybeTipPos with Some p -> p | None -> Position.FromMaxIndex raws - log |> logBatchRead direction maxItems stream.name t (!responseCount,events) pos.index ru - return pos, events } + log |> logQuery direction maxItems stream.name t (!responseCount,raws) pos.index ru + return pos, decoded } type [] Token = { stream: CollectionStream; pos: Position } module Token = @@ -639,7 +634,6 @@ open FSharp.Control open Microsoft.Azure.Documents open Serilog open System -open System.Collections.Generic [] module Internal = @@ -647,12 +641,13 @@ module Internal = type InternalSyncResult = Written of Storage.StreamToken | ConflictUnknown of Storage.StreamToken | Conflict of Storage.StreamToken * IIndexedEvent[] [] - type LoadFromTokenResult = Unchanged | Found of Storage.StreamToken * IIndexedEvent[] + type LoadFromTokenResult<'event> = Unchanged | Found of Storage.StreamToken * 'event[] /// Defines policies for retrying with respect to transient failures calling CosmosDb (as opposed to application level concurrency conflicts) type EqxConnection(client: IDocumentClient, ?readRetryPolicy: IRetryPolicy, ?writeRetryPolicy: IRetryPolicy) = member __.Client = client - member __.ReadRetryPolicy = readRetryPolicy + member __.TipRetryPolicy = readRetryPolicy + member __.QueryRetryPolicy = readRetryPolicy member __.WriteRetryPolicy = writeRetryPolicy /// Defines the policies in force regarding how to constrain query responses @@ -670,51 +665,39 @@ type EqxBatchingPolicy member __.MaxRequests = maxRequests type EqxGateway(conn : EqxConnection, batching : EqxBatchingPolicy) = - let eventTypesPredicate resolved = - let acc = System.Collections.Generic.HashSet() - fun (x: IIndexedEvent) -> - acc.Add x.EventType |> ignore - resolved acc - let (|Satisfies|_|) predicate (xs:IIndexedEvent[]) = - match Array.tryFindIndexBack predicate xs with + let (|FromUnfold|_|) (tryDecode: #IEvent -> 'event option) (isOrigin: 'event -> bool) (xs:#IEvent[]) : Option<'event[]> = + match Array.tryFindIndexBack (tryDecode >> Option.exists isOrigin) xs with | None -> None - | Some index -> Array.sub xs index (xs.Length - index) |> Some - let loadBackwardsStopping log predicate stream: Async = async { - let! pos, events = Query.walk log conn.Client conn.ReadRetryPolicy batching.MaxItems batching.MaxRequests Direction.Backward stream None predicate + | Some index -> xs |> Seq.skip index |> Seq.choose tryDecode |> Array.ofSeq |> Some + member __.LoadBackwardsStopping log stream (tryDecode,isOrigin): Async = async { + let! pos, events = Query.walk log conn.Client conn.QueryRetryPolicy batching.MaxItems batching.MaxRequests Direction.Backward stream None (tryDecode,isOrigin) Array.Reverse events return Token.create stream pos, events } - member __.LoadBackwardsStopping log predicate stream: Async = - let predicate = eventTypesPredicate predicate - loadBackwardsStopping log predicate stream - member __.Read log batchingOverride stream direction startPos predicate: Async = async { - let batching = defaultArg batchingOverride batching - let! pos, events = Query.walk log conn.Client conn.ReadRetryPolicy batching.MaxItems batching.MaxRequests direction stream startPos predicate + member __.Read log stream direction startPos (tryDecode,isOrigin) : Async = async { + let! pos, events = Query.walk log conn.Client conn.QueryRetryPolicy batching.MaxItems batching.MaxRequests direction stream startPos (tryDecode,isOrigin) return Token.create stream pos, events } - member __.LoadFromProjectionsOrRollingSnapshots log predicate (stream,maybePos): Async = async { - let! res = Index.tryLoad log None(* TODO conn.ReadRetryPolicy*) conn.Client stream maybePos - let predicate = eventTypesPredicate predicate + member __.LoadFromUnfoldsOrRollingSnapshots log (stream,maybePos) (tryDecode,isOrigin): Async = async { + let! res = Tip.tryLoad log conn.TipRetryPolicy conn.Client stream maybePos match res with - | Index.Result.NotFound -> return Token.create stream Position.FromKnownEmpty, Array.empty - | Index.Result.NotModified -> return invalidOp "Not handled" - | Index.Result.Found (pos, Satisfies predicate enoughEvents) -> return Token.create stream pos, enoughEvents - | _ -> return! loadBackwardsStopping log predicate stream } + | Tip.Result.NotFound -> return Token.create stream Position.FromKnownEmpty, Array.empty + | Tip.Result.NotModified -> return invalidOp "Not handled" + | Tip.Result.Found (pos, FromUnfold tryDecode isOrigin span) -> return Token.create stream pos, span + | _ -> return! __.LoadBackwardsStopping log stream (tryDecode,isOrigin) } member __.GetPosition(log, stream, ?pos): Async = async { - let! res = Index.tryLoad log None(* TODO conn.ReadRetryPolicy*) conn.Client stream pos + let! res = Tip.tryLoad log conn.TipRetryPolicy conn.Client stream pos match res with - | Index.Result.NotFound -> return Token.create stream Position.FromKnownEmpty - | Index.Result.NotModified -> return Token.create stream pos.Value - | Index.Result.Found (pos, _projectionsAndEvents) -> return Token.create stream pos } - member __.LoadFromToken log (stream,pos) predicate: Async = async { - let predicate = eventTypesPredicate predicate - let! res = Index.tryLoad log None(* TODO conn.ReadRetryPolicy*) conn.Client stream (Some pos) + | Tip.Result.NotFound -> return Token.create stream Position.FromKnownEmpty + | Tip.Result.NotModified -> return Token.create stream pos.Value + | Tip.Result.Found (pos, _unfoldsAndEvents) -> return Token.create stream pos } + member __.LoadFromToken(log, (stream,pos), (tryDecode, isOrigin)): Async> = async { + let! res = Tip.tryLoad log conn.TipRetryPolicy conn.Client stream (Some pos) match res with - | Index.Result.NotFound -> return LoadFromTokenResult.Found (Token.create stream Position.FromKnownEmpty,Array.empty) - | Index.Result.NotModified -> return LoadFromTokenResult.Unchanged - | Index.Result.Found (pos, Satisfies predicate enoughEvents) -> return LoadFromTokenResult.Found (Token.create stream pos, enoughEvents) - | _ -> - let! res = __.Read log None stream Direction.Forward (Some pos) (fun _ -> false) - return LoadFromTokenResult.Found res } - member __.Sync log stream (expectedVersion, batch: Store.WipBatch): Async = async { + | Tip.Result.NotFound -> return LoadFromTokenResult.Found (Token.create stream Position.FromKnownEmpty,Array.empty) + | Tip.Result.NotModified -> return LoadFromTokenResult.Unchanged + | Tip.Result.Found (pos, FromUnfold tryDecode isOrigin span) -> return LoadFromTokenResult.Found (Token.create stream pos, span) + | _ -> let! res = __.Read log stream Direction.Forward (Some pos) (tryDecode,isOrigin) + return LoadFromTokenResult.Found res } + member __.Sync log stream (expectedVersion, batch: Store.Tip): Async = async { let! wr = Sync.batch log conn.WriteRetryPolicy conn.Client stream (expectedVersion,batch) match wr with | Sync.Result.Conflict (pos',events) -> return InternalSyncResult.Conflict (Token.create stream pos',events) @@ -724,22 +707,17 @@ type EqxGateway(conn : EqxConnection, batching : EqxBatchingPolicy) = type private Category<'event, 'state>(gateway : EqxGateway, codec : UnionCodec.IUnionEncoder<'event, byte[]>) = let tryDecode (x: #IEvent) = codec.TryDecode { caseName = x.EventType; payload = x.Data } let (|TryDecodeFold|) (fold: 'state -> 'event seq -> 'state) initial (events: IIndexedEvent seq) : 'state = Seq.choose tryDecode events |> fold initial - let respond (fold: 'state -> 'event seq -> 'state) initial events : 'state = - fold initial (Seq.choose tryDecode events) - member __.Load includeProjections collectionStream fold initial predicate (log : ILogger): Async = async { + member __.Load includeUnfolds collectionStream fold initial isOrigin (log : ILogger): Async = async { let! token, events = - if not includeProjections then gateway.LoadBackwardsStopping log predicate collectionStream - else gateway.LoadFromProjectionsOrRollingSnapshots log predicate (collectionStream,None) - return token, respond fold initial events } - member __.LoadFromToken (Token.Unpack streamPos, state: 'state as current) fold predicate (log : ILogger): Async = async { - let! res = gateway.LoadFromToken log streamPos predicate + if not includeUnfolds then gateway.LoadBackwardsStopping log collectionStream (tryDecode,isOrigin) + else gateway.LoadFromUnfoldsOrRollingSnapshots log (collectionStream,None) (tryDecode,isOrigin) + return token, fold initial events } + member __.LoadFromToken (Token.Unpack streamPos, state: 'state as current) fold isOrigin (log : ILogger): Async = async { + let! res = gateway.LoadFromToken(log, streamPos, (tryDecode,isOrigin)) match res with | LoadFromTokenResult.Unchanged -> return current - | LoadFromTokenResult.Found (token',events) -> return token', respond fold state events } - member __.Sync (Token.Unpack (stream,pos), state as current) (project: 'state -> 'event seq -> 'event seq) - (expectedVersion : int64 option, events, state') - fold predicate log - : Async> = async { + | LoadFromTokenResult.Found (token', events') -> return token', fold state events' } + member __.Sync(Token.Unpack (stream,pos), state as current, expectedVersion, events, unfold, fold, isOrigin, log): Async> = async { let encodeEvent (x : 'event) : IEvent = let e = codec.Encode x { new IEvent with @@ -747,14 +725,14 @@ type private Category<'event, 'state>(gateway : EqxGateway, codec : UnionCodec.I member __.Data = e.payload member __.Meta = null } let state' = fold state (Seq.ofList events) - let eventsEncoded, projectionsEncoded = Seq.map encodeEvent events |> Array.ofSeq, Seq.map encodeEvent (project state' events) + let eventsEncoded, projectionsEncoded = Seq.map encodeEvent events |> Array.ofSeq, Seq.map encodeEvent (unfold state' events) let baseIndex = pos.index + int64 (List.length events) - let projections = Sync.mkProjections baseIndex projectionsEncoded + let projections = Sync.mkUnfold baseIndex projectionsEncoded let batch = Sync.mkBatch stream eventsEncoded projections let! res = gateway.Sync log stream (expectedVersion,batch) match res with - | InternalSyncResult.Conflict (token',events') -> return Storage.SyncResult.Conflict (async { return token', respond fold state events' }) - | InternalSyncResult.ConflictUnknown _token' -> return Storage.SyncResult.Conflict (__.LoadFromToken current fold predicate log) + | InternalSyncResult.Conflict (token',TryDecodeFold fold state events') -> return Storage.SyncResult.Conflict (async { return token', events' }) + | InternalSyncResult.ConflictUnknown _token' -> return Storage.SyncResult.Conflict (__.LoadFromToken current fold isOrigin log) | InternalSyncResult.Written token' -> return Storage.SyncResult.Written (token', state') } module Caching = @@ -817,17 +795,17 @@ module Caching = CategoryTee<'event,'state>(category, addOrUpdateSlidingExpirationCacheEntry) :> _ type private Folder<'event, 'state> - ( category : Category<'event, 'state>, fold: 'state -> 'event seq -> 'state, initial: 'state, - predicate : System.Collections.Generic.HashSet -> bool, - mkCollectionStream : string -> Store.CollectionStream, - // Whether or not a projection function is supplied controls whether reads consult the index or not - ?project: ('state -> 'event seq -> 'event seq), + ( category: Category<'event, 'state>, fold: 'state -> 'event seq -> 'state, initial: 'state, + isOrigin: 'event -> bool, + mkCollectionStream: string -> Store.CollectionStream, + // Whether or not an `unfold` function is supplied controls whether reads do a point read before querying + ?unfold: ('state -> 'event list -> 'event seq), ?readCache) = interface ICategory<'event, 'state> with member __.Load streamName (log : ILogger): Async = let collStream = mkCollectionStream streamName - let batched = category.Load (Option.isSome project) collStream fold initial predicate log - let cached tokenAndState = category.LoadFromToken tokenAndState fold predicate log + let batched = category.Load (Option.isSome unfold) collStream fold initial isOrigin log + let cached tokenAndState = category.LoadFromToken tokenAndState fold isOrigin log match readCache with | None -> batched | Some (cache : Caching.Cache, prefix : string) -> @@ -836,8 +814,8 @@ type private Folder<'event, 'state> | Some tokenAndState -> cached tokenAndState member __.TrySync (log : ILogger) (Token.Unpack (_stream,pos) as streamToken,state) (events : 'event list) : Async> = async { - let! syncRes = category.Sync (streamToken,state) (defaultArg project (fun _ _ -> Seq.empty)) (Some pos.index, events, fold state events) fold predicate log - match syncRes with + let! res = category.Sync((streamToken,state), Some pos.index, events, (defaultArg unfold (fun _ _ -> Seq.empty)), fold, isOrigin, log) + match res with | Storage.SyncResult.Conflict resync -> return Storage.SyncResult.Conflict resync | Storage.SyncResult.Written (token',state') -> return Storage.SyncResult.Written (token',state') } @@ -855,22 +833,21 @@ type EqxStore(gateway: EqxGateway, collections: EqxCollections) = [] type CachingStrategy = - /// Retain a single set of State, together with the associated etags + /// Retain a single 'state per streamName, together with the associated etag /// NB while a strategy like EventStore.Caching.SlidingWindowPrefixed is obviously easy to implement, the recommended approach is to - /// track all relevant data in the state, and/or have the `project` function ensure all relevant events get indexed quickly + /// track all relevant data in the state, and/or have the `unfold` function ensure _all_ relevant events get held in the `u`nfolds in tip | SlidingWindow of Caching.Cache * window: TimeSpan [] type AccessStrategy<'event,'state> = - /// Require a configurable Set of Event Types to have been accumulated from a) projections + b) searching backward in the event stream - /// until `resolved` deems it so; fold foward based on those - /// When saving, `project` the 'state to seed the set of events that `resolved` will see first - | Projections of resolved: (ISet -> bool) * project: ('state -> 'event seq) + /// Allow events that pass the `isOrigin` test to be used in lieu of folding all the events from the start of the stream + /// When saving, `unfold` the 'state, saving in the Tip + | Unfolded of isOrigin: ('event -> bool) * unfold: ('state -> 'event seq) /// Simplified version of projection that only has a single Projection Event Type /// Provides equivalent performance to Projections, just simplified function signatures - | Projection of eventType: string * ('state -> 'event) - /// Simplified version - | AnyKnownEventType of eventTypes: System.Collections.Generic.ISet + | Snapshot of isValid: ('event -> bool) * generate: ('state -> 'event) + /// Trust every event type as being an origin + | AnyKnownEventType type EqxStreamBuilder<'event, 'state>(store : EqxStore, codec, fold, initial, ?access, ?caching) = member __.Create streamName : Equinox.IStream<'event, 'state> = @@ -878,21 +855,14 @@ type EqxStreamBuilder<'event, 'state>(store : EqxStore, codec, fold, initial, ?a match caching with | None -> None | Some (CachingStrategy.SlidingWindow(cache, _)) -> Some(cache, null) - let predicate, projectOption = + let isOrigin, projectOption = match access with | None -> (fun _ -> false), None - | Some (AccessStrategy.Projections (predicate,project)) -> - predicate, - Some (fun state _events -> project state) - | Some (AccessStrategy.Projection (et,compact)) -> - (fun (ets: System.Collections.Generic.HashSet) -> ets.Contains et), - Some (fun state _events -> seq [compact state]) - | Some (AccessStrategy.AnyKnownEventType knownEventTypes) -> - (fun (ets: System.Collections.Generic.HashSet) -> knownEventTypes.Overlaps ets), - Some (fun _ events -> Seq.last events |> Seq.singleton) + | Some (AccessStrategy.Unfolded (isOrigin, unfold)) -> isOrigin, Some (fun state _events -> unfold state) + | Some (AccessStrategy.Snapshot (isValid,generate)) -> isValid, Some (fun state _events -> seq [generate state]) + | Some (AccessStrategy.AnyKnownEventType) -> (fun _ -> true), Some (fun _ events -> Seq.last events |> Seq.singleton) let category = Category<'event, 'state>(store.Gateway, codec) - let folder = Folder<'event, 'state>(category, fold, initial, predicate, store.Collections.CollectionForStream, ?project=projectOption, ?readCache = readCacheOption) - + let folder = Folder<'event, 'state>(category, fold, initial, isOrigin, store.Collections.CollectionForStream, ?unfold=projectOption, ?readCache = readCacheOption) let category : ICategory<_,_> = match caching with | None -> folder :> _ @@ -965,7 +935,7 @@ type EqxConnector match tags with None -> () | Some tags -> for key, value in tags do yield sprintf "%s=%s" key value } let sanitizedName = name.Replace('\'','_').Replace(':','_') // sic; Align with logging for ES Adapter let client = new Client.DocumentClient(uri, key, connPolicy, Nullable(defaultArg defaultConsistencyLevel ConsistencyLevel.Session)) - log.ForContext("Uri", uri).Information("Eqx connecting to Cosmos with Connection Name {connectionName}", sanitizedName) + log.ForContext("Uri", uri).Information("EqxCosmos connecting to Cosmos with Connection Name {connectionName}", sanitizedName) do! client.OpenAsync() |> Async.AwaitTaskCorrect return client :> IDocumentClient } @@ -1030,7 +1000,7 @@ type EqxContext match maxCount with | Some limit -> maxCountPredicate limit | None -> fun _ -> false - return! gateway.Read logger None stream direction startPos isOrigin } + return! gateway.Read logger stream direction startPos (Some,isOrigin) } /// Establishes the current position of the stream in as effficient a manner as possible /// (The ideal situation is that the preceding token is supplied as input in order to avail of 1RU low latency state checks) diff --git a/tests/Equinox.Cosmos.Integration/CosmosCoreIntegration.fs b/tests/Equinox.Cosmos.Integration/CosmosCoreIntegration.fs index 8e0622993..63da90de1 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosCoreIntegration.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosCoreIntegration.fs @@ -101,7 +101,7 @@ type Tests(testOutputHelper) = // If a fail triggers a rerun, we need to dump the previous log entries captured capture.Clear() let! pos = Events.getNextIndex ctx streamName - test <@ [EqxAct.IndexNotFound] = capture.ExternalCalls @> + test <@ [EqxAct.TipNotFound] = capture.ExternalCalls @> 0L =! pos verifyRequestChargesMax 1 // for a 404 by definition capture.Clear() @@ -132,7 +132,7 @@ type Tests(testOutputHelper) = capture.Clear() let! res = Events.getNextIndex ctx streamName - test <@ [EqxAct.Index] = capture.ExternalCalls @> + test <@ [EqxAct.Tip] = capture.ExternalCalls @> verifyRequestChargesMax 2 capture.Clear() pos =! res @@ -148,12 +148,12 @@ type Tests(testOutputHelper) = capture.Clear() let! pos = ctx.Sync(stream,?position=None) - test <@ [EqxAct.Index] = capture.ExternalCalls @> + test <@ [EqxAct.Tip] = capture.ExternalCalls @> verifyRequestChargesMax 50 // 41 observed // for a 200, you'll pay a lot (we omitted to include the position that NonIdempotentAppend yielded) capture.Clear() let! _pos = ctx.Sync(stream,pos) - test <@ [EqxAct.IndexNotModified] = capture.ExternalCalls @> + test <@ [EqxAct.TipNotModified] = capture.ExternalCalls @> verifyRequestChargesMax 1 // for a 302 by definition - when an etag IfNotMatch is honored, you only pay one RU capture.Clear() } @@ -205,7 +205,7 @@ type Tests(testOutputHelper) = verifyCorrectEvents 1L expected res - test <@ List.replicate 2 EqxAct.SliceForward @ [EqxAct.BatchForward] = capture.ExternalCalls @> + test <@ List.replicate 2 EqxAct.ResponseForward @ [EqxAct.QueryForward] = capture.ExternalCalls @> verifyRequestChargesMax 8 // observed 6.14 // was 3 } @@ -222,7 +222,7 @@ type Tests(testOutputHelper) = verifyCorrectEvents 1L expected res // 2 items atm - test <@ [EqxAct.SliceForward; EqxAct.SliceForward; EqxAct.BatchForward] = capture.ExternalCalls @> + test <@ [EqxAct.ResponseForward; EqxAct.ResponseForward; EqxAct.QueryForward] = capture.ExternalCalls @> verifyRequestChargesMax 7 // observed 6.14 // was 6 } @@ -239,7 +239,7 @@ type Tests(testOutputHelper) = verifyCorrectEvents 1L expected res // TODO [implement and] prove laziness - test <@ List.replicate 2 EqxAct.SliceForward @ [EqxAct.BatchForward] = capture.ExternalCalls @> + test <@ List.replicate 2 EqxAct.ResponseForward @ [EqxAct.QueryForward] = capture.ExternalCalls @> verifyRequestChargesMax 10 // observed 8.99 // was 3 } @@ -259,7 +259,7 @@ type Tests(testOutputHelper) = verifyCorrectEventsBackward 4L expected res - test <@ List.replicate 3 EqxAct.SliceBackward @ [EqxAct.BatchBackward] = capture.ExternalCalls @> + test <@ List.replicate 3 EqxAct.ResponseBackward @ [EqxAct.QueryBackward] = capture.ExternalCalls @> verifyRequestChargesMax 10 // observed 8.98 // was 3 } diff --git a/tests/Equinox.Cosmos.Integration/CosmosFixturesInfrastructure.fs b/tests/Equinox.Cosmos.Integration/CosmosFixturesInfrastructure.fs index 152a4ade1..98ed85c4c 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosFixturesInfrastructure.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosFixturesInfrastructure.fs @@ -52,35 +52,38 @@ module SerilogHelpers = | _ -> None open Equinox.Cosmos [] - type EqxAct = Append | Resync | Conflict | SliceForward | SliceBackward | BatchForward | BatchBackward | Index | IndexNotFound | IndexNotModified | SliceWaste - let (|EqxAction|) (evt : Equinox.Cosmos.Log.Event) = - match evt with - | Log.WriteSuccess _ -> EqxAct.Append - | Log.WriteResync _ -> EqxAct.Resync - | Log.WriteConflict _ -> EqxAct.Conflict - | Log.Slice (Direction.Forward,{count = 0}) -> EqxAct.SliceWaste // TODO remove, see comment where these are emitted - | Log.Slice (Direction.Forward,_) -> EqxAct.SliceForward - | Log.Slice (Direction.Backward,{count = 0}) -> EqxAct.SliceWaste // TODO remove, see comment where these are emitted - | Log.Slice (Direction.Backward,_) -> EqxAct.SliceBackward - | Log.Batch (Direction.Forward,_,_) -> EqxAct.BatchForward - | Log.Batch (Direction.Backward,_,_) -> EqxAct.BatchBackward - | Log.Index _ -> EqxAct.Index - | Log.IndexNotFound _ -> EqxAct.IndexNotFound - | Log.IndexNotModified _ -> EqxAct.IndexNotModified + type EqxAct = + | Tip | TipNotFound | TipNotModified + | ResponseForward | ResponseBackward | ResponseWaste + | QueryForward | QueryBackward + | Append | Resync | Conflict + let (|EqxAction|) = function + | Log.Tip _ -> EqxAct.Tip + | Log.TipNotFound _ -> EqxAct.TipNotFound + | Log.TipNotModified _ -> EqxAct.TipNotModified + | Log.Response (Direction.Forward, {count = 0}) -> EqxAct.ResponseWaste // TODO remove, see comment where these are emitted + | Log.Response (Direction.Forward,_) -> EqxAct.ResponseForward + | Log.Response (Direction.Backward, {count = 0}) -> EqxAct.ResponseWaste // TODO remove, see comment where these are emitted + | Log.Response (Direction.Backward,_) -> EqxAct.ResponseBackward + | Log.Query (Direction.Forward,_,_) -> EqxAct.QueryForward + | Log.Query (Direction.Backward,_,_) -> EqxAct.QueryBackward + | Log.SyncSuccess _ -> EqxAct.Append + | Log.SyncResync _ -> EqxAct.Resync + | Log.SyncConflict _ -> EqxAct.Conflict let inline (|Stats|) ({ ru = ru }: Equinox.Cosmos.Log.Measurement) = ru - let (|CosmosReadRc|CosmosWriteRc|CosmosResyncRc|CosmosSliceRc|) = function - | Log.Index (Stats s) - | Log.IndexNotFound (Stats s) - | Log.IndexNotModified (Stats s) - | Log.Batch (_,_, (Stats s)) -> CosmosReadRc s - | Log.WriteSuccess (Stats s) - | Log.WriteConflict (Stats s) -> CosmosWriteRc s - | Log.WriteResync (Stats s) -> CosmosResyncRc s + let (|CosmosReadRc|CosmosWriteRc|CosmosResyncRc|CosmosResponseRc|) = function + | Log.Tip (Stats s) + | Log.TipNotFound (Stats s) + | Log.TipNotModified (Stats s) // slices are rolled up into batches so be sure not to double-count - | Log.Slice (_,Stats s) -> CosmosSliceRc s + | Log.Response (_,Stats s) -> CosmosResponseRc s + | Log.Query (_,_, (Stats s)) -> CosmosReadRc s + | Log.SyncSuccess (Stats s) + | Log.SyncConflict (Stats s) -> CosmosWriteRc s + | Log.SyncResync (Stats s) -> CosmosResyncRc s /// Facilitates splitting between events with direct charges vs synthetic events Equinox generates to avoid double counting let (|CosmosRequestCharge|EquinoxChargeRollup|) = function - | CosmosSliceRc _ -> + | CosmosResponseRc _ -> EquinoxChargeRollup | CosmosReadRc rc | CosmosWriteRc rc | CosmosResyncRc rc as e -> CosmosRequestCharge (e,rc) diff --git a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs index 1ccbbc205..494a7a953 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs @@ -21,12 +21,12 @@ module Cart = let projection = "Compacted",snd snapshot let createServiceWithProjection connection batchSize log = let store = createEqxStore connection batchSize - let resolveStream = EqxStreamBuilder(store, codec, fold, initial, AccessStrategy.Projection projection).Create + let resolveStream = EqxStreamBuilder(store, codec, fold, initial, AccessStrategy.Snapshot snapshot).Create Backend.Cart.Service(log, resolveStream) let createServiceWithProjectionAndCaching connection batchSize log cache = let store = createEqxStore connection batchSize let sliding20m = CachingStrategy.SlidingWindow (cache, TimeSpan.FromMinutes 20.) - let resolveStream = EqxStreamBuilder(store, codec, fold, initial, AccessStrategy.Projection projection, sliding20m).Create + let resolveStream = EqxStreamBuilder(store, codec, fold, initial, AccessStrategy.Snapshot snapshot, sliding20m).Create Backend.Cart.Service(log, resolveStream) module ContactPreferences = @@ -37,7 +37,7 @@ module ContactPreferences = let resolveStream = EqxStreamBuilder(gateway, codec, fold, initial).Create Backend.ContactPreferences.Service(log, resolveStream) let createService createGateway log = - let resolveStream = EqxStreamBuilder(createGateway 1, codec, fold, initial, AccessStrategy.AnyKnownEventType (System.Collections.Generic.HashSet ["contactPreferencesChanged"])).Create + let resolveStream = EqxStreamBuilder(createGateway 1, codec, fold, initial, AccessStrategy.AnyKnownEventType).Create Backend.ContactPreferences.Service(log, resolveStream) #nowarn "1182" // From hereon in, we may have some 'unused' privates (the tests) @@ -69,7 +69,7 @@ type Tests(testOutputHelper) = // The command processing should trigger only a single read and a single write call let addRemoveCount = 6 do! addAndThenRemoveItemsManyTimesExceptTheLastOne context cartId skuId service addRemoveCount - test <@ [EqxAct.SliceWaste; EqxAct.BatchBackward; EqxAct.Append] = capture.ExternalCalls @> + test <@ [EqxAct.ResponseWaste; EqxAct.QueryBackward; EqxAct.Append] = capture.ExternalCalls @> // Restart the counting capture.Clear() @@ -80,7 +80,7 @@ type Tests(testOutputHelper) = // Need to read 4 batches to read 11 events in batches of 3 let expectedBatches = ceil(float expectedEventCount/float batchSize) |> int - test <@ List.replicate (expectedBatches-1) EqxAct.SliceBackward @ [EqxAct.SliceBackward; EqxAct.BatchBackward] = capture.ExternalCalls @> + test <@ List.replicate (expectedBatches-1) EqxAct.ResponseBackward @ [EqxAct.ResponseBackward; EqxAct.QueryBackward] = capture.ExternalCalls @> } [] @@ -163,7 +163,7 @@ type Tests(testOutputHelper) = && [EqxAct.Resync] = c2 @> } - let singleBatchBackwards = [EqxAct.SliceBackward; EqxAct.BatchBackward] + let singleBatchBackwards = [EqxAct.ResponseBackward; EqxAct.QueryBackward] let batchBackwardsAndAppend = singleBatchBackwards @ [EqxAct.Append] [] @@ -186,7 +186,7 @@ type Tests(testOutputHelper) = let! result = service.Read email test <@ value = result @> - test <@ [EqxAct.Index; EqxAct.Append; EqxAct.Index] = capture.ExternalCalls @> + test <@ [EqxAct.Tip; EqxAct.Append; EqxAct.Tip] = capture.ExternalCalls @> } [] @@ -203,17 +203,17 @@ type Tests(testOutputHelper) = let! _ = service2.Read cartId // ... should see a single read as we are writes are cached - test <@ [EqxAct.IndexNotFound; EqxAct.Append; EqxAct.Index] = capture.ExternalCalls @> + test <@ [EqxAct.TipNotFound; EqxAct.Append; EqxAct.Tip] = capture.ExternalCalls @> // Add two more - the roundtrip should only incur a single read capture.Clear() do! addAndThenRemoveItemsManyTimes context cartId skuId service1 1 - test <@ [EqxAct.Index; EqxAct.Append] = capture.ExternalCalls @> + test <@ [EqxAct.Tip; EqxAct.Append] = capture.ExternalCalls @> // While we now have 12 events, we should be able to read them with a single call capture.Clear() let! _ = service2.Read cartId - test <@ [EqxAct.Index] = capture.ExternalCalls @> + test <@ [EqxAct.Tip] = capture.ExternalCalls @> } [] @@ -231,17 +231,17 @@ type Tests(testOutputHelper) = let! _ = service2.Read cartId // ... should see a single Cached Indexed read given writes are cached and writer emits etag - test <@ [EqxAct.IndexNotFound; EqxAct.Append; EqxAct.IndexNotModified] = capture.ExternalCalls @> + test <@ [EqxAct.TipNotFound; EqxAct.Append; EqxAct.TipNotModified] = capture.ExternalCalls @> // Add two more - the roundtrip should only incur a single read, which should be cached by virtue of being a second one in successono capture.Clear() do! addAndThenRemoveItemsManyTimes context cartId skuId service1 1 - test <@ [EqxAct.IndexNotModified; EqxAct.Append] = capture.ExternalCalls @> + test <@ [EqxAct.TipNotModified; EqxAct.Append] = capture.ExternalCalls @> // While we now have 12 events, we should be able to read them with a single call capture.Clear() let! _ = service2.Read cartId let! _ = service2.Read cartId // First is cached because writer emits etag, second remains cached - test <@ [EqxAct.IndexNotModified; EqxAct.IndexNotModified] = capture.ExternalCalls @> + test <@ [EqxAct.TipNotModified; EqxAct.TipNotModified] = capture.ExternalCalls @> } \ No newline at end of file diff --git a/tests/Equinox.Cosmos.Integration/JsonConverterTests.fs b/tests/Equinox.Cosmos.Integration/JsonConverterTests.fs index 06b795722..721209113 100644 --- a/tests/Equinox.Cosmos.Integration/JsonConverterTests.fs +++ b/tests/Equinox.Cosmos.Integration/JsonConverterTests.fs @@ -36,7 +36,7 @@ type Base64ZipUtf8Tests() = [] let ``serializes, achieving compression`` () = let encoded = unionEncoder.Encode(A { embed = String('x',5000) }) - let e : Store.Projection = + let e : Store.Unfold = { i = 42L t = encoded.caseName d = encoded.payload @@ -53,14 +53,14 @@ type Base64ZipUtf8Tests() = if hasNulls then () else let encoded = unionEncoder.Encode value - let e : Store.Projection = + let e : Store.Unfold = { i = 42L t = encoded.caseName d = encoded.payload m = null } let ser = JsonConvert.SerializeObject(e) test <@ ser.Contains("\"d\":\"") @> - let des = JsonConvert.DeserializeObject(ser) - let d : Equinox.UnionCodec.EncodedUnion<_> = { caseName = des.t; payload=des.d } + let des = JsonConvert.DeserializeObject(ser) + let d : Equinox.UnionCodec.EncodedUnion<_> = { caseName = des.t; payload = des.d } let decoded = unionEncoder.Decode d test <@ value = decoded @> \ No newline at end of file From 63bd10213f1d074ea30700cb73cde6dae139a512 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 27 Nov 2018 22:23:06 +0000 Subject: [PATCH 45/66] Add RU counts to batching tests --- .../CosmosFixturesInfrastructure.fs | 6 ++-- .../CosmosIntegration.fs | 35 +++++++++++++------ 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/tests/Equinox.Cosmos.Integration/CosmosFixturesInfrastructure.fs b/tests/Equinox.Cosmos.Integration/CosmosFixturesInfrastructure.fs index 98ed85c4c..9cc24fd40 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosFixturesInfrastructure.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosFixturesInfrastructure.fs @@ -54,16 +54,14 @@ module SerilogHelpers = [] type EqxAct = | Tip | TipNotFound | TipNotModified - | ResponseForward | ResponseBackward | ResponseWaste + | ResponseForward | ResponseBackward | QueryForward | QueryBackward | Append | Resync | Conflict let (|EqxAction|) = function | Log.Tip _ -> EqxAct.Tip | Log.TipNotFound _ -> EqxAct.TipNotFound | Log.TipNotModified _ -> EqxAct.TipNotModified - | Log.Response (Direction.Forward, {count = 0}) -> EqxAct.ResponseWaste // TODO remove, see comment where these are emitted | Log.Response (Direction.Forward,_) -> EqxAct.ResponseForward - | Log.Response (Direction.Backward, {count = 0}) -> EqxAct.ResponseWaste // TODO remove, see comment where these are emitted | Log.Response (Direction.Backward,_) -> EqxAct.ResponseBackward | Log.Query (Direction.Forward,_,_) -> EqxAct.QueryForward | Log.Query (Direction.Backward,_,_) -> EqxAct.QueryBackward @@ -107,7 +105,7 @@ module SerilogHelpers = interface Serilog.Core.ILogEventSink with member __.Emit logEvent = writeSerilogEvent logEvent member __.Clear () = captured.Clear() member __.ChooseCalls chooser = captured |> Seq.choose chooser |> List.ofSeq - member __.ExternalCalls = __.ChooseCalls (function EqxEvent (EqxAction act) (*when act <> EqxAct.Waste*) -> Some act | _ -> None) + member __.ExternalCalls = __.ChooseCalls (function EqxEvent (EqxAction act) -> Some act | _ -> None) member __.RequestCharges = __.ChooseCalls (function EqxEvent (CosmosRequestCharge e) -> Some e | _ -> None) type TestsWithLogCapture(testOutputHelper) = diff --git a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs index 494a7a953..49a21caf0 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs @@ -57,30 +57,43 @@ type Tests(testOutputHelper) = let addAndThenRemoveItemsManyTimesExceptTheLastOne context cartId skuId service count = addAndThenRemoveItems true context cartId skuId service count + let verifyRequestChargesMax rus = + let tripRequestCharges = [ for e, c in capture.RequestCharges -> sprintf "%A" e, c ] + test <@ float rus >= Seq.sum (Seq.map snd tripRequestCharges) @> + [] let ``Can roundtrip against Cosmos, correctly batching the reads [without using the Index for reads]`` context skuId = Async.RunSynchronously <| async { let! conn = connectToSpecifiedCosmosOrSimulator log - let batchSize = 3 - let service = Cart.createServiceWithoutOptimization conn batchSize log + let maxItemsPerRequest = 2 + let maxEventsPerBatch = 1 + let service = Cart.createServiceWithoutOptimization conn maxItemsPerRequest log capture.Clear() // for re-runs of the test let cartId = Guid.NewGuid() |> CartId // The command processing should trigger only a single read and a single write call - let addRemoveCount = 6 - do! addAndThenRemoveItemsManyTimesExceptTheLastOne context cartId skuId service addRemoveCount - test <@ [EqxAct.ResponseWaste; EqxAct.QueryBackward; EqxAct.Append] = capture.ExternalCalls @> - // Restart the counting - capture.Clear() + let addRemoveCount = 2 + let eventsPerAction = addRemoveCount * 2 - 1 + let batches = 4 + for i in [1..batches] do + do! addAndThenRemoveItemsManyTimesExceptTheLastOne context cartId skuId service addRemoveCount + let expectedBatchesOf2Items = + match i with + | 1 -> 1 // it does cost a single trip to determine there are 0 items + | i -> ceil(float (i-1) * float eventsPerAction / float maxItemsPerRequest / float maxEventsPerBatch) |> int + test <@ List.replicate expectedBatchesOf2Items EqxAct.ResponseBackward @ [EqxAct.QueryBackward; EqxAct.Append] = capture.ExternalCalls @> + verifyRequestChargesMax 39 // 37.15 + capture.Clear() // Validate basic operation; Key side effect: Log entries will be emitted to `capture` let! state = service.Read cartId - let expectedEventCount = 2 * addRemoveCount - 1 + let expectedEventCount = batches * eventsPerAction test <@ addRemoveCount = match state with { items = [{ quantity = quantity }] } -> quantity | _ -> failwith "nope" @> - // Need to read 4 batches to read 11 events in batches of 3 - let expectedBatches = ceil(float expectedEventCount/float batchSize) |> int - test <@ List.replicate (expectedBatches-1) EqxAct.ResponseBackward @ [EqxAct.ResponseBackward; EqxAct.QueryBackward] = capture.ExternalCalls @> + // Need 6 trips of 2 maxItemsPerRequest to read 12 events + test <@ let expectedResponses = ceil(float expectedEventCount/float maxItemsPerRequest/float maxEventsPerBatch) |> int + List.replicate expectedResponses EqxAct.ResponseBackward @ [EqxAct.QueryBackward] = capture.ExternalCalls @> + verifyRequestChargesMax 20 // 18.47 } [] From 3cd9d04d4888fb3a7716b5a55dcb9b6984400a12 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 27 Nov 2018 22:24:31 +0000 Subject: [PATCH 46/66] Add laziness and tests to getAll[Backwards] --- src/Equinox.Cosmos/Cosmos.fs | 63 +++++++++++++++++-- .../CosmosCoreIntegration.fs | 40 ++++++++---- 2 files changed, 86 insertions(+), 17 deletions(-) diff --git a/src/Equinox.Cosmos/Cosmos.fs b/src/Equinox.Cosmos/Cosmos.fs index fe02d679f..0b3d75e55 100644 --- a/src/Equinox.Cosmos/Cosmos.fs +++ b/src/Equinox.Cosmos/Cosmos.fs @@ -615,6 +615,52 @@ module private Tip = log |> logQuery direction maxItems stream.name t (!responseCount,raws) pos.index ru return pos, decoded } + let walkLazy<'event> (log : ILogger) client retryPolicy maxItems maxRequests direction (stream: CollectionStream) startPos + (tryDecode : IIndexedEvent -> 'event option, isOrigin: 'event -> bool) + : AsyncSeq<'event[]> = asyncSeq { + let responseCount = ref 0 + use query = mkQuery client maxItems stream direction startPos + let pullSlice = handleResponse direction stream startPos + let retryingLoggingReadSlice query = Log.withLoggedRetries retryPolicy "readAttempt" (pullSlice query) + let log = log |> Log.prop "batchSize" maxItems |> Log.prop "stream" stream.name + let mutable ru = 0. + let allSlices = ResizeArray() + let startTicks = System.Diagnostics.Stopwatch.GetTimestamp() + try + let readlog = log |> Log.prop "direction" direction + let mutable ok = true + while ok do + incr responseCount + + match maxRequests with + | Some mpbr when !responseCount >= mpbr -> readlog.Information "batch Limit exceeded"; invalidOp "batch Limit exceeded" + | _ -> () + + let batchLog = readlog |> Log.prop "batchIndex" !responseCount + let! (slice,_pos,rus) = retryingLoggingReadSlice query batchLog + ru <- ru + rus + allSlices.AddRange(slice) + + let acc = ResizeArray() + for x in slice do + match tryDecode x with + | Some e when isOrigin e -> + let used, residual = slice |> calculateUsedVersusDroppedPayload x.Index + log.Information("EqxCosmos Stop stream={stream} at={index} {case} used={used} residual={residual}", + stream.name, x.Index, x.EventType, used, residual) + ok <- false + acc.Add e + | Some e -> acc.Add e + | None -> () + yield acc.ToArray() + ok <- ok && query.HasMoreResults + finally + let endTicks = System.Diagnostics.Stopwatch.GetTimestamp() + let t = StopwatchInterval(startTicks, endTicks) + + query.Dispose() + log |> logQuery direction maxItems stream.name t (!responseCount,allSlices.ToArray()) -1L ru } + type [] Token = { stream: CollectionStream; pos: Position } module Token = let create stream pos : Storage.StreamToken = { value = box { stream = stream; pos = pos } } @@ -676,6 +722,8 @@ type EqxGateway(conn : EqxConnection, batching : EqxBatchingPolicy) = member __.Read log stream direction startPos (tryDecode,isOrigin) : Async = async { let! pos, events = Query.walk log conn.Client conn.QueryRetryPolicy batching.MaxItems batching.MaxRequests direction stream startPos (tryDecode,isOrigin) return Token.create stream pos, events } + member __.ReadLazy (batching: EqxBatchingPolicy) log stream direction startPos (tryDecode,isOrigin) : AsyncSeq<'event[]> = + Query.walkLazy log conn.Client conn.QueryRetryPolicy batching.MaxItems batching.MaxRequests direction stream startPos (tryDecode,isOrigin) member __.LoadFromUnfoldsOrRollingSnapshots log (stream,maybePos) (tryDecode,isOrigin): Async = async { let! res = Tip.tryLoad log conn.TipRetryPolicy conn.Client stream maybePos match res with @@ -990,6 +1038,11 @@ type EqxContext member __.CreateStream(streamName) = collections.CollectionForStream streamName + member internal __.GetLazy((stream, startPos), ?batchSize, ?direction) : AsyncSeq = + let direction = defaultArg direction Direction.Forward + let batching = EqxBatchingPolicy(defaultArg batchSize 10) + gateway.ReadLazy batching logger stream direction startPos (Some,fun _ -> false) + member internal __.GetInternal((stream, startPos), ?maxCount, ?direction) = async { let direction = defaultArg direction Direction.Forward if maxCount = Some 0 then @@ -1011,10 +1064,8 @@ type EqxContext /// Reads in batches of `batchSize` from the specified `Position`, allowing the reader to efficiently walk away from a running query /// ... NB as long as they Dispose! - member __.Walk(stream, batchSize, ?position, ?direction) : AsyncSeq = asyncSeq { - let! _pos,data = __.GetInternal((stream, position), batchSize, ?direction=direction) - // TODO add laziness - return AsyncSeq.ofSeq data } + member __.Walk(stream, batchSize, ?position, ?direction) : AsyncSeq = + __.GetLazy((stream, position), batchSize, ?direction=direction) /// Reads all Events from a `Position` in a given `direction` member __.Read(stream, ?position, ?maxCount, ?direction) : Async = @@ -1092,8 +1143,8 @@ module Events = /// reading in batches of the specified size. /// Returns an empty sequence if the stream is empty or if the sequence number is smaller than the smallest /// sequence number in the stream. - let getAllBackwards (ctx: EqxContext) (streamName: string) (MaxPosition index: int64) (maxCount: int): AsyncSeq = - ctx.Walk(ctx.CreateStream streamName, maxCount, ?position=index, direction=Direction.Backward) + let getAllBackwards (ctx: EqxContext) (streamName: string) (MaxPosition index: int64) (batchSize: int): AsyncSeq = + ctx.Walk(ctx.CreateStream streamName, batchSize, ?position=index, direction=Direction.Backward) /// Returns an async array of events in the stream backwards starting from the specified sequence number, /// number of events to read is specified by batchSize diff --git a/tests/Equinox.Cosmos.Integration/CosmosCoreIntegration.fs b/tests/Equinox.Cosmos.Integration/CosmosCoreIntegration.fs index 63da90de1..31c144fc9 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosCoreIntegration.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosCoreIntegration.fs @@ -229,18 +229,20 @@ type Tests(testOutputHelper) = [] let getAll (TestStream streamName) = Async.RunSynchronously <| async { let! conn = connectToSpecifiedCosmosOrSimulator log - let ctx = mkContextWithItemLimit conn (Some 2) + let ctx = mkContextWithItemLimit conn (Some 1) let! expected = add6EventsIn2Batches ctx streamName + capture.Clear() - let! res = Events.get ctx streamName 1L 4 // Events.getAll >> AsyncSeq.concatSeq |> AsyncSeq.toArrayAsync - let expected = expected |> Array.tail |> Array.take 4 + let! res = Events.getAll ctx streamName 0L 1 |> AsyncSeq.concatSeq |> AsyncSeq.takeWhileInclusive (fun _ -> false) |> AsyncSeq.toArrayAsync + let expected = expected |> Array.take 1 - verifyCorrectEvents 1L expected res - - // TODO [implement and] prove laziness - test <@ List.replicate 2 EqxAct.ResponseForward @ [EqxAct.QueryForward] = capture.ExternalCalls @> - verifyRequestChargesMax 10 // observed 8.99 // was 3 + verifyCorrectEvents 0L expected res + test <@ [EqxAct.ResponseForward; EqxAct.QueryForward] = capture.ExternalCalls @> + let queryRoundTripsAndItemCounts = function EqxEvent (Log.Query (Direction.Forward, responses, { count = c })) -> Some (responses,c) | _ -> None + // validate that, despite only requesting max 1 item, we only needed one trip (which contained only one item) + [1,1] =! capture.ChooseCalls queryRoundTripsAndItemCounts + verifyRequestChargesMax 4 // 3.07 // was 3 // 2.94 } (* Backward *) @@ -263,8 +265,24 @@ type Tests(testOutputHelper) = verifyRequestChargesMax 10 // observed 8.98 // was 3 } - // TODO AsyncSeq version - // TODO 2 batches backward test - // TODO mine other integration tests \ No newline at end of file + [] + let getAllBackwards (TestStream streamName) = Async.RunSynchronously <| async { + let! conn = connectToSpecifiedCosmosOrSimulator log + let ctx = mkContextWithItemLimit conn (Some 2) + + let! expected = add6EventsIn2Batches ctx streamName + capture.Clear() + + let! res = Events.getAllBackwards ctx streamName 10L 2 |> AsyncSeq.concatSeq |> AsyncSeq.takeWhileInclusive (fun x -> x.Index <> 2L) |> AsyncSeq.toArrayAsync + let expected = expected |> Array.skip 2 + + verifyCorrectEventsBackward 5L expected res + // only 2 batches of 2 items triggered + test <@ List.replicate 2 EqxAct.ResponseBackward @ [EqxAct.QueryBackward] = capture.ExternalCalls @> + // validate that we didnt trigger loading of the last item + let queryRoundTripsAndItemCounts = function EqxEvent (Log.Query (Direction.Backward, responses, { count = c })) -> Some (responses,c) | _ -> None + [2,4] =! capture.ChooseCalls queryRoundTripsAndItemCounts + verifyRequestChargesMax 7 // observed 6.03 // was 3 // 2.95 + } \ No newline at end of file From 4561a809f0c76275dc90f1c0e96e3c6d7b8051dc Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 29 Nov 2018 11:57:14 +0000 Subject: [PATCH 47/66] Unify Tip vs Batch formats (#54) --- src/Equinox.Cosmos/Cosmos.fs | 138 ++++++++---------- .../CosmosCoreIntegration.fs | 2 +- .../JsonConverterTests.fs | 15 +- 3 files changed, 69 insertions(+), 86 deletions(-) diff --git a/src/Equinox.Cosmos/Cosmos.fs b/src/Equinox.Cosmos/Cosmos.fs index 0b3d75e55..f85bc7c26 100644 --- a/src/Equinox.Cosmos/Cosmos.fs +++ b/src/Equinox.Cosmos/Cosmos.fs @@ -37,7 +37,7 @@ open Newtonsoft.Json /// A 'normal' (frozen, not Tip) Batch of Events (without any Unfolds) type [] - Event = + Batch = { /// DocDb-mandated Partition Key, must be maintained within the document /// Not actually required if running in single partition mode, but for simplicity, we always write it p: string // "{streamName}" @@ -52,15 +52,32 @@ type [] /// as it will do: 1. read 2. merge 3. write merged version contingent on the _etag not having changed [] _etag: string + /// When we encounter the Tip, we're interested in the 'real i', which is kept in -_i for now + [] + _i: int64 - /// Same as `id`; necessitated by fact that it's not presently possible to do an ORDER BY on the row key + /// base 'i' value for the Events held herein i: int64 // {index} - /// Creation datetime (as opposed to system-defined _lastUpdated which is touched by triggers, replication etc.) - c: System.DateTimeOffset // ISO 8601 + /// The events at this offset in the stream + e: BatchEvent[] } + /// Unless running in single partion mode (which would restrict us to 10GB per collection) + /// we need to nominate a partition key that will be in every document + static member PartitionKeyField = "p" + /// As one cannot sort by the implicit `id` field, we have an indexed `i` field for sort and range query use + static member IndexedFields = [Batch.PartitionKeyField; "i"] + /// If we encounter the tip (id=-1) doc, we're interested in its etag so we can re-sync for 1 RU + member x.TryToPosition() = + if x.id <> Tip.WellKnownDocumentId then None + else Some { index = x._i+x.e.LongLength; etag = match x._etag with null -> None | x -> Some x } +/// A single event from the array held in a batch +and [] + BatchEvent = + { /// Creation datetime (as opposed to system-defined _lastUpdated which is touched by triggers, replication etc.) + t: System.DateTimeOffset // ISO 8601 /// The Case (Event Type), used to drive deserialization - t: string // required + c: string // required /// Event body, as UTF-8 encoded json ready to be injected into the Json being rendered for DocDb [)>] @@ -70,23 +87,13 @@ type [] [)>] [] m: byte[] } // optional - /// Unless running in single partion mode (which would restrict us to 10GB per collection) - /// we need to nominate a partition key that will be in every document - static member PartitionKeyField = "p" - /// As one cannot sort by the implicit `id` field, we have an indexed `i` field for sort and range query use - static member IndexedFields = [Event.PartitionKeyField; "i"] - /// If we encounter a -1 doc, we're interested in its etag so we can re-read for one RU - member x.TryToPosition() = - if x.id <> Tip.WellKnownDocumentId then None - else Some { index = (let ``x.e.LongLength`` = 1L in x.i+``x.e.LongLength``); etag = match x._etag with null -> None | x -> Some x } /// The Special 'Pending' Batch Format /// NB this Type does double duty as /// a) transport for when we read it /// b) a way of encoding a batch that the stored procedure will write in to the actual document (`i` is -1 until Stored Proc computes it) /// The stored representation has the following differences vs a 'normal' (frozen/completed) Batch -/// a) `id` and `i` = `-1` as WIP document currently always is -/// b) events are retained as in an `e` array, not top level fields -/// c) contains unfolds (`c`) +/// a) `id` = `-1` +/// b) contains unfolds (`u`) and [] Tip = { /// Partition key, as per Batch @@ -107,35 +114,18 @@ and [] e: BatchEvent[] /// Compaction/Snapshot/Projection events - c: Unfold[] } + u: Unfold[] } /// arguably this should be a high nember to reflect fact it is the freshest ? static member WellKnownDocumentId = "-1" /// Create Position from Tip record context (facilitating 1 RU reads) member x.ToPosition() = { index = x._i+x.e.LongLength; etag = match x._etag with null -> None | x -> Some x } -/// A single event from the array held in a batch -and [] - BatchEvent = - { /// Creation date (as opposed to system-defined _lastUpdated which is touched by triggers, replication etc.) - c: System.DateTimeOffset // ISO 8601 - - /// The Event Type, used to drive deserialization - t: string // required - - /// Event body, as UTF-8 encoded json ready to be injected into the Json being rendered for DocDb - [)>] - d: byte[] // required - - /// Optional metadata, as UTF-8 encoded json, ready to emit directly (null, not written if missing) - [)>] - [] - m: byte[] } // optional /// Compaction/Snapshot/Projection Event based on the state at a given point in time `i` and Unfold = { /// Base: Stream Position (Version) of State from which this Unfold Event was generated i: int64 /// The Case (Event Type) of this compaction/snapshot, used to drive deserialization - t: string // required + c: string // required /// Event body - Json -> UTF-8 -> Deflate -> Base64 [)>] @@ -152,7 +142,7 @@ type Enum() = { new IIndexedEvent with member __.Index = b._i + int64 offset member __.IsUnfold = false - member __.EventType = x.t + member __.EventType = x.c member __.Data = x.d member __.Meta = x.m }) static member Events(i: int64, e: BatchEvent[]) = @@ -160,26 +150,20 @@ type Enum() = { new IIndexedEvent with member __.Index = i + int64 offset member __.IsUnfold = false - member __.EventType = x.t + member __.EventType = x.c member __.Data = x.d member __.Meta = x.m }) - static member Event(x: Event) = - Seq.singleton - { new IIndexedEvent with - member __.Index = x.i - member __.IsUnfold = false - member __.EventType = x.t - member __.Data = x.d - member __.Meta = x.m } - static member Unfolds(xs: Unfold[]) = seq { + static member Events(b: Batch) = + Enum.Events (b.i, b.e) + static member Unfolds (xs: Unfold[]) = seq { for x in xs -> { new IIndexedEvent with member __.Index = x.i member __.IsUnfold = true - member __.EventType = x.t + member __.EventType = x.c member __.Data = x.d member __.Meta = x.m } } - static member EventsAndUnfolds(x:Tip): IIndexedEvent seq = - Enum.Unfolds x.c + static member EventsAndUnfolds(x: Tip): IIndexedEvent seq = + Enum.Unfolds x.u /// Reference to Collection and name that will be used as the location for the stream type [] CollectionStream = { collectionUri: System.Uri; name: string } with @@ -289,12 +273,12 @@ module Sync = // NB don't nest in a private module, or serialization will fail miserably ;) [] type SyncResponse = { etag: string; nextI: int64; conflicts: BatchEvent[] } - let [] sprocName = "EquinoxSync-SingleEvents-021" // NB need to renumber for any breaking change + let [] sprocName = "EquinoxSync-SingleArray-001" // NB need to renumber for any breaking change let [] sprocBody = """ // Manages the merging of the supplied Request Batch, fulfilling one of the following end-states // 1 Verify no current WIP batch, the incoming `req` becomes the WIP batch (the caller is entrusted to provide a valid and complete set of inputs, or it's GIGO) -// 2 Current WIP batch has space to accommodate the incoming projections (req.c) and events (req.e) - merge them in, replacing any superseded projections +// 2 Current WIP batch has space to accommodate the incoming projections (req.u) and events (req.e) - merge them in, replacing any superseded projections // 3. Current WIP batch would become too large - remove WIP state from active document by replacing the well known id with a correct one; proceed as per 1 function sync(req, expectedVersion) { if (!req) throw new Error("Missing req argument"); @@ -332,7 +316,7 @@ function sync(req, expectedVersion) { if (current && current.e.length + req.e.length > 10) { current._i = current._i + current.e.length; current.e = req.e; - current.c = req.c; + current.u = req.u; // as we've mutated the document in a manner that can conflict with other writers, out write needs to be contingent on no competing updates having taken place finalize(current); @@ -342,7 +326,7 @@ function sync(req, expectedVersion) { // Append the new events into the current batch Array.prototype.push.apply(current.e, req.e); // Replace all the projections - current.c = req.c; + current.u = req.u; // TODO: should remove only projections being superseded // as we've mutated the document in a manner that can conflict with other writers, out write needs to be contingent on no competing updates having taken place @@ -364,10 +348,12 @@ function sync(req, expectedVersion) { p: req.p, id: eventI.toString(), i: eventI, - c: e.c, - t: e.t, - d: e.d, - m: e.m + e: [ { + c: e.c, + t: e.t, + d: e.d, + m: e.m + }] }; const isAccepted = collection.createDocument(collectionLink, doc, function (err) { if (err) throw err; @@ -407,16 +393,16 @@ function sync(req, expectedVersion) { let private logged client (stream: CollectionStream) (expectedVersion, req: Tip) (log : ILogger) : Async = async { let verbose = log.IsEnabled Events.LogEventLevel.Debug - let log = if verbose then log |> Log.propEvents (Enum.Events req) |> Log.propDataUnfolds req.c else log + let log = if verbose then log |> Log.propEvents (Enum.Events req) |> Log.propDataUnfolds req.u else log let (Log.BatchLen bytes), count = Enum.Events req, req.e.Length let log = log |> Log.prop "bytes" bytes let writeLog = log |> Log.prop "stream" stream.name |> Log.prop "expectedVersion" expectedVersion - |> Log.prop "count" req.e.Length |> Log.prop "ucount" req.c.Length - let! t, (ru,result) = run client stream (expectedVersion,req) |> Stopwatch.Time + |> Log.prop "count" req.e.Length |> Log.prop "ucount" req.u.Length + let! t, (ru,result) = run client stream (expectedVersion, req) |> Stopwatch.Time let resultLog = let mkMetric ru : Log.Measurement = { stream = stream.name; interval = t; bytes = bytes; count = count; ru = ru } - let logConflict () = writeLog.Information("EqxCosmos Sync: Conflict writing {eventTypes}", [| for x in req.e -> x.t |]) + let logConflict () = writeLog.Information("EqxCosmos Sync: Conflict writing {eventTypes}", [| for x in req.e -> x.c |]) match result with | Result.Written pos -> log |> Log.event (Log.SyncSuccess (mkMetric ru)) |> Log.prop "nextExpectedVersion" pos @@ -427,7 +413,7 @@ function sync(req, expectedVersion) { logConflict () let log = if verbose then log |> Log.prop "nextExpectedVersion" pos |> Log.propData "conflicts" xs else log log |> Log.event (Log.SyncResync(mkMetric ru)) |> Log.prop "conflict" true - resultLog.Information("EqxCosmos {action:l} {count}+{ucount} {ms}ms rc={ru}", "Sync", req.e.Length, req.c.Length, (let e = t.Elapsed in e.TotalMilliseconds), ru) + resultLog.Information("EqxCosmos {action:l} {count}+{ucount} {ms}ms rc={ru}", "Sync", req.e.Length, req.u.Length, (let e = t.Elapsed in e.TotalMilliseconds), ru) return result } let batch (log : ILogger) retryPolicy client pk batch: Async = @@ -435,10 +421,10 @@ function sync(req, expectedVersion) { Log.withLoggedRetries retryPolicy "writeAttempt" call log let mkBatch (stream: Store.CollectionStream) (events: IEvent[]) unfolds: Tip = { p = stream.name; id = Store.Tip.WellKnownDocumentId; _i = -1L(*Server-managed*); _etag = null - e = [| for e in events -> { c = DateTimeOffset.UtcNow; t = e.EventType; d = e.Data; m = e.Meta } |] - c = Array.ofSeq unfolds } + e = [| for e in events -> { t = DateTimeOffset.UtcNow; c = e.EventType; d = e.Data; m = e.Meta } |] + u = Array.ofSeq unfolds } let mkUnfold baseIndex (unfolds: IEvent seq) : Store.Unfold seq = - unfolds |> Seq.mapi (fun offset x -> { i = baseIndex + int64 offset; t = x.EventType; d = x.Data; m = x.Meta } : Store.Unfold) + unfolds |> Seq.mapi (fun offset x -> { i = baseIndex + int64 offset; c = x.EventType; d = x.Data; m = x.Meta } : Store.Unfold) module Initialization = open System.Collections.ObjectModel @@ -449,7 +435,7 @@ function sync(req, expectedVersion) { let createCollection (client: IDocumentClient) (dbUri: Uri) collName ru = async { let pkd = PartitionKeyDefinition() - pkd.Paths.Add(sprintf "/%s" Store.Event.PartitionKeyField) + pkd.Paths.Add(sprintf "/%s" Store.Batch.PartitionKeyField) let colld = DocumentCollection(Id = collName, PartitionKey = pkd) colld.IndexingPolicy.IndexingMode <- IndexingMode.Consistent @@ -458,7 +444,7 @@ function sync(req, expectedVersion) { // Given how long and variable the blacklist would be, we whitelist instead colld.IndexingPolicy.ExcludedPaths <- Collection [|ExcludedPath(Path="/*")|] // NB its critical to index the nominated PartitionKey field defined above or there will be runtime errors - colld.IndexingPolicy.IncludedPaths <- Collection [| for k in Store.Event.IndexedFields -> IncludedPath(Path=sprintf "/%s/?" k) |] + colld.IndexingPolicy.IncludedPaths <- Collection [| for k in Store.Batch.IndexedFields -> IncludedPath(Path=sprintf "/%s/?" k) |] let! coll = client.CreateDocumentCollectionIfNotExistsAsync(dbUri, colld, Client.RequestOptions(OfferThroughput=Nullable ru)) |> Async.AwaitTaskCorrect return coll.Resource.Id } @@ -493,9 +479,9 @@ module private Tip = (log 0 0 Log.TipNotFound).Information("EqxCosmos {action:l} {res} {ms}ms rc={ru}", "Tip", 404, (let e = t.Elapsed in e.TotalMilliseconds), ru) | ReadResult.Found doc -> let log = - let (Log.BatchLen bytes), count = Enum.Unfolds doc.c, doc.c.Length + let (Log.BatchLen bytes), count = Enum.Unfolds doc.u, doc.u.Length log bytes count Log.Tip - let log = if (not << log.IsEnabled) Events.LogEventLevel.Debug then log else log |> Log.propDataUnfolds doc.c |> Log.prop "etag" doc._etag + let log = if (not << log.IsEnabled) Events.LogEventLevel.Debug then log else log |> Log.propDataUnfolds doc.u |> Log.prop "etag" doc._etag log.Information("EqxCosmos {action:l} {res} {ms}ms rc={ru}", "Tip", 200, (let e = t.Elapsed in e.TotalMilliseconds), ru) return ru, res } type [] Result = NotModified | NotFound | Found of Position * IIndexedEvent[] @@ -518,15 +504,15 @@ module private Tip = let f = if direction = Direction.Forward then "c.i >= @id ORDER BY c.i ASC" else "c.i < @id ORDER BY c.i DESC" SqlQuerySpec("SELECT * FROM c WHERE c.i != -1 AND " + f, SqlParameterCollection [SqlParameter("@id", p.index)]) let feedOptions = new Client.FeedOptions(PartitionKey=PartitionKey(stream.name), MaxItemCount=Nullable maxItems) - client.CreateDocumentQuery(stream.collectionUri, querySpec, feedOptions).AsDocumentQuery() + client.CreateDocumentQuery(stream.collectionUri, querySpec, feedOptions).AsDocumentQuery() // Unrolls the Batches in a response - note when reading backwards, the events are emitted in reverse order of index - let private handleResponse direction (stream: CollectionStream) (startPos: Position option) (query: IDocumentQuery) (log: ILogger) + let private handleResponse direction (stream: CollectionStream) (startPos: Position option) (query: IDocumentQuery) (log: ILogger) : Async = async { let! ct = Async.CancellationToken - let! t, (res : Client.FeedResponse) = query.ExecuteNextAsync(ct) |> Async.AwaitTaskCorrect |> Stopwatch.Time + let! t, (res : Client.FeedResponse) = query.ExecuteNextAsync(ct) |> Async.AwaitTaskCorrect |> Stopwatch.Time let batches, ru = Array.ofSeq res, res.RequestCharge - let events = batches |> Seq.collect Enum.Event |> Array.ofSeq + let events = batches |> Seq.collect Enum.Events |> Array.ofSeq let (Log.BatchLen bytes), count = events, events.Length let reqMetric : Log.Measurement = { stream = stream.name; interval = t; bytes = bytes; count = count; ru = ru } // TODO investigate whether there is a way to avoid the potential cost (or whether there is significance to it) of these null responses @@ -539,9 +525,9 @@ module private Tip = let maybePosition = batches |> Array.tryPick (fun x -> x.TryToPosition()) return events, maybePosition, ru } - let private run (log : ILogger) (readSlice: IDocumentQuery -> ILogger -> Async) + let private run (log : ILogger) (readSlice: IDocumentQuery -> ILogger -> Async) (maxPermittedBatchReads: int option) - (query: IDocumentQuery) + (query: IDocumentQuery) : AsyncSeq = let rec loop batchCount : AsyncSeq = asyncSeq { match maxPermittedBatchReads with diff --git a/tests/Equinox.Cosmos.Integration/CosmosCoreIntegration.fs b/tests/Equinox.Cosmos.Integration/CosmosCoreIntegration.fs index 31c144fc9..f7002401a 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosCoreIntegration.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosCoreIntegration.fs @@ -176,7 +176,7 @@ type Tests(testOutputHelper) = let! res = Events.append ctx streamName 0L expected test <@ AppendResult.Ok 1L = res @> test <@ [EqxAct.Append] = capture.ExternalCalls @> - verifyRequestChargesMax 12 // was 10, observed 10.57 + verifyRequestChargesMax 14 // observed 12.73 // was 10 capture.Clear() // Try overwriting it (a competing consumer would see the same) diff --git a/tests/Equinox.Cosmos.Integration/JsonConverterTests.fs b/tests/Equinox.Cosmos.Integration/JsonConverterTests.fs index 721209113..5d99c734e 100644 --- a/tests/Equinox.Cosmos.Integration/JsonConverterTests.fs +++ b/tests/Equinox.Cosmos.Integration/JsonConverterTests.fs @@ -21,12 +21,9 @@ type VerbatimUtf8Tests() = [] let ``encodes correctly`` () = let encoded = mkUnionEncoder().Encode(A { embed = "\"" }) - let e : Store.Event = - { p = "streamName"; id = string 0; i = 0L; _etag=null - c = DateTimeOffset.MinValue - t = encoded.caseName - d = encoded.payload - m = null } + let e : Store.Batch = + { p = "streamName"; id = string 0; i = 0L; _i = 0L; _etag = null + e = [| { t = DateTimeOffset.MinValue; c = encoded.caseName; d = encoded.payload; m = null } |] } let res = JsonConvert.SerializeObject(e) test <@ res.Contains """"d":{"embed":"\""}""" @> @@ -38,7 +35,7 @@ type Base64ZipUtf8Tests() = let encoded = unionEncoder.Encode(A { embed = String('x',5000) }) let e : Store.Unfold = { i = 42L - t = encoded.caseName + c = encoded.caseName d = encoded.payload m = null } let res = JsonConvert.SerializeObject e @@ -55,12 +52,12 @@ type Base64ZipUtf8Tests() = let encoded = unionEncoder.Encode value let e : Store.Unfold = { i = 42L - t = encoded.caseName + c = encoded.caseName d = encoded.payload m = null } let ser = JsonConvert.SerializeObject(e) test <@ ser.Contains("\"d\":\"") @> let des = JsonConvert.DeserializeObject(ser) - let d : Equinox.UnionCodec.EncodedUnion<_> = { caseName = des.t; payload = des.d } + let d : Equinox.UnionCodec.EncodedUnion<_> = { caseName = des.c; payload = des.d } let decoded = unionEncoder.Decode d test <@ value = decoded @> \ No newline at end of file From 7e719d52d2894894f1a930ab5c0122bd24c1dd0d Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 29 Nov 2018 12:18:01 +0000 Subject: [PATCH 48/66] Multi-event Batches (#48) * Multi-event batches; Tip now a Batch too * Handle startIndex within multi-item batches --- src/Equinox.Cosmos/Cosmos.fs | 221 +++++++++--------- .../CosmosCoreIntegration.fs | 106 +++++---- .../CosmosFixtures.fs | 3 + .../CosmosIntegration.fs | 12 +- .../JsonConverterTests.fs | 2 +- 5 files changed, 184 insertions(+), 160 deletions(-) diff --git a/src/Equinox.Cosmos/Cosmos.fs b/src/Equinox.Cosmos/Cosmos.fs index f85bc7c26..ce4093ba3 100644 --- a/src/Equinox.Cosmos/Cosmos.fs +++ b/src/Equinox.Cosmos/Cosmos.fs @@ -40,6 +40,7 @@ type [] Batch = { /// DocDb-mandated Partition Key, must be maintained within the document /// Not actually required if running in single partition mode, but for simplicity, we always write it + [] // Not requested in queries p: string // "{streamName}" /// DocDb-mandated unique row key; needs to be unique within any partition it is maintained; must be string @@ -52,24 +53,24 @@ type [] /// as it will do: 1. read 2. merge 3. write merged version contingent on the _etag not having changed [] _etag: string - /// When we encounter the Tip, we're interested in the 'real i', which is kept in -_i for now - [] - _i: int64 /// base 'i' value for the Events held herein i: int64 // {index} + // `i` value for successor batch (to facilitate identifying which Batch a given startPos is within) + n: int64 // {index} + /// The events at this offset in the stream e: BatchEvent[] } /// Unless running in single partion mode (which would restrict us to 10GB per collection) /// we need to nominate a partition key that will be in every document static member PartitionKeyField = "p" /// As one cannot sort by the implicit `id` field, we have an indexed `i` field for sort and range query use - static member IndexedFields = [Batch.PartitionKeyField; "i"] + static member IndexedFields = [Batch.PartitionKeyField; "i"; "n"] /// If we encounter the tip (id=-1) doc, we're interested in its etag so we can re-sync for 1 RU member x.TryToPosition() = if x.id <> Tip.WellKnownDocumentId then None - else Some { index = x._i+x.e.LongLength; etag = match x._etag with null -> None | x -> Some x } + else Some { index = x.n; etag = match x._etag with null -> None | x -> Some x } /// A single event from the array held in a batch and [] BatchEvent = @@ -87,6 +88,7 @@ and [] [)>] [] m: byte[] } // optional + /// The Special 'Pending' Batch Format /// NB this Type does double duty as /// a) transport for when we read it @@ -96,7 +98,8 @@ and [] /// b) contains unfolds (`u`) and [] Tip = - { /// Partition key, as per Batch + { [] // Not requested in queries + /// Partition key, as per Batch p: string // "{streamName}" /// Document Id within partition, as per Batch id: string // "{-1}" - Well known IdConstant used while this remains the pending batch @@ -108,7 +111,10 @@ and [] _etag: string /// base 'i' value for the Events held herein - _i: int64 + i: int64 + + /// `i` value for successor batch (to facilitate identifying which Batch a given startPos is within) + n: int64 // {index} /// Events e: BatchEvent[] @@ -118,7 +124,7 @@ and [] /// arguably this should be a high nember to reflect fact it is the freshest ? static member WellKnownDocumentId = "-1" /// Create Position from Tip record context (facilitating 1 RU reads) - member x.ToPosition() = { index = x._i+x.e.LongLength; etag = match x._etag with null -> None | x -> Some x } + member x.ToPosition() = { index = x.n; etag = match x._etag with null -> None | x -> Some x } /// Compaction/Snapshot/Projection Event based on the state at a given point in time `i` and Unfold = { /// Base: Stream Position (Version) of State from which this Unfold Event was generated @@ -140,21 +146,31 @@ type Enum() = static member Events(b: Tip) = b.e |> Seq.mapi (fun offset x -> { new IIndexedEvent with - member __.Index = b._i + int64 offset + member __.Index = b.i + int64 offset member __.IsUnfold = false member __.EventType = x.c member __.Data = x.d member __.Meta = x.m }) - static member Events(i: int64, e: BatchEvent[]) = - e |> Seq.mapi (fun offset x -> - { new IIndexedEvent with - member __.Index = i + int64 offset - member __.IsUnfold = false - member __.EventType = x.c - member __.Data = x.d - member __.Meta = x.m }) - static member Events(b: Batch) = - Enum.Events (b.i, b.e) + static member Events(i: int64, e: BatchEvent[], startIndex, backward) = seq { + let isValidGivenStartIndex backward si i = + match si with + | Some si when backward -> i < si + | Some si -> i >= si + | _ -> true + for offset in 0..e.Length-1 do + let index = i + int64 offset + if isValidGivenStartIndex backward startIndex index then + let x = e.[offset] + yield { + new IIndexedEvent with + member __.Index = index + member __.IsUnfold = false + member __.EventType = x.c + member __.Data = x.d + member __.Meta = x.m } } + static member Events(b: Batch, startIndex, backward) = + Enum.Events(b.i, b.e, startIndex, backward) + |> if backward then System.Linq.Enumerable.Reverse else id static member Unfolds (xs: Unfold[]) = seq { for x in xs -> { new IIndexedEvent with member __.Index = x.i @@ -163,7 +179,10 @@ type Enum() = member __.Data = x.d member __.Meta = x.m } } static member EventsAndUnfolds(x: Tip): IIndexedEvent seq = - Enum.Unfolds x.u + Enum.Events x + |> Seq.append (Enum.Unfolds x.u) + // where Index is equal, unfolds get delivered after the events so the fold semantics can be 'idempotent' + |> Seq.sortBy (fun x -> x.Index, x.IsUnfold) /// Reference to Collection and name that will be used as the location for the stream type [] CollectionStream = { collectionUri: System.Uri; name: string } with @@ -272,35 +291,34 @@ module private DocDb = module Sync = // NB don't nest in a private module, or serialization will fail miserably ;) [] - type SyncResponse = { etag: string; nextI: int64; conflicts: BatchEvent[] } - let [] sprocName = "EquinoxSync-SingleArray-001" // NB need to renumber for any breaking change + type SyncResponse = { etag: string; n: int64; conflicts: BatchEvent[] } + let [] sprocName = "EquinoxSync002" // NB need to renumber for any breaking change let [] sprocBody = """ // Manages the merging of the supplied Request Batch, fulfilling one of the following end-states -// 1 Verify no current WIP batch, the incoming `req` becomes the WIP batch (the caller is entrusted to provide a valid and complete set of inputs, or it's GIGO) -// 2 Current WIP batch has space to accommodate the incoming projections (req.u) and events (req.e) - merge them in, replacing any superseded projections -// 3. Current WIP batch would become too large - remove WIP state from active document by replacing the well known id with a correct one; proceed as per 1 -function sync(req, expectedVersion) { +// 1 Verify no current Tip batch, the incoming `req` becomes the Tip batch (the caller is entrusted to provide a valid and complete set of inputs, or it's GIGO) +// 2 Current Tip batch has space to accommodate the incoming unfolds (req.u) and events (req.e) - merge them in, replacing any superseded unfolds +// 3 Current Tip batch would become too large - remove Tip-specific state from active doc by replacing the well known id with a correct one; proceed as per 1 +function sync(req, expectedVersion, maxEvents) { if (!req) throw new Error("Missing req argument"); const collection = getContext().getCollection(); const collectionLink = collection.getSelfLink(); const response = getContext().getResponse(); - // Locate the WIP (-1) batch (which may not exist) - const wipDocId = collection.getAltLink() + "/docs/" + req.id; - const isAccepted = collection.readDocument(wipDocId, {}, function (err, current) { + // Locate the Tip (-1) batch (which may not exist) + const tipDocId = collection.getAltLink() + "/docs/" + req.id; + const isAccepted = collection.readDocument(tipDocId, {}, function (err, current) { // Verify we dont have a conflicting write if (expectedVersion === -1) { executeUpsert(current); } else if (!current && expectedVersion !== 0) { - // If there is no WIP page, the writer has no possible reason for writing at an index other than zero - response.setBody({ etag: null, nextI: 0, conflicts: [] }); - } else if (current && expectedVersion !== current._i + current.e.length) { + // If there is no Tip page, the writer has no possible reason for writing at an index other than zero + response.setBody({ etag: null, n: 0, conflicts: [] }); + } else if (current && expectedVersion !== current.n) { // Where possible, we extract conflicting events from e and/or c in order to avoid another read cycle // yielding [] triggers the client to go loading the events itself - const conflicts = expectedVersion < current._i ? [] : current.e.slice(expectedVersion - current._i); - const nextI = current._i + current.e.length; - response.setBody({ etag: current._etag, nextI: nextI, conflicts: conflicts }); + const conflicts = expectedVersion < current.i ? [] : current.e.slice(expectedVersion - current.i); + response.setBody({ etag: current._etag, n: current.n, conflicts: conflicts }); } else { executeUpsert(current); } @@ -310,62 +328,43 @@ function sync(req, expectedVersion) { function executeUpsert(current) { function callback(err, doc) { if (err) throw err; - response.setBody({ etag: doc._etag, nextI: doc._i + doc.e.length, conflicts: null }); + response.setBody({ etag: doc._etag, n: doc.n, conflicts: null }); } - // If we have hit a sensible limit for a slice in the WIP document, trim the events - if (current && current.e.length + req.e.length > 10) { - current._i = current._i + current.e.length; - current.e = req.e; - current.u = req.u; + // `i` is established when first written; `n` needs to stay in step with i+batch.e.length + function pos(batch, i) { + batch.i = i + batch.n = batch.i + batch.e.length; + return batch; + } + // If we have hit a sensible limit for a slice, swap to a new one + if (current && current.e.length + req.e.length > maxEvents) { + // remove the well-known `id` value identifying the batch as being the Tip + current.id = current.i.toString(); + // ... As it's no longer a Tip batch, we definitely don't want unfolds taking up space + delete current.u; + + // TODO Carry forward `u` items not present in `batch`, together with supporting catchup events from preceding batches // as we've mutated the document in a manner that can conflict with other writers, out write needs to be contingent on no competing updates having taken place - finalize(current); - const isAccepted = collection.replaceDocument(current._self, current, { etag: current._etag }, callback); - if (!isAccepted) throw new Error("Unable to restart WIP batch."); + const tipUpdateAccepted = collection.replaceDocument(current._self, current, { etag: current._etag }, callback); + if (!tipUpdateAccepted) throw new Error("Unable to remove Tip markings."); + + const isAccepted = collection.createDocument(collectionLink, pos(req,current.n), { disableAutomaticIdGeneration: true }, callback); + if (!isAccepted) throw new Error("Unable to create Tip batch."); } else if (current) { // Append the new events into the current batch Array.prototype.push.apply(current.e, req.e); - // Replace all the projections + // Replace all the unfolds // TODO: should remove only unfolds being superseded current.u = req.u; - // TODO: should remove only projections being superseded // as we've mutated the document in a manner that can conflict with other writers, out write needs to be contingent on no competing updates having taken place - finalize(current); - const isAccepted = collection.replaceDocument(current._self, current, { etag: current._etag }, callback); - if (!isAccepted) throw new Error("Unable to replace WIP batch."); + const isAccepted = collection.replaceDocument(current._self, pos(current, current.i), { etag: current._etag }, callback); + if (!isAccepted) throw new Error("Unable to replace Tip batch."); } else { - current = req; - current._i = 0; - // concurrency control is by virtue of fact that any conflicting writer will encounter a primary key violation (which will result in a retry) - finalize(current); - const isAccepted = collection.createDocument(collectionLink, current, { disableAutomaticIdGeneration: true }, callback); - if (!isAccepted) throw new Error("Unable to create WIP batch."); - } - for (i = 0; i < req.e.length; i++) { - const e = req.e[i]; - const eventI = current._i + current.e.length - req.e.length + i; - const doc = { - p: req.p, - id: eventI.toString(), - i: eventI, - e: [ { - c: e.c, - t: e.t, - d: e.d, - m: e.m - }] - }; - const isAccepted = collection.createDocument(collectionLink, doc, function (err) { - if (err) throw err; - }); - if (!isAccepted) throw new Error("Unable to add event " + doc.i); + const isAccepted = collection.createDocument(collectionLink, pos(req,0), { disableAutomaticIdGeneration: true }, callback); + if (!isAccepted) throw new Error("Unable to create Tip batch."); } } - - function finalize(current) { - current.i = -1; - current.id = current.i.toString(); - } }""" [] @@ -374,23 +373,23 @@ function sync(req, expectedVersion) { | Conflict of Position * events: IIndexedEvent[] | ConflictUnknown of Position - let private run (client: IDocumentClient) (stream: CollectionStream) (expectedVersion: int64 option, req: Tip) + let private run (client: IDocumentClient) (stream: CollectionStream) (expectedVersion: int64 option, req: Tip, maxEvents: int) : Async = async { let sprocLink = sprintf "%O/sprocs/%s" stream.collectionUri sprocName let opts = Client.RequestOptions(PartitionKey=PartitionKey(stream.name)) let! ct = Async.CancellationToken let ev = match expectedVersion with Some ev -> Position.FromI ev | None -> Position.FromAppendAtEnd let! (res : Client.StoredProcedureResponse) = - client.ExecuteStoredProcedureAsync(sprocLink, opts, ct, box req, box ev.index) |> Async.AwaitTaskCorrect + client.ExecuteStoredProcedureAsync(sprocLink, opts, ct, box req, box ev.index, box maxEvents) |> Async.AwaitTaskCorrect - let newPos = { index = res.Response.nextI; etag = Option.ofObj res.Response.etag } + let newPos = { index = res.Response.n; etag = Option.ofObj res.Response.etag } return res.RequestCharge, res.Response.conflicts |> function | null -> Result.Written newPos | [||] when newPos.index = 0L -> Result.Conflict (newPos, Array.empty) | [||] -> Result.ConflictUnknown newPos - | xs -> Result.Conflict (newPos, Enum.Events (ev.index, xs) |> Array.ofSeq) } + | xs -> Result.Conflict (newPos, Enum.Events(ev.index, xs, None, false) |> Array.ofSeq) } - let private logged client (stream: CollectionStream) (expectedVersion, req: Tip) (log : ILogger) + let private logged client (stream: CollectionStream) (expectedVersion, req: Tip, maxEvents) (log : ILogger) : Async = async { let verbose = log.IsEnabled Events.LogEventLevel.Debug let log = if verbose then log |> Log.propEvents (Enum.Events req) |> Log.propDataUnfolds req.u else log @@ -399,7 +398,7 @@ function sync(req, expectedVersion) { let writeLog = log |> Log.prop "stream" stream.name |> Log.prop "expectedVersion" expectedVersion |> Log.prop "count" req.e.Length |> Log.prop "ucount" req.u.Length - let! t, (ru,result) = run client stream (expectedVersion, req) |> Stopwatch.Time + let! t, (ru,result) = run client stream (expectedVersion, req, maxEvents) |> Stopwatch.Time let resultLog = let mkMetric ru : Log.Measurement = { stream = stream.name; interval = t; bytes = bytes; count = count; ru = ru } let logConflict () = writeLog.Information("EqxCosmos Sync: Conflict writing {eventTypes}", [| for x in req.e -> x.c |]) @@ -420,7 +419,7 @@ function sync(req, expectedVersion) { let call = logged client pk batch Log.withLoggedRetries retryPolicy "writeAttempt" call log let mkBatch (stream: Store.CollectionStream) (events: IEvent[]) unfolds: Tip = - { p = stream.name; id = Store.Tip.WellKnownDocumentId; _i = -1L(*Server-managed*); _etag = null + { p = stream.name; id = Store.Tip.WellKnownDocumentId; n = -1L(*Server-managed*); i = -1L(*Server-managed*); _etag = null e = [| for e in events -> { t = DateTimeOffset.UtcNow; c = e.EventType; d = e.Data; m = e.Meta } |] u = Array.ofSeq unfolds } let mkUnfold baseIndex (unfolds: IEvent seq) : Store.Unfold seq = @@ -498,11 +497,12 @@ module private Tip = open Microsoft.Azure.Documents.Linq let private mkQuery (client : IDocumentClient) maxItems (stream: CollectionStream) (direction: Direction) (startPos: Position option) = let querySpec = + let fields = "c.id, c.i, c._etag, c.n, c.e" match startPos with - | None -> SqlQuerySpec("SELECT * FROM c WHERE c.i!=-1 ORDER BY c.i " + if direction = Direction.Forward then "ASC" else "DESC") + | None -> SqlQuerySpec(sprintf "SELECT %s FROM c ORDER BY c.i " fields + if direction = Direction.Forward then "ASC" else "DESC") | Some p -> - let f = if direction = Direction.Forward then "c.i >= @id ORDER BY c.i ASC" else "c.i < @id ORDER BY c.i DESC" - SqlQuerySpec("SELECT * FROM c WHERE c.i != -1 AND " + f, SqlParameterCollection [SqlParameter("@id", p.index)]) + let f = if direction = Direction.Forward then "c.n > @id ORDER BY c.i ASC" else "c.i < @id ORDER BY c.i DESC" + SqlQuerySpec(sprintf "SELECT %s FROM c WHERE " fields + f, SqlParameterCollection [SqlParameter("@id", p.index)]) let feedOptions = new Client.FeedOptions(PartitionKey=PartitionKey(stream.name), MaxItemCount=Nullable maxItems) client.CreateDocumentQuery(stream.collectionUri, querySpec, feedOptions).AsDocumentQuery() @@ -512,14 +512,14 @@ module private Tip = let! ct = Async.CancellationToken let! t, (res : Client.FeedResponse) = query.ExecuteNextAsync(ct) |> Async.AwaitTaskCorrect |> Stopwatch.Time let batches, ru = Array.ofSeq res, res.RequestCharge - let events = batches |> Seq.collect Enum.Events |> Array.ofSeq + let startIndex = match startPos with Some { index = i } -> Some i | _ -> None + let events = batches |> Seq.collect (fun b -> Enum.Events(b, startIndex, backward=(direction = Direction.Backward))) |> Array.ofSeq let (Log.BatchLen bytes), count = events, events.Length let reqMetric : Log.Measurement = { stream = stream.name; interval = t; bytes = bytes; count = count; ru = ru } - // TODO investigate whether there is a way to avoid the potential cost (or whether there is significance to it) of these null responses - let log = if batches.Length = 0 && count = 0 && ru = 0. then log else let evt = Log.Response (direction, reqMetric) in log |> Log.event evt + let log = let evt = Log.Response (direction, reqMetric) in log |> Log.event evt let log = if (not << log.IsEnabled) Events.LogEventLevel.Debug then log else log |> Log.propEvents events let index = if count = 0 then Nullable () else Nullable <| Seq.min (seq { for x in batches -> x.i }) - (log |> Log.prop "startIndex" (match startPos with Some { index = i } -> Nullable i | _ -> Nullable()) |> Log.prop "bytes" bytes) + (log |> Log.prop "startIndex" (match startIndex with Some i -> Nullable i | None -> Nullable()) |> Log.prop "bytes" bytes) .Information("EqxCosmos {action:l} {count}/{batches} {direction} {ms}ms i={index} rc={ru}", "Response", count, batches.Length, direction, (let e = t.Elapsed in e.TotalMilliseconds), index, ru) let maybePosition = batches |> Array.tryPick (fun x -> x.TryToPosition()) @@ -541,15 +541,13 @@ module private Tip = yield! loop (batchCount + 1) } loop 0 - let private logQuery direction batchSize streamName interval (responsesCount, events : IIndexedEvent []) nextI (ru: float) (log : ILogger) = + let private logQuery direction batchSize streamName interval (responsesCount, events : IIndexedEvent []) n (ru: float) (log : ILogger) = let (Log.BatchLen bytes), count = events, events.Length let reqMetric : Log.Measurement = { stream = streamName; interval = interval; bytes = bytes; count = count; ru = ru } let action = match direction with Direction.Forward -> "QueryF" | Direction.Backward -> "QueryB" - // TODO investigate whether there is a way to avoid the potential cost (or whether there is significance to it) of these null responses - let log = if count = 0 && ru = 0. then log else let evt = Log.Event.Query (direction, responsesCount, reqMetric) in log |> Log.event evt - (log |> Log.prop "bytes" bytes |> Log.prop "batchSize" batchSize).Information( - "EqxCosmos {action:l} {stream} v{nextI} {count}/{responses} {ms}ms rc={ru}", - action, streamName, nextI, count, responsesCount, (let e = interval.Elapsed in e.TotalMilliseconds), ru) + (log |> Log.prop "bytes" bytes |> Log.prop "batchSize" batchSize |> Log.event (Log.Event.Query (direction, responsesCount, reqMetric))).Information( + "EqxCosmos {action:l} {stream} v{n} {count}/{responses} {ms}ms rc={ru}", + action, streamName, n, count, responsesCount, (let e = interval.Elapsed in e.TotalMilliseconds), ru) let private calculateUsedVersusDroppedPayload stopIndex (xs: IIndexedEvent[]) : int * int = let mutable used, dropped = 0, 0 @@ -676,25 +674,29 @@ module Internal = type LoadFromTokenResult<'event> = Unchanged | Found of Storage.StreamToken * 'event[] /// Defines policies for retrying with respect to transient failures calling CosmosDb (as opposed to application level concurrency conflicts) -type EqxConnection(client: IDocumentClient, ?readRetryPolicy: IRetryPolicy, ?writeRetryPolicy: IRetryPolicy) = +type EqxConnection(client: IDocumentClient, ?readRetryPolicy: IRetryPolicy, ?writeRetryPolicy) = member __.Client = client member __.TipRetryPolicy = readRetryPolicy member __.QueryRetryPolicy = readRetryPolicy member __.WriteRetryPolicy = writeRetryPolicy -/// Defines the policies in force regarding how to constrain query responses +/// Defines the policies in force regarding how to a) split up calls b) limit the number of events per slice type EqxBatchingPolicy ( // Max items to request in query response. Defaults to 10. ?defaultMaxItems : int, // Dynamic version of `defaultMaxItems`, allowing one to react to dynamic configuration changes. Default to using `defaultMaxItems` ?getDefaultMaxItems : unit -> int, /// Maximum number of trips to permit when slicing the work into multiple responses based on `MaxSlices`. Default: unlimited. - ?maxRequests) = + ?maxRequests, + /// Maximum number of events to accumualte within the `WipBatch` before switching to a new one when adding Events. Defaults to 10. + ?maxEventsPerSlice) = let getdefaultMaxItems = defaultArg getDefaultMaxItems (fun () -> defaultArg defaultMaxItems 10) /// Limit for Maximum number of `Batch` records in a single query batch response member __.MaxItems = getdefaultMaxItems () - /// Maximum number of trips to permit when slicing the work into multiple responses based on `MaxSlices` + /// Maximum number of trips to permit when slicing the work into multiple responses based on `MaxItems` member __.MaxRequests = maxRequests + /// Maximum number of events to accumulate within the `Tip` before switching to a new one when adding Events + member __.MaxEventsPerSlice = defaultArg maxEventsPerSlice 10 type EqxGateway(conn : EqxConnection, batching : EqxBatchingPolicy) = let (|FromUnfold|_|) (tryDecode: #IEvent -> 'event option) (isOrigin: 'event -> bool) (xs:#IEvent[]) : Option<'event[]> = @@ -732,7 +734,7 @@ type EqxGateway(conn : EqxConnection, batching : EqxBatchingPolicy) = | _ -> let! res = __.Read log stream Direction.Forward (Some pos) (tryDecode,isOrigin) return LoadFromTokenResult.Found res } member __.Sync log stream (expectedVersion, batch: Store.Tip): Async = async { - let! wr = Sync.batch log conn.WriteRetryPolicy conn.Client stream (expectedVersion,batch) + let! wr = Sync.batch log conn.WriteRetryPolicy conn.Client stream (expectedVersion,batch,batching.MaxItems) match wr with | Sync.Result.Conflict (pos',events) -> return InternalSyncResult.Conflict (Token.create stream pos',events) | Sync.Result.ConflictUnknown pos' -> return InternalSyncResult.ConflictUnknown (Token.create stream pos') @@ -1006,9 +1008,13 @@ type EqxContext /// Defaults to 10 ?defaultMaxItems, /// Alternate way of specifying defaultMaxItems which facilitates reading it from a cached dynamic configuration - ?getDefaultMaxItems) = + ?getDefaultMaxItems, + /// Threshold defining the number of events a slice is allowed to hold before switching to a new Batch is triggered. + /// Defaults to 1 + ?maxEventsPerSlice) = let getDefaultMaxItems = match getDefaultMaxItems with Some f -> f | None -> fun () -> defaultArg defaultMaxItems 10 - let batching = EqxBatchingPolicy(getDefaultMaxItems=getDefaultMaxItems) + let maxEventsPerSlice = defaultArg maxEventsPerSlice 1 + let batching = EqxBatchingPolicy(getDefaultMaxItems=getDefaultMaxItems, maxEventsPerSlice=maxEventsPerSlice) let gateway = EqxGateway(conn, batching) let maxCountPredicate count = @@ -1026,7 +1032,8 @@ type EqxContext member internal __.GetLazy((stream, startPos), ?batchSize, ?direction) : AsyncSeq = let direction = defaultArg direction Direction.Forward - let batching = EqxBatchingPolicy(defaultArg batchSize 10) + let batchSize = defaultArg batchSize batching.MaxItems * maxEventsPerSlice + let batching = EqxBatchingPolicy(if batchSize < maxEventsPerSlice then 1 else batchSize/maxEventsPerSlice) gateway.ReadLazy batching logger stream direction startPos (Some,fun _ -> false) member internal __.GetInternal((stream, startPos), ?maxCount, ?direction) = async { diff --git a/tests/Equinox.Cosmos.Integration/CosmosCoreIntegration.fs b/tests/Equinox.Cosmos.Integration/CosmosCoreIntegration.fs index f7002401a..29864744e 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosCoreIntegration.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosCoreIntegration.fs @@ -32,7 +32,7 @@ type Tests(testOutputHelper) = incr testIterations sprintf "events-%O-%i" name !testIterations let mkContextWithItemLimit conn defaultBatchSize = - EqxContext(conn,collections,log,?defaultMaxItems=defaultBatchSize) + EqxContext(conn,collections,log,?defaultMaxItems=defaultBatchSize,maxEventsPerSlice=10) let mkContext conn = mkContextWithItemLimit conn None let verifyRequestChargesMax rus = @@ -48,7 +48,7 @@ type Tests(testOutputHelper) = let! res = Events.append ctx streamName index <| EventData.Create(0,1) test <@ AppendResult.Ok 1L = res @> test <@ [EqxAct.Append] = capture.ExternalCalls @> - verifyRequestChargesMax 14 // observed 12.03 // was 10 + verifyRequestChargesMax 10 // Clear the counters capture.Clear() @@ -85,7 +85,6 @@ type Tests(testOutputHelper) = let xs, baseIndex = if direction = Direction.Forward then xs, baseIndex else Array.rev xs, baseIndex - int64 (Array.length expected) + 1L - test <@ expected.Length = xs.Length @> test <@ [for i in 0..expected.Length - 1 -> baseIndex + int64 i] = [for r in xs -> r.Index] @> test <@ [for e in expected -> e.EventType] = [ for r in xs -> r.EventType ] @> for i,x,y in Seq.mapi2 (fun i x y -> i,x,y) [for e in expected -> e.Data] [for r in xs -> r.Data] do @@ -107,28 +106,25 @@ type Tests(testOutputHelper) = capture.Clear() let mutable pos = 0L - let ae = false // TODO fix bug for appendBatchSize in [4; 5; 9] do - if ae then - let! res = Events.appendAtEnd ctx streamName <| EventData.Create (int pos,appendBatchSize) - pos <- pos + int64 appendBatchSize - //let! res = Events.append ctx streamName pos (Array.replicate appendBatchSize event) - test <@ [EqxAct.Append] = capture.ExternalCalls @> - pos =! res - else - let! res = Events.append ctx streamName pos <| EventData.Create (int pos,appendBatchSize) - pos <- pos + int64 appendBatchSize - //let! res = Events.append ctx streamName pos (Array.replicate appendBatchSize event) - test <@ [EqxAct.Append] = capture.ExternalCalls @> - AppendResult.Ok pos =! res - verifyRequestChargesMax 50 // was 20, observed 41.64 // 15.59 observed + let! res = Events.appendAtEnd ctx streamName <| EventData.Create (int pos,appendBatchSize) + test <@ [EqxAct.Append] = capture.ExternalCalls @> + pos <- pos + int64 appendBatchSize + pos =! res + verifyRequestChargesMax 20 // 15.59 observed + capture.Clear() + + let! res = Events.getNextIndex ctx streamName + test <@ [EqxAct.Tip] = capture.ExternalCalls @> + verifyRequestChargesMax 2 + pos =! res capture.Clear() let! res = Events.appendAtEnd ctx streamName <| EventData.Create (int pos,42) pos <- pos + 42L pos =! res test <@ [EqxAct.Append] = capture.ExternalCalls @> - verifyRequestChargesMax 180 // observed 167.32 // was 20 + verifyRequestChargesMax 20 capture.Clear() let! res = Events.getNextIndex ctx streamName @@ -140,11 +136,10 @@ type Tests(testOutputHelper) = // Demonstrate benefit/mechanism for using the Position-based API to avail of the etag tracking let stream = ctx.CreateStream streamName - let max = 2000 // observed to time out server side // WAS 5000 - let extrasCount = match extras with x when x * 100 > max -> max | x when x < 1 -> 1 | x -> x*100 + let extrasCount = match extras with x when x > 50 -> 5000 | x when x < 1 -> 1 | x -> x*100 let! _pos = ctx.NonIdempotentAppend(stream, EventData.Create (int pos,extrasCount)) test <@ [EqxAct.Append] = capture.ExternalCalls @> - verifyRequestChargesMax 7000 // 6867.7 observed // was 300 // 278 observed + verifyRequestChargesMax 300 // 278 observed capture.Clear() let! pos = ctx.Sync(stream,?position=None) @@ -176,7 +171,7 @@ type Tests(testOutputHelper) = let! res = Events.append ctx streamName 0L expected test <@ AppendResult.Ok 1L = res @> test <@ [EqxAct.Append] = capture.ExternalCalls @> - verifyRequestChargesMax 14 // observed 12.73 // was 10 + verifyRequestChargesMax 10 capture.Clear() // Try overwriting it (a competing consumer would see the same) @@ -186,7 +181,7 @@ type Tests(testOutputHelper) = | AppendResult.Conflict (1L, e) -> verifyCorrectEvents 0L expected e | x -> x |> failwithf "Unexpected %A" test <@ [EqxAct.Resync] = capture.ExternalCalls @> - verifyRequestChargesMax 5 // observed 4.21 // was 4 + verifyRequestChargesMax 5 // 4.02 capture.Clear() } @@ -205,29 +200,28 @@ type Tests(testOutputHelper) = verifyCorrectEvents 1L expected res - test <@ List.replicate 2 EqxAct.ResponseForward @ [EqxAct.QueryForward] = capture.ExternalCalls @> - verifyRequestChargesMax 8 // observed 6.14 // was 3 + test <@ [EqxAct.ResponseForward; EqxAct.QueryForward] = capture.ExternalCalls @> + verifyRequestChargesMax 4 // 3.14 // was 3 before introduction of multi-event batches } [] let ``get (in 2 batches)`` (TestStream streamName) = Async.RunSynchronously <| async { let! conn = connectToSpecifiedCosmosOrSimulator log - let ctx = mkContextWithItemLimit conn (Some 2) + let ctx = mkContextWithItemLimit conn (Some 1) let! expected = add6EventsIn2Batches ctx streamName - let expected = Array.tail expected |> Array.take 3 + let expected = expected |> Array.take 3 - let! res = Events.get ctx streamName 1L 3 + let! res = Events.get ctx streamName 0L 3 - verifyCorrectEvents 1L expected res + verifyCorrectEvents 0L expected res // 2 items atm test <@ [EqxAct.ResponseForward; EqxAct.ResponseForward; EqxAct.QueryForward] = capture.ExternalCalls @> - verifyRequestChargesMax 7 // observed 6.14 // was 6 - } + verifyRequestChargesMax 6 } // 5.77 [] - let getAll (TestStream streamName) = Async.RunSynchronously <| async { + let ``get Lazy`` (TestStream streamName) = Async.RunSynchronously <| async { let! conn = connectToSpecifiedCosmosOrSimulator log let ctx = mkContextWithItemLimit conn (Some 1) @@ -242,7 +236,7 @@ type Tests(testOutputHelper) = let queryRoundTripsAndItemCounts = function EqxEvent (Log.Query (Direction.Forward, responses, { count = c })) -> Some (responses,c) | _ -> None // validate that, despite only requesting max 1 item, we only needed one trip (which contained only one item) [1,1] =! capture.ChooseCalls queryRoundTripsAndItemCounts - verifyRequestChargesMax 4 // 3.07 // was 3 // 2.94 + verifyRequestChargesMax 3 // 2.97 } (* Backward *) @@ -250,39 +244,55 @@ type Tests(testOutputHelper) = [] let getBackwards (TestStream streamName) = Async.RunSynchronously <| async { let! conn = connectToSpecifiedCosmosOrSimulator log - let ctx = mkContextWithItemLimit conn (Some 2) + let ctx = mkContextWithItemLimit conn (Some 1) let! expected = add6EventsIn2Batches ctx streamName // We want to skip reading the last - let expected = Array.take 5 expected + let expected = Array.take 5 expected |> Array.tail - let! res = Events.getBackwards ctx streamName 4L 5 + let! res = Events.getBackwards ctx streamName 4L 4 verifyCorrectEventsBackward 4L expected res - test <@ List.replicate 3 EqxAct.ResponseBackward @ [EqxAct.QueryBackward] = capture.ExternalCalls @> - verifyRequestChargesMax 10 // observed 8.98 // was 3 + test <@ [EqxAct.ResponseBackward; EqxAct.QueryBackward] = capture.ExternalCalls @> + verifyRequestChargesMax 3 } - // TODO 2 batches backward test + [] + let ``getBackwards (2 batches)`` (TestStream streamName) = Async.RunSynchronously <| async { + let! conn = connectToSpecifiedCosmosOrSimulator log + let ctx = mkContextWithItemLimit conn (Some 1) + + let! expected = add6EventsIn2Batches ctx streamName + + // We want to skip reading the last two, which means getting both, but disregarding some of the second batch + let expected = Array.take 4 expected + + let! res = Events.getBackwards ctx streamName 3L 4 + + verifyCorrectEventsBackward 3L expected res + + test <@ List.replicate 2 EqxAct.ResponseBackward @ [EqxAct.QueryBackward] = capture.ExternalCalls @> + verifyRequestChargesMax 6 // 5.77 + } [] - let getAllBackwards (TestStream streamName) = Async.RunSynchronously <| async { + let ``getBackwards Lazy`` (TestStream streamName) = Async.RunSynchronously <| async { let! conn = connectToSpecifiedCosmosOrSimulator log - let ctx = mkContextWithItemLimit conn (Some 2) + let ctx = mkContextWithItemLimit conn (Some 1) let! expected = add6EventsIn2Batches ctx streamName capture.Clear() - let! res = Events.getAllBackwards ctx streamName 10L 2 |> AsyncSeq.concatSeq |> AsyncSeq.takeWhileInclusive (fun x -> x.Index <> 2L) |> AsyncSeq.toArrayAsync - let expected = expected |> Array.skip 2 + let! res = Events.getAllBackwards ctx streamName 10L 1 |> AsyncSeq.concatSeq |> AsyncSeq.takeWhileInclusive (fun x -> x.Index <> 2L) |> AsyncSeq.toArrayAsync + let expected = expected |> Array.skip 2 // omit index 0, 1 as we vote to finish at 2L verifyCorrectEventsBackward 5L expected res - // only 2 batches of 2 items triggered - test <@ List.replicate 2 EqxAct.ResponseBackward @ [EqxAct.QueryBackward] = capture.ExternalCalls @> - // validate that we didnt trigger loading of the last item + // only 1 request of 1 item triggered + test <@ [EqxAct.ResponseBackward; EqxAct.QueryBackward] = capture.ExternalCalls @> + // validate that, despite only requesting max 1 item, we only needed one trip, bearing 5 items (from which one item was omitted) let queryRoundTripsAndItemCounts = function EqxEvent (Log.Query (Direction.Backward, responses, { count = c })) -> Some (responses,c) | _ -> None - [2,4] =! capture.ChooseCalls queryRoundTripsAndItemCounts - verifyRequestChargesMax 7 // observed 6.03 // was 3 // 2.95 + [1,5] =! capture.ChooseCalls queryRoundTripsAndItemCounts + verifyRequestChargesMax 3 // 2.98 } \ No newline at end of file diff --git a/tests/Equinox.Cosmos.Integration/CosmosFixtures.fs b/tests/Equinox.Cosmos.Integration/CosmosFixtures.fs index 467c1bf57..b6515d81d 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosFixtures.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosFixtures.fs @@ -34,4 +34,7 @@ let collections = let createEqxStore connection batchSize = let gateway = EqxGateway(connection, EqxBatchingPolicy(defaultMaxItems=batchSize)) + EqxStore(gateway, collections) +let createEqxStoreWithMaxEventsPerSlice connection batchSize maxEventsPerSlice = + let gateway = EqxGateway(connection, EqxBatchingPolicy(defaultMaxItems=batchSize, maxEventsPerSlice=maxEventsPerSlice)) EqxStore(gateway, collections) \ No newline at end of file diff --git a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs index 49a21caf0..1ca91754f 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs @@ -18,6 +18,10 @@ module Cart = let store = createEqxStore connection batchSize let resolveStream = EqxStreamBuilder(store, codec, fold, initial).Create Backend.Cart.Service(log, resolveStream) + let createServiceWithoutOptimizationAndMaxItems connection batchSize maxEventsPerSlice log = + let store = createEqxStoreWithMaxEventsPerSlice connection batchSize maxEventsPerSlice + let resolveStream = EqxStreamBuilder(store, codec, fold, initial).Create + Backend.Cart.Service(log, resolveStream) let projection = "Compacted",snd snapshot let createServiceWithProjection connection batchSize log = let store = createEqxStore connection batchSize @@ -66,8 +70,8 @@ type Tests(testOutputHelper) = let! conn = connectToSpecifiedCosmosOrSimulator log let maxItemsPerRequest = 2 - let maxEventsPerBatch = 1 - let service = Cart.createServiceWithoutOptimization conn maxItemsPerRequest log + let maxEventsPerBatch = 3 + let service = Cart.createServiceWithoutOptimizationAndMaxItems conn maxItemsPerRequest maxEventsPerBatch log capture.Clear() // for re-runs of the test let cartId = Guid.NewGuid() |> CartId @@ -82,7 +86,7 @@ type Tests(testOutputHelper) = | 1 -> 1 // it does cost a single trip to determine there are 0 items | i -> ceil(float (i-1) * float eventsPerAction / float maxItemsPerRequest / float maxEventsPerBatch) |> int test <@ List.replicate expectedBatchesOf2Items EqxAct.ResponseBackward @ [EqxAct.QueryBackward; EqxAct.Append] = capture.ExternalCalls @> - verifyRequestChargesMax 39 // 37.15 + verifyRequestChargesMax 25 // 20.59 capture.Clear() // Validate basic operation; Key side effect: Log entries will be emitted to `capture` @@ -93,7 +97,7 @@ type Tests(testOutputHelper) = // Need 6 trips of 2 maxItemsPerRequest to read 12 events test <@ let expectedResponses = ceil(float expectedEventCount/float maxItemsPerRequest/float maxEventsPerBatch) |> int List.replicate expectedResponses EqxAct.ResponseBackward @ [EqxAct.QueryBackward] = capture.ExternalCalls @> - verifyRequestChargesMax 20 // 18.47 + verifyRequestChargesMax 7 // 5.93 } [] diff --git a/tests/Equinox.Cosmos.Integration/JsonConverterTests.fs b/tests/Equinox.Cosmos.Integration/JsonConverterTests.fs index 5d99c734e..24e207fb9 100644 --- a/tests/Equinox.Cosmos.Integration/JsonConverterTests.fs +++ b/tests/Equinox.Cosmos.Integration/JsonConverterTests.fs @@ -22,7 +22,7 @@ type VerbatimUtf8Tests() = let ``encodes correctly`` () = let encoded = mkUnionEncoder().Encode(A { embed = "\"" }) let e : Store.Batch = - { p = "streamName"; id = string 0; i = 0L; _i = 0L; _etag = null + { p = "streamName"; id = string 0; i = -1L; n = -1L; _etag = null e = [| { t = DateTimeOffset.MinValue; c = encoded.caseName; d = encoded.payload; m = null } |] } let res = JsonConvert.SerializeObject(e) test <@ res.Contains """"d":{"embed":"\""}""" @> From 2a39579526647458efec703d2f004dfeaee594b3 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 29 Nov 2018 15:50:33 +0000 Subject: [PATCH 49/66] Tidy names+namespaces --- samples/Store/Integration/CartIntegration.fs | 6 +- .../ContactPreferencesIntegration.fs | 6 +- .../Store/Integration/FavoritesIntegration.fs | 4 +- samples/Store/Integration/LogIntegration.fs | 4 +- src/Equinox.Cosmos/Cosmos.fs | 448 +++++++++--------- .../CosmosCoreIntegration.fs | 64 +-- .../CosmosFixtures.fs | 2 +- .../CosmosFixturesInfrastructure.fs | 45 +- .../CosmosIntegration.fs | 14 +- .../JsonConverterTests.fs | 2 +- 10 files changed, 306 insertions(+), 289 deletions(-) diff --git a/samples/Store/Integration/CartIntegration.fs b/samples/Store/Integration/CartIntegration.fs index 3001083f0..291feb431 100644 --- a/samples/Store/Integration/CartIntegration.fs +++ b/samples/Store/Integration/CartIntegration.fs @@ -1,6 +1,6 @@ module Samples.Store.Integration.CartIntegration -open Equinox.Cosmos.Builder +open Equinox.Cosmos open Equinox.Cosmos.Integration open Equinox.EventStore open Equinox.MemoryStore @@ -24,9 +24,9 @@ let resolveGesStreamWithoutCustomAccessStrategy gateway = GesResolver(gateway, codec, fold, initial).Resolve let resolveEqxStreamWithProjection gateway = - EqxStreamBuilder(gateway, codec, fold, initial, AccessStrategy.Snapshot snapshot).Create + EqxResolver(gateway, codec, fold, initial, AccessStrategy.Snapshot snapshot).Resolve let resolveEqxStreamWithoutCustomAccessStrategy gateway = - EqxStreamBuilder(gateway, codec, fold, initial).Create + EqxResolver(gateway, codec, fold, initial).Resolve let addAndThenRemoveItemsManyTimesExceptTheLastOne context cartId skuId (service: Backend.Cart.Service) count = service.FlowAsync(cartId, fun _ctx execute -> diff --git a/samples/Store/Integration/ContactPreferencesIntegration.fs b/samples/Store/Integration/ContactPreferencesIntegration.fs index dae2aa140..571d079bb 100644 --- a/samples/Store/Integration/ContactPreferencesIntegration.fs +++ b/samples/Store/Integration/ContactPreferencesIntegration.fs @@ -1,6 +1,6 @@ module Samples.Store.Integration.ContactPreferencesIntegration -open Equinox.Cosmos.Builder +open Equinox.Cosmos open Equinox.Cosmos.Integration open Equinox.EventStore open Equinox.MemoryStore @@ -22,9 +22,9 @@ let resolveStreamGesWithoutAccessStrategy gateway = GesResolver(gateway defaultBatchSize, codec, fold, initial).Resolve let resolveStreamEqxWithKnownEventTypeSemantics gateway = - EqxStreamBuilder(gateway 1, codec, fold, initial, AccessStrategy.AnyKnownEventType).Create + EqxResolver(gateway 1, codec, fold, initial, AccessStrategy.AnyKnownEventType).Resolve let resolveStreamEqxWithoutCustomAccessStrategy gateway = - EqxStreamBuilder(gateway defaultBatchSize, codec, fold, initial).Create + EqxResolver(gateway defaultBatchSize, codec, fold, initial).Resolve type Tests(testOutputHelper) = let testOutput = TestOutputAdapter testOutputHelper diff --git a/samples/Store/Integration/FavoritesIntegration.fs b/samples/Store/Integration/FavoritesIntegration.fs index 3bd07e6f5..c3a382278 100644 --- a/samples/Store/Integration/FavoritesIntegration.fs +++ b/samples/Store/Integration/FavoritesIntegration.fs @@ -1,6 +1,6 @@ module Samples.Store.Integration.FavoritesIntegration -open Equinox.Cosmos.Builder +open Equinox.Cosmos open Equinox.Cosmos.Integration open Equinox.EventStore open Equinox.MemoryStore @@ -22,7 +22,7 @@ let createServiceGes gateway log = Backend.Favorites.Service(log, resolveStream) let createServiceEqx gateway log = - let resolveStream = EqxStreamBuilder(gateway, codec, fold, initial, AccessStrategy.Snapshot snapshot).Create + let resolveStream = EqxResolver(gateway, codec, fold, initial, AccessStrategy.Snapshot snapshot).Resolve Backend.Favorites.Service(log, resolveStream) type Tests(testOutputHelper) = diff --git a/samples/Store/Integration/LogIntegration.fs b/samples/Store/Integration/LogIntegration.fs index 57445d470..879d219d3 100644 --- a/samples/Store/Integration/LogIntegration.fs +++ b/samples/Store/Integration/LogIntegration.fs @@ -23,7 +23,7 @@ module EquinoxEsInterop = | Log.Batch (Direction.Backward,c,m) -> "LoadB", m, Some c { action = action; stream = metric.stream; interval = metric.interval; bytes = metric.bytes; count = metric.count; batches = batches } module EquinoxCosmosInterop = - open Equinox.Cosmos + open Equinox.Cosmos.Store [] type FlatMetric = { action: string; stream: string; interval: StopwatchInterval; bytes: int; count: int; responses: int option; ru: float } with override __.ToString() = sprintf "%s-Stream=%s %s-Elapsed=%O Ru=%O" __.action __.stream __.action __.interval.Elapsed __.ru @@ -62,7 +62,7 @@ type SerilogMetricsExtractor(emit : string -> unit) = logEvent.Properties |> Seq.tryPick (function | KeyValue (k, SerilogScalar (:? Equinox.EventStore.Log.Event as m)) -> Some <| Choice1Of3 (k,m) - | KeyValue (k, SerilogScalar (:? Equinox.Cosmos.Log.Event as m)) -> Some <| Choice2Of3 (k,m) + | KeyValue (k, SerilogScalar (:? Equinox.Cosmos.Store.Log.Event as m)) -> Some <| Choice2Of3 (k,m) | _ -> None) |> Option.defaultValue (Choice3Of3 ()) let handleLogEvent logEvent = diff --git a/src/Equinox.Cosmos/Cosmos.fs b/src/Equinox.Cosmos/Cosmos.fs index ce4093ba3..7819f2b3e 100644 --- a/src/Equinox.Cosmos/Cosmos.fs +++ b/src/Equinox.Cosmos/Cosmos.fs @@ -1,39 +1,25 @@ -namespace Equinox.Cosmos.Events +namespace Equinox.Cosmos.Store -/// Common form for either a Domain Event or an Unfolded Event -type IEvent = - /// The Event Type, used to drive deserialization - abstract member EventType : string - /// Event body, as UTF-8 encoded json ready to be injected into the Json being rendered for DocDb - abstract member Data : byte[] - /// Optional metadata (null, or same as d, not written if missing) - abstract member Meta : byte[] +open Newtonsoft.Json +open System -/// Represents a Domain Event or Unfold, together with it's Index in the event sequence -type IIndexedEvent = - inherit IEvent - /// The index into the event sequence of this event - abstract member Index : int64 - /// Indicates whether this is a Domain Event or an Unfolded Event based on the state inferred from the events up to `Index` - abstract member IsUnfold: bool +/// A single Domain Event from the array held in a Batch +type [] + Event = + { /// Creation datetime (as opposed to system-defined _lastUpdated which is touched by triggers, replication etc.) + t: DateTimeOffset // ISO 8601 -/// Position and Etag to which an operation is relative -type [] Position = { index: int64; etag: string option } with - /// If we have strong reason to suspect a stream is empty, we won't have an etag (and Writer Stored Procedure special cases this) - static member internal FromKnownEmpty = Position.FromI 0L - /// NB very inefficient compared to FromDocument or using one already returned to you - static member internal FromI(i: int64) = { index = i; etag = None } - /// Just Do It mode - static member internal FromAppendAtEnd = Position.FromI -1L // sic - needs to yield -1 - /// NB very inefficient compared to FromDocument or using one already returned to you - static member internal FromMaxIndex(xs: IIndexedEvent[]) = - if Array.isEmpty xs then Position.FromKnownEmpty - else Position.FromI (1L + Seq.max (seq { for x in xs -> x.Index })) + /// The Case (Event Type); used to drive deserialization + c: string // required -namespace Equinox.Cosmos.Store + /// Event body, as UTF-8 encoded json ready to be injected into the Json being rendered for DocDb + [)>] + d: byte[] // required -open Equinox.Cosmos.Events -open Newtonsoft.Json + /// Optional metadata, as UTF-8 encoded json, ready to emit directly (null, not written if missing) + [)>] + [] + m: byte[] } // optional /// A 'normal' (frozen, not Tip) Batch of Events (without any Unfolds) type [] @@ -60,43 +46,35 @@ type [] // `i` value for successor batch (to facilitate identifying which Batch a given startPos is within) n: int64 // {index} - /// The events at this offset in the stream - e: BatchEvent[] } + /// The Domain Events (as opposed to Unfolded Events, see Tip) at this offset in the stream + e: Event[] } /// Unless running in single partion mode (which would restrict us to 10GB per collection) /// we need to nominate a partition key that will be in every document - static member PartitionKeyField = "p" + static member internal PartitionKeyField = "p" /// As one cannot sort by the implicit `id` field, we have an indexed `i` field for sort and range query use - static member IndexedFields = [Batch.PartitionKeyField; "i"; "n"] - /// If we encounter the tip (id=-1) doc, we're interested in its etag so we can re-sync for 1 RU - member x.TryToPosition() = - if x.id <> Tip.WellKnownDocumentId then None - else Some { index = x.n; etag = match x._etag with null -> None | x -> Some x } -/// A single event from the array held in a batch -and [] - BatchEvent = - { /// Creation datetime (as opposed to system-defined _lastUpdated which is touched by triggers, replication etc.) - t: System.DateTimeOffset // ISO 8601 + static member internal IndexedFields = [Batch.PartitionKeyField; "i"; "n"] + +/// Compaction/Snapshot/Projection Event based on the state at a given point in time `i` +type Unfold = + { /// Base: Stream Position (Version) of State from which this Unfold Event was generated + i: int64 - /// The Case (Event Type), used to drive deserialization + /// The Case (Event Type) of this compaction/snapshot, used to drive deserialization c: string // required - /// Event body, as UTF-8 encoded json ready to be injected into the Json being rendered for DocDb - [)>] + /// Event body - Json -> UTF-8 -> Deflate -> Base64 + [)>] d: byte[] // required - /// Optional metadata, as UTF-8 encoded json, ready to emit directly (null, not written if missing) - [)>] + /// Optional metadata, same encoding as `d` (can be null; not written if missing) + [)>] [] m: byte[] } // optional -/// The Special 'Pending' Batch Format -/// NB this Type does double duty as -/// a) transport for when we read it -/// b) a way of encoding a batch that the stored procedure will write in to the actual document (`i` is -1 until Stored Proc computes it) -/// The stored representation has the following differences vs a 'normal' (frozen/completed) Batch -/// a) `id` = `-1` -/// b) contains unfolds (`u`) -and [] +/// The special-case 'Pending' Batch Format used to read the currently active (and mutable) document +/// Stored representation has the following diffs vs a 'normal' (frozen/completed) Batch: a) `id` = `-1` b) contains unfolds (`u`) +/// NB the type does double duty as a) model for when we read it b) encoding a batch being sent to the stored proc +type [] Tip = { [] // Not requested in queries /// Partition key, as per Batch @@ -116,34 +94,68 @@ and [] /// `i` value for successor batch (to facilitate identifying which Batch a given startPos is within) n: int64 // {index} - /// Events - e: BatchEvent[] + /// Domain Events, will eventually move out to a Batch + e: Event[] - /// Compaction/Snapshot/Projection events + /// Compaction/Snapshot/Projection events - owned and managed by the sync stored proc u: Unfold[] } /// arguably this should be a high nember to reflect fact it is the freshest ? - static member WellKnownDocumentId = "-1" - /// Create Position from Tip record context (facilitating 1 RU reads) - member x.ToPosition() = { index = x.n; etag = match x._etag with null -> None | x -> Some x } -/// Compaction/Snapshot/Projection Event based on the state at a given point in time `i` -and Unfold = - { /// Base: Stream Position (Version) of State from which this Unfold Event was generated - i: int64 + static member internal WellKnownDocumentId = "-1" - /// The Case (Event Type) of this compaction/snapshot, used to drive deserialization - c: string // required +/// Position and Etag to which an operation is relative +type [] + Position = { index: int64; etag: string option } - /// Event body - Json -> UTF-8 -> Deflate -> Base64 - [)>] - d: byte[] // required +/// Common form for either a Domain Event or an Unfolded Event +type IEvent = + /// The Event Type, used to drive deserialization + abstract member EventType : string + /// Event body, as UTF-8 encoded json ready to be injected into the Json being rendered for DocDb + abstract member Data : byte[] + /// Optional metadata (null, or same as d, not written if missing) + abstract member Meta : byte[] - /// Optional metadata, same encoding as `d` (can be null; not written if missing) - [)>] - [] - m: byte[] } // optional +/// Represents a Domain Event or Unfold, together with it's Index in the event sequence +type IIndexedEvent = + inherit IEvent + /// The index into the event sequence of this event + abstract member Index : int64 + /// Indicates this is not a Domain Event, but actually an Unfolded Event based on the state inferred from the events up to `Index` + abstract member IsUnfold: bool + +/// An Event about to be written, see IEvent for further information +type EventData = + { eventType : string; data : byte[]; meta : byte[] } + interface IEvent with member __.EventType = __.eventType member __.Data = __.data member __.Meta = __.meta + static member Create(eventType, data, ?meta) = { eventType = eventType; data = data; meta = defaultArg meta null} + +module internal Position = + /// NB very inefficient compared to FromDocument or using one already returned to you + let fromI (i: int64) = { index = i; etag = None } + /// If we have strong reason to suspect a stream is empty, we won't have an etag (and Writer Stored Procedure special cases this) + let fromKnownEmpty = fromI 0L + /// Just Do It mode + let fromAppendAtEnd = fromI -1L // sic - needs to yield -1 + /// NB very inefficient compared to FromDocument or using one already returned to you + let fromMaxIndex (xs: IIndexedEvent[]) = + if Array.isEmpty xs then fromKnownEmpty + else fromI (1L + Seq.max (seq { for x in xs -> x.Index })) + /// Create Position from Tip record context (facilitating 1 RU reads) + let fromTip (x: Tip) = { index = x.n; etag = match x._etag with null -> None | x -> Some x } + /// If we encounter the tip (id=-1) doc, we're interested in its etag so we can re-sync for 1 RU + let tryFromBatch (x: Batch) = + if x.id <> Tip.WellKnownDocumentId then None + else Some { index = x.n; etag = match x._etag with null -> None | x -> Some x } -type Enum() = - static member Events(b: Tip) = +[] +type Direction = Forward | Backward override this.ToString() = match this with Forward -> "Forward" | Backward -> "Backward" + +/// Reference to Collection and name that will be used as the location for the stream +type [] + CollectionStream = { collectionUri: System.Uri; name: string } //with + +type internal Enum() = + static member internal Events(b: Tip) = b.e |> Seq.mapi (fun offset x -> { new IIndexedEvent with member __.Index = b.i + int64 offset @@ -151,15 +163,16 @@ type Enum() = member __.EventType = x.c member __.Data = x.d member __.Meta = x.m }) - static member Events(i: int64, e: BatchEvent[], startIndex, backward) = seq { - let isValidGivenStartIndex backward si i = - match si with - | Some si when backward -> i < si - | Some si -> i >= si + static member Events(i: int64, e: Event[], startPos : Position option, direction) = seq { + // If we're loading from a nominated position, we need to discard items in the batch before/after the start on the start page + let isValidGivenStartPos i = + match startPos with + | Some sp when direction = Direction.Backward -> i < sp.index + | Some sp -> i >= sp.index | _ -> true for offset in 0..e.Length-1 do let index = i + int64 offset - if isValidGivenStartIndex backward startIndex index then + if isValidGivenStartPos index then let x = e.[offset] yield { new IIndexedEvent with @@ -168,9 +181,9 @@ type Enum() = member __.EventType = x.c member __.Data = x.d member __.Meta = x.m } } - static member Events(b: Batch, startIndex, backward) = - Enum.Events(b.i, b.e, startIndex, backward) - |> if backward then System.Linq.Enumerable.Reverse else id + static member internal Events(b: Batch, startPos, direction) = + Enum.Events(b.i, b.e, startPos, direction) + |> if direction = Direction.Backward then System.Linq.Enumerable.Reverse else id static member Unfolds (xs: Unfold[]) = seq { for x in xs -> { new IIndexedEvent with member __.Index = x.i @@ -184,26 +197,10 @@ type Enum() = // where Index is equal, unfolds get delivered after the events so the fold semantics can be 'idempotent' |> Seq.sortBy (fun x -> x.Index, x.IsUnfold) -/// Reference to Collection and name that will be used as the location for the stream -type [] CollectionStream = { collectionUri: System.Uri; name: string } with - static member Create(collectionUri, name) = { collectionUri = collectionUri; name = name } - -namespace Equinox.Cosmos +type IRetryPolicy = abstract member Execute: (int -> Async<'T>) -> Async<'T> -open Equinox -open Equinox.Cosmos.Events -open Equinox.Cosmos.Store open Equinox.Store -open FSharp.Control -open Microsoft.Azure.Documents open Serilog -open System - -[] -type Direction = Forward | Backward with - override this.ToString() = match this with Forward -> "Forward" | Backward -> "Backward" - -type IRetryPolicy = abstract member Execute: (int -> Async<'T>) -> Async<'T> module Log = [] @@ -229,6 +226,8 @@ module Log = log.ForContext(name, sprintf "[%s]" (String.concat ",\n\r" items)) let propEvents = propData "events" let propDataUnfolds = Enum.Unfolds >> propData "unfolds" + let propStartPos (value : Position) log = prop "startPos" value.index log + let propMaybeStartPos (value : Position option) log = match value with None -> log | Some value -> propStartPos value log let withLoggedRetries<'t> (retryPolicy: IRetryPolicy option) (contextLabel : string) (f : ILogger -> Async<'t>) log: Async<'t> = match retryPolicy with @@ -248,6 +247,8 @@ module Log = let (|EventLen|) (x: #IEvent) = let (BlobLen bytes), (BlobLen metaBytes) = x.Data, x.Meta in bytes+metaBytes let (|BatchLen|) = Seq.sumBy (|EventLen|) +open Microsoft.Azure.Documents + [] module private DocDb = /// Extracts the innermost exception from a nested hierarchy of Aggregate Exceptions @@ -291,7 +292,7 @@ module private DocDb = module Sync = // NB don't nest in a private module, or serialization will fail miserably ;) [] - type SyncResponse = { etag: string; n: int64; conflicts: BatchEvent[] } + type SyncResponse = { etag: string; n: int64; conflicts: Event[] } let [] sprocName = "EquinoxSync002" // NB need to renumber for any breaking change let [] sprocBody = """ @@ -378,7 +379,7 @@ function sync(req, expectedVersion, maxEvents) { let sprocLink = sprintf "%O/sprocs/%s" stream.collectionUri sprocName let opts = Client.RequestOptions(PartitionKey=PartitionKey(stream.name)) let! ct = Async.CancellationToken - let ev = match expectedVersion with Some ev -> Position.FromI ev | None -> Position.FromAppendAtEnd + let ev = match expectedVersion with Some ev -> Position.fromI ev | None -> Position.fromAppendAtEnd let! (res : Client.StoredProcedureResponse) = client.ExecuteStoredProcedureAsync(sprocLink, opts, ct, box req, box ev.index, box maxEvents) |> Async.AwaitTaskCorrect @@ -387,11 +388,11 @@ function sync(req, expectedVersion, maxEvents) { | null -> Result.Written newPos | [||] when newPos.index = 0L -> Result.Conflict (newPos, Array.empty) | [||] -> Result.ConflictUnknown newPos - | xs -> Result.Conflict (newPos, Enum.Events(ev.index, xs, None, false) |> Array.ofSeq) } + | xs -> Result.Conflict (newPos, Enum.Events(ev.index, xs, None, Direction.Forward) |> Array.ofSeq) } let private logged client (stream: CollectionStream) (expectedVersion, req: Tip, maxEvents) (log : ILogger) : Async = async { - let verbose = log.IsEnabled Events.LogEventLevel.Debug + let verbose = log.IsEnabled Serilog.Events.LogEventLevel.Debug let log = if verbose then log |> Log.propEvents (Enum.Events req) |> Log.propDataUnfolds req.u else log let (Log.BatchLen bytes), count = Enum.Events req, req.e.Length let log = log |> Log.prop "bytes" bytes @@ -418,12 +419,12 @@ function sync(req, expectedVersion, maxEvents) { let batch (log : ILogger) retryPolicy client pk batch: Async = let call = logged client pk batch Log.withLoggedRetries retryPolicy "writeAttempt" call log - let mkBatch (stream: Store.CollectionStream) (events: IEvent[]) unfolds: Tip = - { p = stream.name; id = Store.Tip.WellKnownDocumentId; n = -1L(*Server-managed*); i = -1L(*Server-managed*); _etag = null + let mkBatch (stream: CollectionStream) (events: IEvent[]) unfolds: Tip = + { p = stream.name; id = Tip.WellKnownDocumentId; n = -1L(*Server-managed*); i = -1L(*Server-managed*); _etag = null e = [| for e in events -> { t = DateTimeOffset.UtcNow; c = e.EventType; d = e.Data; m = e.Meta } |] u = Array.ofSeq unfolds } - let mkUnfold baseIndex (unfolds: IEvent seq) : Store.Unfold seq = - unfolds |> Seq.mapi (fun offset x -> { i = baseIndex + int64 offset; c = x.EventType; d = x.Data; m = x.Meta } : Store.Unfold) + let mkUnfold baseIndex (unfolds: IEvent seq) : Unfold seq = + unfolds |> Seq.mapi (fun offset x -> { i = baseIndex + int64 offset; c = x.EventType; d = x.Data; m = x.Meta } : Unfold) module Initialization = open System.Collections.ObjectModel @@ -434,7 +435,7 @@ function sync(req, expectedVersion, maxEvents) { let createCollection (client: IDocumentClient) (dbUri: Uri) collName ru = async { let pkd = PartitionKeyDefinition() - pkd.Paths.Add(sprintf "/%s" Store.Batch.PartitionKeyField) + pkd.Paths.Add(sprintf "/%s" Batch.PartitionKeyField) let colld = DocumentCollection(Id = collName, PartitionKey = pkd) colld.IndexingPolicy.IndexingMode <- IndexingMode.Consistent @@ -443,7 +444,7 @@ function sync(req, expectedVersion, maxEvents) { // Given how long and variable the blacklist would be, we whitelist instead colld.IndexingPolicy.ExcludedPaths <- Collection [|ExcludedPath(Path="/*")|] // NB its critical to index the nominated PartitionKey field defined above or there will be runtime errors - colld.IndexingPolicy.IncludedPaths <- Collection [| for k in Store.Batch.IndexedFields -> IncludedPath(Path=sprintf "/%s/?" k) |] + colld.IndexingPolicy.IncludedPaths <- Collection [| for k in Batch.IndexedFields -> IncludedPath(Path=sprintf "/%s/?" k) |] let! coll = client.CreateDocumentCollectionIfNotExistsAsync(dbUri, colld, Client.RequestOptions(OfferThroughput=Nullable ru)) |> Async.AwaitTaskCorrect return coll.Resource.Id } @@ -461,7 +462,7 @@ function sync(req, expectedVersion, maxEvents) { //let! _aux = createAux client dbUri collName auxRu return! createProc log client collUri } -module private Tip = +module internal Tip = let private get (client: IDocumentClient) (stream: CollectionStream, maybePos: Position option) = let coll = DocDbCollection(client, stream.collectionUri) let ac = match maybePos with Some { etag=Some etag } -> Client.AccessCondition(Type=Client.AccessConditionType.IfNoneMatch, Condition=etag) | _ -> null @@ -491,38 +492,38 @@ module private Tip = match res with | ReadResult.NotModified -> return Result.NotModified | ReadResult.NotFound -> return Result.NotFound - | ReadResult.Found doc -> return Result.Found (doc.ToPosition(), Enum.EventsAndUnfolds doc |> Array.ofSeq) } + | ReadResult.Found doc -> return Result.Found (Position.fromTip doc, Enum.EventsAndUnfolds doc |> Array.ofSeq) } - module private Query = + module internal Query = open Microsoft.Azure.Documents.Linq - let private mkQuery (client : IDocumentClient) maxItems (stream: CollectionStream) (direction: Direction) (startPos: Position option) = + open FSharp.Control + let private mkQuery (client : IDocumentClient) maxItems (stream: CollectionStream) (direction: Direction) startPos = let querySpec = let fields = "c.id, c.i, c._etag, c.n, c.e" match startPos with | None -> SqlQuerySpec(sprintf "SELECT %s FROM c ORDER BY c.i " fields + if direction = Direction.Forward then "ASC" else "DESC") - | Some p -> - let f = if direction = Direction.Forward then "c.n > @id ORDER BY c.i ASC" else "c.i < @id ORDER BY c.i DESC" - SqlQuerySpec(sprintf "SELECT %s FROM c WHERE " fields + f, SqlParameterCollection [SqlParameter("@id", p.index)]) + | Some { index = positionSoExclusiveWhenBackward } -> + let f = if direction = Direction.Forward then "c.n > @startPos ORDER BY c.i ASC" else "c.i < @startPos ORDER BY c.i DESC" + SqlQuerySpec(sprintf "SELECT %s FROM c WHERE " fields + f, SqlParameterCollection [SqlParameter("@startPos", positionSoExclusiveWhenBackward)]) let feedOptions = new Client.FeedOptions(PartitionKey=PartitionKey(stream.name), MaxItemCount=Nullable maxItems) client.CreateDocumentQuery(stream.collectionUri, querySpec, feedOptions).AsDocumentQuery() // Unrolls the Batches in a response - note when reading backwards, the events are emitted in reverse order of index - let private handleResponse direction (stream: CollectionStream) (startPos: Position option) (query: IDocumentQuery) (log: ILogger) + let private handleResponse direction (stream: CollectionStream) startPos (query: IDocumentQuery) (log: ILogger) : Async = async { let! ct = Async.CancellationToken let! t, (res : Client.FeedResponse) = query.ExecuteNextAsync(ct) |> Async.AwaitTaskCorrect |> Stopwatch.Time let batches, ru = Array.ofSeq res, res.RequestCharge - let startIndex = match startPos with Some { index = i } -> Some i | _ -> None - let events = batches |> Seq.collect (fun b -> Enum.Events(b, startIndex, backward=(direction = Direction.Backward))) |> Array.ofSeq + let events = batches |> Seq.collect (fun b -> Enum.Events(b, startPos, direction)) |> Array.ofSeq let (Log.BatchLen bytes), count = events, events.Length let reqMetric : Log.Measurement = { stream = stream.name; interval = t; bytes = bytes; count = count; ru = ru } let log = let evt = Log.Response (direction, reqMetric) in log |> Log.event evt let log = if (not << log.IsEnabled) Events.LogEventLevel.Debug then log else log |> Log.propEvents events let index = if count = 0 then Nullable () else Nullable <| Seq.min (seq { for x in batches -> x.i }) - (log |> Log.prop "startIndex" (match startIndex with Some i -> Nullable i | None -> Nullable()) |> Log.prop "bytes" bytes) + (log |> (match startPos with Some pos -> Log.propStartPos pos | None -> id) |> Log.prop "bytes" bytes) .Information("EqxCosmos {action:l} {count}/{batches} {direction} {ms}ms i={index} rc={ru}", "Response", count, batches.Length, direction, (let e = t.Elapsed in e.TotalMilliseconds), index, ru) - let maybePosition = batches |> Array.tryPick (fun x -> x.TryToPosition()) + let maybePosition = batches |> Array.tryPick Position.tryFromBatch return events, maybePosition, ru } let private run (log : ILogger) (readSlice: IDocumentQuery -> ILogger -> Async) @@ -544,8 +545,9 @@ module private Tip = let private logQuery direction batchSize streamName interval (responsesCount, events : IIndexedEvent []) n (ru: float) (log : ILogger) = let (Log.BatchLen bytes), count = events, events.Length let reqMetric : Log.Measurement = { stream = streamName; interval = interval; bytes = bytes; count = count; ru = ru } + let evt = Log.Event.Query (direction, responsesCount, reqMetric) let action = match direction with Direction.Forward -> "QueryF" | Direction.Backward -> "QueryB" - (log |> Log.prop "bytes" bytes |> Log.prop "batchSize" batchSize |> Log.event (Log.Event.Query (direction, responsesCount, reqMetric))).Information( + (log |> Log.prop "bytes" bytes |> Log.prop "batchSize" batchSize |> Log.event evt).Information( "EqxCosmos {action:l} {stream} v{n} {count}/{responses} {ms}ms rc={ru}", action, streamName, n, count, responsesCount, (let e = interval.Elapsed in e.TotalMilliseconds), ru) @@ -594,7 +596,7 @@ module private Tip = let! t, (events, maybeTipPos, ru) = mergeBatches log batches |> Stopwatch.Time query.Dispose() let raws, decoded = (Array.map fst events), (events |> Seq.choose snd |> Array.ofSeq) - let pos = match maybeTipPos with Some p -> p | None -> Position.FromMaxIndex raws + let pos = match maybeTipPos with Some p -> p | None -> Position.fromMaxIndex raws log |> logQuery direction maxItems stream.name t (!responseCount,raws) pos.index ru return pos, decoded } @@ -647,34 +649,32 @@ module private Tip = type [] Token = { stream: CollectionStream; pos: Position } module Token = - let create stream pos : Storage.StreamToken = { value = box { stream = stream; pos = pos } } - let (|Unpack|) (token: Storage.StreamToken) : CollectionStream*Position = let t = unbox token.value in t.stream,t.pos + let create stream pos : Equinox.Store.StreamToken = { value = box { stream = stream; pos = pos } } + let (|Unpack|) (token: Equinox.Store.StreamToken) : CollectionStream*Position = let t = unbox token.value in t.stream,t.pos let supersedes (Unpack (_,currentPos)) (Unpack (_,xPos)) = let currentVersion, newVersion = currentPos.index, xPos.index let currentETag, newETag = currentPos.etag, xPos.etag newVersion > currentVersion || currentETag <> newETag -namespace Equinox.Cosmos.Builder +[] +module Internal = + [] + type InternalSyncResult = Written of Equinox.Store.StreamToken | ConflictUnknown of Equinox.Store.StreamToken | Conflict of Equinox.Store.StreamToken * IIndexedEvent[] + + [] + type LoadFromTokenResult<'event> = Unchanged | Found of Equinox.Store.StreamToken * 'event[] + +namespace Equinox.Cosmos open Equinox -open Equinox.Cosmos.Events // NB needs to be shadow by Equinox.Cosmos -open Equinox.Cosmos +open Equinox.Cosmos.Store open Equinox.Store.Infrastructure open FSharp.Control -open Microsoft.Azure.Documents open Serilog open System -[] -module Internal = - [] - type InternalSyncResult = Written of Storage.StreamToken | ConflictUnknown of Storage.StreamToken | Conflict of Storage.StreamToken * IIndexedEvent[] - - [] - type LoadFromTokenResult<'event> = Unchanged | Found of Storage.StreamToken * 'event[] - /// Defines policies for retrying with respect to transient failures calling CosmosDb (as opposed to application level concurrency conflicts) -type EqxConnection(client: IDocumentClient, ?readRetryPolicy: IRetryPolicy, ?writeRetryPolicy) = +type EqxConnection(client: Microsoft.Azure.Documents.IDocumentClient, ?readRetryPolicy: IRetryPolicy, ?writeRetryPolicy) = member __.Client = client member __.TipRetryPolicy = readRetryPolicy member __.QueryRetryPolicy = readRetryPolicy @@ -703,37 +703,37 @@ type EqxGateway(conn : EqxConnection, batching : EqxBatchingPolicy) = match Array.tryFindIndexBack (tryDecode >> Option.exists isOrigin) xs with | None -> None | Some index -> xs |> Seq.skip index |> Seq.choose tryDecode |> Array.ofSeq |> Some - member __.LoadBackwardsStopping log stream (tryDecode,isOrigin): Async = async { + member __.LoadBackwardsStopping log stream (tryDecode,isOrigin): Async = async { let! pos, events = Query.walk log conn.Client conn.QueryRetryPolicy batching.MaxItems batching.MaxRequests Direction.Backward stream None (tryDecode,isOrigin) Array.Reverse events return Token.create stream pos, events } - member __.Read log stream direction startPos (tryDecode,isOrigin) : Async = async { + member __.Read log stream direction startPos (tryDecode,isOrigin) : Async = async { let! pos, events = Query.walk log conn.Client conn.QueryRetryPolicy batching.MaxItems batching.MaxRequests direction stream startPos (tryDecode,isOrigin) return Token.create stream pos, events } member __.ReadLazy (batching: EqxBatchingPolicy) log stream direction startPos (tryDecode,isOrigin) : AsyncSeq<'event[]> = Query.walkLazy log conn.Client conn.QueryRetryPolicy batching.MaxItems batching.MaxRequests direction stream startPos (tryDecode,isOrigin) - member __.LoadFromUnfoldsOrRollingSnapshots log (stream,maybePos) (tryDecode,isOrigin): Async = async { + member __.LoadFromUnfoldsOrRollingSnapshots log (stream,maybePos) (tryDecode,isOrigin): Async = async { let! res = Tip.tryLoad log conn.TipRetryPolicy conn.Client stream maybePos match res with - | Tip.Result.NotFound -> return Token.create stream Position.FromKnownEmpty, Array.empty + | Tip.Result.NotFound -> return Token.create stream Position.fromKnownEmpty, Array.empty | Tip.Result.NotModified -> return invalidOp "Not handled" | Tip.Result.Found (pos, FromUnfold tryDecode isOrigin span) -> return Token.create stream pos, span | _ -> return! __.LoadBackwardsStopping log stream (tryDecode,isOrigin) } - member __.GetPosition(log, stream, ?pos): Async = async { + member __.GetPosition(log, stream, ?pos): Async = async { let! res = Tip.tryLoad log conn.TipRetryPolicy conn.Client stream pos match res with - | Tip.Result.NotFound -> return Token.create stream Position.FromKnownEmpty + | Tip.Result.NotFound -> return Token.create stream Position.fromKnownEmpty | Tip.Result.NotModified -> return Token.create stream pos.Value | Tip.Result.Found (pos, _unfoldsAndEvents) -> return Token.create stream pos } member __.LoadFromToken(log, (stream,pos), (tryDecode, isOrigin)): Async> = async { let! res = Tip.tryLoad log conn.TipRetryPolicy conn.Client stream (Some pos) match res with - | Tip.Result.NotFound -> return LoadFromTokenResult.Found (Token.create stream Position.FromKnownEmpty,Array.empty) + | Tip.Result.NotFound -> return LoadFromTokenResult.Found (Token.create stream Position.fromKnownEmpty,Array.empty) | Tip.Result.NotModified -> return LoadFromTokenResult.Unchanged | Tip.Result.Found (pos, FromUnfold tryDecode isOrigin span) -> return LoadFromTokenResult.Found (Token.create stream pos, span) | _ -> let! res = __.Read log stream Direction.Forward (Some pos) (tryDecode,isOrigin) return LoadFromTokenResult.Found res } - member __.Sync log stream (expectedVersion, batch: Store.Tip): Async = async { + member __.Sync log stream (expectedVersion, batch: Tip): Async = async { let! wr = Sync.batch log conn.WriteRetryPolicy conn.Client stream (expectedVersion,batch,batching.MaxItems) match wr with | Sync.Result.Conflict (pos',events) -> return InternalSyncResult.Conflict (Token.create stream pos',events) @@ -743,23 +743,18 @@ type EqxGateway(conn : EqxConnection, batching : EqxBatchingPolicy) = type private Category<'event, 'state>(gateway : EqxGateway, codec : UnionCodec.IUnionEncoder<'event, byte[]>) = let tryDecode (x: #IEvent) = codec.TryDecode { caseName = x.EventType; payload = x.Data } let (|TryDecodeFold|) (fold: 'state -> 'event seq -> 'state) initial (events: IIndexedEvent seq) : 'state = Seq.choose tryDecode events |> fold initial - member __.Load includeUnfolds collectionStream fold initial isOrigin (log : ILogger): Async = async { + member __.Load includeUnfolds collectionStream fold initial isOrigin (log : ILogger): Async = async { let! token, events = if not includeUnfolds then gateway.LoadBackwardsStopping log collectionStream (tryDecode,isOrigin) else gateway.LoadFromUnfoldsOrRollingSnapshots log (collectionStream,None) (tryDecode,isOrigin) return token, fold initial events } - member __.LoadFromToken (Token.Unpack streamPos, state: 'state as current) fold isOrigin (log : ILogger): Async = async { + member __.LoadFromToken (Token.Unpack streamPos, state: 'state as current) fold isOrigin (log : ILogger): Async = async { let! res = gateway.LoadFromToken(log, streamPos, (tryDecode,isOrigin)) match res with | LoadFromTokenResult.Unchanged -> return current | LoadFromTokenResult.Found (token', events') -> return token', fold state events' } - member __.Sync(Token.Unpack (stream,pos), state as current, expectedVersion, events, unfold, fold, isOrigin, log): Async> = async { - let encodeEvent (x : 'event) : IEvent = - let e = codec.Encode x - { new IEvent with - member __.EventType = e.caseName - member __.Data = e.payload - member __.Meta = null } + member __.Sync(Token.Unpack (stream,pos), state as current, expectedVersion, events, unfold, fold, isOrigin, log): Async> = async { + let encodeEvent (x : 'event) : IEvent = let e = codec.Encode x in EventData.Create(e.caseName,e.payload) :> _ let state' = fold state (Seq.ofList events) let eventsEncoded, projectionsEncoded = Seq.map encodeEvent events |> Array.ofSeq, Seq.map encodeEvent (unfold state' events) let baseIndex = pos.index + int64 (List.length events) @@ -767,14 +762,14 @@ type private Category<'event, 'state>(gateway : EqxGateway, codec : UnionCodec.I let batch = Sync.mkBatch stream eventsEncoded projections let! res = gateway.Sync log stream (expectedVersion,batch) match res with - | InternalSyncResult.Conflict (token',TryDecodeFold fold state events') -> return Storage.SyncResult.Conflict (async { return token', events' }) - | InternalSyncResult.ConflictUnknown _token' -> return Storage.SyncResult.Conflict (__.LoadFromToken current fold isOrigin log) - | InternalSyncResult.Written token' -> return Storage.SyncResult.Written (token', state') } + | InternalSyncResult.Conflict (token',TryDecodeFold fold state events') -> return Store.SyncResult.Conflict (async { return token', events' }) + | InternalSyncResult.ConflictUnknown _token' -> return Store.SyncResult.Conflict (__.LoadFromToken current fold isOrigin log) + | InternalSyncResult.Written token' -> return Store.SyncResult.Written (token', state') } module Caching = open System.Runtime.Caching [] - type CacheEntry<'state>(initialToken : Storage.StreamToken, initialState :'state) = + type CacheEntry<'state>(initialToken : Store.StreamToken, initialState :'state) = let mutable currentToken, currentState = initialToken, initialState member __.UpdateIfNewer (other : CacheEntry<'state>) = lock __ <| fun () -> @@ -782,7 +777,7 @@ module Caching = if otherToken |> Token.supersedes currentToken then currentToken <- otherToken currentState <- otherState - member __.Value : Storage.StreamToken * 'state = + member __.Value : Store.StreamToken * 'state = lock __ <| fun () -> currentToken, currentState @@ -803,29 +798,29 @@ module Caching = | x -> failwithf "TryGet Incompatible cache entry %A" x /// Forwards all state changes in all streams of an ICategory to a `tee` function - type CategoryTee<'event, 'state>(inner: ICategory<'event, 'state>, tee : string -> Storage.StreamToken * 'state -> unit) = + type CategoryTee<'event, 'state>(inner: Store.ICategory<'event, 'state, CollectionStream>, tee : string -> Store.StreamToken * 'state -> unit) = let intercept streamName tokenAndState = tee streamName tokenAndState tokenAndState let interceptAsync load streamName = async { let! tokenAndState = load return intercept streamName tokenAndState } - interface ICategory<'event, 'state> with - member __.Load (streamName : string) (log : ILogger) : Async = - interceptAsync (inner.Load streamName log) streamName + interface Store.ICategory<'event, 'state, CollectionStream> with + member __.Load stream (log : ILogger) : Async = + interceptAsync (inner.Load stream log) stream.name member __.TrySync (log : ILogger) (Token.Unpack (stream,_) as streamToken,state) (events : 'event list) - : Async> = async { + : Async> = async { let! syncRes = inner.TrySync log (streamToken, state) events match syncRes with - | Storage.SyncResult.Conflict resync -> return Storage.SyncResult.Conflict (interceptAsync resync stream.name) - | Storage.SyncResult.Written (token', state') ->return Storage.SyncResult.Written (intercept stream.name (token', state')) } + | Store.SyncResult.Conflict resync -> return Store.SyncResult.Conflict (interceptAsync resync stream.name) + | Store.SyncResult.Written (token', state') ->return Store.SyncResult.Written (intercept stream.name (token', state')) } let applyCacheUpdatesWithSlidingExpiration (cache: Cache) (prefix: string) (slidingExpiration : TimeSpan) - (category: ICategory<'event, 'state>) - : ICategory<'event, 'state> = + (category: Store.ICategory<'event, 'state, CollectionStream>) + : Store.ICategory<'event, 'state, CollectionStream> = let policy = new CacheItemPolicy(SlidingExpiration = slidingExpiration) let addOrUpdateSlidingExpirationCacheEntry streamName = CacheEntry >> cache.UpdateIfNewer policy (prefix + streamName) CategoryTee<'event,'state>(category, addOrUpdateSlidingExpirationCacheEntry) :> _ @@ -833,34 +828,35 @@ module Caching = type private Folder<'event, 'state> ( category: Category<'event, 'state>, fold: 'state -> 'event seq -> 'state, initial: 'state, isOrigin: 'event -> bool, - mkCollectionStream: string -> Store.CollectionStream, // Whether or not an `unfold` function is supplied controls whether reads do a point read before querying ?unfold: ('state -> 'event list -> 'event seq), ?readCache) = - interface ICategory<'event, 'state> with - member __.Load streamName (log : ILogger): Async = - let collStream = mkCollectionStream streamName + interface Store.ICategory<'event, 'state, CollectionStream> with + member __.Load collStream (log : ILogger): Async = let batched = category.Load (Option.isSome unfold) collStream fold initial isOrigin log let cached tokenAndState = category.LoadFromToken tokenAndState fold isOrigin log match readCache with | None -> batched | Some (cache : Caching.Cache, prefix : string) -> - match cache.TryGet(prefix + streamName) with + match cache.TryGet(prefix + collStream.name) with | None -> batched | Some tokenAndState -> cached tokenAndState member __.TrySync (log : ILogger) (Token.Unpack (_stream,pos) as streamToken,state) (events : 'event list) - : Async> = async { + : Async> = async { let! res = category.Sync((streamToken,state), Some pos.index, events, (defaultArg unfold (fun _ _ -> Seq.empty)), fold, isOrigin, log) match res with - | Storage.SyncResult.Conflict resync -> return Storage.SyncResult.Conflict resync - | Storage.SyncResult.Written (token',state') -> return Storage.SyncResult.Written (token',state') } + | Store.SyncResult.Conflict resync -> return Store.SyncResult.Conflict resync + | Store.SyncResult.Written (token',state') -> return Store.SyncResult.Written (token',state') } /// Defines a process for mapping from a Stream Name to the appropriate storage area, allowing control over segregation / co-locating of data -type EqxCollections(selectDatabaseAndCollection : string -> string*string) = - new (databaseId, collectionId) = EqxCollections(fun _streamName -> databaseId, collectionId) - member __.CollectionForStream streamName = - let databaseId, collectionId = selectDatabaseAndCollection streamName - Store.CollectionStream.Create(Client.UriFactory.CreateDocumentCollectionUri(databaseId, collectionId), streamName) +type EqxCollections(categoryAndIdToDatabaseCollectionAndStream : string -> string -> string*string*string) = + new (databaseId, collectionId) = + // TOCONSIDER - this works to support the Core.Events APIs + let genStreamName categoryName streamId = if categoryName = null then streamId else sprintf "%s-%s" categoryName streamId + EqxCollections(fun categoryName streamId -> databaseId, collectionId, genStreamName categoryName streamId) + member __.CollectionForStream (categoryName,id) : CollectionStream = + let databaseId, collectionId, streamName = categoryAndIdToDatabaseCollectionAndStream categoryName id + { collectionUri = Microsoft.Azure.Documents.Client.UriFactory.CreateDocumentCollectionUri(databaseId, collectionId); name = streamName } /// Pairs a Gateway, defining the retry policies for CosmosDb with an EqxCollections to type EqxStore(gateway: EqxGateway, collections: EqxCollections) = @@ -885,27 +881,38 @@ type AccessStrategy<'event,'state> = /// Trust every event type as being an origin | AnyKnownEventType -type EqxStreamBuilder<'event, 'state>(store : EqxStore, codec, fold, initial, ?access, ?caching) = - member __.Create streamName : Equinox.IStream<'event, 'state> = - let readCacheOption = - match caching with - | None -> None - | Some (CachingStrategy.SlidingWindow(cache, _)) -> Some(cache, null) - let isOrigin, projectOption = - match access with - | None -> (fun _ -> false), None - | Some (AccessStrategy.Unfolded (isOrigin, unfold)) -> isOrigin, Some (fun state _events -> unfold state) - | Some (AccessStrategy.Snapshot (isValid,generate)) -> isValid, Some (fun state _events -> seq [generate state]) - | Some (AccessStrategy.AnyKnownEventType) -> (fun _ -> true), Some (fun _ events -> Seq.last events |> Seq.singleton) - let category = Category<'event, 'state>(store.Gateway, codec) - let folder = Folder<'event, 'state>(category, fold, initial, isOrigin, store.Collections.CollectionForStream, ?unfold=projectOption, ?readCache = readCacheOption) - let category : ICategory<_,_> = - match caching with - | None -> folder :> _ - | Some (CachingStrategy.SlidingWindow(cache, window)) -> - Caching.applyCacheUpdatesWithSlidingExpiration cache null window folder - - Equinox.Stream.create category streamName +type EqxResolver<'event, 'state>(store : EqxStore, codec, fold, initial, ?access, ?caching) = + let readCacheOption = + match caching with + | None -> None + | Some (CachingStrategy.SlidingWindow(cache, _)) -> Some(cache, null) + let isOrigin, projectOption = + match access with + | None -> (fun _ -> false), None + | Some (AccessStrategy.Unfolded (isOrigin, unfold)) -> isOrigin, Some (fun state _events -> unfold state) + | Some (AccessStrategy.Snapshot (isValid,generate)) -> isValid, Some (fun state _events -> seq [generate state]) + | Some (AccessStrategy.AnyKnownEventType) -> (fun _ -> true), Some (fun _ events -> Seq.last events |> Seq.singleton) + let cosmosCat = Category<'event, 'state>(store.Gateway, codec) + let folder = Folder<'event, 'state>(cosmosCat, fold, initial, isOrigin, ?unfold=projectOption, ?readCache = readCacheOption) + let category : Store.ICategory<_,_,CollectionStream> = + match caching with + | None -> folder :> _ + | Some (CachingStrategy.SlidingWindow(cache, window)) -> + Caching.applyCacheUpdatesWithSlidingExpiration cache null window folder + + let mkStreamName = store.Collections.CollectionForStream + let resolve = Equinox.Store.Stream.create category + + member __.Resolve = function + | Target.CatId (categoryName,streamId) -> + resolve <| mkStreamName (categoryName, streamId) + | Target.CatIdEmpty (categoryName,streamId) -> + let stream = mkStreamName (categoryName, streamId) + Store.Stream.ofMemento (Token.create stream Position.fromKnownEmpty,initial) (resolve stream) + | Target.DeprecatedRawName _ as x -> failwithf "Stream name not supported: %A" x + + member __.FromMemento(Token.Unpack (stream,_pos) as streamToken,state) = + Store.Stream.ofMemento (streamToken,state) (resolve stream) [] type Discovery = @@ -928,6 +935,7 @@ type ConnectionMode = // More efficient than Gateway, but suboptimal | DirectHttps +open Microsoft.Azure.Documents type EqxConnector ( requestTimeout: TimeSpan, maxRetryAttemptsOnThrottledRequests: int, maxRetryWaitTimeInSeconds: int, log : ILogger, @@ -985,8 +993,7 @@ type EqxConnector namespace Equinox.Cosmos.Core open Equinox.Cosmos -open Equinox.Cosmos.Builder -open Equinox.Cosmos.Events +open Equinox.Cosmos.Store open FSharp.Control /// Outcome of appending events, specifying the new and/or conflicting events, together with the updated Target write position @@ -1028,7 +1035,7 @@ type EqxContext let! (Token.Unpack (_,pos')), data = res return pos', data } - member __.CreateStream(streamName) = collections.CollectionForStream streamName + member __.CreateStream(streamName) = collections.CollectionForStream(null, streamName) member internal __.GetLazy((stream, startPos), ?batchSize, ?direction) : AsyncSeq = let direction = defaultArg direction Direction.Forward @@ -1040,7 +1047,7 @@ type EqxContext let direction = defaultArg direction Direction.Forward if maxCount = Some 0 then // Search semantics include the first hit so we need to special case this anyway - return Token.create stream (defaultArg startPos Position.FromKnownEmpty), Array.empty + return Token.create stream (defaultArg startPos Position.fromKnownEmpty), Array.empty else let isOrigin = match maxCount with @@ -1070,14 +1077,14 @@ type EqxContext let batch = Sync.mkBatch stream events Seq.empty let! res = gateway.Sync logger stream (Some position.index,batch) match res with - | Builder.Internal.InternalSyncResult.Written (Token.Unpack (_,pos)) -> return AppendResult.Ok pos - | Builder.Internal.InternalSyncResult.Conflict (Token.Unpack (_,pos),events) -> return AppendResult.Conflict (pos, events) - | Builder.Internal.InternalSyncResult.ConflictUnknown (Token.Unpack (_,pos)) -> return AppendResult.ConflictUnknown pos } + | InternalSyncResult.Written (Token.Unpack (_,pos)) -> return AppendResult.Ok pos + | InternalSyncResult.Conflict (Token.Unpack (_,pos),events) -> return AppendResult.Conflict (pos, events) + | InternalSyncResult.ConflictUnknown (Token.Unpack (_,pos)) -> return AppendResult.ConflictUnknown pos } /// Low level, non-idempotent call appending events to a stream without a concurrency control mechanism in play /// NB Should be used sparingly; Equinox.Handler enables building equivalent equivalent idempotent handling with minimal code. member __.NonIdempotentAppend(stream, events: IEvent[]) : Async = async { - let! res = __.Sync(stream, Position.FromAppendAtEnd, events) + let! res = __.Sync(stream, Position.fromAppendAtEnd, events) match res with | AppendResult.Ok token -> return token | x -> return x |> sprintf "Conflict despite it being disabled %A" |> invalidOp } @@ -1100,10 +1107,13 @@ module Events = return xs } let (|MinPosition|) = function | 0L -> None - | i -> Some (Position.FromI i) + | i -> Some (Position.fromI i) let (|MaxPosition|) = function | int64.MaxValue -> None - | i -> Some (Position.FromI (i + 1L)) + | i -> Some (Position.fromI (i + 1L)) + + /// Creates an Event record, suitable for supplying to Append et al + let create eventType data meta = Store.EventData.Create(eventType, data, meta) :> IEvent /// Returns an async sequence of events in the stream starting at the specified sequence number, /// reading in batches of the specified size. @@ -1123,7 +1133,7 @@ module Events = /// If the specified expected sequence number does not match the stream, the events are not appended /// and a failure is returned. let append (ctx: EqxContext) (streamName: string) (index: int64) (events: IEvent[]): Async> = - ctx.Sync(ctx.CreateStream streamName, Position.FromI index, events) |> stripSyncResult + ctx.Sync(ctx.CreateStream streamName, Position.fromI index, events) |> stripSyncResult /// Appends a batch of events to a stream at the the present Position without any conflict checks. /// NB typically, it is recommended to ensure idempotency of operations by using the `append` and related API as diff --git a/tests/Equinox.Cosmos.Integration/CosmosCoreIntegration.fs b/tests/Equinox.Cosmos.Integration/CosmosCoreIntegration.fs index 29864744e..388fb2d12 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosCoreIntegration.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosCoreIntegration.fs @@ -1,8 +1,8 @@ module Equinox.Cosmos.Integration.CoreIntegration -open Equinox.Cosmos.Integration.Infrastructure open Equinox.Cosmos open Equinox.Cosmos.Core +open Equinox.Cosmos.Integration.Infrastructure open FSharp.Control open Newtonsoft.Json.Linq open Swensen.Unquote @@ -12,15 +12,13 @@ open System.Text #nowarn "1182" // From hereon in, we may have some 'unused' privates (the tests) -type EventData = { eventType:string; data: byte[] } with - interface Events.IEvent with - member __.EventType = __.eventType - member __.Data = __.data - member __.Meta = Encoding.UTF8.GetBytes("{\"m\":\"m\"}") - static member private Create(i, ?eventType, ?json) : Events.IEvent = - { eventType = sprintf "%s:%d" (defaultArg eventType "test_event") i - data = System.Text.Encoding.UTF8.GetBytes(defaultArg json "{\"d\":\"d\"}") } :> _ - static member Create(i, c) = Array.init c (fun x -> EventData.Create(x+i)) +type TestEvents() = + static member private Create(i, ?eventType, ?json) = + Events.create + (sprintf "%s:%d" (defaultArg eventType "test_event") i) + (Encoding.UTF8.GetBytes(defaultArg json "{\"d\":\"d\"}")) + (Encoding.UTF8.GetBytes "{\"m\":\"m\"}") + static member Create(i, c) = Array.init c (fun x -> TestEvents.Create(x+i)) type Tests(testOutputHelper) = inherit TestsWithLogCapture(testOutputHelper) @@ -45,14 +43,14 @@ type Tests(testOutputHelper) = let ctx = mkContext conn let index = 0L - let! res = Events.append ctx streamName index <| EventData.Create(0,1) + let! res = Events.append ctx streamName index <| TestEvents.Create(0,1) test <@ AppendResult.Ok 1L = res @> test <@ [EqxAct.Append] = capture.ExternalCalls @> verifyRequestChargesMax 10 // Clear the counters capture.Clear() - let! res = Events.append ctx streamName 1L <| EventData.Create(1,5) + let! res = Events.append ctx streamName 1L <| TestEvents.Create(1,5) test <@ AppendResult.Ok 6L = res @> test <@ [EqxAct.Append] = capture.ExternalCalls @> // We didnt request small batches or splitting so it's not dramatically more expensive to write N events @@ -71,26 +69,26 @@ type Tests(testOutputHelper) = let add6EventsIn2Batches ctx streamName = async { let index = 0L - let! res = Events.append ctx streamName index <| EventData.Create(0,1) + let! res = Events.append ctx streamName index <| TestEvents.Create(0,1) test <@ AppendResult.Ok 1L = res @> - let! res = Events.append ctx streamName 1L <| EventData.Create(1,5) + let! res = Events.append ctx streamName 1L <| TestEvents.Create(1,5) test <@ AppendResult.Ok 6L = res @> // Only start counting RUs from here capture.Clear() - return EventData.Create(0,6) + return TestEvents.Create(0,6) } - let verifyCorrectEventsEx direction baseIndex (expected: Events.IEvent []) (xs: Events.IIndexedEvent[]) = + let verifyCorrectEventsEx direction baseIndex (expected: Store.IEvent []) (xs: Store.IIndexedEvent[]) = let xs, baseIndex = - if direction = Direction.Forward then xs, baseIndex + if direction = Equinox.Cosmos.Store.Direction.Forward then xs, baseIndex else Array.rev xs, baseIndex - int64 (Array.length expected) + 1L test <@ [for i in 0..expected.Length - 1 -> baseIndex + int64 i] = [for r in xs -> r.Index] @> test <@ [for e in expected -> e.EventType] = [ for r in xs -> r.EventType ] @> for i,x,y in Seq.mapi2 (fun i x y -> i,x,y) [for e in expected -> e.Data] [for r in xs -> r.Data] do verifyUtf8JsonEquals i x y - let verifyCorrectEventsBackward = verifyCorrectEventsEx Direction.Backward - let verifyCorrectEvents = verifyCorrectEventsEx Direction.Forward + let verifyCorrectEventsBackward = verifyCorrectEventsEx Equinox.Cosmos.Store.Direction.Backward + let verifyCorrectEvents = verifyCorrectEventsEx Equinox.Cosmos.Store.Direction.Forward [] let ``appendAtEnd and getNextIndex`` (extras, TestStream streamName) = Async.RunSynchronously <| async { @@ -107,7 +105,7 @@ type Tests(testOutputHelper) = let mutable pos = 0L for appendBatchSize in [4; 5; 9] do - let! res = Events.appendAtEnd ctx streamName <| EventData.Create (int pos,appendBatchSize) + let! res = Events.appendAtEnd ctx streamName <| TestEvents.Create (int pos,appendBatchSize) test <@ [EqxAct.Append] = capture.ExternalCalls @> pos <- pos + int64 appendBatchSize pos =! res @@ -120,7 +118,7 @@ type Tests(testOutputHelper) = pos =! res capture.Clear() - let! res = Events.appendAtEnd ctx streamName <| EventData.Create (int pos,42) + let! res = Events.appendAtEnd ctx streamName <| TestEvents.Create (int pos,42) pos <- pos + 42L pos =! res test <@ [EqxAct.Append] = capture.ExternalCalls @> @@ -137,7 +135,7 @@ type Tests(testOutputHelper) = let stream = ctx.CreateStream streamName let extrasCount = match extras with x when x > 50 -> 5000 | x when x < 1 -> 1 | x -> x*100 - let! _pos = ctx.NonIdempotentAppend(stream, EventData.Create (int pos,extrasCount)) + let! _pos = ctx.NonIdempotentAppend(stream, TestEvents.Create (int pos,extrasCount)) test <@ [EqxAct.Append] = capture.ExternalCalls @> verifyRequestChargesMax 300 // 278 observed capture.Clear() @@ -159,7 +157,7 @@ type Tests(testOutputHelper) = let ctx = mkContext conn // Attempt to write, skipping Index 0 - let! res = Events.append ctx streamName 1L <| EventData.Create(0,1) + let! res = Events.append ctx streamName 1L <| TestEvents.Create(0,1) test <@ [EqxAct.Resync] = capture.ExternalCalls @> // The response aligns with a normal conflict in that it passes the entire set of conflicting events () test <@ AppendResult.Conflict (0L,[||]) = res @> @@ -167,15 +165,15 @@ type Tests(testOutputHelper) = capture.Clear() // Now write at the correct position - let expected = EventData.Create(1,1) + let expected = TestEvents.Create(1,1) let! res = Events.append ctx streamName 0L expected test <@ AppendResult.Ok 1L = res @> test <@ [EqxAct.Append] = capture.ExternalCalls @> - verifyRequestChargesMax 10 + verifyRequestChargesMax 11 // 10.33 capture.Clear() // Try overwriting it (a competing consumer would see the same) - let! res = Events.append ctx streamName 0L <| EventData.Create(-42,2) + let! res = Events.append ctx streamName 0L <| TestEvents.Create(-42,2) // This time we get passed the conflicting events - we pay a little for that, but that's unavoidable match res with | AppendResult.Conflict (1L, e) -> verifyCorrectEvents 0L expected e @@ -233,7 +231,9 @@ type Tests(testOutputHelper) = verifyCorrectEvents 0L expected res test <@ [EqxAct.ResponseForward; EqxAct.QueryForward] = capture.ExternalCalls @> - let queryRoundTripsAndItemCounts = function EqxEvent (Log.Query (Direction.Forward, responses, { count = c })) -> Some (responses,c) | _ -> None + let queryRoundTripsAndItemCounts = function + | EqxEvent (Equinox.Cosmos.Store.Log.Event.Query (Equinox.Cosmos.Store.Direction.Forward, responses, { count = c })) -> Some (responses,c) + | _ -> None // validate that, despite only requesting max 1 item, we only needed one trip (which contained only one item) [1,1] =! capture.ChooseCalls queryRoundTripsAndItemCounts verifyRequestChargesMax 3 // 2.97 @@ -285,14 +285,20 @@ type Tests(testOutputHelper) = let! expected = add6EventsIn2Batches ctx streamName capture.Clear() - let! res = Events.getAllBackwards ctx streamName 10L 1 |> AsyncSeq.concatSeq |> AsyncSeq.takeWhileInclusive (fun x -> x.Index <> 2L) |> AsyncSeq.toArrayAsync + let! res = + Events.getAllBackwards ctx streamName 10L 1 + |> AsyncSeq.concatSeq + |> AsyncSeq.takeWhileInclusive (fun x -> x.Index <> 2L) + |> AsyncSeq.toArrayAsync let expected = expected |> Array.skip 2 // omit index 0, 1 as we vote to finish at 2L verifyCorrectEventsBackward 5L expected res // only 1 request of 1 item triggered test <@ [EqxAct.ResponseBackward; EqxAct.QueryBackward] = capture.ExternalCalls @> // validate that, despite only requesting max 1 item, we only needed one trip, bearing 5 items (from which one item was omitted) - let queryRoundTripsAndItemCounts = function EqxEvent (Log.Query (Direction.Backward, responses, { count = c })) -> Some (responses,c) | _ -> None + let queryRoundTripsAndItemCounts = function + | EqxEvent (Equinox.Cosmos.Store.Log.Event.Query (Equinox.Cosmos.Store.Direction.Backward, responses, { count = c })) -> Some (responses,c) + | _ -> None [1,5] =! capture.ChooseCalls queryRoundTripsAndItemCounts verifyRequestChargesMax 3 // 2.98 } \ No newline at end of file diff --git a/tests/Equinox.Cosmos.Integration/CosmosFixtures.fs b/tests/Equinox.Cosmos.Integration/CosmosFixtures.fs index b6515d81d..6b030b164 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosFixtures.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosFixtures.fs @@ -1,7 +1,7 @@ [] module Equinox.Cosmos.Integration.CosmosFixtures -open Equinox.Cosmos.Builder +open Equinox.Cosmos open System module Option = diff --git a/tests/Equinox.Cosmos.Integration/CosmosFixturesInfrastructure.fs b/tests/Equinox.Cosmos.Integration/CosmosFixturesInfrastructure.fs index 9cc24fd40..d2dcbed15 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosFixturesInfrastructure.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosFixturesInfrastructure.fs @@ -50,7 +50,8 @@ module SerilogHelpers = let (|SerilogScalar|_|) : Serilog.Events.LogEventPropertyValue -> obj option = function | (:? ScalarValue as x) -> Some x.Value | _ -> None - open Equinox.Cosmos + open Equinox.Cosmos.Store + open Equinox.Cosmos.Store.Log [] type EqxAct = | Tip | TipNotFound | TipNotModified @@ -58,36 +59,36 @@ module SerilogHelpers = | QueryForward | QueryBackward | Append | Resync | Conflict let (|EqxAction|) = function - | Log.Tip _ -> EqxAct.Tip - | Log.TipNotFound _ -> EqxAct.TipNotFound - | Log.TipNotModified _ -> EqxAct.TipNotModified - | Log.Response (Direction.Forward,_) -> EqxAct.ResponseForward - | Log.Response (Direction.Backward,_) -> EqxAct.ResponseBackward - | Log.Query (Direction.Forward,_,_) -> EqxAct.QueryForward - | Log.Query (Direction.Backward,_,_) -> EqxAct.QueryBackward - | Log.SyncSuccess _ -> EqxAct.Append - | Log.SyncResync _ -> EqxAct.Resync - | Log.SyncConflict _ -> EqxAct.Conflict - let inline (|Stats|) ({ ru = ru }: Equinox.Cosmos.Log.Measurement) = ru + | Event.Tip _ -> EqxAct.Tip + | Event.TipNotFound _ -> EqxAct.TipNotFound + | Event.TipNotModified _ -> EqxAct.TipNotModified + | Event.Response (Direction.Forward,_) -> EqxAct.ResponseForward + | Event.Response (Direction.Backward,_) -> EqxAct.ResponseBackward + | Event.Query (Direction.Forward,_,_) -> EqxAct.QueryForward + | Event.Query (Direction.Backward,_,_) -> EqxAct.QueryBackward + | Event.SyncSuccess _ -> EqxAct.Append + | Event.SyncResync _ -> EqxAct.Resync + | Event.SyncConflict _ -> EqxAct.Conflict + let inline (|Stats|) ({ ru = ru }: Equinox.Cosmos.Store.Log.Measurement) = ru let (|CosmosReadRc|CosmosWriteRc|CosmosResyncRc|CosmosResponseRc|) = function - | Log.Tip (Stats s) - | Log.TipNotFound (Stats s) - | Log.TipNotModified (Stats s) + | Event.Tip (Stats s) + | Event.TipNotFound (Stats s) + | Event.TipNotModified (Stats s) // slices are rolled up into batches so be sure not to double-count - | Log.Response (_,Stats s) -> CosmosResponseRc s - | Log.Query (_,_, (Stats s)) -> CosmosReadRc s - | Log.SyncSuccess (Stats s) - | Log.SyncConflict (Stats s) -> CosmosWriteRc s - | Log.SyncResync (Stats s) -> CosmosResyncRc s + | Event.Response (_,Stats s) -> CosmosResponseRc s + | Event.Query (_,_, (Stats s)) -> CosmosReadRc s + | Event.SyncSuccess (Stats s) + | Event.SyncConflict (Stats s) -> CosmosWriteRc s + | Event.SyncResync (Stats s) -> CosmosResyncRc s /// Facilitates splitting between events with direct charges vs synthetic events Equinox generates to avoid double counting let (|CosmosRequestCharge|EquinoxChargeRollup|) = function | CosmosResponseRc _ -> EquinoxChargeRollup | CosmosReadRc rc | CosmosWriteRc rc | CosmosResyncRc rc as e -> CosmosRequestCharge (e,rc) - let (|EqxEvent|_|) (logEvent : LogEvent) : Equinox.Cosmos.Log.Event option = + let (|EqxEvent|_|) (logEvent : LogEvent) : Equinox.Cosmos.Store.Log.Event option = logEvent.Properties.Values |> Seq.tryPick (function - | SerilogScalar (:? Equinox.Cosmos.Log.Event as e) -> Some e + | SerilogScalar (:? Equinox.Cosmos.Store.Log.Event as e) -> Some e | _ -> None) let (|HasProp|_|) (name : string) (e : LogEvent) : LogEventPropertyValue option = diff --git a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs index 1ca91754f..ef40a1e43 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs @@ -1,8 +1,8 @@ module Equinox.Cosmos.Integration.CosmosIntegration open Domain +open Equinox.Cosmos open Equinox.Cosmos.Integration.Infrastructure -open Equinox.Cosmos.Builder open Swensen.Unquote open System.Threading open System @@ -16,21 +16,21 @@ module Cart = let codec = genCodec() let createServiceWithoutOptimization connection batchSize log = let store = createEqxStore connection batchSize - let resolveStream = EqxStreamBuilder(store, codec, fold, initial).Create + let resolveStream = EqxResolver(store, codec, fold, initial).Resolve Backend.Cart.Service(log, resolveStream) let createServiceWithoutOptimizationAndMaxItems connection batchSize maxEventsPerSlice log = let store = createEqxStoreWithMaxEventsPerSlice connection batchSize maxEventsPerSlice - let resolveStream = EqxStreamBuilder(store, codec, fold, initial).Create + let resolveStream = EqxResolver(store, codec, fold, initial).Resolve Backend.Cart.Service(log, resolveStream) let projection = "Compacted",snd snapshot let createServiceWithProjection connection batchSize log = let store = createEqxStore connection batchSize - let resolveStream = EqxStreamBuilder(store, codec, fold, initial, AccessStrategy.Snapshot snapshot).Create + let resolveStream = EqxResolver(store, codec, fold, initial, AccessStrategy.Snapshot snapshot).Resolve Backend.Cart.Service(log, resolveStream) let createServiceWithProjectionAndCaching connection batchSize log cache = let store = createEqxStore connection batchSize let sliding20m = CachingStrategy.SlidingWindow (cache, TimeSpan.FromMinutes 20.) - let resolveStream = EqxStreamBuilder(store, codec, fold, initial, AccessStrategy.Snapshot snapshot, sliding20m).Create + let resolveStream = EqxResolver(store, codec, fold, initial, AccessStrategy.Snapshot snapshot, sliding20m).Resolve Backend.Cart.Service(log, resolveStream) module ContactPreferences = @@ -38,10 +38,10 @@ module ContactPreferences = let codec = genCodec() let createServiceWithoutOptimization createGateway defaultBatchSize log _ignoreWindowSize _ignoreCompactionPredicate = let gateway = createGateway defaultBatchSize - let resolveStream = EqxStreamBuilder(gateway, codec, fold, initial).Create + let resolveStream = EqxResolver(gateway, codec, fold, initial).Resolve Backend.ContactPreferences.Service(log, resolveStream) let createService createGateway log = - let resolveStream = EqxStreamBuilder(createGateway 1, codec, fold, initial, AccessStrategy.AnyKnownEventType).Create + let resolveStream = EqxResolver(createGateway 1, codec, fold, initial, AccessStrategy.AnyKnownEventType).Resolve Backend.ContactPreferences.Service(log, resolveStream) #nowarn "1182" // From hereon in, we may have some 'unused' privates (the tests) diff --git a/tests/Equinox.Cosmos.Integration/JsonConverterTests.fs b/tests/Equinox.Cosmos.Integration/JsonConverterTests.fs index 24e207fb9..6ced20f0a 100644 --- a/tests/Equinox.Cosmos.Integration/JsonConverterTests.fs +++ b/tests/Equinox.Cosmos.Integration/JsonConverterTests.fs @@ -20,7 +20,7 @@ type VerbatimUtf8Tests() = [] let ``encodes correctly`` () = - let encoded = mkUnionEncoder().Encode(A { embed = "\"" }) + let encoded = unionEncoder.Encode(A { embed = "\"" }) let e : Store.Batch = { p = "streamName"; id = string 0; i = -1L; n = -1L; _etag = null e = [| { t = DateTimeOffset.MinValue; c = encoded.caseName; d = encoded.payload; m = null } |] } From 6f2b2f5b4c0259c4a9e47f8476fb7233558bcdea Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 6 Dec 2018 16:00:10 +0000 Subject: [PATCH 50/66] Rebased cosmos support for web host and CLI (#55) --- build.ps1 | 6 +- samples/Infrastructure/Infrastructure.fsproj | 2 + samples/Infrastructure/Log.fs | 43 +++++++++++++ samples/Infrastructure/Services.fs | 4 ++ samples/Infrastructure/Storage.fs | 64 +++++++++++++++++++- samples/Web/Program.fs | 3 + samples/Web/Startup.fs | 6 ++ src/Equinox.Cosmos/Cosmos.fs | 12 ++-- tools/Equinox.Tool/Program.fs | 54 ++++++++++++++++- 9 files changed, 183 insertions(+), 11 deletions(-) create mode 100644 samples/Infrastructure/Log.fs diff --git a/build.ps1 b/build.ps1 index 3b27d0148..9762133db 100644 --- a/build.ps1 +++ b/build.ps1 @@ -20,8 +20,8 @@ $env:EQUINOX_INTEGRATION_SKIP_EVENTSTORE=[string]$skipEs if ($skipEs) { warn "Skipping EventStore tests" } function cliCosmos($arghs) { - Write-Host "dotnet run cli/Equinox.Cli cosmos -s -d $cosmosDatabase -c $cosmosCollection $arghs" - dotnet run -p cli/Equinox.Cli -f netcoreapp2.1 cosmos -s $cosmosServer -d $cosmosDatabase -c $cosmosCollection @arghs + Write-Host "dotnet run cli/Equinox.Cli -- $arghs cosmos -s -d $cosmosDatabase -c $cosmosCollection" + dotnet run -p cli/Equinox.Cli -f netcoreapp2.1 -- @arghs cosmos -s $cosmosServer -d $cosmosDatabase -c $cosmosCollection } if ($skipCosmos) { @@ -30,7 +30,7 @@ if ($skipCosmos) { warn "Skipping Provisioning Cosmos" } else { warn "Provisioning cosmos..." - cliCosmos @("provision", "-ru", "1000") + cliCosmos @("init", "-ru", "1000") $deprovisionCosmos=$true } $env:EQUINOX_INTEGRATION_SKIP_COSMOS=[string]$skipCosmos diff --git a/samples/Infrastructure/Infrastructure.fsproj b/samples/Infrastructure/Infrastructure.fsproj index 53f3122df..a3b43d63b 100644 --- a/samples/Infrastructure/Infrastructure.fsproj +++ b/samples/Infrastructure/Infrastructure.fsproj @@ -11,10 +11,12 @@ + + diff --git a/samples/Infrastructure/Log.fs b/samples/Infrastructure/Log.fs new file mode 100644 index 000000000..834011cf9 --- /dev/null +++ b/samples/Infrastructure/Log.fs @@ -0,0 +1,43 @@ +module Samples.Infrastructure.Log + +open Serilog.Events + +[] +module SerilogHelpers = + open Equinox.Cosmos.Store + let inline (|Stats|) ({ interval = i; ru = ru }: Log.Measurement) = ru, let e = i.Elapsed in int64 e.TotalMilliseconds + + let (|CosmosReadRc|CosmosWriteRc|CosmosResyncRc|CosmosResponseRc|) = function + | Log.Tip (Stats s) + | Log.TipNotFound (Stats s) + | Log.TipNotModified (Stats s) + | Log.Query (_,_, (Stats s)) -> CosmosReadRc s + // slices are rolled up into batches so be sure not to double-count + | Log.Response (_,(Stats s)) -> CosmosResponseRc s + | Log.SyncSuccess (Stats s) + | Log.SyncConflict (Stats s) -> CosmosWriteRc s + | Log.SyncResync (Stats s) -> CosmosResyncRc s + let (|SerilogScalar|_|) : LogEventPropertyValue -> obj option = function + | (:? ScalarValue as x) -> Some x.Value + | _ -> None + let (|CosmosMetric|_|) (logEvent : LogEvent) : Log.Event option = + match logEvent.Properties.TryGetValue("cosmosEvt") with + | true, SerilogScalar (:? Log.Event as e) -> Some e + | _ -> None + type RuCounter = + { mutable rux100: int64; mutable count: int64; mutable ms: int64 } + static member Create() = { rux100 = 0L; count = 0L; ms = 0L } + member __.Ingest (ru, ms) = + System.Threading.Interlocked.Increment(&__.count) |> ignore + System.Threading.Interlocked.Add(&__.rux100, int64 (ru*100.)) |> ignore + System.Threading.Interlocked.Add(&__.ms, ms) |> ignore + type RuCounterSink() = + static member val Read = RuCounter.Create() + static member val Write = RuCounter.Create() + static member val Resync = RuCounter.Create() + interface Serilog.Core.ILogEventSink with + member __.Emit logEvent = logEvent |> function + | CosmosMetric (CosmosReadRc stats) -> RuCounterSink.Read.Ingest stats + | CosmosMetric (CosmosWriteRc stats) -> RuCounterSink.Write.Ingest stats + | CosmosMetric (CosmosResyncRc stats) -> RuCounterSink.Resync.Ingest stats + | _ -> () \ No newline at end of file diff --git a/samples/Infrastructure/Services.fs b/samples/Infrastructure/Services.fs index 4bd84cab6..4ec5eb6df 100644 --- a/samples/Infrastructure/Services.fs +++ b/samples/Infrastructure/Services.fs @@ -18,6 +18,10 @@ type StreamResolver(storage) = | Storage.StorageConfig.Es (gateway, cache, unfolds) -> let accessStrategy = if unfolds then Equinox.EventStore.AccessStrategy.RollingSnapshots snapshot |> Some else None Equinox.EventStore.GesResolver<'event,'state>(gateway, codec, fold, initial, ?access = accessStrategy, ?caching = cache).Resolve + | Storage.StorageConfig.Cosmos (gateway, cache, unfolds, databaseId, connectionId) -> + let store = Equinox.Cosmos.EqxStore(gateway, Equinox.Cosmos.EqxCollections(databaseId, connectionId)) + let accessStrategy = if unfolds then Equinox.Cosmos.AccessStrategy.Snapshot snapshot |> Some else None + Equinox.Cosmos.EqxResolver<'event,'state>(store, codec, fold, initial, ?access = accessStrategy, ?caching = cache).Resolve type ServiceBuilder(storageConfig, handlerLog) = let resolver = StreamResolver(storageConfig) diff --git a/samples/Infrastructure/Storage.fs b/samples/Infrastructure/Storage.fs index 57ebc8613..873d829df 100644 --- a/samples/Infrastructure/Storage.fs +++ b/samples/Infrastructure/Storage.fs @@ -9,7 +9,7 @@ type [] MemArguments = interface IArgParserTemplate with member a.Usage = a |> function | VerboseStore -> "Include low level Store logging." -and [] EsArguments = +type [] EsArguments = | [] VerboseStore | [] Timeout of float | [] Retries of int @@ -28,12 +28,35 @@ and [] EsArguments = | Password _ -> "specify a Password (default: changeit)." | ConcurrentOperationsLimit _ -> "max concurrent operations in flight (default: 5000)." | HeartbeatTimeout _ -> "specify heartbeat timeout in seconds (default: 1.5)." +type [] CosmosArguments = + | [] VerboseStore + | [] ConnectionMode of Equinox.Cosmos.ConnectionMode + | [] Timeout of float + | [] Retries of int + | [] Connection of string + | [] Database of string + | [] Collection of string + | [] RetriesWaitTime of int + | [] PageSize of int + interface IArgParserTemplate with + member a.Usage = a |> function + | VerboseStore -> "Include low level Store logging." + | ConnectionMode _ -> "Override the connection mode (default: DirectTcp)." + | Timeout _ -> "specify operation timeout in seconds (default: 5)." + | Retries _ -> "specify operation retries (default: 1)." + | Connection _ -> "specify a connection string for a Cosmos account (defaults: envvar:EQUINOX_COSMOS_CONNECTION, Cosmos Emulator)." + | Database _ -> "specify a database name for Cosmos account (defaults: envvar:EQUINOX_COSMOS_DATABASE, test)." + | Collection _ -> "specify a collection name for Cosmos account (defaults: envvar:EQUINOX_COSMOS_COLLECTION, test)." + | RetriesWaitTime _ -> "specify max wait-time for retry when being throttled by Cosmos in seconds (default: 5)" + | PageSize _ -> "Specify maximum number of events to record on a page before switching to a new one (default: 1)" + let defaultBatchSize = 500 [] type StorageConfig = | Memory of Equinox.MemoryStore.VolatileStore | Es of Equinox.EventStore.GesGateway * Equinox.EventStore.CachingStrategy option * unfolds: bool + | Cosmos of Equinox.Cosmos.EqxGateway * Equinox.Cosmos.CachingStrategy option * unfolds: bool * databaseId: string * collectionId: string module MemoryStore = let config () = @@ -69,4 +92,41 @@ module EventStore = let c = Caching.Cache("Cli", sizeMb = 50) CachingStrategy.SlidingWindow (c, TimeSpan.FromMinutes 20.) |> Some else None - StorageConfig.Es ((createGateway conn defaultBatchSize), cacheStrategy, unfolds) \ No newline at end of file + StorageConfig.Es ((createGateway conn defaultBatchSize), cacheStrategy, unfolds) + +module Cosmos = + open Equinox.Cosmos + + /// Standing up an Equinox instance is necessary to run for test purposes; You'll need to either: + /// 1) replace connection below with a connection string or Uri+Key for an initialized Equinox instance with a database and collection named "equinox-test" + /// 2) Set the 3x environment variables and create a local Equinox using cli/Equinox.cli/bin/Release/net461/Equinox.Cli ` + /// cosmos -s $env:EQUINOX_COSMOS_CONNECTION -d $env:EQUINOX_COSMOS_DATABASE -c $env:EQUINOX_COSMOS_COLLECTION provision -ru 1000 + let private connect (log: ILogger) mode discovery operationTimeout (maxRetryForThrottling, maxRetryWaitTime) = + EqxConnector(log=log, mode=mode, requestTimeout=operationTimeout, maxRetryAttemptsOnThrottledRequests=maxRetryForThrottling, maxRetryWaitTimeInSeconds=maxRetryWaitTime) + .Connect("equinox-cli", discovery) + let private createGateway connection (maxItems,maxEvents) = EqxGateway(connection, EqxBatchingPolicy(defaultMaxItems=maxItems, maxEventsPerSlice=maxEvents)) + let conn (log: ILogger, storeLog) (sargs : ParseResults) = + let read key = Environment.GetEnvironmentVariable key |> Option.ofObj + + let (Discovery.UriAndKey (connUri,_)) as discovery = + sargs.GetResult(Connection, defaultArg (read "EQUINOX_COSMOS_CONNECTION") "AccountEndpoint=https://localhost:8081;AccountKey=C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==;") + |> Discovery.FromConnectionString + + let dbName = sargs.GetResult(Database, defaultArg (read "EQUINOX_COSMOS_DATABASE") "equinox-test") + let collName = sargs.GetResult(Collection, defaultArg (read "EQUINOX_COSMOS_COLLECTION") "equinox-test") + let timeout = sargs.GetResult(Timeout,5.) |> float |> TimeSpan.FromSeconds + let mode = sargs.GetResult(ConnectionMode,ConnectionMode.DirectTcp) + let (retries, maxRetryWaitTime) as operationThrottling = sargs.GetResult(Retries, 1), sargs.GetResult(RetriesWaitTime, 5) + let pageSize = sargs.GetResult(PageSize,1) + log.Information("Using CosmosDb {mode} Connection {connection} Database: {database} Collection: {collection} maxEventsPerSlice: {pageSize}. " + + "Request timeout: {timeout} with {retries} retries; throttling MaxRetryWaitTime {maxRetryWaitTime}", + mode, connUri, dbName, collName, pageSize, timeout, retries, maxRetryWaitTime) + dbName, collName, pageSize, connect storeLog mode discovery timeout operationThrottling |> Async.RunSynchronously + let config (log: ILogger, storeLog) (cache, unfolds) (sargs : ParseResults) = + let dbName, collName, pageSize, conn = conn (log, storeLog) sargs + let cacheStrategy = + if cache then + let c = Caching.Cache("Cli", sizeMb = 50) + CachingStrategy.SlidingWindow (c, TimeSpan.FromMinutes 20.) |> Some + else None + StorageConfig.Cosmos (createGateway conn (defaultBatchSize,pageSize), cacheStrategy, unfolds, dbName, collName) \ No newline at end of file diff --git a/samples/Web/Program.fs b/samples/Web/Program.fs index 8b749f709..72cc1a9b6 100644 --- a/samples/Web/Program.fs +++ b/samples/Web/Program.fs @@ -4,6 +4,7 @@ open Argu open Microsoft.AspNetCore open Microsoft.AspNetCore.Hosting open Microsoft.Extensions.DependencyInjection +open Samples.Infrastructure.Log open Serilog module Program = @@ -27,6 +28,8 @@ module Program = .MinimumLevel.Override("Microsoft", Serilog.Events.LogEventLevel.Warning) .Enrich.FromLogContext() .WriteTo.Console() + // TOCONSIDER log and reset every minute or something ? + .WriteTo.Sink(RuCounterSink()) let c = let maybeSeq = if args.Contains LocalSeq then Some "http://localhost:5341" else None match maybeSeq with None -> c | Some endpoint -> c.WriteTo.Seq(endpoint) diff --git a/samples/Web/Startup.fs b/samples/Web/Startup.fs index b601efdb7..4837a84ef 100644 --- a/samples/Web/Startup.fs +++ b/samples/Web/Startup.fs @@ -18,6 +18,7 @@ type Arguments = | [] Unfolds | [] Memory of ParseResults | [] Es of ParseResults + | [] Cosmos of ParseResults interface IArgParserTemplate with member a.Usage = a |> function | VerboseConsole -> "Include low level Domain and Store logging in screen output." @@ -26,6 +27,7 @@ type Arguments = | Unfolds -> "employ a store-appropriate Rolling Snapshots and/or Unfolding strategy." | Memory _ -> "specify In-Memory Volatile Store (Default store)." | Es _ -> "specify storage in EventStore (--help for options)." + | Cosmos _ -> "specify storage in CosmosDb (--help for options)." type App = class end @@ -54,6 +56,10 @@ type Startup() = let storeLog = createStoreLog <| sargs.Contains EsArguments.VerboseStore log.Information("EventStore Storage options: {options:l}", options) EventStore.config (log,storeLog) (cache, unfolds) sargs, storeLog + | Some (Cosmos sargs) -> + let storeLog = createStoreLog <| sargs.Contains CosmosArguments.VerboseStore + log.Information("CosmosDb Storage options: {options:l}", options) + Cosmos.config (log,storeLog) (cache, unfolds) sargs, storeLog | _ | Some (Memory _) -> log.Fatal("Web App is using Volatile Store; Storage options: {options:l}", options) MemoryStore.config (), log diff --git a/src/Equinox.Cosmos/Cosmos.fs b/src/Equinox.Cosmos/Cosmos.fs index 7819f2b3e..b88073de4 100644 --- a/src/Equinox.Cosmos/Cosmos.fs +++ b/src/Equinox.Cosmos/Cosmos.fs @@ -953,6 +953,7 @@ type EqxConnector /// Additional strings identifying the context of this connection; should provide enough context to disambiguate all potential connections to a cluster /// NB as this will enter server and client logs, it should not contain sensitive information ?tags : (string*string) seq) = + do if log = null then nullArg "log" let connPolicy = let cp = Client.ConnectionPolicy.Default @@ -1010,7 +1011,7 @@ type EqxContext /// Database + Collection selector collections: EqxCollections, /// Logger to write to - see https://github.com/serilog/serilog/wiki/Provided-Sinks for how to wire to your logger - logger : Serilog.ILogger, + log : Serilog.ILogger, /// Optional maximum number of Store.Batch records to retrieve as a set (how many Events are placed therein is controlled by maxEventsPerSlice). /// Defaults to 10 ?defaultMaxItems, @@ -1019,6 +1020,7 @@ type EqxContext /// Threshold defining the number of events a slice is allowed to hold before switching to a new Batch is triggered. /// Defaults to 1 ?maxEventsPerSlice) = + do if log = null then nullArg "log" let getDefaultMaxItems = match getDefaultMaxItems with Some f -> f | None -> fun () -> defaultArg defaultMaxItems 10 let maxEventsPerSlice = defaultArg maxEventsPerSlice 1 let batching = EqxBatchingPolicy(getDefaultMaxItems=getDefaultMaxItems, maxEventsPerSlice=maxEventsPerSlice) @@ -1041,7 +1043,7 @@ type EqxContext let direction = defaultArg direction Direction.Forward let batchSize = defaultArg batchSize batching.MaxItems * maxEventsPerSlice let batching = EqxBatchingPolicy(if batchSize < maxEventsPerSlice then 1 else batchSize/maxEventsPerSlice) - gateway.ReadLazy batching logger stream direction startPos (Some,fun _ -> false) + gateway.ReadLazy batching log stream direction startPos (Some,fun _ -> false) member internal __.GetInternal((stream, startPos), ?maxCount, ?direction) = async { let direction = defaultArg direction Direction.Forward @@ -1053,13 +1055,13 @@ type EqxContext match maxCount with | Some limit -> maxCountPredicate limit | None -> fun _ -> false - return! gateway.Read logger stream direction startPos (Some,isOrigin) } + return! gateway.Read log stream direction startPos (Some,isOrigin) } /// Establishes the current position of the stream in as effficient a manner as possible /// (The ideal situation is that the preceding token is supplied as input in order to avail of 1RU low latency state checks) member __.Sync(stream, ?position: Position) : Async = async { //let indexed predicate = load fold initial (coll.Gateway.IndexedOrBatched log predicate (stream,None)) - let! (Token.Unpack (_,pos')) = gateway.GetPosition(logger, stream, ?pos=position) + let! (Token.Unpack (_,pos')) = gateway.GetPosition(log, stream, ?pos=position) return pos' } /// Reads in batches of `batchSize` from the specified `Position`, allowing the reader to efficiently walk away from a running query @@ -1075,7 +1077,7 @@ type EqxContext /// Callers should implement appropriate idempotent handling, or use Equinox.Handler for that purpose member __.Sync(stream, position, events: IEvent[]) : Async> = async { let batch = Sync.mkBatch stream events Seq.empty - let! res = gateway.Sync logger stream (Some position.index,batch) + let! res = gateway.Sync log stream (Some position.index,batch) match res with | InternalSyncResult.Written (Token.Unpack (_,pos)) -> return AppendResult.Ok pos | InternalSyncResult.Conflict (Token.Unpack (_,pos),events) -> return AppendResult.Conflict (pos, events) diff --git a/tools/Equinox.Tool/Program.fs b/tools/Equinox.Tool/Program.fs index 9aea3b18e..364e356d2 100644 --- a/tools/Equinox.Tool/Program.fs +++ b/tools/Equinox.Tool/Program.fs @@ -4,6 +4,7 @@ open Argu open Domain.Infrastructure open Equinox.Tool.Infrastructure open Microsoft.Extensions.DependencyInjection +open Samples.Infrastructure.Log open Samples.Infrastructure.Storage open Serilog open Serilog.Events @@ -18,6 +19,7 @@ type Arguments = | [] LocalSeq | [] LogFile of string | [] Run of ParseResults + | [] Initialize of ParseResults interface IArgParserTemplate with member a.Usage = a |> function | Verbose -> "Include low level logging regarding specific test runs." @@ -25,6 +27,14 @@ type Arguments = | LocalSeq -> "Configures writing to a local Seq endpoint at http://localhost:5341, see https://getseq.net" | LogFile _ -> "specify a log file to write the result breakdown into (default: eqx.log)." | Run _ -> "Run a load test" + | Initialize _ -> "Initialize a store" +and []InitArguments = + | [] Rus of int + | [] Cosmos of ParseResults + interface IArgParserTemplate with + member a.Usage = a |> function + | Rus _ -> "Specify RU/s level to provision for the Application Collection." + | Cosmos _ -> "Cosmos Connection parameters." and []WebArguments = | [] Endpoint of string interface IArgParserTemplate with @@ -42,6 +52,7 @@ and [] | [] ReportIntervalS of int | [] Memory of ParseResults | [] Es of ParseResults + | [] Cosmos of ParseResults | [] Web of ParseResults interface IArgParserTemplate with member a.Usage = a |> function @@ -55,12 +66,14 @@ and [] | ReportIntervalS _ -> "specify reporting intervals in seconds (default: 10)." | Memory _ -> "target in-process Transient Memory Store (Default if not other target specified)." | Es _ -> "Run transactions in-process against EventStore." + | Cosmos _ -> "Run transactions in-process against CosmosDb." | Web _ -> "Run transactions against a Web endpoint." and Test = Favorite | SaveForLater | Todo let createStoreLog verbose verboseConsole maybeSeqEndpoint = let c = LoggerConfiguration().Destructure.FSharpTypes() let c = if verbose then c.MinimumLevel.Debug() else c + let c = c.WriteTo.Sink(RuCounterSink()) let c = c.WriteTo.Console((if verbose && verboseConsole then LogEventLevel.Debug else LogEventLevel.Warning), theme = Sinks.SystemConsole.Themes.AnsiConsoleTheme.Code) let c = match maybeSeqEndpoint with None -> c | Some endpoint -> c.WriteTo.Seq(endpoint) c.CreateLogger() :> ILogger @@ -98,6 +111,10 @@ module LoadTest = let storeLog = createStoreLog <| sargs.Contains EsArguments.VerboseStore log.Information("Running transactions in-process against EventStore with storage options: {options:l}", options) storeLog, EventStore.config (log,storeLog) (cache, unfolds) sargs |> Some, None + | Some (Cosmos sargs) -> + let storeLog = createStoreLog <| sargs.Contains CosmosArguments.VerboseStore + log.Information("Running transactions in-process against CosmosDb with storage options: {options:l}", options) + storeLog, Cosmos.config (log,storeLog) (cache, unfolds) sargs |> Some, None | _ | Some (Memory _) -> log.Warning("Running transactions in-process against Volatile Store with storage options: {options:l}", options) createStoreLog false, MemoryStore.config () |> Some, None @@ -137,11 +154,37 @@ module LoadTest = resultFile.Information("Aggregate: {aggregate}", r) log.Information("Run completed; Current memory allocation: {bytes:n2} MiB", (GC.GetTotalMemory(true) |> float) / 1024./1024.) + match storeConfig with + | Some (StorageConfig.Cosmos _) -> + let stats = + [ "Read", RuCounterSink.Read + "Write", RuCounterSink.Write + "Resync", RuCounterSink.Resync ] + let mutable totalCount, totalRc, totalMs = 0L, 0., 0L + let logActivity name count rc lat = + log.Information("{name}: {count:n0} requests costing {ru:n0} RU (average: {avg:n2}); Average latency: {lat:n0}ms", + name, count, rc, (if count = 0L then Double.NaN else rc/float count), (if count = 0L then Double.NaN else float lat/float count)) + for name, stat in stats do + let ru = float stat.rux100 / 100. + totalCount <- totalCount + stat.count + totalRc <- totalRc + ru + totalMs <- totalMs + stat.ms + logActivity name stat.count ru stat.ms + logActivity "TOTAL" totalCount totalRc totalMs + let measures : (string * (TimeSpan -> float)) list = + [ "s", fun x -> x.TotalSeconds + "m", fun x -> x.TotalMinutes + "h", fun x -> x.TotalHours ] + let logPeriodicRate name count ru = log.Information("rp{name} {count:n0} = ~{ru:n0} RU", name, count, ru) + let duration = args.GetResult(DurationM,1.) |> TimeSpan.FromMinutes + for uom, f in measures do let d = f duration in if d <> 0. then logPeriodicRate uom (float totalCount/d |> int64) (totalRc/d) + | _ -> () + let createDomainLog verbose verboseConsole maybeSeqEndpoint = let c = LoggerConfiguration().Destructure.FSharpTypes().Enrich.FromLogContext() let c = if verbose then c.MinimumLevel.Debug() else c let c = c.WriteTo.Sink(RuCounterSink()) - let c = c.WriteTo.Console((if verboseConsole then LogEventLevel.Debug else LogEventLevel.Warning), theme = Sinks.SystemConsole.Themes.AnsiConsoleTheme.Code) + let c = c.WriteTo.Console((if verboseConsole then LogEventLevel.Debug else LogEventLevel.Information), theme = Sinks.SystemConsole.Themes.AnsiConsoleTheme.Code) let c = match maybeSeqEndpoint with None -> c | Some endpoint -> c.WriteTo.Seq(endpoint) c.CreateLogger() @@ -156,6 +199,15 @@ let main argv = let verbose = args.Contains Verbose let log = createDomainLog verbose verboseConsole maybeSeq match args.GetSubCommand() with + | Initialize iargs -> + let rus = iargs.GetResult(Rus) + match iargs.TryGetSubCommand() with + | Some (InitArguments.Cosmos sargs) -> + let storeLog = createStoreLog (sargs.Contains CosmosArguments.VerboseStore) verboseConsole maybeSeq + let dbName, collName, (_pageSize: int), conn = Cosmos.conn (log,storeLog) sargs + log.Information("Configuring CosmosDb Collection with Throughput Provision: {rus:n0} RU/s", rus) + Equinox.Cosmos.Store.Sync.Initialization.initialize log conn.Client dbName collName rus |> Async.RunSynchronously + | _ -> failwith "please specify a `cosmos` endpoint" | Run rargs -> let reportFilename = args.GetResult(LogFile,programName+".log") |> fun n -> System.IO.FileInfo(n).FullName LoadTest.run log (verbose,verboseConsole,maybeSeq) reportFilename rargs From 1ead911cd47e820461cdc71a3e3c69cf6febebd3 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 13 Dec 2018 10:02:22 +0000 Subject: [PATCH 51/66] Remove tip isa Batch semantics (#58) --- src/Equinox.Cosmos/Cosmos.fs | 72 ++++++++----------- .../CosmosCoreIntegration.fs | 18 +++-- .../CosmosIntegration.fs | 6 ++ 3 files changed, 48 insertions(+), 48 deletions(-) diff --git a/src/Equinox.Cosmos/Cosmos.fs b/src/Equinox.Cosmos/Cosmos.fs index b88073de4..d94ccef06 100644 --- a/src/Equinox.Cosmos/Cosmos.fs +++ b/src/Equinox.Cosmos/Cosmos.fs @@ -293,13 +293,14 @@ module Sync = // NB don't nest in a private module, or serialization will fail miserably ;) [] type SyncResponse = { etag: string; n: int64; conflicts: Event[] } - let [] sprocName = "EquinoxSync002" // NB need to renumber for any breaking change + let [] sprocName = "EquinoxNoTipEvents" // NB need to renumber for any breaking change let [] sprocBody = """ // Manages the merging of the supplied Request Batch, fulfilling one of the following end-states -// 1 Verify no current Tip batch, the incoming `req` becomes the Tip batch (the caller is entrusted to provide a valid and complete set of inputs, or it's GIGO) -// 2 Current Tip batch has space to accommodate the incoming unfolds (req.u) and events (req.e) - merge them in, replacing any superseded unfolds -// 3 Current Tip batch would become too large - remove Tip-specific state from active doc by replacing the well known id with a correct one; proceed as per 1 +// 1 perform expectedVersion verification (can request inhibiting of check by supplying -1) +// 2a Verify no current Tip; if so - incoming req.e and defines the 'next' position / unfolds +// 2b If we already have a tip, move position forward, replace unfolds +// 3 insert a new document containing the events as part of the same batch of work function sync(req, expectedVersion, maxEvents) { if (!req) throw new Error("Missing req argument"); const collection = getContext().getCollection(); @@ -316,10 +317,7 @@ function sync(req, expectedVersion, maxEvents) { // If there is no Tip page, the writer has no possible reason for writing at an index other than zero response.setBody({ etag: null, n: 0, conflicts: [] }); } else if (current && expectedVersion !== current.n) { - // Where possible, we extract conflicting events from e and/or c in order to avoid another read cycle - // yielding [] triggers the client to go loading the events itself - const conflicts = expectedVersion < current.i ? [] : current.e.slice(expectedVersion - current.i); - response.setBody({ etag: current._etag, n: current.n, conflicts: conflicts }); + response.setBody({ etag: current._etag, n: current.n, conflicts: [] }); } else { executeUpsert(current); } @@ -331,40 +329,27 @@ function sync(req, expectedVersion, maxEvents) { if (err) throw err; response.setBody({ etag: doc._etag, n: doc.n, conflicts: null }); } - // `i` is established when first written; `n` needs to stay in step with i+batch.e.length - function pos(batch, i) { - batch.i = i - batch.n = batch.i + batch.e.length; - return batch; - } - // If we have hit a sensible limit for a slice, swap to a new one - if (current && current.e.length + req.e.length > maxEvents) { - // remove the well-known `id` value identifying the batch as being the Tip - current.id = current.i.toString(); - // ... As it's no longer a Tip batch, we definitely don't want unfolds taking up space - delete current.u; - - // TODO Carry forward `u` items not present in `batch`, together with supporting catchup events from preceding batches - - // as we've mutated the document in a manner that can conflict with other writers, out write needs to be contingent on no competing updates having taken place - const tipUpdateAccepted = collection.replaceDocument(current._self, current, { etag: current._etag }, callback); - if (!tipUpdateAccepted) throw new Error("Unable to remove Tip markings."); - - const isAccepted = collection.createDocument(collectionLink, pos(req,current.n), { disableAutomaticIdGeneration: true }, callback); - if (!isAccepted) throw new Error("Unable to create Tip batch."); - } else if (current) { - // Append the new events into the current batch - Array.prototype.push.apply(current.e, req.e); - // Replace all the unfolds // TODO: should remove only unfolds being superseded - current.u = req.u; + var tip; + if (!current) { + tip = { p: req.p, id: req.id, i: req.e.length, n: req.e.length, e: [], u: req.u }; + const tipAccepted = collection.createDocument(collectionLink, tip, { disableAutomaticIdGeneration: true }, callback); + if (!tipAccepted) throw new Error("Unable to create Tip."); + } else { + // TODO Carry forward `u` items not in `req`, together with supporting catchup events from preceding batches + const n = current.n + req.e.length; + tip = { p: current.p, id: current.id, i: n, n: n, e: [], u: req.u }; // as we've mutated the document in a manner that can conflict with other writers, out write needs to be contingent on no competing updates having taken place - const isAccepted = collection.replaceDocument(current._self, pos(current, current.i), { etag: current._etag }, callback); - if (!isAccepted) throw new Error("Unable to replace Tip batch."); - } else { - const isAccepted = collection.createDocument(collectionLink, pos(req,0), { disableAutomaticIdGeneration: true }, callback); - if (!isAccepted) throw new Error("Unable to create Tip batch."); + const tipAccepted = collection.replaceDocument(current._self, tip, { etag: current._etag }, callback); + if (!tipAccepted) throw new Error("Unable to replace Tip."); } + // For now, always do an Insert, as Change Feed mechanism does not yet afford us a way to + // a) guarantee an item per write (can be squashed) + // b) with metadata sufficient for us to determine the items added (only etags, no way to convey i/n in feed item) + const i = tip.n - req.e.length; + const batch = { p: tip.p, id: i.toString(), i: i, n: tip.n, e: req.e }; + const batchAccepted = collection.createDocument(collectionLink, batch, { disableAutomaticIdGeneration: true }); + if (!batchAccepted) throw new Error("Unable to insert Batch."); } }""" @@ -499,12 +484,13 @@ module internal Tip = open FSharp.Control let private mkQuery (client : IDocumentClient) maxItems (stream: CollectionStream) (direction: Direction) startPos = let querySpec = - let fields = "c.id, c.i, c._etag, c.n, c.e" + let root = sprintf "SELECT c.id, c.i, c._etag, c.n, c.e FROM c WHERE c.id!=\"%s\"" Tip.WellKnownDocumentId + let tail = sprintf "ORDER BY c.i %s" (if direction = Direction.Forward then "ASC" else "DESC") match startPos with - | None -> SqlQuerySpec(sprintf "SELECT %s FROM c ORDER BY c.i " fields + if direction = Direction.Forward then "ASC" else "DESC") + | None -> SqlQuerySpec(sprintf "%s %s" root tail) | Some { index = positionSoExclusiveWhenBackward } -> - let f = if direction = Direction.Forward then "c.n > @startPos ORDER BY c.i ASC" else "c.i < @startPos ORDER BY c.i DESC" - SqlQuerySpec(sprintf "SELECT %s FROM c WHERE " fields + f, SqlParameterCollection [SqlParameter("@startPos", positionSoExclusiveWhenBackward)]) + let cond = if direction = Direction.Forward then "c.n > @startPos" else "c.i < @startPos" + SqlQuerySpec(sprintf "%s AND %s %s" root cond tail, SqlParameterCollection [SqlParameter("@startPos", positionSoExclusiveWhenBackward)]) let feedOptions = new Client.FeedOptions(PartitionKey=PartitionKey(stream.name), MaxItemCount=Nullable maxItems) client.CreateDocumentQuery(stream.collectionUri, querySpec, feedOptions).AsDocumentQuery() diff --git a/tests/Equinox.Cosmos.Integration/CosmosCoreIntegration.fs b/tests/Equinox.Cosmos.Integration/CosmosCoreIntegration.fs index 388fb2d12..1ced8d174 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosCoreIntegration.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosCoreIntegration.fs @@ -46,7 +46,7 @@ type Tests(testOutputHelper) = let! res = Events.append ctx streamName index <| TestEvents.Create(0,1) test <@ AppendResult.Ok 1L = res @> test <@ [EqxAct.Append] = capture.ExternalCalls @> - verifyRequestChargesMax 10 + verifyRequestChargesMax 12 // 11.78 // WAS 10 // Clear the counters capture.Clear() @@ -169,16 +169,24 @@ type Tests(testOutputHelper) = let! res = Events.append ctx streamName 0L expected test <@ AppendResult.Ok 1L = res @> test <@ [EqxAct.Append] = capture.ExternalCalls @> - verifyRequestChargesMax 11 // 10.33 + verifyRequestChargesMax 12 // 11.35 WAS 11 // 10.33 capture.Clear() // Try overwriting it (a competing consumer would see the same) let! res = Events.append ctx streamName 0L <| TestEvents.Create(-42,2) // This time we get passed the conflicting events - we pay a little for that, but that's unavoidable match res with +#if EVENTS_IN_TIP | AppendResult.Conflict (1L, e) -> verifyCorrectEvents 0L expected e +#else + | AppendResult.ConflictUnknown 1L -> () +#endif | x -> x |> failwithf "Unexpected %A" +#if EVENTS_IN_TIP test <@ [EqxAct.Resync] = capture.ExternalCalls @> +#else + test <@ [EqxAct.Conflict] = capture.ExternalCalls @> +#endif verifyRequestChargesMax 5 // 4.02 capture.Clear() } @@ -236,7 +244,7 @@ type Tests(testOutputHelper) = | _ -> None // validate that, despite only requesting max 1 item, we only needed one trip (which contained only one item) [1,1] =! capture.ChooseCalls queryRoundTripsAndItemCounts - verifyRequestChargesMax 3 // 2.97 + verifyRequestChargesMax 4 // 3.02 // WAS 3 // 2.97 } (* Backward *) @@ -256,7 +264,7 @@ type Tests(testOutputHelper) = verifyCorrectEventsBackward 4L expected res test <@ [EqxAct.ResponseBackward; EqxAct.QueryBackward] = capture.ExternalCalls @> - verifyRequestChargesMax 3 + verifyRequestChargesMax 4 // 3.04 // WAS 3 } [] @@ -300,5 +308,5 @@ type Tests(testOutputHelper) = | EqxEvent (Equinox.Cosmos.Store.Log.Event.Query (Equinox.Cosmos.Store.Direction.Backward, responses, { count = c })) -> Some (responses,c) | _ -> None [1,5] =! capture.ChooseCalls queryRoundTripsAndItemCounts - verifyRequestChargesMax 3 // 2.98 + verifyRequestChargesMax 4 // 3.04 // WAS 3 // 2.98 } \ No newline at end of file diff --git a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs index ef40a1e43..23db4fa03 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs @@ -175,9 +175,15 @@ type Tests(testOutputHelper) = && has sku21 21 && has sku22 22 @> // Intended conflicts pertained let conflict = function EqxAct.Conflict | EqxAct.Resync as x -> Some x | _ -> None +#if EVENTS_IN_TIP test <@ let c2 = List.choose conflict capture2.ExternalCalls [EqxAct.Resync] = List.choose conflict capture1.ExternalCalls && [EqxAct.Resync] = c2 @> +#else + test <@ let c2 = List.choose conflict capture2.ExternalCalls + [EqxAct.Conflict] = List.choose conflict capture1.ExternalCalls + && [EqxAct.Conflict] = c2 @> +#endif } let singleBatchBackwards = [EqxAct.ResponseBackward; EqxAct.QueryBackward] From 606efe86d19ac80c37ea12d9aa99f8e491065480 Mon Sep 17 00:00:00 2001 From: Eirik Tsarpalis Date: Wed, 12 Dec 2018 12:36:09 +0000 Subject: [PATCH 52/66] Add AsyncCacheCell --- src/Equinox.EventStore/Infrastructure.fs | 22 ++++++++++++++ .../CacheCellTests.fs | 30 +++++++++++++++++++ .../Equinox.Cosmos.Integration.fsproj | 1 + 3 files changed, 53 insertions(+) create mode 100644 tests/Equinox.Cosmos.Integration/CacheCellTests.fs diff --git a/src/Equinox.EventStore/Infrastructure.fs b/src/Equinox.EventStore/Infrastructure.fs index 22ce54431..e9cd67da6 100644 --- a/src/Equinox.EventStore/Infrastructure.fs +++ b/src/Equinox.EventStore/Infrastructure.fs @@ -118,6 +118,28 @@ type Stopwatch = return tr, result } +/// Asynchronous Lazy<'T> that guarantees workflow will be executed at most once. +type AsyncLazy<'T>(workflow : Async<'T>) = + let task = lazy(Async.StartAsTask workflow) + member __.AwaitValue() = Async.AwaitTaskCorrect task.Value + +/// Generic async lazy caching implementation that admits expiration/recomputation semantics +type AsyncCacheCell<'T>(workflow : Async<'T>, isExpired : 'T -> bool) = + let mutable currentCell = AsyncLazy workflow + + /// Gets or asynchronously recomputes a cached value depending on expiry and availability + member __.AwaitValue() = async { + let cell = currentCell + let! current = cell.AwaitValue() + if isExpired current then + // avoid unneccessary recomputation in cases where competing threads detect expiry; + // the first write attempt wins, and everybody else reads off that value. + let _ = System.Threading.Interlocked.CompareExchange(¤tCell, AsyncLazy workflow, cell) + return! currentCell.AwaitValue() + else + return current + } + [] module Regex = open System.Text.RegularExpressions diff --git a/tests/Equinox.Cosmos.Integration/CacheCellTests.fs b/tests/Equinox.Cosmos.Integration/CacheCellTests.fs new file mode 100644 index 000000000..11096acde --- /dev/null +++ b/tests/Equinox.Cosmos.Integration/CacheCellTests.fs @@ -0,0 +1,30 @@ +module Equinox.Cosmos.Integration.CacheCellTests + +open Equinox.Store.Infrastructure +open Swensen.Unquote +open System.Threading +open Xunit + +[] +let ``AsyncLazy correctness`` () = async { + // ensure that the encapsulated computation fires only once + let count = ref 0 + let cell = AsyncLazy (async { return Interlocked.Increment count }) + let! accessResult = [|1 .. 100|] |> Array.map (fun i -> cell.AwaitValue ()) |> Async.Parallel + test <@ accessResult |> Array.forall ((=) 1) @> } + +[] +let ``AsyncCacheCell correctness`` () = async { + // ensure that the encapsulated computation fires only once + // and that expiry functions as expected + let state = ref 0 + let expectedValue = ref 1 + let cell = AsyncCacheCell (async { return Interlocked.Increment state }, fun value -> value <> !expectedValue) + + let! accessResult = [|1 .. 100|] |> Array.map (fun i -> cell.AwaitValue ()) |> Async.Parallel + test <@ accessResult |> Array.forall ((=) 1) @> + + incr expectedValue + + let! accessResult = [|1 .. 100|] |> Array.map (fun i -> cell.AwaitValue ()) |> Async.Parallel + test <@ accessResult |> Array.forall ((=) 2) @> } diff --git a/tests/Equinox.Cosmos.Integration/Equinox.Cosmos.Integration.fsproj b/tests/Equinox.Cosmos.Integration/Equinox.Cosmos.Integration.fsproj index 796526db0..d8440534f 100644 --- a/tests/Equinox.Cosmos.Integration/Equinox.Cosmos.Integration.fsproj +++ b/tests/Equinox.Cosmos.Integration/Equinox.Cosmos.Integration.fsproj @@ -13,6 +13,7 @@ + From ed6feb95601ddc7efcffa6e45c8d984bc89bed02 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 12 Dec 2018 13:59:34 +0000 Subject: [PATCH 53/66] Stop caching failures --- src/Equinox.EventStore/Infrastructure.fs | 24 ++++++--- .../CacheCellTests.fs | 54 +++++++++++++++++-- 2 files changed, 67 insertions(+), 11 deletions(-) diff --git a/src/Equinox.EventStore/Infrastructure.fs b/src/Equinox.EventStore/Infrastructure.fs index e9cd67da6..34ed46f72 100644 --- a/src/Equinox.EventStore/Infrastructure.fs +++ b/src/Equinox.EventStore/Infrastructure.fs @@ -122,22 +122,32 @@ type Stopwatch = type AsyncLazy<'T>(workflow : Async<'T>) = let task = lazy(Async.StartAsTask workflow) member __.AwaitValue() = Async.AwaitTaskCorrect task.Value + member internal __.PeekInternalTask = task /// Generic async lazy caching implementation that admits expiration/recomputation semantics +/// If `workflow` fails, all readers entering while the load/refresh is in progress will share the failure type AsyncCacheCell<'T>(workflow : Async<'T>, isExpired : 'T -> bool) = let mutable currentCell = AsyncLazy workflow + let update cell = async { + // avoid unneccessary recomputation in cases where competing threads detect expiry; + // the first write attempt wins, and everybody else reads off that value. + let _ = System.Threading.Interlocked.CompareExchange(¤tCell, AsyncLazy workflow, cell) + return! currentCell.AwaitValue() + } /// Gets or asynchronously recomputes a cached value depending on expiry and availability member __.AwaitValue() = async { let cell = currentCell - let! current = cell.AwaitValue() - if isExpired current then - // avoid unneccessary recomputation in cases where competing threads detect expiry; - // the first write attempt wins, and everybody else reads off that value. - let _ = System.Threading.Interlocked.CompareExchange(¤tCell, AsyncLazy workflow, cell) - return! currentCell.AwaitValue() + let currentState = cell.PeekInternalTask + // If the last attempt completed, but failed, we need to treat it as expired; for TMI on this, see https://stackoverflow.com/a/33946166/11635 + if currentState.IsValueCreated && currentState.Value.IsCompleted && currentState.Value.Status <> System.Threading.Tasks.TaskStatus.RanToCompletion then + return! update cell else - return current + let! current = cell.AwaitValue() + if isExpired current then + return! update cell + else + return current } [] diff --git a/tests/Equinox.Cosmos.Integration/CacheCellTests.fs b/tests/Equinox.Cosmos.Integration/CacheCellTests.fs index 11096acde..b10577173 100644 --- a/tests/Equinox.Cosmos.Integration/CacheCellTests.fs +++ b/tests/Equinox.Cosmos.Integration/CacheCellTests.fs @@ -4,6 +4,7 @@ open Equinox.Store.Infrastructure open Swensen.Unquote open System.Threading open Xunit +open System [] let ``AsyncLazy correctness`` () = async { @@ -15,16 +16,61 @@ let ``AsyncLazy correctness`` () = async { [] let ``AsyncCacheCell correctness`` () = async { - // ensure that the encapsulated computation fires only once - // and that expiry functions as expected + // ensure that the encapsulated computation fires only once and that expiry functions as expected let state = ref 0 let expectedValue = ref 1 let cell = AsyncCacheCell (async { return Interlocked.Increment state }, fun value -> value <> !expectedValue) - let! accessResult = [|1 .. 100|] |> Array.map (fun i -> cell.AwaitValue ()) |> Async.Parallel + let! accessResult = [|1 .. 100|] |> Array.map (fun _i -> cell.AwaitValue ()) |> Async.Parallel test <@ accessResult |> Array.forall ((=) 1) @> incr expectedValue - let! accessResult = [|1 .. 100|] |> Array.map (fun i -> cell.AwaitValue ()) |> Async.Parallel + let! accessResult = [|1 .. 100|] |> Array.map (fun _i -> cell.AwaitValue ()) |> Async.Parallel test <@ accessResult |> Array.forall ((=) 2) @> } + +[] +let ``AsyncCacheCell correctness with throwing`` initiallyThrowing = async { + // ensure that the encapsulated computation fires only once and that expiry functions as expected + let state = ref 0 + let expectedValue = ref 1 + let mutable throwing = initiallyThrowing + let update = async { + let r = Interlocked.Increment state + if throwing then + do! Async.Sleep 2000 + invalidOp "fails" + return r + } + + let cell = AsyncCacheCell (update, fun value -> value <> !expectedValue) + + // If the runner is throwing, we want to be sure it doesn't place us in a failed state forever, per the semantics of Lazy + // However, we _do_ want to be sure that the function only runs once + if initiallyThrowing then + let! accessResult = [|1 .. 10|] |> Array.map (fun _ -> cell.AwaitValue () |> Async.Catch) |> Async.Parallel + test <@ accessResult |> Array.forall (function Choice2Of2 (:? InvalidOperationException) -> true | _ -> false) @> + throwing <- false + else + let! r = cell.AwaitValue() + test <@ 1 = r @> + + incr expectedValue + + let! accessResult = [|1 .. 100|] |> Array.map (fun _ -> cell.AwaitValue ()) |> Async.Parallel + test <@ accessResult |> Array.forall ((=) 2) @> + + // invalidate the cached value + incr expectedValue + // but make the comptutation ultimately fail + throwing <- true + // All share the failure + let! accessResult = [|1 .. 10|] |> Array.map (fun _ -> cell.AwaitValue () |> Async.Catch) |> Async.Parallel + test <@ accessResult |> Array.forall (function Choice2Of2 (:? InvalidOperationException) -> true | _ -> false) @> + // Restore normality + throwing <- false + + incr expectedValue + + let! accessResult = [|1 .. 10|] |> Array.map (fun _ -> cell.AwaitValue ()) |> Async.Parallel + test <@ accessResult |> Array.forall ((=) 4) @> } \ No newline at end of file From 22fffb4d4041935bd07b937e978ee717eb246661 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 13 Dec 2018 09:57:41 +0000 Subject: [PATCH 54/66] Add stored proc auto provisioning per collection #59 --- build.ps1 | 3 +- src/Equinox.Cosmos/Cosmos.fs | 103 ++++++++++++++--------- src/Equinox.EventStore/Infrastructure.fs | 32 +++++-- tools/Equinox.Tool/Program.fs | 9 +- 4 files changed, 98 insertions(+), 49 deletions(-) diff --git a/build.ps1 b/build.ps1 index 9762133db..ea8521a34 100644 --- a/build.ps1 +++ b/build.ps1 @@ -30,7 +30,8 @@ if ($skipCosmos) { warn "Skipping Provisioning Cosmos" } else { warn "Provisioning cosmos..." - cliCosmos @("init", "-ru", "1000") + # -P: inhibit creation of stored proc (everything in the repo should work without it due to auto-provisioning) + cliCosmos @("init", "-ru", "400", "-P") $deprovisionCosmos=$true } $env:EQUINOX_INTEGRATION_SKIP_COSMOS=[string]$skipCosmos diff --git a/src/Equinox.Cosmos/Cosmos.fs b/src/Equinox.Cosmos/Cosmos.fs index d94ccef06..25f097115 100644 --- a/src/Equinox.Cosmos/Cosmos.fs +++ b/src/Equinox.Cosmos/Cosmos.fs @@ -151,8 +151,7 @@ module internal Position = type Direction = Forward | Backward override this.ToString() = match this with Forward -> "Forward" | Backward -> "Backward" /// Reference to Collection and name that will be used as the location for the stream -type [] - CollectionStream = { collectionUri: System.Uri; name: string } //with +type [] CollectionStream = { collectionUri: Uri; name: string } type internal Enum() = static member internal Events(b: Tip) = @@ -293,8 +292,8 @@ module Sync = // NB don't nest in a private module, or serialization will fail miserably ;) [] type SyncResponse = { etag: string; n: int64; conflicts: Event[] } - let [] sprocName = "EquinoxNoTipEvents" // NB need to renumber for any breaking change - let [] sprocBody = """ + let [] private sprocName = "EquinoxNoTipEvents" // NB need to renumber for any breaking change + let [] private sprocBody = """ // Manages the merging of the supplied Request Batch, fulfilling one of the following end-states // 1 perform expectedVersion verification (can request inhibiting of check by supplying -1) @@ -413,12 +412,10 @@ function sync(req, expectedVersion, maxEvents) { module Initialization = open System.Collections.ObjectModel - let createDatabase (client:IDocumentClient) dbName = async { + let createDatabaseIfNotExists (client:IDocumentClient) dbName = let opts = Client.RequestOptions(ConsistencyLevel = Nullable ConsistencyLevel.Session) - let! db = client.CreateDatabaseIfNotExistsAsync(Database(Id=dbName), options = opts) |> Async.AwaitTaskCorrect - return db.Resource.Id } - - let createCollection (client: IDocumentClient) (dbUri: Uri) collName ru = async { + client.CreateDatabaseIfNotExistsAsync(Database(Id=dbName), options = opts) |> Async.AwaitTaskCorrect |> Async.Ignore + let createCollectionIfNotExists (client: IDocumentClient) (dbName,collName) ru = async { let pkd = PartitionKeyDefinition() pkd.Paths.Add(sprintf "/%s" Batch.PartitionKeyField) let colld = DocumentCollection(Id = collName, PartitionKey = pkd) @@ -430,22 +427,17 @@ function sync(req, expectedVersion, maxEvents) { colld.IndexingPolicy.ExcludedPaths <- Collection [|ExcludedPath(Path="/*")|] // NB its critical to index the nominated PartitionKey field defined above or there will be runtime errors colld.IndexingPolicy.IncludedPaths <- Collection [| for k in Batch.IndexedFields -> IncludedPath(Path=sprintf "/%s/?" k) |] - let! coll = client.CreateDocumentCollectionIfNotExistsAsync(dbUri, colld, Client.RequestOptions(OfferThroughput=Nullable ru)) |> Async.AwaitTaskCorrect - return coll.Resource.Id } - - let createProc (log: ILogger) (client: IDocumentClient) (collectionUri: Uri) = async { - let def = new StoredProcedure(Id = sprocName, Body = sprocBody) - log.Information("Creating stored procedure {sprocId}", def.Id) - // TODO ifnotexist semantics - return! client.CreateStoredProcedureAsync(collectionUri, def) |> Async.AwaitTaskCorrect |> Async.Ignore } - - let initialize log (client : IDocumentClient) dbName collName ru = async { - let! dbId = createDatabase client dbName - let dbUri = Client.UriFactory.CreateDatabaseUri dbId - let! collId = createCollection client dbUri collName ru - let collUri = Client.UriFactory.CreateDocumentCollectionUri (dbName, collId) - //let! _aux = createAux client dbUri collName auxRu - return! createProc log client collUri } + let dbUri = Client.UriFactory.CreateDatabaseUri dbName + return! client.CreateDocumentCollectionIfNotExistsAsync(dbUri, colld, Client.RequestOptions(OfferThroughput=Nullable ru)) |> Async.AwaitTaskCorrect |> Async.Ignore } + let private createStoredProcIfNotExists (client: IDocumentClient) (collectionUri: Uri) (name, body): Async = async { + try let! r = client.CreateStoredProcedureAsync(collectionUri, StoredProcedure(Id = name, Body = body)) |> Async.AwaitTaskCorrect + return r.RequestCharge + with DocDbException ((DocDbStatusCode sc) as e) when sc = System.Net.HttpStatusCode.Conflict -> return e.RequestCharge } + let createSyncStoredProcIfNotExists (log: ILogger option) client collUri = async { + let! t, ru = createStoredProcIfNotExists client collUri (sprocName,sprocBody) |> Stopwatch.Time + match log with + | None -> () + | Some log -> log.Information("Created stored procedure {sprocId} rc={ru} t={ms}", sprocName, ru, (let e = t.Elapsed in e.TotalMilliseconds)) } module internal Tip = let private get (client: IDocumentClient) (stream: CollectionStream, maybePos: Position option) = @@ -658,6 +650,7 @@ open Equinox.Store.Infrastructure open FSharp.Control open Serilog open System +open System.Collections.Concurrent /// Defines policies for retrying with respect to transient failures calling CosmosDb (as opposed to application level concurrency conflicts) type EqxConnection(client: Microsoft.Azure.Documents.IDocumentClient, ?readRetryPolicy: IRetryPolicy, ?writeRetryPolicy) = @@ -719,6 +712,8 @@ type EqxGateway(conn : EqxConnection, batching : EqxBatchingPolicy) = | Tip.Result.Found (pos, FromUnfold tryDecode isOrigin span) -> return LoadFromTokenResult.Found (Token.create stream pos, span) | _ -> let! res = __.Read log stream Direction.Forward (Some pos) (tryDecode,isOrigin) return LoadFromTokenResult.Found res } + member __.CreateSyncStoredProcIfNotExists log = + Sync.Initialization.createSyncStoredProcIfNotExists log conn.Client member __.Sync log stream (expectedVersion, batch: Tip): Async = async { let! wr = Sync.batch log conn.WriteRetryPolicy conn.Client stream (expectedVersion,batch,batching.MaxItems) match wr with @@ -834,20 +829,37 @@ type private Folder<'event, 'state> | Store.SyncResult.Conflict resync -> return Store.SyncResult.Conflict resync | Store.SyncResult.Written (token',state') -> return Store.SyncResult.Written (token',state') } +/// Holds Database/Collection pair, coordinating initialization activities +type private EqxCollection(databaseId, collectionId, ?initCollection : Uri -> Async) = + let collectionUri = Microsoft.Azure.Documents.Client.UriFactory.CreateDocumentCollectionUri(databaseId, collectionId) + let initGuard = initCollection |> Option.map (fun init -> AsyncCacheCell(init collectionUri)) + + member __.CollectionUri = collectionUri + member internal __.InitializationGate = match initGuard with Some g when g.PeekIsValid() |> not -> Some g.AwaitValue | _ -> None + /// Defines a process for mapping from a Stream Name to the appropriate storage area, allowing control over segregation / co-locating of data -type EqxCollections(categoryAndIdToDatabaseCollectionAndStream : string -> string -> string*string*string) = +type EqxCollections(categoryAndIdToDatabaseCollectionAndStream : string -> string -> string*string*string, ?disableInitialization) = + // Index of database*collection -> Initialization Context + let collections = ConcurrentDictionary() new (databaseId, collectionId) = // TOCONSIDER - this works to support the Core.Events APIs let genStreamName categoryName streamId = if categoryName = null then streamId else sprintf "%s-%s" categoryName streamId EqxCollections(fun categoryName streamId -> databaseId, collectionId, genStreamName categoryName streamId) - member __.CollectionForStream (categoryName,id) : CollectionStream = + + member internal __.Resolve(categoryName, id, init) : CollectionStream * (unit -> Async) option = let databaseId, collectionId, streamName = categoryAndIdToDatabaseCollectionAndStream categoryName id - { collectionUri = Microsoft.Azure.Documents.Client.UriFactory.CreateDocumentCollectionUri(databaseId, collectionId); name = streamName } + let init = match disableInitialization with Some true -> None | _ -> Some init + + let coll = collections.GetOrAdd((databaseId,collectionId), fun (db,coll) -> EqxCollection(db, coll, ?initCollection = init)) + { collectionUri = coll.CollectionUri; name = streamName },coll.InitializationGate -/// Pairs a Gateway, defining the retry policies for CosmosDb with an EqxCollections to -type EqxStore(gateway: EqxGateway, collections: EqxCollections) = +/// Pairs a Gateway, defining the retry policies for CosmosDb with an EqxCollections defining mappings from (category,id) to (database,collection,streamName) +type EqxStore(gateway: EqxGateway, collections: EqxCollections, ?resolverLog) = + let init = gateway.CreateSyncStoredProcIfNotExists resolverLog member __.Gateway = gateway member __.Collections = collections + member internal __.ResolveCollStream(categoryName, id) : CollectionStream * (unit -> Async) option = + collections.Resolve(categoryName, id, init) [] type CachingStrategy = @@ -886,19 +898,27 @@ type EqxResolver<'event, 'state>(store : EqxStore, codec, fold, initial, ?access | Some (CachingStrategy.SlidingWindow(cache, window)) -> Caching.applyCacheUpdatesWithSlidingExpiration cache null window folder - let mkStreamName = store.Collections.CollectionForStream - let resolve = Equinox.Store.Stream.create category + let resolveStream (streamId, maybeCollectionInitializationGate) = + { new Store.IStream<'event, 'state> with + member __.Load log = category.Load streamId log + member __.TrySync (log: ILogger) (token: Store.StreamToken, originState: 'state) (events: 'event list) = + match maybeCollectionInitializationGate with + | None -> category.TrySync log (token, originState) events + | Some init -> async { + do! init () + return! category.TrySync log (token, originState) events } } member __.Resolve = function | Target.CatId (categoryName,streamId) -> - resolve <| mkStreamName (categoryName, streamId) + store.ResolveCollStream(categoryName, streamId) |> resolveStream | Target.CatIdEmpty (categoryName,streamId) -> - let stream = mkStreamName (categoryName, streamId) - Store.Stream.ofMemento (Token.create stream Position.fromKnownEmpty,initial) (resolve stream) + let collStream, maybeInit = store.ResolveCollStream(categoryName, streamId) + Store.Stream.ofMemento (Token.create collStream Position.fromKnownEmpty,initial) (resolveStream (collStream, maybeInit)) | Target.DeprecatedRawName _ as x -> failwithf "Stream name not supported: %A" x member __.FromMemento(Token.Unpack (stream,_pos) as streamToken,state) = - Store.Stream.ofMemento (streamToken,state) (resolve stream) + let skipInitialization = None + Store.Stream.ofMemento (streamToken,state) (resolveStream (stream,skipInitialization)) [] type Discovery = @@ -1023,7 +1043,8 @@ type EqxContext let! (Token.Unpack (_,pos')), data = res return pos', data } - member __.CreateStream(streamName) = collections.CollectionForStream(null, streamName) + member __.ResolveStream(streamName) = collections.Resolve(null, streamName, gateway.CreateSyncStoredProcIfNotExists (Some log)) + member __.CreateStream(streamName) = __.ResolveStream streamName |> fst member internal __.GetLazy((stream, startPos), ?batchSize, ?direction) : AsyncSeq = let direction = defaultArg direction Direction.Forward @@ -1061,7 +1082,13 @@ type EqxContext /// Appends the supplied batch of events, subject to a consistency check based on the `position` /// Callers should implement appropriate idempotent handling, or use Equinox.Handler for that purpose - member __.Sync(stream, position, events: IEvent[]) : Async> = async { + member __.Sync(stream : CollectionStream, position, events: IEvent[]) : Async> = async { + // Writes go through the stored proc, which we need to provision per-collection + // Having to do this here in this way is far from ideal, but work on caching, external snapshots and caching is likely + // to move this about before we reach a final destination in any case + match __.ResolveStream stream.name |> snd with + | None -> () + | Some init -> do! init () let batch = Sync.mkBatch stream events Seq.empty let! res = gateway.Sync log stream (Some position.index,batch) match res with diff --git a/src/Equinox.EventStore/Infrastructure.fs b/src/Equinox.EventStore/Infrastructure.fs index 34ed46f72..4f53a4279 100644 --- a/src/Equinox.EventStore/Infrastructure.fs +++ b/src/Equinox.EventStore/Infrastructure.fs @@ -126,30 +126,44 @@ type AsyncLazy<'T>(workflow : Async<'T>) = /// Generic async lazy caching implementation that admits expiration/recomputation semantics /// If `workflow` fails, all readers entering while the load/refresh is in progress will share the failure -type AsyncCacheCell<'T>(workflow : Async<'T>, isExpired : 'T -> bool) = +type AsyncCacheCell<'T>(workflow : Async<'T>, ?isExpired : 'T -> bool) = let mutable currentCell = AsyncLazy workflow + + let initializationFailed (value: System.Threading.Tasks.Task<_>) = + // for TMI on this, see https://stackoverflow.com/a/33946166/11635 + value.IsCompleted && value.Status <> System.Threading.Tasks.TaskStatus.RanToCompletion + let update cell = async { // avoid unneccessary recomputation in cases where competing threads detect expiry; - // the first write attempt wins, and everybody else reads off that value. + // the first write attempt wins, and everybody else reads off that value let _ = System.Threading.Interlocked.CompareExchange(¤tCell, AsyncLazy workflow, cell) return! currentCell.AwaitValue() } + /// Enables callers to short-circuit the gate by checking whether a value has been computed + member __.PeekIsValid() = + let cell = currentCell + let currentState = cell.PeekInternalTask + if not currentState.IsValueCreated then false else + + let value = currentState.Value + not (initializationFailed value) + && (match isExpired with Some f -> not (f value.Result) | _ -> false) + /// Gets or asynchronously recomputes a cached value depending on expiry and availability member __.AwaitValue() = async { let cell = currentCell let currentState = cell.PeekInternalTask - // If the last attempt completed, but failed, we need to treat it as expired; for TMI on this, see https://stackoverflow.com/a/33946166/11635 - if currentState.IsValueCreated && currentState.Value.IsCompleted && currentState.Value.Status <> System.Threading.Tasks.TaskStatus.RanToCompletion then + // If the last attempt completed, but failed, we need to treat it as expired + if currentState.IsValueCreated && initializationFailed currentState.Value then return! update cell else let! current = cell.AwaitValue() - if isExpired current then - return! update cell - else - return current + match isExpired with + | Some f when f current -> return! update cell + | _ -> return current } - + [] module Regex = open System.Text.RegularExpressions diff --git a/tools/Equinox.Tool/Program.fs b/tools/Equinox.Tool/Program.fs index 364e356d2..ea79205b0 100644 --- a/tools/Equinox.Tool/Program.fs +++ b/tools/Equinox.Tool/Program.fs @@ -30,10 +30,12 @@ type Arguments = | Initialize _ -> "Initialize a store" and []InitArguments = | [] Rus of int + | [] SkipStoredProc | [] Cosmos of ParseResults interface IArgParserTemplate with member a.Usage = a |> function | Rus _ -> "Specify RU/s level to provision for the Application Collection." + | SkipStoredProc -> "Inhibit creation of stored procedure in cited Collection." | Cosmos _ -> "Cosmos Connection parameters." and []WebArguments = | [] Endpoint of string @@ -206,7 +208,12 @@ let main argv = let storeLog = createStoreLog (sargs.Contains CosmosArguments.VerboseStore) verboseConsole maybeSeq let dbName, collName, (_pageSize: int), conn = Cosmos.conn (log,storeLog) sargs log.Information("Configuring CosmosDb Collection with Throughput Provision: {rus:n0} RU/s", rus) - Equinox.Cosmos.Store.Sync.Initialization.initialize log conn.Client dbName collName rus |> Async.RunSynchronously + Async.RunSynchronously <| async { + do! Equinox.Cosmos.Store.Sync.Initialization.createDatabaseIfNotExists conn.Client dbName + do! Equinox.Cosmos.Store.Sync.Initialization.createCollectionIfNotExists conn.Client (dbName,collName) rus + let collectionUri = Microsoft.Azure.Documents.Client.UriFactory.CreateDocumentCollectionUri(dbName,collName) + if not (iargs.Contains SkipStoredProc) then + do! Equinox.Cosmos.Store.Sync.Initialization.createSyncStoredProcIfNotExists (Some (upcast log)) conn.Client collectionUri } | _ -> failwith "please specify a `cosmos` endpoint" | Run rargs -> let reportFilename = args.GetResult(LogFile,programName+".log") |> fun n -> System.IO.FileInfo(n).FullName From 05c312f6e65764bff71972637a3f11ed8466b69f Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 14 Dec 2018 12:15:42 +0000 Subject: [PATCH 55/66] Add initAux CLI command re #60 --- build.ps1 | 9 +++++++-- src/Equinox.Cosmos/Cosmos.fs | 33 +++++++++++++++++++------------ tools/Equinox.Tool/Program.fs | 37 ++++++++++++++++++++++++++++------- 3 files changed, 58 insertions(+), 21 deletions(-) diff --git a/build.ps1 b/build.ps1 index ea8521a34..49d61f363 100644 --- a/build.ps1 +++ b/build.ps1 @@ -7,6 +7,7 @@ param( [Alias("cd")][string] $cosmosDatabase=$env:EQUINOX_COSMOS_DATABASE, [Alias("cc")][string] $cosmosCollection=$env:EQUINOX_COSMOS_COLLECTION, [Alias("scp")][switch][bool] $skipProvisionCosmos=$skipCosmos -or -not $cosmosServer -or -not $cosmosDatabase -or -not $cosmosCollection, + [Alias("ca")][switch][bool] $cosmosProvisionAux, [Alias("scd")][switch][bool] $skipDeprovisionCosmos=$skipProvisionCosmos, [string] $additionalMsBuildArgs="-t:Build" ) @@ -29,10 +30,14 @@ if ($skipCosmos) { } elseif ($skipProvisionCosmos) { warn "Skipping Provisioning Cosmos" } else { - warn "Provisioning cosmos..." + warn "Provisioning cosmos (without stored procedure)..." # -P: inhibit creation of stored proc (everything in the repo should work without it due to auto-provisioning) - cliCosmos @("init", "-ru", "400", "-P") + cliCosmos @("init", "-ru", "400", "-P") $deprovisionCosmos=$true + if ($cosmosProvisionAux) { + warn "Provisioning cosmos aux collection for projector..." + cliCosmos @("initAux", "-ru", "400") + } } $env:EQUINOX_INTEGRATION_SKIP_COSMOS=[string]$skipCosmos diff --git a/src/Equinox.Cosmos/Cosmos.fs b/src/Equinox.Cosmos/Cosmos.fs index 25f097115..034b4c2d9 100644 --- a/src/Equinox.Cosmos/Cosmos.fs +++ b/src/Equinox.Cosmos/Cosmos.fs @@ -415,29 +415,38 @@ function sync(req, expectedVersion, maxEvents) { let createDatabaseIfNotExists (client:IDocumentClient) dbName = let opts = Client.RequestOptions(ConsistencyLevel = Nullable ConsistencyLevel.Session) client.CreateDatabaseIfNotExistsAsync(Database(Id=dbName), options = opts) |> Async.AwaitTaskCorrect |> Async.Ignore - let createCollectionIfNotExists (client: IDocumentClient) (dbName,collName) ru = async { + let private createCollectionIfNotExists (client: IDocumentClient) dbName (def: DocumentCollection, ru) = async { + let dbUri = Client.UriFactory.CreateDatabaseUri dbName + return! client.CreateDocumentCollectionIfNotExistsAsync(dbUri, def, Client.RequestOptions(OfferThroughput=Nullable ru)) |> Async.AwaitTaskCorrect |> Async.Ignore } + let private createStoredProcIfNotExists (client: IDocumentClient) (collectionUri: Uri) (name, body): Async = async { + try let! r = client.CreateStoredProcedureAsync(collectionUri, StoredProcedure(Id = name, Body = body)) |> Async.AwaitTaskCorrect + return r.RequestCharge + with DocDbException ((DocDbStatusCode sc) as e) when sc = System.Net.HttpStatusCode.Conflict -> return e.RequestCharge } + let createBatchAndTipCollectionIfNotExists (client: IDocumentClient) (dbName,collName) ru : Async = let pkd = PartitionKeyDefinition() pkd.Paths.Add(sprintf "/%s" Batch.PartitionKeyField) - let colld = DocumentCollection(Id = collName, PartitionKey = pkd) + let def = DocumentCollection(Id = collName, PartitionKey = pkd) - colld.IndexingPolicy.IndexingMode <- IndexingMode.Consistent - colld.IndexingPolicy.Automatic <- true + def.IndexingPolicy.IndexingMode <- IndexingMode.Consistent + def.IndexingPolicy.Automatic <- true // Can either do a blacklist or a whitelist // Given how long and variable the blacklist would be, we whitelist instead - colld.IndexingPolicy.ExcludedPaths <- Collection [|ExcludedPath(Path="/*")|] + def.IndexingPolicy.ExcludedPaths <- Collection [|ExcludedPath(Path="/*")|] // NB its critical to index the nominated PartitionKey field defined above or there will be runtime errors - colld.IndexingPolicy.IncludedPaths <- Collection [| for k in Batch.IndexedFields -> IncludedPath(Path=sprintf "/%s/?" k) |] - let dbUri = Client.UriFactory.CreateDatabaseUri dbName - return! client.CreateDocumentCollectionIfNotExistsAsync(dbUri, colld, Client.RequestOptions(OfferThroughput=Nullable ru)) |> Async.AwaitTaskCorrect |> Async.Ignore } - let private createStoredProcIfNotExists (client: IDocumentClient) (collectionUri: Uri) (name, body): Async = async { - try let! r = client.CreateStoredProcedureAsync(collectionUri, StoredProcedure(Id = name, Body = body)) |> Async.AwaitTaskCorrect - return r.RequestCharge - with DocDbException ((DocDbStatusCode sc) as e) when sc = System.Net.HttpStatusCode.Conflict -> return e.RequestCharge } + def.IndexingPolicy.IncludedPaths <- Collection [| for k in Batch.IndexedFields -> IncludedPath(Path=sprintf "/%s/?" k) |] + createCollectionIfNotExists client dbName (def, ru) let createSyncStoredProcIfNotExists (log: ILogger option) client collUri = async { let! t, ru = createStoredProcIfNotExists client collUri (sprocName,sprocBody) |> Stopwatch.Time match log with | None -> () | Some log -> log.Information("Created stored procedure {sprocId} rc={ru} t={ms}", sprocName, ru, (let e = t.Elapsed in e.TotalMilliseconds)) } + let createAuxCollectionIfNotExists (client: IDocumentClient) (dbName,collName) ru : Async = + let def = DocumentCollection(Id = collName) + // for now, we are leaving the default IndexingPolicy mode wrt fields to index and in which manner as default: autoindexing all fields + def.IndexingPolicy.IndexingMode <- IndexingMode.Lazy + // Expire Projector documentId to Kafka offsets mapping records after one year + def.DefaultTimeToLive <- Nullable (365 * 60 * 60 * 24) + createCollectionIfNotExists client dbName (def, ru) module internal Tip = let private get (client: IDocumentClient) (stream: CollectionStream, maybePos: Position option) = diff --git a/tools/Equinox.Tool/Program.fs b/tools/Equinox.Tool/Program.fs index ea79205b0..d6444bd64 100644 --- a/tools/Equinox.Tool/Program.fs +++ b/tools/Equinox.Tool/Program.fs @@ -19,7 +19,8 @@ type Arguments = | [] LocalSeq | [] LogFile of string | [] Run of ParseResults - | [] Initialize of ParseResults + | [] Init of ParseResults + | [] InitAux of ParseResults interface IArgParserTemplate with member a.Usage = a |> function | Verbose -> "Include low level logging regarding specific test runs." @@ -27,7 +28,8 @@ type Arguments = | LocalSeq -> "Configures writing to a local Seq endpoint at http://localhost:5341, see https://getseq.net" | LogFile _ -> "specify a log file to write the result breakdown into (default: eqx.log)." | Run _ -> "Run a load test" - | Initialize _ -> "Initialize a store" + | Init _ -> "Initialize store (presently only relevant for `cosmos`, where it creates database+collection+stored proc if not already present)." + | InitAux _ -> "Initialize auxilliary store (presently only relevant for `cosmos`, when you intend to run the [presently closed source] Projector)." and []InitArguments = | [] Rus of int | [] SkipStoredProc @@ -37,6 +39,15 @@ and []InitArguments = | Rus _ -> "Specify RU/s level to provision for the Application Collection." | SkipStoredProc -> "Inhibit creation of stored procedure in cited Collection." | Cosmos _ -> "Cosmos Connection parameters." +and []InitAuxArguments = + | [] Rus of int + | [] Suffix of string + | [] Cosmos of ParseResults + interface IArgParserTemplate with + member a.Usage = a |> function + | Rus _ -> "Specify RU/s level to provision for the Application Collection." + | Suffix _ -> "Specify Collection Name suffix (default: `-aux`)." + | Cosmos _ -> "Cosmos Connection parameters." and []WebArguments = | [] Endpoint of string interface IArgParserTemplate with @@ -201,24 +212,36 @@ let main argv = let verbose = args.Contains Verbose let log = createDomainLog verbose verboseConsole maybeSeq match args.GetSubCommand() with - | Initialize iargs -> - let rus = iargs.GetResult(Rus) + | Init iargs -> + let rus = iargs.GetResult(InitArguments.Rus) match iargs.TryGetSubCommand() with | Some (InitArguments.Cosmos sargs) -> let storeLog = createStoreLog (sargs.Contains CosmosArguments.VerboseStore) verboseConsole maybeSeq let dbName, collName, (_pageSize: int), conn = Cosmos.conn (log,storeLog) sargs - log.Information("Configuring CosmosDb Collection with Throughput Provision: {rus:n0} RU/s", rus) + log.Information("Configuring CosmosDb Collection {collName} with Throughput Provision: {rus:n0} RU/s", collName, rus) Async.RunSynchronously <| async { do! Equinox.Cosmos.Store.Sync.Initialization.createDatabaseIfNotExists conn.Client dbName - do! Equinox.Cosmos.Store.Sync.Initialization.createCollectionIfNotExists conn.Client (dbName,collName) rus + do! Equinox.Cosmos.Store.Sync.Initialization.createBatchAndTipCollectionIfNotExists conn.Client (dbName,collName) rus let collectionUri = Microsoft.Azure.Documents.Client.UriFactory.CreateDocumentCollectionUri(dbName,collName) if not (iargs.Contains SkipStoredProc) then do! Equinox.Cosmos.Store.Sync.Initialization.createSyncStoredProcIfNotExists (Some (upcast log)) conn.Client collectionUri } | _ -> failwith "please specify a `cosmos` endpoint" + | InitAux iargs -> + let rus = iargs.GetResult(InitAuxArguments.Rus) + match iargs.TryGetSubCommand() with + | Some (InitAuxArguments.Cosmos sargs) -> + let storeLog = createStoreLog (sargs.Contains CosmosArguments.VerboseStore) verboseConsole maybeSeq + let dbName, collName, (_pageSize: int), conn = Cosmos.conn (log,storeLog) sargs + let collName = collName + (iargs.GetResult(InitAuxArguments.Suffix,"-aux")) + log.Information("Configuring CosmosDb Aux Collection {collName} with Throughput Provision: {rus:n0} RU/s", collName, rus) + Async.RunSynchronously <| async { + do! Equinox.Cosmos.Store.Sync.Initialization.createDatabaseIfNotExists conn.Client dbName + do! Equinox.Cosmos.Store.Sync.Initialization.createAuxCollectionIfNotExists conn.Client (dbName,collName) rus } + | _ -> failwith "please specify a `cosmos` endpoint" | Run rargs -> let reportFilename = args.GetResult(LogFile,programName+".log") |> fun n -> System.IO.FileInfo(n).FullName LoadTest.run log (verbose,verboseConsole,maybeSeq) reportFilename rargs - | _ -> failwith "Please specify a valid subcommand :- init or run" + | _ -> failwith "Please specify a valid subcommand :- init, initAux or run" 0 with e -> printfn "%s" e.Message From 494402e20d7ed5107faa0552c17dd1e2fa8c5577 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 14 Dec 2018 15:49:46 +0000 Subject: [PATCH 56/66] Hide command from help message to avoid confusion --- tools/Equinox.Tool/Program.fs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/Equinox.Tool/Program.fs b/tools/Equinox.Tool/Program.fs index d6444bd64..9a2dd00b5 100644 --- a/tools/Equinox.Tool/Program.fs +++ b/tools/Equinox.Tool/Program.fs @@ -20,7 +20,8 @@ type Arguments = | [] LogFile of string | [] Run of ParseResults | [] Init of ParseResults - | [] InitAux of ParseResults + | [] // this command is not useful unless you have access to the [presently closed-source] Equinox.Cosmos.Projector + [] InitAux of ParseResults interface IArgParserTemplate with member a.Usage = a |> function | Verbose -> "Include low level logging regarding specific test runs." From eb7d6699e06a0594c21ccf2249462c3e77e97a20 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 14 Dec 2018 11:10:34 +0000 Subject: [PATCH 57/66] Misc cleanup from self-review --- build.proj | 3 +- samples/Store/Domain/Cart.fs | 2 +- samples/Store/Integration/LogIntegration.fs | 2 +- src/StoredProcedure.js | 154 ------------------ .../CosmosCoreIntegration.fs | 4 +- .../Equinox.Cosmos.Integration.fsproj | 4 +- 6 files changed, 8 insertions(+), 161 deletions(-) delete mode 100644 src/StoredProcedure.js diff --git a/build.proj b/build.proj index d754ad9c4..fd1713fa0 100644 --- a/build.proj +++ b/build.proj @@ -28,10 +28,11 @@ + - + \ No newline at end of file diff --git a/samples/Store/Domain/Cart.fs b/samples/Store/Domain/Cart.fs index 9e665c876..d53cad63f 100644 --- a/samples/Store/Domain/Cart.fs +++ b/samples/Store/Domain/Cart.fs @@ -29,7 +29,7 @@ module Folds = let toSnapshot (s: State) : Events.Compaction.State = { items = [| for i in s.items -> { skuId = i.skuId; quantity = i.quantity; returnsWaived = i.returnsWaived } |] } let ofCompacted (s: Events.Compaction.State) : State = - { items = if s.items = null then [] else [ for i in s.items -> { skuId = i.skuId; quantity = i.quantity; returnsWaived = i.returnsWaived } ] } + { items = [ for i in s.items -> { skuId = i.skuId; quantity = i.quantity; returnsWaived = i.returnsWaived } ] } let initial = { items = [] } let evolve (state : State) event = let updateItems f = { state with items = f state.items } diff --git a/samples/Store/Integration/LogIntegration.fs b/samples/Store/Integration/LogIntegration.fs index 879d219d3..eb352142e 100644 --- a/samples/Store/Integration/LogIntegration.fs +++ b/samples/Store/Integration/LogIntegration.fs @@ -1,8 +1,8 @@ module Samples.Store.Integration.LogIntegration open Domain -open Equinox.Store open Equinox.Cosmos.Integration +open Equinox.Store open Swensen.Unquote open System open System.Collections.Concurrent diff --git a/src/StoredProcedure.js b/src/StoredProcedure.js deleted file mode 100644 index 3574724be..000000000 --- a/src/StoredProcedure.js +++ /dev/null @@ -1,154 +0,0 @@ -function write2 (partitionkey, events, expectedVersion, pendingEvents, projections) { - - if (events === undefined || events==null) events = []; - if (expectedVersion === undefined) expectedVersion = -2; - if (pendingEvents === undefined) pendingEvents = null; - if (projections === undefined || projections==null) projections = {}; - - var collection = getContext().getCollection(); - var collectionLink = collection.getSelfLink(); - - tryQueryAndUpdate(); - - // Recursively queries for a document by id w/ support for continuation tokens. - // Calls tryUpdate(document) as soon as the query returns a document. - function tryQueryAndUpdate(continuation) { - var query = {query: "select * from root r where r.id = @id and r.k = @k", parameters: [{name: "@id", value: "-1"},{name: "@p", value: partitionkey}]}; - var requestOptions = {continuation: continuation}; - - var isAccepted = collection.queryDocuments(collectionLink, query, requestOptions, function (err, documents, responseOptions) { - if (err) throw new Error("Error" + err.message); - - if (documents.length > 0) { - // If the document is found, update it. - // There is no need to check for a continuation token since we are querying for a single document. - tryUpdate(documents[0], false); - } else if (responseOptions.continuation) { - // Else if the query came back empty, but with a continuation token; repeat the query w/ the token. - // It is highly unlikely for this to happen when performing a query by id; but is included to serve as an example for larger queries. - tryQueryAndUpdate(responseOptions.continuation); - } else { - // Else the snapshot does not exist; create snapshot - var doc = {p:partitionkey, id:"-1", latest:-1, projections:{"_lowWatermark":{"base":-1}}}; - tryUpdate(doc, true); - } - }); - - // If we hit execution bounds - throw an exception. - // This is highly unlikely given that this is a query by id; but is included to serve as an example for larger queries. - if (!isAccepted) { - throw new Error("The stored procedure timed out."); - } - } - - function insertEvents() - { - for (i=0; i0) { - if (doc.pendingEvents==null) - doc.pendingEvents = {"base": parseInt(events[0].id)-1, "value": events}; - else - Array.prototype.push.apply(doc.pendingEvents.value, events); - } - } - - // The kernel function - function tryUpdate(doc, isCreate) { - - // DocumentDB supports optimistic concurrency control via HTTP ETag. - var requestOptions = {etag: doc._etag}; - - // Step 1: Insert events to DB - if (expectedVersion==-2) { - // thor mode - var i; - for (i=0; i parseInt(value.id)>newBase); - doc.pendingEvents.base = newBase; - if (doc.pendingEvents.value.length==0) - delete doc["pendingEvents"]; - } - } - - // Step 6: Replace existing snapshot document or create the first snapshot document for this partition key - if (!isCreate) - { - var isAccepted = collection.replaceDocument(doc._self, doc, requestOptions, function (err, updatedDocument, responseOptions) { - if (err) throw new Error("Error" + err.message); - console.log("etag of replaced document is: %s", updatedDocument._etag); - getContext().getResponse().setBody(updatedDocument._etag); - }); - - // If we hit execution bounds - throw an exception. - if (!isAccepted) { - throw new Error("Unable to replace snapshot document."); - } - } - else { - var isAccepted = collection.createDocument(collectionLink, doc, function (err, documentCreated) { - if (err) throw new Error ("Create doc " + JSON.stringify(doc) + " failed"); - console.log("etag of created document is: %s", documentCreated._etag); - getContext().getResponse().setBody(documentCreated._etag); - }); - - // If we hit execution bounds - throw an exception. - if (!isAccepted) { - throw new Error("Unable to create snapshot document."); - } - } - } - } diff --git a/tests/Equinox.Cosmos.Integration/CosmosCoreIntegration.fs b/tests/Equinox.Cosmos.Integration/CosmosCoreIntegration.fs index 1ced8d174..867d6f3d7 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosCoreIntegration.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosCoreIntegration.fs @@ -46,7 +46,7 @@ type Tests(testOutputHelper) = let! res = Events.append ctx streamName index <| TestEvents.Create(0,1) test <@ AppendResult.Ok 1L = res @> test <@ [EqxAct.Append] = capture.ExternalCalls @> - verifyRequestChargesMax 12 // 11.78 // WAS 10 + verifyRequestChargesMax 13 // 12.88 // WAS 10 // Clear the counters capture.Clear() @@ -122,7 +122,7 @@ type Tests(testOutputHelper) = pos <- pos + 42L pos =! res test <@ [EqxAct.Append] = capture.ExternalCalls @> - verifyRequestChargesMax 20 + verifyRequestChargesMax 23 // 22.2 // WAS 20 capture.Clear() let! res = Events.getNextIndex ctx streamName diff --git a/tests/Equinox.Cosmos.Integration/Equinox.Cosmos.Integration.fsproj b/tests/Equinox.Cosmos.Integration/Equinox.Cosmos.Integration.fsproj index d8440534f..02085c658 100644 --- a/tests/Equinox.Cosmos.Integration/Equinox.Cosmos.Integration.fsproj +++ b/tests/Equinox.Cosmos.Integration/Equinox.Cosmos.Integration.fsproj @@ -17,8 +17,8 @@ - - + + From 2ad42369660cc7de369a2551ca2756eed9aaa741 Mon Sep 17 00:00:00 2001 From: Michael Liao Date: Mon, 17 Dec 2018 13:55:56 -0500 Subject: [PATCH 58/66] loosened up constraints for newtonsoft for net461 --- samples/Web/Web.fsproj | 1 + src/Equinox.Codec/Equinox.Codec.fsproj | 1 + src/Equinox.Cosmos/Equinox.Cosmos.fsproj | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/samples/Web/Web.fsproj b/samples/Web/Web.fsproj index 4bbda7057..455fc62cb 100644 --- a/samples/Web/Web.fsproj +++ b/samples/Web/Web.fsproj @@ -19,6 +19,7 @@ + diff --git a/src/Equinox.Codec/Equinox.Codec.fsproj b/src/Equinox.Codec/Equinox.Codec.fsproj index 20a780134..78ac349b8 100644 --- a/src/Equinox.Codec/Equinox.Codec.fsproj +++ b/src/Equinox.Codec/Equinox.Codec.fsproj @@ -20,6 +20,7 @@ + diff --git a/src/Equinox.Cosmos/Equinox.Cosmos.fsproj b/src/Equinox.Cosmos/Equinox.Cosmos.fsproj index c4fbd86eb..4f64ddbbc 100644 --- a/src/Equinox.Cosmos/Equinox.Cosmos.fsproj +++ b/src/Equinox.Cosmos/Equinox.Cosmos.fsproj @@ -1,4 +1,4 @@ - + netstandard2.0;net461 @@ -29,6 +29,7 @@ + From 7bc04387c8245759b3c59e0f230b391677addaba Mon Sep 17 00:00:00 2001 From: Michael Liao Date: Mon, 17 Dec 2018 14:14:31 -0500 Subject: [PATCH 59/66] updated netstandard dependency constraint for newtonsoft --- src/Equinox.Codec/Equinox.Codec.fsproj | 2 +- src/Equinox.Cosmos/Equinox.Cosmos.fsproj | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Equinox.Codec/Equinox.Codec.fsproj b/src/Equinox.Codec/Equinox.Codec.fsproj index 78ac349b8..784fe5529 100644 --- a/src/Equinox.Codec/Equinox.Codec.fsproj +++ b/src/Equinox.Codec/Equinox.Codec.fsproj @@ -19,7 +19,7 @@ - + diff --git a/src/Equinox.Cosmos/Equinox.Cosmos.fsproj b/src/Equinox.Cosmos/Equinox.Cosmos.fsproj index 4f64ddbbc..cd4936da3 100644 --- a/src/Equinox.Cosmos/Equinox.Cosmos.fsproj +++ b/src/Equinox.Cosmos/Equinox.Cosmos.fsproj @@ -17,8 +17,8 @@ - + @@ -28,8 +28,8 @@ - - + + From bc0d5c0806b5c298a4abbdaf495e4eaaa32251b6 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 19 Dec 2018 09:34:02 +0000 Subject: [PATCH 60/66] Add note re Equinox.Cosmos Newtonsoft.Json dependency downgrade --- samples/Web/Web.fsproj | 1 - src/Equinox.Codec/Equinox.Codec.fsproj | 3 +-- src/Equinox.Cosmos/Equinox.Cosmos.fsproj | 7 +++++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/samples/Web/Web.fsproj b/samples/Web/Web.fsproj index 455fc62cb..4bbda7057 100644 --- a/samples/Web/Web.fsproj +++ b/samples/Web/Web.fsproj @@ -19,7 +19,6 @@ - diff --git a/src/Equinox.Codec/Equinox.Codec.fsproj b/src/Equinox.Codec/Equinox.Codec.fsproj index 784fe5529..20a780134 100644 --- a/src/Equinox.Codec/Equinox.Codec.fsproj +++ b/src/Equinox.Codec/Equinox.Codec.fsproj @@ -19,8 +19,7 @@ - - + diff --git a/src/Equinox.Cosmos/Equinox.Cosmos.fsproj b/src/Equinox.Cosmos/Equinox.Cosmos.fsproj index cd4936da3..425a61fbf 100644 --- a/src/Equinox.Cosmos/Equinox.Cosmos.fsproj +++ b/src/Equinox.Cosmos/Equinox.Cosmos.fsproj @@ -17,8 +17,8 @@ - + @@ -28,8 +28,11 @@ - + + From b906e108ec98638c60617bae71ba6c0c609d2d83 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 19 Dec 2018 09:50:56 +0000 Subject: [PATCH 61/66] Update readme to reflect imminent merge --- README.md | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 47110122f..3abd338fe 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Equinox provides a unified programming model for event sourced processing agains Current supported backends are: - [EventStore](https://eventstore.org/) - this codebase itself has been in production since 2017 (commit history reflects usage), with elements dating back to 2016. -- [Azure Cosmos DB](https://docs.microsoft.com/en-us/azure/cosmos-db/) (See [`cosmos` branch](https://github.com/jet/equinox/tree/cosmos) - will converge with `master` very shortly). +- [Azure Cosmos DB](https://docs.microsoft.com/en-us/azure/cosmos-db) - contains code dating back to 2016, however [the storage model](https://github.com/jet/equinox/wiki/Cosmos-Storage-Model) was arrived at based on intensive benchmarking squash-merged in [#42](https://github.com/jet/equinox/pull/42). - (For integration test purposes only) Volatile in-memory store. The underlying patterns have their roots in the [DDD-CQRS-ES](https://groups.google.com/forum/#!forum/dddcqrs) community, and the hard work and generosity of countless folks there presenting, explaining, writing and hacking over the years. It would be unfair to single out even a small number of people despite the immense credit that is due. @@ -33,7 +33,7 @@ _If you're looking to learn more about and/or discuss Event Sourcing and it's my - NB while this works well, and can deliver excellent performance (especially when allied with the Cache), [it's not a panacea, as noted in this excellent EventStore article on the topic](https://eventstore.org/docs/event-sourcing-basics/rolling-snapshots/index.html) - **`Equinox.Cosmos` 'Tip with Unfolds' schema**: In contrast to `Equinox.EventStore`'s `Access.RollingSnapshots`, when using `Equinox.Cosmos`, optimized command processing is managed via the `Tip`; a document per stream with a well-known identity enabling syncs via point-reads by virtue of the fact that the document maintains: a) the present Position of the stream - i.e. the index at which the next events will be appended - b) compressed [_unfolds_]((https://github.com/jet/equinox/wiki/Cosmos-Storage-Model) + b) (compressed) [_unfolds_]((https://github.com/jet/equinox/wiki/Cosmos-Storage-Model) c) (optionally) events since those unfolded events ([presently removed](https://github.com/jet/equinox/pull/58), but [should return](https://github.com/jet/equinox/wiki/Roadmap)) This yields many of the benefits of the in-stream Rolling Snapshots approach while reducing latency, RU provisioning requirement, and Request Charges:- @@ -51,8 +51,8 @@ The Equinox components within this repository are delivered as a series of multi - `Equinox.Codec` (Nuget: `Equinox.Codec`, depends on `TypeShape`, (optionally) `Newtonsoft.Json >= 11.0.2` but can support any serializer): [a scheme for the serializing Events modelled as an F# Discriminated Union with the following capabilities](https://eiriktsarpalis.wordpress.com/2018/10/30/a-contract-pattern-for-schemaless-datastores/): - independent of any specific serializer - allows tagging of Discriminated Union cases in a versionable manner with low-dependency `DataMember(Name=` tags using [TypeShape](https://github.com/eiriktsarpalis/TypeShape)'s [`UnionContractEncoder`](https://github.com/eiriktsarpalis/TypeShape/blob/master/tests/TypeShape.Tests/UnionContractTests.fs) -- `Equinox.Cosmos` (Nuget: `Equinox.Cosmos`, depends on `DocumentDb.Client`, `System.Runtime.Caching`, `FSharp.Control.AsyncSeq`): Production-strength Azure CosmosDb Adapter with integrated transactionally-consistent snapshotting, facilitating optimal read performance in terms of latency and RU costs, instrumented to the degree necessitated by Jet's production monitoring requirements. -- `Equinox.EventStore` (Nuget: `Equinox.EventStore`, depends on `EventStore.Client[Api.NetCore] >= 4`, `System.Runtime.Caching`, `FSharp.Control.AsyncSeq`): Production-strength [EventStore](http://geteventstore.com) Adapter instrumented to the degree necessitated by Jet's production monitoring requirements +- `Equinox.Cosmos` (Nuget: `Equinox.Cosmos`, depends on `System.Runtime.Caching`, `FSharp.Control.AsyncSeq`, `TypeShape`, `Microsoft.Azure.DocumentDb[Core]`): Production-strength Azure CosmosDb Adapter with integrated transactionally-consistent snapshotting, facilitating optimal read performance in terms of latency and RU costs, instrumented to the degree necessitated by Jet's production monitoring requirements. +- `Equinox.EventStore` (Nuget: `Equinox.EventStore`, depends on `EventStore.Client[Api.NetCore] >= 4`, `System.Runtime.Caching`, `FSharp.Control.AsyncSeq`, `TypeShape`): Production-strength [EventStore](https://eventstore.org/) Adapter instrumented to the degree necessitated by Jet's production monitoring requirements - `Equinox.MemoryStore` (Nuget: `Equinox.MemoryStore`): In-memory store for integration testing/performance baselining/providing out-of-the-box zero dependency storage for examples. - `samples/Store` (in this repo): Example domain types reflecting examples of how one applies Equinox to a diverse set of stream-based models - `samples/TodoBackend` (in this repo): Standard https://todobackend.com compliant backend @@ -68,22 +68,18 @@ The repo is versioned based on [SemVer 2.0](https://semver.org/spec/v2.0.0.html) Please raise GitHub issues for any questions so others can benefit from the discussion. -We are getting very close to that point and are extremely excited by that. But we're not there yet; this is intentionally a soft launch. - -For now, the core focus of work here will be on converging the `cosmos` branch, which will bring changes, clarifications, simplifications and features, that all need to be integrated into the production systems built on it, before we can consider broader-based additive changes and/or significantly increasing the API surface area. - The aim in the medium term (and the hope from the inception of this work) is to run Equinox as a proper Open Source project at the point where there is enough time for maintainers to do that properly. -Unfortunately, in the interim, the barrier for contributions will unfortunately be inordinately high in the short term: +We are getting very close to that point and are extremely excited by that. But we're not there yet; this is intentionally a soft launch. -- bugfixes with good test coverage are always welcome - PRs yield MyGet-hosted NuGets and in general we'll seek to move them to NuGet prerelease and then NuGet release packages with relatively short timelines. +Unfortunately, in the interim, the barrier for contributions will unfortunately be inordinately high: +- bugfixes with good test coverage are always welcome - PRs yield [MyGet-hosted NuGets](https://www.myget.org/F/jet/api/v3/index.json); in general we'll seek to move them to NuGet prerelease and then NuGet release packages with relatively short timelines. - minor improvements / tweaks, subject to discussing in a GitHub issue first to see if it fits, but no promises at this time, even if the ideas are fantastic and necessary :sob: - tests, examples and scenarios are always welcome; Equinox is intended to address a very broad base of usage patterns; Please note that the emphasis will always be (in order) 1. providing advice on how to achieve your aims without changing Equinox 2. how to open up an appropriate extension point in Equinox 3. (when all else fails), add to the complexity of the system by adding API surface area or logic. -- we will likely punt on non-IO perf improvements until such point as Cosmos support is converged into `master` -- Naming is hard; there is definitely room for improvement. There likely will be a set of controlled deprecations, switching to names, and then removing the old ones. However, PRs other than for discussion purposes probably don't make sense right now. +- Naming is hard; there is definitely room for improvement. There may at some point be a set of controlled deprecations, switching to names, and then removing the old ones. However it should be noted that widespread renaming is not on the cards due to the number of downstream systems that would be affected. ## BUILDING From 63c287c88a07bd58486155ad7b0998a9366b8d8d Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 19 Dec 2018 10:17:25 +0000 Subject: [PATCH 62/66] Tidy Newtonsoft.Json dependencies+docs --- README.md | 4 ++-- src/Equinox.Codec/Equinox.Codec.fsproj | 6 +++++- src/Equinox.Cosmos/Equinox.Cosmos.fsproj | 5 ----- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 3abd338fe..3c9fc509f 100644 --- a/README.md +++ b/README.md @@ -48,10 +48,10 @@ _If you're looking to learn more about and/or discuss Event Sourcing and it's my The Equinox components within this repository are delivered as a series of multi-targeted Nuget packages targeting `net461` (F# 3.1+) and `netstandard2.0` (F# 4.5+) profiles; each of the constituent elements is designed to be easily swappable as dictated by the task at hand. Each of the components can be inlined or customized easily:- - `Equinox.Handler` (Nuget: `Equinox`, depends on `Serilog` (but no specific Serilog sinks, i.e. you can forward to `NLog` etc)): Store-agnostic decision flow runner that manages the optimistic concurrency protocol -- `Equinox.Codec` (Nuget: `Equinox.Codec`, depends on `TypeShape`, (optionally) `Newtonsoft.Json >= 11.0.2` but can support any serializer): [a scheme for the serializing Events modelled as an F# Discriminated Union with the following capabilities](https://eiriktsarpalis.wordpress.com/2018/10/30/a-contract-pattern-for-schemaless-datastores/): +- `Equinox.Codec` (Nuget: `Equinox.Codec`, depends on `TypeShape`, (`Newtonsoft.Json` `>= 10.0.3` on `net461`, `>= 11.0.2` on `netstandard2.0` but can support any serializer): [a scheme for the serializing Events modelled as an F# Discriminated Union with the following capabilities](https://eiriktsarpalis.wordpress.com/2018/10/30/a-contract-pattern-for-schemaless-datastores/): - independent of any specific serializer - allows tagging of Discriminated Union cases in a versionable manner with low-dependency `DataMember(Name=` tags using [TypeShape](https://github.com/eiriktsarpalis/TypeShape)'s [`UnionContractEncoder`](https://github.com/eiriktsarpalis/TypeShape/blob/master/tests/TypeShape.Tests/UnionContractTests.fs) -- `Equinox.Cosmos` (Nuget: `Equinox.Cosmos`, depends on `System.Runtime.Caching`, `FSharp.Control.AsyncSeq`, `TypeShape`, `Microsoft.Azure.DocumentDb[Core]`): Production-strength Azure CosmosDb Adapter with integrated transactionally-consistent snapshotting, facilitating optimal read performance in terms of latency and RU costs, instrumented to the degree necessitated by Jet's production monitoring requirements. +- `Equinox.Cosmos` (Nuget: `Equinox.Cosmos`, depends on `System.Runtime.Caching`, `FSharp.Control.AsyncSeq`, `TypeShape`, `Microsoft.Azure.DocumentDb[.Core]`): Production-strength Azure CosmosDb Adapter with integrated transactionally-consistent snapshotting, facilitating optimal read performance in terms of latency and RU costs, instrumented to the degree necessitated by Jet's production monitoring requirements. - `Equinox.EventStore` (Nuget: `Equinox.EventStore`, depends on `EventStore.Client[Api.NetCore] >= 4`, `System.Runtime.Caching`, `FSharp.Control.AsyncSeq`, `TypeShape`): Production-strength [EventStore](https://eventstore.org/) Adapter instrumented to the degree necessitated by Jet's production monitoring requirements - `Equinox.MemoryStore` (Nuget: `Equinox.MemoryStore`): In-memory store for integration testing/performance baselining/providing out-of-the-box zero dependency storage for examples. - `samples/Store` (in this repo): Example domain types reflecting examples of how one applies Equinox to a diverse set of stream-based models diff --git a/src/Equinox.Codec/Equinox.Codec.fsproj b/src/Equinox.Codec/Equinox.Codec.fsproj index 20a780134..13cefc561 100644 --- a/src/Equinox.Codec/Equinox.Codec.fsproj +++ b/src/Equinox.Codec/Equinox.Codec.fsproj @@ -19,7 +19,11 @@ - + + + diff --git a/src/Equinox.Cosmos/Equinox.Cosmos.fsproj b/src/Equinox.Cosmos/Equinox.Cosmos.fsproj index 425a61fbf..5377af83e 100644 --- a/src/Equinox.Cosmos/Equinox.Cosmos.fsproj +++ b/src/Equinox.Cosmos/Equinox.Cosmos.fsproj @@ -28,11 +28,6 @@ - - - From bb2ac5203f81ac88b7a43334703034c2ca1e89f5 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 19 Dec 2018 10:22:03 +0000 Subject: [PATCH 63/66] Up RU expectation to reflect observation --- tests/Equinox.Cosmos.Integration/CosmosCoreIntegration.fs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/Equinox.Cosmos.Integration/CosmosCoreIntegration.fs b/tests/Equinox.Cosmos.Integration/CosmosCoreIntegration.fs index 867d6f3d7..14ea74bc2 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosCoreIntegration.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosCoreIntegration.fs @@ -46,7 +46,7 @@ type Tests(testOutputHelper) = let! res = Events.append ctx streamName index <| TestEvents.Create(0,1) test <@ AppendResult.Ok 1L = res @> test <@ [EqxAct.Append] = capture.ExternalCalls @> - verifyRequestChargesMax 13 // 12.88 // WAS 10 + verifyRequestChargesMax 14 // 13.73 // WAS 10 // Clear the counters capture.Clear() @@ -161,7 +161,7 @@ type Tests(testOutputHelper) = test <@ [EqxAct.Resync] = capture.ExternalCalls @> // The response aligns with a normal conflict in that it passes the entire set of conflicting events () test <@ AppendResult.Conflict (0L,[||]) = res @> - verifyRequestChargesMax 5 + verifyRequestChargesMax 6 // 5.24 // WAS 5 capture.Clear() // Now write at the correct position @@ -169,7 +169,7 @@ type Tests(testOutputHelper) = let! res = Events.append ctx streamName 0L expected test <@ AppendResult.Ok 1L = res @> test <@ [EqxAct.Append] = capture.ExternalCalls @> - verifyRequestChargesMax 12 // 11.35 WAS 11 // 10.33 + verifyRequestChargesMax 13 // 12.1 WAS 11 // 10.33 capture.Clear() // Try overwriting it (a competing consumer would see the same) From f2dc53a86344edd1e2748f6c90c79b08ca503c6f Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 19 Dec 2018 11:19:01 +0000 Subject: [PATCH 64/66] Fix to handle Codec change in rebase --- tests/Equinox.Cosmos.Integration/JsonConverterTests.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Equinox.Cosmos.Integration/JsonConverterTests.fs b/tests/Equinox.Cosmos.Integration/JsonConverterTests.fs index 6ced20f0a..e7ec7e6a2 100644 --- a/tests/Equinox.Cosmos.Integration/JsonConverterTests.fs +++ b/tests/Equinox.Cosmos.Integration/JsonConverterTests.fs @@ -59,5 +59,5 @@ type Base64ZipUtf8Tests() = test <@ ser.Contains("\"d\":\"") @> let des = JsonConvert.DeserializeObject(ser) let d : Equinox.UnionCodec.EncodedUnion<_> = { caseName = des.c; payload = des.d } - let decoded = unionEncoder.Decode d + let decoded = unionEncoder.TryDecode d |> Option.get test <@ value = decoded @> \ No newline at end of file From 32f874c7281a740b70818bf9ace7930ac634fa53 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 19 Dec 2018 12:03:06 +0000 Subject: [PATCH 65/66] Exclude Mono failures due to multiple json.net versions --- src/Equinox.Codec/Equinox.Codec.fsproj | 3 ++- .../Equinox.Cosmos.Integration.fsproj | 2 +- tests/Equinox.Cosmos.Integration/JsonConverterTests.fs | 9 +++++++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/Equinox.Codec/Equinox.Codec.fsproj b/src/Equinox.Codec/Equinox.Codec.fsproj index 13cefc561..53d0b782e 100644 --- a/src/Equinox.Codec/Equinox.Codec.fsproj +++ b/src/Equinox.Codec/Equinox.Codec.fsproj @@ -21,7 +21,8 @@ + in this case we're relaxing the constraint for net461 so as to not trigger a set of transitive dependency updates at the present time + should also be able to remove [] in JsonConverterTests when this special casing goes away --> diff --git a/tests/Equinox.Cosmos.Integration/Equinox.Cosmos.Integration.fsproj b/tests/Equinox.Cosmos.Integration/Equinox.Cosmos.Integration.fsproj index 02085c658..81bdcff66 100644 --- a/tests/Equinox.Cosmos.Integration/Equinox.Cosmos.Integration.fsproj +++ b/tests/Equinox.Cosmos.Integration/Equinox.Cosmos.Integration.fsproj @@ -5,6 +5,7 @@ Library false 5 + NET461 @@ -34,6 +35,5 @@ - \ No newline at end of file diff --git a/tests/Equinox.Cosmos.Integration/JsonConverterTests.fs b/tests/Equinox.Cosmos.Integration/JsonConverterTests.fs index e7ec7e6a2..f817fa7a4 100644 --- a/tests/Equinox.Cosmos.Integration/JsonConverterTests.fs +++ b/tests/Equinox.Cosmos.Integration/JsonConverterTests.fs @@ -19,6 +19,9 @@ type VerbatimUtf8Tests() = let unionEncoder = mkUnionEncoder () [] +#if NET461 + [] // Likely due to net461 not having consistent json.net refs and no binding redirects +#endif let ``encodes correctly`` () = let encoded = unionEncoder.Encode(A { embed = "\"" }) let e : Store.Batch = @@ -31,6 +34,9 @@ type Base64ZipUtf8Tests() = let unionEncoder = mkUnionEncoder () [] +#if NET461 + [] // Likely due to net461 not having consistent json.net refs and no binding redirects +#endif let ``serializes, achieving compression`` () = let encoded = unionEncoder.Encode(A { embed = String('x',5000) }) let e : Store.Unfold = @@ -42,6 +48,9 @@ type Base64ZipUtf8Tests() = test <@ res.Contains("\"d\":\"") && res.Length < 100 @> [] +#if NET461 + [] // Likely due to net461 not having consistent json.net refs and no binding redirects +#endif let roundtrips value = let hasNulls = match value with From 60160a9a903cda2795ef338f07f9a0dbd48853ae Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Mon, 24 Dec 2018 10:14:05 +0000 Subject: [PATCH 66/66] Cleanup --- Directory.Build.props | 5 +++-- README.md | 17 ++++++++--------- build.proj | 10 +++++----- build.ps1 | 4 ++-- samples/Infrastructure/Storage.fs | 12 ++++++------ samples/Store/Integration/CartIntegration.fs | 4 ++++ samples/Store/Integration/CodecIntegration.fs | 4 ++++ .../ContactPreferencesIntegration.fs | 4 ++++ .../Store/Integration/FavoritesIntegration.fs | 4 ++++ src/Equinox.Cosmos/Equinox.Cosmos.fsproj | 2 ++ .../Equinox.EventStore.fsproj | 2 +- .../CosmosIntegration.fs | 3 ++- 12 files changed, 45 insertions(+), 26 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index bebac5421..5b9b3b1cd 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -20,8 +20,9 @@ false - - $(NoWarn);FS2003;NU5105 + + + $(NoWarn);FS2003;NU5105;FS0988 diff --git a/README.md b/README.md index 3c9fc509f..dbbf241b8 100644 --- a/README.md +++ b/README.md @@ -26,14 +26,14 @@ _If you're looking to learn more about and/or discuss Event Sourcing and it's my - Logging is both high performance and pluggable (using [Serilog](https://github.com/serilog/serilog) to your hosting context (we feed log info to Splunk and the metrics embedded in the LogEvent Properties to Prometheus; see relevant tests for examples) - Extracted from working software; currently used for all data storage within Jet's API gateway and Cart processing. - Significant test coverage for core facilities, and per Storage system. -- **`Equinox.EventStore` Transactionally-consistent Rolling Snapshots**: Command processing can be optimized by employing in-stream 'compaction' events in service of the following ends: +- **`Equinox.EventStore` In-stream Rolling Snapshots**: Command processing can be optimized by employing 'compaction' events in service of the following ends: - no additional roundtrips to the store needed at either the Load or Sync points in the flow - support, (via `UnionContractEncoder`) for the maintenance of multiple co-existing compaction schemas in a given stream (A snapshot isa Event) - compaction events typically do not get deleted (consistent with how EventStore works), although it is safe to do so in concept - NB while this works well, and can deliver excellent performance (especially when allied with the Cache), [it's not a panacea, as noted in this excellent EventStore article on the topic](https://eventstore.org/docs/event-sourcing-basics/rolling-snapshots/index.html) - **`Equinox.Cosmos` 'Tip with Unfolds' schema**: In contrast to `Equinox.EventStore`'s `Access.RollingSnapshots`, when using `Equinox.Cosmos`, optimized command processing is managed via the `Tip`; a document per stream with a well-known identity enabling syncs via point-reads by virtue of the fact that the document maintains: a) the present Position of the stream - i.e. the index at which the next events will be appended - b) (compressed) [_unfolds_]((https://github.com/jet/equinox/wiki/Cosmos-Storage-Model) + b) (compressed) [_unfolds_](https://github.com/jet/equinox/wiki/Cosmos-Storage-Model) c) (optionally) events since those unfolded events ([presently removed](https://github.com/jet/equinox/pull/58), but [should return](https://github.com/jet/equinox/wiki/Roadmap)) This yields many of the benefits of the in-stream Rolling Snapshots approach while reducing latency, RU provisioning requirement, and Request Charges:- @@ -48,21 +48,21 @@ _If you're looking to learn more about and/or discuss Event Sourcing and it's my The Equinox components within this repository are delivered as a series of multi-targeted Nuget packages targeting `net461` (F# 3.1+) and `netstandard2.0` (F# 4.5+) profiles; each of the constituent elements is designed to be easily swappable as dictated by the task at hand. Each of the components can be inlined or customized easily:- - `Equinox.Handler` (Nuget: `Equinox`, depends on `Serilog` (but no specific Serilog sinks, i.e. you can forward to `NLog` etc)): Store-agnostic decision flow runner that manages the optimistic concurrency protocol -- `Equinox.Codec` (Nuget: `Equinox.Codec`, depends on `TypeShape`, (`Newtonsoft.Json` `>= 10.0.3` on `net461`, `>= 11.0.2` on `netstandard2.0` but can support any serializer): [a scheme for the serializing Events modelled as an F# Discriminated Union with the following capabilities](https://eiriktsarpalis.wordpress.com/2018/10/30/a-contract-pattern-for-schemaless-datastores/): +- `Equinox.Codec` (Nuget: `Equinox.Codec`, depends on `TypeShape`, `Newtonsoft.Json` (`>= 10.0.3` on `net461`, `>= 11.0.2` on `netstandard2.0` but can support any serializer): [a scheme for the serializing Events modelled as an F# Discriminated Union with the following capabilities](https://eiriktsarpalis.wordpress.com/2018/10/30/a-contract-pattern-for-schemaless-datastores/): - independent of any specific serializer - - allows tagging of Discriminated Union cases in a versionable manner with low-dependency `DataMember(Name=` tags using [TypeShape](https://github.com/eiriktsarpalis/TypeShape)'s [`UnionContractEncoder`](https://github.com/eiriktsarpalis/TypeShape/blob/master/tests/TypeShape.Tests/UnionContractTests.fs) -- `Equinox.Cosmos` (Nuget: `Equinox.Cosmos`, depends on `System.Runtime.Caching`, `FSharp.Control.AsyncSeq`, `TypeShape`, `Microsoft.Azure.DocumentDb[.Core]`): Production-strength Azure CosmosDb Adapter with integrated transactionally-consistent snapshotting, facilitating optimal read performance in terms of latency and RU costs, instrumented to the degree necessitated by Jet's production monitoring requirements. -- `Equinox.EventStore` (Nuget: `Equinox.EventStore`, depends on `EventStore.Client[Api.NetCore] >= 4`, `System.Runtime.Caching`, `FSharp.Control.AsyncSeq`, `TypeShape`): Production-strength [EventStore](https://eventstore.org/) Adapter instrumented to the degree necessitated by Jet's production monitoring requirements + - allows tagging of F# Discriminated Union cases in a versionable manner with low-dependency `DataMember(Name=` tags using [TypeShape](https://github.com/eiriktsarpalis/TypeShape)'s [`UnionContractEncoder`](https://github.com/eiriktsarpalis/TypeShape/blob/master/tests/TypeShape.Tests/UnionContractTests.fs) +- `Equinox.Tool` (Nuget: `dotnet tool install Equinox.Tool -g`): Tool incorporating a benchmark scenario runner, facilitating running representative load tests composed of transactions in `samples/Store` and `samples/TodoBackend` against any nominated store; this allows perf tuning and measurement in terms of both latency and transaction charge aspects. - `Equinox.MemoryStore` (Nuget: `Equinox.MemoryStore`): In-memory store for integration testing/performance baselining/providing out-of-the-box zero dependency storage for examples. +- `Equinox.EventStore` (Nuget: `Equinox.EventStore`, depends on `EventStore.Client[Api.NetCore] >= 4`, `System.Runtime.Caching`, `FSharp.Control.AsyncSeq`): Production-strength [EventStore](https://eventstore.org/) Adapter instrumented to the degree necessitated by Jet's production monitoring requirements +- `Equinox.Cosmos` (Nuget: `Equinox.Cosmos`, depends on `System.Runtime.Caching`, `FSharp.Control.AsyncSeq`, `Microsoft.Azure.DocumentDb[.Core]`): Production-strength Azure CosmosDb Adapter with integrated 'unfolds' feature, facilitating optimal read performance in terms of latency and RU costs, instrumented to the degree necessitated by Jet's production monitoring requirements. - `samples/Store` (in this repo): Example domain types reflecting examples of how one applies Equinox to a diverse set of stream-based models - `samples/TodoBackend` (in this repo): Standard https://todobackend.com compliant backend -- `Equinox.Tool` (Nuget: `dotnet tool install Equinox.Tool -g`): Tool incorporating a benchmark scenario runner, facilitating running representative load tests composed of transactions in `samples/Store` and `samples/TodoBackend` against any nominated store; this allows perf tuning and measurement in terms of both latency and transaction charge aspects. ## Versioning ## About Versioning -The repo is versioned based on [SemVer 2.0](https://semver.org/spec/v2.0.0.html) using the tiny-but-mighty [MinVer](https://github.com/adamralph/minver) from @adamralph. [See here](https://github.com/adamralph/minver#how-it-works) for more information on how it works. +The repo is versioned based on [SemVer 2.0](https://semver.org/spec/v2.0.0.html) using the tiny-but-mighty [MinVer](https://github.com/adamralph/minver) from [@adamralph](https://github.com/adamralph). [See here](https://github.com/adamralph/minver#how-it-works) for more information on how it works. ## CONTRIBUTING @@ -174,7 +174,6 @@ The CLI can drive the Store and TodoBackend samples in the `samples/Web` ASP.NET eqx run -t saveforlater -f 200 web ### run CosmosDb benchmark (when provisioned) - $env:EQUINOX_COSMOS_CONNECTION="AccountEndpoint=https://....;AccountKey=....=;" $env:EQUINOX_COSMOS_DATABASE="equinox-test" $env:EQUINOX_COSMOS_COLLECTION="equinox-test" diff --git a/build.proj b/build.proj index fd1713fa0..96eeef436 100644 --- a/build.proj +++ b/build.proj @@ -15,11 +15,11 @@ - - - - - + + + + + diff --git a/build.ps1 b/build.ps1 index 49d61f363..54842ad50 100644 --- a/build.ps1 +++ b/build.ps1 @@ -21,8 +21,8 @@ $env:EQUINOX_INTEGRATION_SKIP_EVENTSTORE=[string]$skipEs if ($skipEs) { warn "Skipping EventStore tests" } function cliCosmos($arghs) { - Write-Host "dotnet run cli/Equinox.Cli -- $arghs cosmos -s -d $cosmosDatabase -c $cosmosCollection" - dotnet run -p cli/Equinox.Cli -f netcoreapp2.1 -- @arghs cosmos -s $cosmosServer -d $cosmosDatabase -c $cosmosCollection + Write-Host "dotnet run tools/Equinox.Tool -- $arghs cosmos -s -d $cosmosDatabase -c $cosmosCollection" + dotnet run -p tools/Equinox.Tool -f netcoreapp2.1 -- @arghs cosmos -s $cosmosServer -d $cosmosDatabase -c $cosmosCollection } if ($skipCosmos) { diff --git a/samples/Infrastructure/Storage.fs b/samples/Infrastructure/Storage.fs index 873d829df..cabf2bdf2 100644 --- a/samples/Infrastructure/Storage.fs +++ b/samples/Infrastructure/Storage.fs @@ -9,7 +9,7 @@ type [] MemArguments = interface IArgParserTemplate with member a.Usage = a |> function | VerboseStore -> "Include low level Store logging." -type [] EsArguments = +and [] EsArguments = | [] VerboseStore | [] Timeout of float | [] Retries of int @@ -73,7 +73,7 @@ module EventStore = heartbeatTimeout=heartbeatTimeout, concurrentOperationsLimit = col, log=(if log.IsEnabled(Serilog.Events.LogEventLevel.Debug) then Logger.SerilogVerbose log else Logger.SerilogNormal log), tags=["M", Environment.MachineName; "I", Guid.NewGuid() |> string]) - .Establish("equinox-tool", Discovery.GossipDns dnsQuery, ConnectionStrategy.ClusterTwinPreferSlaveReads) + .Establish("equinox-samples", Discovery.GossipDns dnsQuery, ConnectionStrategy.ClusterTwinPreferSlaveReads) let private createGateway connection batchSize = GesGateway(connection, GesBatchingPolicy(maxBatchSize = batchSize)) let config (log: ILogger, storeLog) (cache, unfolds) (sargs : ParseResults) = let host = sargs.GetResult(Host,"localhost") @@ -89,7 +89,7 @@ module EventStore = let conn = connect storeLog (host, heartbeatTimeout, concurrentOperationsLimit) creds operationThrottling |> Async.RunSynchronously let cacheStrategy = if cache then - let c = Caching.Cache("Cli", sizeMb = 50) + let c = Caching.Cache("equinox-samples", sizeMb = 50) CachingStrategy.SlidingWindow (c, TimeSpan.FromMinutes 20.) |> Some else None StorageConfig.Es ((createGateway conn defaultBatchSize), cacheStrategy, unfolds) @@ -99,11 +99,11 @@ module Cosmos = /// Standing up an Equinox instance is necessary to run for test purposes; You'll need to either: /// 1) replace connection below with a connection string or Uri+Key for an initialized Equinox instance with a database and collection named "equinox-test" - /// 2) Set the 3x environment variables and create a local Equinox using cli/Equinox.cli/bin/Release/net461/Equinox.Cli ` + /// 2) Set the 3x environment variables and create a local Equinox using tools/Equinox.Tool/bin/Release/net461/eqx.exe ` /// cosmos -s $env:EQUINOX_COSMOS_CONNECTION -d $env:EQUINOX_COSMOS_DATABASE -c $env:EQUINOX_COSMOS_COLLECTION provision -ru 1000 let private connect (log: ILogger) mode discovery operationTimeout (maxRetryForThrottling, maxRetryWaitTime) = EqxConnector(log=log, mode=mode, requestTimeout=operationTimeout, maxRetryAttemptsOnThrottledRequests=maxRetryForThrottling, maxRetryWaitTimeInSeconds=maxRetryWaitTime) - .Connect("equinox-cli", discovery) + .Connect("equinox-samples", discovery) let private createGateway connection (maxItems,maxEvents) = EqxGateway(connection, EqxBatchingPolicy(defaultMaxItems=maxItems, maxEventsPerSlice=maxEvents)) let conn (log: ILogger, storeLog) (sargs : ParseResults) = let read key = Environment.GetEnvironmentVariable key |> Option.ofObj @@ -126,7 +126,7 @@ module Cosmos = let dbName, collName, pageSize, conn = conn (log, storeLog) sargs let cacheStrategy = if cache then - let c = Caching.Cache("Cli", sizeMb = 50) + let c = Caching.Cache("equinox-tool", sizeMb = 50) CachingStrategy.SlidingWindow (c, TimeSpan.FromMinutes 20.) |> Some else None StorageConfig.Cosmos (createGateway conn (defaultBatchSize,pageSize), cacheStrategy, unfolds, dbName, collName) \ No newline at end of file diff --git a/samples/Store/Integration/CartIntegration.fs b/samples/Store/Integration/CartIntegration.fs index 291feb431..16c451eee 100644 --- a/samples/Store/Integration/CartIntegration.fs +++ b/samples/Store/Integration/CartIntegration.fs @@ -5,6 +5,7 @@ open Equinox.Cosmos.Integration open Equinox.EventStore open Equinox.MemoryStore open Swensen.Unquote +open Xunit #nowarn "1182" // From hereon in, we may have some 'unused' privates (the tests) @@ -47,6 +48,9 @@ type Tests(testOutputHelper) = } [] +#if NET461 + [] // Likely due to net461 not having consistent json.net refs and no binding redirects +#endif let ``Can roundtrip in Memory, correctly folding the events`` args = Async.RunSynchronously <| async { let log, store = createLog (), createMemoryStore () let service = createServiceMem log store diff --git a/samples/Store/Integration/CodecIntegration.fs b/samples/Store/Integration/CodecIntegration.fs index 89eea866e..9fa894a87 100644 --- a/samples/Store/Integration/CodecIntegration.fs +++ b/samples/Store/Integration/CodecIntegration.fs @@ -4,6 +4,7 @@ module Samples.Store.Integration.CodecIntegration open Domain open Swensen.Unquote open TypeShape.UnionContract +open Xunit let serializationSettings = Newtonsoft.Json.Converters.FSharp.Settings.CreateCorrect(converters= @@ -42,6 +43,9 @@ let render = function let codec = genCodec() [] +#if NET461 + [] // Likely due to net461 not having consistent json.net refs and no binding redirects +#endif let ``Can roundtrip, rendering correctly`` (x: SimpleDu) = let serialized = codec.Encode x render x =! System.Text.Encoding.UTF8.GetString(serialized.payload) diff --git a/samples/Store/Integration/ContactPreferencesIntegration.fs b/samples/Store/Integration/ContactPreferencesIntegration.fs index 571d079bb..91a527301 100644 --- a/samples/Store/Integration/ContactPreferencesIntegration.fs +++ b/samples/Store/Integration/ContactPreferencesIntegration.fs @@ -5,6 +5,7 @@ open Equinox.Cosmos.Integration open Equinox.EventStore open Equinox.MemoryStore open Swensen.Unquote +open Xunit #nowarn "1182" // From hereon in, we may have some 'unused' privates (the tests) @@ -38,6 +39,9 @@ type Tests(testOutputHelper) = test <@ value = actual @> } [] +#if NET461 + [] // Likely due to net461 not having consistent json.net refs and no binding redirects +#endif let ``Can roundtrip in Memory, correctly folding the events`` args = Async.RunSynchronously <| async { let service = let log, store = createLog (), createMemoryStore () in createServiceMem log store do! act service args diff --git a/samples/Store/Integration/FavoritesIntegration.fs b/samples/Store/Integration/FavoritesIntegration.fs index c3a382278..bbf603124 100644 --- a/samples/Store/Integration/FavoritesIntegration.fs +++ b/samples/Store/Integration/FavoritesIntegration.fs @@ -5,6 +5,7 @@ open Equinox.Cosmos.Integration open Equinox.EventStore open Equinox.MemoryStore open Swensen.Unquote +open Xunit #nowarn "1182" // From hereon in, we may have some 'unused' privates (the tests) @@ -40,6 +41,9 @@ type Tests(testOutputHelper) = test <@ Array.isEmpty items @> } [] +#if NET461 + [] // Likely due to net461 not having consistent json.net refs and no binding redirects +#endif let ``Can roundtrip in Memory, correctly folding the events`` args = Async.RunSynchronously <| async { let store = createMemoryStore () let service = let log = createLog () in createServiceMem log store diff --git a/src/Equinox.Cosmos/Equinox.Cosmos.fsproj b/src/Equinox.Cosmos/Equinox.Cosmos.fsproj index 5377af83e..4572b39b5 100644 --- a/src/Equinox.Cosmos/Equinox.Cosmos.fsproj +++ b/src/Equinox.Cosmos/Equinox.Cosmos.fsproj @@ -23,6 +23,8 @@ + + diff --git a/src/Equinox.EventStore/Equinox.EventStore.fsproj b/src/Equinox.EventStore/Equinox.EventStore.fsproj index 92af6061c..27f1a0810 100644 --- a/src/Equinox.EventStore/Equinox.EventStore.fsproj +++ b/src/Equinox.EventStore/Equinox.EventStore.fsproj @@ -7,7 +7,7 @@ false true true - NET461 + $(DefineConstants);NET461 diff --git a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs index 23db4fa03..e2742db14 100644 --- a/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs +++ b/tests/Equinox.Cosmos.Integration/CosmosIntegration.fs @@ -12,7 +12,8 @@ let genCodec<'Union when 'Union :> TypeShape.UnionContract.IUnionContract>() = Equinox.UnionCodec.JsonUtf8.Create<'Union>(serializationSettings) module Cart = - let fold, initial, snapshot = Domain.Cart.Folds.fold, Domain.Cart.Folds.initial, Domain.Cart.Folds.snapshot + let fold, initial = Domain.Cart.Folds.fold, Domain.Cart.Folds.initial + let snapshot = Domain.Cart.Folds.isOrigin, Domain.Cart.Folds.compact let codec = genCodec() let createServiceWithoutOptimization connection batchSize log = let store = createEqxStore connection batchSize