Skip to content

Commit

Permalink
Fix: CosmosDB EntriesIterator (#8367)
Browse files Browse the repository at this point in the history 
* Fix: CosmosDB EntriesIterator

* Fix lint

* Update emulator configuration

* CR Fixes
  • Loading branch information
@N-o-Z
N-o-Z authored Nov 14, 2024
1 parent 068019a commit caa565e
Show file tree
Hide file tree
Showing 7 changed files with 137 additions and 126 deletions.
20 changes: 10 additions & 10 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,9 @@ require (
github.com/vbauerster/mpb/v5 v5.4.0
github.com/xitongsys/parquet-go v1.6.2
github.com/xitongsys/parquet-go-source v0.0.0-20230607234618-40034c8066df
golang.org/x/crypto v0.24.0
golang.org/x/crypto v0.27.0
golang.org/x/oauth2 v0.15.0
golang.org/x/term v0.21.0
golang.org/x/term v0.24.0
google.golang.org/api v0.152.0
google.golang.org/protobuf v1.33.0
gopkg.in/natefinch/lumberjack.v2 v2.0.0
Expand All @@ -58,14 +58,14 @@ require (

require (
cloud.google.com/go/compute v1.23.3 // indirect
golang.org/x/sync v0.7.0
golang.org/x/sync v0.8.0
)

require (
cloud.google.com/go/compute/metadata v0.2.3
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0
github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.6
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0
github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v1.2.0
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0
github.com/IBM/pgxpoolprometheus v1.1.1
github.com/Masterminds/sprig/v3 v3.2.3
Expand Down Expand Up @@ -105,7 +105,7 @@ require (
cloud.google.com/go/iam v1.1.5 // indirect
dario.cat/mergo v1.0.0 // indirect
github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
Expand Down Expand Up @@ -248,9 +248,9 @@ require (
go.uber.org/atomic v1.11.0
golang.org/x/exp v0.0.0-20231127185646-65229373498e
golang.org/x/mod v0.17.0 // indirect
golang.org/x/net v0.26.0
golang.org/x/sys v0.21.0 // indirect
golang.org/x/text v0.16.0 // indirect
golang.org/x/net v0.29.0
golang.org/x/sys v0.25.0 // indirect
golang.org/x/text v0.18.0 // indirect
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
google.golang.org/appengine v1.6.8 // indirect
Expand Down
40 changes: 20 additions & 20 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,19 +94,19 @@ github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9mo
github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbLiiGY6sx7f9i+X3m1CHdd5c6Rdw=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.0.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.1/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 h1:E+OJmp2tPvt1W+amx48v1eqbjDYsgN+RzP4q16yV5eM=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1/go.mod h1:a6xsAQUZg+VsS3TJ05SRp524Hs4pZ/AeFSr5ENf0Yjo=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 h1:JZg6HRh6W6U4OLl6lk7BZ7BLisIzM9dG1R50zUk9C/M=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0/go.mod h1:YL1xnZ6QejvQHWJrX/AvhFl4WW4rqHVoKspWNVwFk0M=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.0.0/go.mod h1:+6sju8gk8FRmSajX3Oz4G5Gm7P+mbqE9FVaXXFYTkCM=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0/go.mod h1:bhXu1AjYL+wutSL/kpSq6s7733q2Rb0yuot9Zgfqa/0=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0 h1:U2rTu3Ef+7w9FHKIAXM6ZyqF3UOWJZ12zIm8zECAFfg=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg=
github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.6 h1:oBqQLSI1pZwGOdXJAoJJSzmff9tlfD4KroVfjQQmd0g=
github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.6/go.mod h1:Beh5cHIXJ0oWEDWk9lNFtuklCojLLQ5hl+LqSNTTs0I=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg=
github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v1.2.0 h1:1y5G4XTBTEt0nKNFtM7j6CxqkY5fxSuJb/mD8Zf0gPc=
github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v1.2.0/go.mod h1:1Dp+C8Sly0hnhX8k5zDuw72Z2ehd9Lv+pkLFn8dgXMA=
github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2TzfVZ1pCb5vxm4BtZPUdYWe/Xo8=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 h1:jBQA3cKT4L2rWMpgE7Yt3Hwh2aUj8KXjIGLxjHeYNNo=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0/go.mod h1:4OG6tQ9EOP/MT0NMjDlRzWoVFxfu9rN9B2X+tlSVktg=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0 h1:Ma67P/GGprNwsslzEH6+Kb8nybI8jpDTm4Wmzu2ReK8=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0/go.mod h1:c+Lifp3EDEamAkPVzMooRNOK6CZjNSdEnf1A7jsI9u4=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.1/go.mod h1:eZ4g6GUvXiGulfIbbhh1Xr4XwUYaYaWMqzGD/284wCA=
Expand Down Expand Up @@ -1172,8 +1172,8 @@ golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29/go.mod h1:IxCIyHEi3zRg3s0
golang.org/x/crypto v0.0.0-20220511200225-c6db032c6c88/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A=
golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70=
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
Expand Down Expand Up @@ -1281,8 +1281,8 @@ golang.org/x/net v0.0.0-20220401154927-543a649e0bdd/go.mod h1:CfG3xpIq0wQ8r1q4Su
golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo=
golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
Expand Down Expand Up @@ -1317,8 +1317,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
Expand Down Expand Up @@ -1414,15 +1414,15 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA=
golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0=
golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM=
golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
Expand All @@ -1434,8 +1434,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224=
golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
Expand Down
143 changes: 74 additions & 69 deletions pkg/kv/cosmosdb/main_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,22 +2,50 @@ package cosmosdb_test

import (
"context"
"crypto/tls"
"log"
"net/http"
"os"
"testing"
"time"

"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos"
"github.com/treeverse/lakefs/pkg/kv"
"github.com/treeverse/lakefs/pkg/kv/cosmosdb"
"github.com/treeverse/lakefs/pkg/kv/kvparams"
"github.com/treeverse/lakefs/pkg/kv/kvtest"
"github.com/treeverse/lakefs/pkg/testutil"
)

var testParams *kvparams.CosmosDB
var (
testParams *kvparams.CosmosDB
client *azcosmos.Client
throughput = azcosmos.NewManualThroughputProperties(400)
)

func TestCosmosDB(t *testing.T) {
t.Skip("CosmosDB tests are flaky due to the emulator. If you plan on running those, make sure to assign at least 3CPUs and" +
" 4GB of memory to the container running the emulator.")
kvtest.DriverTest(t, func(t testing.TB, ctx context.Context) kv.Store {
t.Helper()

databaseClient, err := client.NewDatabase(testParams.Database)
if err != nil {
log.Fatalf("creating database client: %v", err)
}

testParams.Container = "test-container" + testutil.UniqueName()
log.Printf("Creating container %s", testParams.Container)
resp2, err := databaseClient.CreateContainer(ctx, azcosmos.ContainerProperties{
ID: testParams.Container,
PartitionKeyDefinition: azcosmos.PartitionKeyDefinition{
Paths: []string{"/partitionKey"},
},
}, &azcosmos.CreateContainerOptions{ThroughputProperties: &throughput})
if err != nil {
print(resp2.RawResponse)
t.Fatalf("creating container: %v", err)
}

store, err := kv.Open(ctx, kvparams.Config{CosmosDB: testParams, Type: cosmosdb.DriverName})
if err != nil {
t.Fatalf("failed to open kv '%s' store: %s", cosmosdb.DriverName, err)
Expand All @@ -28,72 +56,49 @@ func TestCosmosDB(t *testing.T) {
}

func TestMain(m *testing.M) {
code := m.Run()
os.Exit(code)
databaseURI, cleanupFunc, err := testutil.GetCosmosDBInstance()
if err != nil {
log.Fatalf("Could not connect to Docker: %s", err)
}
defer cleanupFunc()

//uncomment the below to run the tests against a CosmosDB emulator
testParams = &kvparams.CosmosDB{
Endpoint: databaseURI,
Key: "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==",
Database: "test-db",
Client: &http.Client{Timeout: 30 * time.Second, Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec // ignore self-signed cert for local testing using the emulator
}},
StrongConsistency: false,
}

cred, err := azcosmos.NewKeyCredential(testParams.Key)
if err != nil {
log.Fatalf("creating key: %v", err)
}
// Create a CosmosDB client
client, err = azcosmos.NewClientWithKey(testParams.Endpoint, cred, &azcosmos.ClientOptions{
ClientOptions: azcore.ClientOptions{
Transport: &http.Client{Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}},
},
})
if err != nil {
log.Fatalf("creating client using access key: %v", err)
}

//databaseURI, cleanupFunc, err := testutil.GetCosmosDBInstance()
//if err != nil {
// log.Fatalf("Could not connect to Docker: %s", err)
//}
//
//testParams = &kvparams.CosmosDB{
// Endpoint: databaseURI,
// Key: "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==",
// Database: "test-db",
// Container: "test-container",
// Client: &http.Client{Timeout: 30 * time.Second, Transport: &http.Transport{
// TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec // ignore self-signed cert for local testing using the emulator
// }},
// StrongConsistency: false,
//}
//
//cred, err := azcosmos.NewKeyCredential(testParams.Key)
//if err != nil {
// log.Fatalf("creating key: %v", err)
//}
//// Create a CosmosDB client
//client, err := azcosmos.NewClientWithKey(testParams.Endpoint, cred, &azcosmos.ClientOptions{
// ClientOptions: azcore.ClientOptions{
// Transport: &http.Client{Transport: &http.Transport{
// TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
// }},
// },
//})
//if err != nil {
// log.Fatalf("creating client using access key: %v", err)
//}
//
//log.Printf("Creating database %s", testParams.Database)
//
//ctx := context.Background()
//throughput := azcosmos.NewManualThroughputProperties(4000)
//resp, err := client.CreateDatabase(ctx, azcosmos.DatabaseProperties{ID: testParams.Database},
// &azcosmos.CreateDatabaseOptions{ThroughputProperties: &throughput})
//if err != nil {
// print(resp.RawResponse)
// log.Fatalf("creating database: %v", err)
//}
//
//databaseClient, err := client.NewDatabase(testParams.Database)
//if err != nil {
// log.Fatalf("creating database client: %v", err)
//}
//
//log.Printf("Creating container %s", testParams.Container)
//resp2, err := databaseClient.CreateContainer(ctx, azcosmos.ContainerProperties{
// ID: testParams.Container,
// PartitionKeyDefinition: azcosmos.PartitionKeyDefinition{
// Paths: []string{"/partitionKey"},
// },
//}, &azcosmos.CreateContainerOptions{ThroughputProperties: &throughput})
//if err != nil {
// print(resp2.RawResponse)
// log.Fatalf("creating container: %v", err)
//}
//
//code := m.Run()
//cleanupFunc()
//os.Exit(code)
log.Printf("Creating database %s", testParams.Database)
ctx := context.Background()
resp, err := client.CreateDatabase(ctx,
azcosmos.DatabaseProperties{ID: testParams.Database},
&azcosmos.CreateDatabaseOptions{ThroughputProperties: &throughput},
)
if err != nil {
print(resp.RawResponse)
log.Fatalf("creating database: %v", err)
}

code := m.Run()
os.Exit(code)
}
18 changes: 13 additions & 5 deletions pkg/kv/cosmosdb/store.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -458,7 +458,18 @@ func (e *EntriesIterator) Next() bool {
// The reason we switch the batch size is to avoid issues like https://github.com/treeverse/lakeFS/issues/7864
// as opposed to the exponential backoff approach in dynamoDB here we use a dynamic page size and let Cosmos DB manage paging.
func (e *EntriesIterator) handleBatchSizeChange() error {
e.startKey = e.entry.Key
if e.entry != nil {
e.startKey = e.entry.Key
} else {
e.store.logger.WithFields(logging.Fields{
"batchSize": e.batchSize,
"partitionKey": string(e.partitionKey),
"currEntryIdx": e.currEntryIdx,
"startKey": e.startKey,
"currPage.Items len": len(e.currPage.Items),
}).Warning("handleBatchSizeChange called when e.entry is nil")
e.startKey = nil
}
e.batchSize = dynamicPageSize
return e.runQuery(false)
}
Expand All @@ -480,10 +491,7 @@ func (e *EntriesIterator) SeekGE(key []byte) {
}
return bytes.Compare(key, currentKey) <= 0
})
if idx == -1 {
// not found, set to the end
e.currEntryIdx = len(e.currPage.Items)
}
// sort.Search states that if condition not met it returns n for list of size n. not found, set to the end
e.currEntryIdx = idx - 1
}

Expand Down
Loading

0 comments on commit caa565e

Please sign in to comment.