From 3a19635ad6d78589d946e144bfde77a3e87498df Mon Sep 17 00:00:00 2001 From: Andres Kepler Date: Tue, 9 Aug 2022 16:15:50 +0300 Subject: [PATCH] Add TiKV and PD resource handling with TiDB cluster --- GNUmakefile | 19 +- go.mod | 4 + go.sum | 8 + mysql/provider.go | 1 + mysql/provider_test.go | 19 + mysql/resource_grant_test.go | 5 +- mysql/resource_ti_config_defaults.go | 348 ++ mysql/resource_ti_config_variable.go | 171 + mysql/resource_ti_config_variable_test.go | 208 + mysql/utils.go | 6 + vendor/github.com/creasty/defaults/.gitignore | 1 + vendor/github.com/creasty/defaults/LICENSE | 22 + vendor/github.com/creasty/defaults/Makefile | 30 + vendor/github.com/creasty/defaults/README.md | 73 + .../github.com/creasty/defaults/defaults.go | 221 ++ vendor/github.com/creasty/defaults/setter.go | 12 + vendor/github.com/tidwall/gjson/LICENSE | 20 + vendor/github.com/tidwall/gjson/README.md | 497 +++ vendor/github.com/tidwall/gjson/SYNTAX.md | 342 ++ vendor/github.com/tidwall/gjson/gjson.go | 3359 +++++++++++++++++ vendor/github.com/tidwall/gjson/logo.png | Bin 0 -> 15936 bytes vendor/github.com/tidwall/match/LICENSE | 20 + vendor/github.com/tidwall/match/README.md | 29 + vendor/github.com/tidwall/match/match.go | 237 ++ vendor/github.com/tidwall/pretty/LICENSE | 20 + vendor/github.com/tidwall/pretty/README.md | 122 + vendor/github.com/tidwall/pretty/pretty.go | 674 ++++ vendor/modules.txt | 12 + 28 files changed, 6474 insertions(+), 6 deletions(-) create mode 100644 mysql/resource_ti_config_defaults.go create mode 100644 mysql/resource_ti_config_variable.go create mode 100644 mysql/resource_ti_config_variable_test.go create mode 100644 vendor/github.com/creasty/defaults/.gitignore create mode 100644 vendor/github.com/creasty/defaults/LICENSE create mode 100644 vendor/github.com/creasty/defaults/Makefile create mode 100644 vendor/github.com/creasty/defaults/README.md create mode 100644 vendor/github.com/creasty/defaults/defaults.go create mode 100644 vendor/github.com/creasty/defaults/setter.go create mode 100644 vendor/github.com/tidwall/gjson/LICENSE create mode 100644 vendor/github.com/tidwall/gjson/README.md create mode 100644 vendor/github.com/tidwall/gjson/SYNTAX.md create mode 100644 vendor/github.com/tidwall/gjson/gjson.go create mode 100644 vendor/github.com/tidwall/gjson/logo.png create mode 100644 vendor/github.com/tidwall/match/LICENSE create mode 100644 vendor/github.com/tidwall/match/README.md create mode 100644 vendor/github.com/tidwall/match/match.go create mode 100644 vendor/github.com/tidwall/pretty/LICENSE create mode 100644 vendor/github.com/tidwall/pretty/README.md create mode 100644 vendor/github.com/tidwall/pretty/pretty.go diff --git a/GNUmakefile b/GNUmakefile index 1cf614892..b7985cce1 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -6,6 +6,8 @@ TERRAFORM_VERSION=0.14.7 TERRAFORM_OS=$(shell uname -s | tr A-Z a-z) TEST_USER=root TEST_PASSWORD=my-secret-pw +ARCH=$(shell uname -m|sed -e's/(amd64|x86_64)/amd64/g' |sed -e's/(arm64|aarch64)/arm64/g') +TIUP_HOME=$(CURDIR)/bin/.tiup default: build @@ -52,12 +54,21 @@ testpercona: testtidb%: $(MAKE) MYSQL_VERSION=$* MYSQL_PORT=34$(shell echo "$*" | tr -d '.') testtidb -testtidb: - -docker run --rm --name test-tidb$(MYSQL_VERSION) -d -p $(MYSQL_PORT):4000 pingcap/tidb:v$(MYSQL_VERSION) +bin/tiup: + test -d "$(CURDIR)/bin" || mkdir -p "$(CURDIR)/bin" + test -d "$(TIUP_HOME)" || mkdir -p "$(TIUP_HOME)" + (curl -sfL "https://tiup-mirrors.pingcap.com/tiup-$(TERRAFORM_OS)-$(ARCH).tar.gz?$(shell date "+%Y%m%d%H%M%S")" -o "$(TIUP_HOME)/tiup-$(TERRAFORM_OS)-$(ARCH).tar.gz" && \ + tar zxf $(TIUP_HOME)/tiup-$(TERRAFORM_OS)-$(ARCH).tar.gz -C $(TIUP_HOME) && \ + chmod 755 $(TIUP_HOME)/tiup && \ + rm "$(TIUP_HOME)/tiup-$(TERRAFORM_OS)-$(ARCH).tar.gz" || exit 1 ) + +testtidb: bin/tiup + $(eval TEMPDIR := $(shell mktemp -d)) + -nohup $(TIUP_HOME)/bin/tiup playground 6.1.0 --without-monitor --db.port $(MYSQL_PORT) > $(TEMPDIR)/tiup.log 2>&1 & echo "$$!" > $(TEMPDIR)/tiup.pid @echo 'Waiting for TiDB...' @while ! mysql -h 127.0.0.1 -P $(MYSQL_PORT) -u "$(TEST_USER)" -e 'SELECT 1' >/dev/null 2>&1; do printf '.'; sleep 1; done ; echo ; echo "Connected!" MYSQL_USERNAME="$(TEST_USER)" MYSQL_PASSWORD="" MYSQL_ENDPOINT=127.0.0.1:$(MYSQL_PORT) $(MAKE) testacc - docker rm -f test-tidb$(MYSQL_VERSION) + kill $$(cat $(TEMPDIR)/tiup.pid) testmariadb%: $(MAKE) MYSQL_VERSION=$* MYSQL_PORT=36$(shell echo "$*" | tr -d '.') testmariadb @@ -69,8 +80,6 @@ testmariadb: MYSQL_USERNAME="$(TEST_USER)" MYSQL_PASSWORD="$(TEST_PASSWORD)" MYSQL_ENDPOINT=127.0.0.1:$(MYSQL_PORT) $(MAKE) testacc docker rm -f test-mariadb$(MYSQL_VERSION) - - vet: @echo "go vet ." @go vet $$(go list ./... | grep -v vendor/) ; if [ $$? -eq 1 ]; then \ diff --git a/go.mod b/go.mod index c9941f231..ed1f135c7 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,12 @@ module github.com/terraform-providers/terraform-provider-mysql require ( + github.com/creasty/defaults v1.6.0 github.com/go-sql-driver/mysql v1.6.0 github.com/gofrs/uuid v4.2.0+incompatible github.com/hashicorp/go-version v1.4.0 github.com/hashicorp/terraform-plugin-sdk/v2 v2.16.0 + github.com/tidwall/gjson v1.14.2 golang.org/x/net v0.0.0-20220517181318-183a9ca12b87 ) @@ -41,6 +43,8 @@ require ( github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/oklog/run v1.1.0 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect github.com/vmihailenco/msgpack/v4 v4.3.12 // indirect github.com/vmihailenco/tagparser v0.1.2 // indirect diff --git a/go.sum b/go.sum index 1aca19011..9be0a72dc 100644 --- a/go.sum +++ b/go.sum @@ -35,6 +35,8 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creasty/defaults v1.6.0 h1:ltuE9cfphUtlrBeomuu8PEyISTXnxqkBIoQfXgv7BSc= +github.com/creasty/defaults v1.6.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -224,6 +226,12 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tidwall/gjson v1.14.2 h1:6BBkirS0rAHjumnjHF6qgy5d2YAJ1TLIaFE2lzfOLqo= +github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= diff --git a/mysql/provider.go b/mysql/provider.go index 46d7c2848..c2e5a11eb 100644 --- a/mysql/provider.go +++ b/mysql/provider.go @@ -140,6 +140,7 @@ func Provider() *schema.Provider { "mysql_sql": resourceSql(), "mysql_user_password": resourceUserPassword(), "mysql_user": resourceUser(), + "mysql_ti_config": resourceTiConfigVariable(), }, ConfigureFunc: providerConfigure, diff --git a/mysql/provider_test.go b/mysql/provider_test.go index dfd94df12..73a61024a 100644 --- a/mysql/provider_test.go +++ b/mysql/provider_test.go @@ -2,6 +2,7 @@ package mysql import ( "context" + "fmt" "os" "strings" "testing" @@ -100,3 +101,21 @@ func testAccPreCheckSkipMariaDB(t *testing.T) { t.Skip("Skip on MariaDB") } } + +func testAccPreCheckSkipNotTiDB(t *testing.T) { + testAccPreCheck(t) + db, err := connectToMySQL(testAccProvider.Meta().(*MySQLConfiguration)) + if err != nil { + return + } + + currentVersionString, err := serverVersionString(db) + if err != nil { + return + } + + if !strings.Contains(currentVersionString, "TiDB") { + msg := fmt.Sprintf("Skip on MySQL %s", currentVersionString) + t.Skip(msg) + } +} diff --git a/mysql/resource_grant_test.go b/mysql/resource_grant_test.go index ee2554bfd..649c73807 100644 --- a/mysql/resource_grant_test.go +++ b/mysql/resource_grant_test.go @@ -109,7 +109,10 @@ func TestAccBroken(t *testing.T) { func TestAccDifferentHosts(t *testing.T) { dbName := fmt.Sprintf("tf-test-%d", rand.Intn(100)) resource.Test(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, + PreCheck: func() { + testAccPreCheck(t) + testAccPreCheckSkipTiDB(t) + }, Providers: testAccProviders, CheckDestroy: testAccGrantCheckDestroy, Steps: []resource.TestStep{ diff --git a/mysql/resource_ti_config_defaults.go b/mysql/resource_ti_config_defaults.go new file mode 100644 index 000000000..987159bee --- /dev/null +++ b/mysql/resource_ti_config_defaults.go @@ -0,0 +1,348 @@ +package mysql + +type defaultConfig struct { + Pd PdConfigurationKeys + TiKv TiKvConfigurationKeys +} + +type PdConfigurationKeys struct { + // The cluster version + ClusterVersion string `json:"cluster-version" default:"IGNOREONDESTROY#"` + Log pdLogKeys `json:"log"` + Schedule pdScheduleKeys `json:"schedule"` + Replication PdReplicationKeys `json:"replication"` + PdServer PdServerKeys `json:"pd-server"` + PdReplicationMode PdReplicationModeKeys `json:"replication-mode"` +} + +type pdLogKeys struct { + // The log level + Level string `json:"level" default:"info"` +} + +type pdScheduleKeys struct { + // Controls the size limit of Region Merge (in MiB) + MaxMergeRegionSize int64 `json:"max-merge-region-size" default:"20"` + // Specifies the maximum numbers of the Region Merge keys + MaxMergeRegionKeys int64 `json:"max-merge-region-keys" default:"200000"` + // Determines the frequency at which replicaChecker checks the health state of a Region + PatrolRegionInterval string `json:"patrol-region-interval" default:"10ms"` + // Determines the time interval of performing split and merge operations on the same Region + SplitMergeInterval string `json:"split-merge-interval" default:"1h"` + // Determines the maximum number of snapshots that a single store can send or receive at the same time + MaxSnapshotCount int64 `json:"max-snapshot-count" default:"64"` + // Determines the maximum number of pending peers in a single store + MaxPendingPeerCount int64 `json:"max-pending-peer-count" default:"64"` + // The downtime after which PD judges that the disconnected store can not be recovered + MaxStoreDownTime string `json:"max-store-down-time" default:"30m"` + // Determines the policy of Leader scheduling + LeaderSchedulePolicy string `json:"leader-schedule-policy" default:"IGNOREONDESTROY#"` + // The number of Leader scheduling tasks performed at the same time + LeaderScheduleLimit int64 `json:"leader-schedule-limit" default:"4"` + // The number of Region scheduling tasks performed at the same time + RegionScheduleLimit int64 `json:"region-schedule-limit" default:"2048"` + // The number of Replica scheduling tasks performed at the same time + ReplicaScheduleLimit int64 `json:"replica-schedule-limit" default:"64"` + // The number of the Region Merge scheduling tasks performed at the same time + MergeScheduleLimit int64 `json:"merge-schedule-limit" default:"8"` + // The number of hot Region scheduling tasks performed at the same time + HotRegionScheduleLimit int64 `json:"hot-region-schedule-limit" default:"4"` + // Determines the threshold at which a Region is considered a hot spot + HotRegionCacheHitsThreshold int64 `json:"hot-region-cache-hits-threshold" default:"3"` + // The threshold ratio below which the capacity of the store is sufficient + HighSpaceRatio float64 `json:"high-space-ratio" default:"0.7"` + // The threshold ratio above which the capacity of the store is insufficient + LowSpaceRatio float64 `json:"low-space-ratio" default:"0.8"` + // Controls the balance buffer size + TolerantSizeRatio int64 `json:"tolerant-size-ratio" default:"0"` + // Determines whether to enable the feature that automatically removes DownReplica + EnableRemoveDownReplica bool `json:"enable-remove-down-replica" default:"false"` + // Determines whether to enable the feature that migrates OfflineReplica + EnableReplaceOfflineReplica bool `json:"enable-replace-offline-replica" default:"false"` + // Determines whether to enable the feature that automatically supplements replicas + EnableMakeUpReplica bool `json:"enable-make-up-replica" default:"false"` + // Determines whether to enable the feature that removes extra replicas + EnableRemoveExtraReplica bool `json:"enable-remove-extra-replica" default:"false"` + // Determines whether to enable isolation level check + EnableLocationReplacement bool `json:"enable-location-replacement" default:"false"` + // Determines whether to enable cross-table merge + EnableCrossTableMerge bool `json:"enable-cross-table-merge" default:"true"` + // Enables one-way merge, which only allows merging with the next adjacent Region + EnableOneWayMerge bool `json:"enable-one-way-merge" default:"false"` +} + +type PdReplicationKeys struct { + // Sets the maximum number of replicas + MaxReplicas int64 `json:"max-replicas" default:"3"` + // The topology information of a TiKV cluster + LocationLabels string `json:"location-labels" default:"[]"` + // Enables Placement Rules + EnablePlacementRules bool `json:"enable-placement-rules" default:"false"` + // Enables the label check + StrictlyMatchLabel bool `json:"strictly-match-label" default:"false"` +} + +type PdServerKeys struct { + // Enables independent Region storage + UseRegionStorage bool `json:"use-region-storage" default:"false"` + // Sets the maximum interval of resetting timestamp (BR) + MaxGapResetTs string `json:"max-gap-reset-ts" default:"IGNOREONDESTROY#"` + // Sets the cluster key type + KeyType string `json:"key-type" default:"IGNOREONDESTROY#"` + // Sets the storage address of the cluster metrics + MetricStorage string `json:"metric-storage" default:"IGNOREONDESTROY#"` + // Sets the dashboard address + DashboardAddress string `json:"dashboard-address" default:"IGNOREONDESTROY#"` +} + +type PdReplicationModeKeys struct { + // Sets the backup mode + ReplicationMode string `json:"replication-mode" default:"IGNOREDESTROY"` +} + +type TiKvConfigurationKeys struct { + Raftstore tikvRaftstoreKeys `json:"raftstore"` + Coprocessor tikvCoprocessorKeys `json:"coprocessor"` + PessimisticTxn tikvPessimisticTxnKeys `json:"pessimistic-txn"` + Readpool tikvReadpoolKeys `json:"readpool"` + Backup tikvBackupKeys `json:"backup"` + Quota tikvQuotaKeys `json:"quota"` + Gc tikvGcKeys `json:"gc"` + Server tikvServerKeys `json:"server"` + Storage tikvStroageKeys `json:"storage"` + Split tikvSplitKeys `json:"split"` + Cdc tikvCdcKeys `json:"cdc"` + Db tikvDbGlobalCfgKeys `json:"defaultDB"` +} + +type tikvQuotaKeys struct { + // The soft limit on the CPU resources used by TiKV foreground to process read and write requests + ForegroundCpuTime int `json:"foreground-cpu-time" default:"0"` + // The soft limit on the bandwidth with which transactions write data + ForegroundWriteBandwidth string `json:"foreground-write-bandwidth" default:"0KB"` + // The soft limit on the bandwidth with which transactions and the Coprocessor read data + ForegroundReadBandwidth string `json:"foreground-read-bandwidth" default:"0KB"` + // The maximum time that a single read or write request is forced to wait before it is processed in the foreground + MaxDelayDuration string `json:"max-delay-duration" default:"500ms"` +} + +type tikvGcKeys struct { + // The threshold at which Region GC is skipped (the number of GC versions/the number of keys) + RatioThreshold int64 `json:"ratio-threshold" default:"0"` + // The number of keys processed in one batch + BatchKeys int64 `json:"batch-keys" default:"0"` + // The maximum bytes that can be written into RocksDB per second + MaxWriteBytesPerSec int64 `json:"max-write-bytes-per-sec" default:"1KB"` + // Whether to enable compaction filter + EnableCompactionFilter bool `json:"enable-compaction-filter" default:"true"` + // Whether to skip the cluster version check of compaction filter (not released) + CompactionFilterSkipVersionCheck bool `json:"compaction-filter-skip-version-check" default:"false"` +} + +type tikvServerKeys struct { + // Limits the memory size that can be used by gRPC + GrpcMemoryPoolQuota string `json:"grpc-memory-pool-quota" default:"IGNOREONDESTROY#"` + // Sets the maximum length of a gRPC message that can be sent + MaxGrpcSendMsgLen int64 `json:"max-grpc-send-msg-len" default:"10485760"` + // Sets the maximum number of Raft messages that are contained in a single gRPC message + RaftMsgMaxBatchSize int64 `json:"raft-msg-max-batch-size" default:"IGNOREONDESTROY#"` +} +type tikvDbGlobalCfgKeys struct { + // The maximum size of total WAL + MaxTotalWalSize int64 `json:"max-total-wal-size" default:"IGNOREONDESTROY#"` + // The number of background threads in RocksDB + MaxBackgroundJobs int64 `json:"max-background-jobs" default:"IGNOREONDESTROY#[max(2, min({number_of_cores}-1,9))]"` + // The maximum number of flush threads in RocksDB + MaxBackgroundFlushes int64 `json:"max-background-flushes" default:"IGNOREONDESTROY#[(max-background-jobs + 3)/4]"` + // The total number of files that RocksDB can open + MaxOpenFiles int64 `json:"max-open-files" default:"40960"` + // The size of readahead during compaction + CompactionReadaheadSize string `json:"compaction-readahead-size" default:"0B"` + // The rate at which OS incrementally synchronizes files to disk while these files are being written asynchronously + BytesPerSync string `json:"bytes-per-sync" default:"1MB"` + // The rate at which OS incrementally synchronizes WAL files to disk while the WAL files are being written + WalBytesPerSync string `json:"wal-bytes-per-sync" default:"512KB"` + // The maximum buffer size used in WritableFileWrite + WritableFileMaxBufferSize string `json:"writable-file-max-buffer-size" default:"1MB"` + DefaultDbConfig tikvDbCfgKeys `json:"default_db_config"` +} + +type tikvDbCfgKeys struct { + // The cache size of a block + BlockCacheSize string `json:"block-cache-size" default:"IGNOREONDESTROY#[{'defaultcf': 'Total_machine_memory * 25%', 'writecf': 'Total_machine_memory * 15%','lockcf': 'Total_machine_memory * 2%'}]"` + // The size of a memtable + WriteBufferSize string `json:"write-buffer-size" default:"IGNOREONDESTROY#[{'defaultcf': '128MB', 'writecf': '128MB','lockcf': '32MB'}]"` + // The maximum number of memtables + MaxWriteBufferNumber int64 `json:"max-write-buffer-number" default:"5"` + // The maximum number of bytes at base level (L1) + MaxBytesForLevelBase string `json:"max-bytes-for-level-base" default:"IGNOREONDESTROY#[{'defaultcf': '512MB', 'writecf': '512MB','lockcf': '128MB'}]"` + // The size of the target file at base level + TargetFileSizeBase string `json:"target-file-size-base" default:"8MB"` + // The maximum number of files at L0 that trigger compaction + Level0FileNumCompactionTrigger int64 `json:"level0-file-num-compaction-trigger" default:"IGNOREONDESTROY#[{'defaultcf': 4, 'writecf': 4, 'lockcf': 1}]"` + // The maximum number of files at L0 that trigger write stall + Level0SlowdownWritesTrigger int64 `json:"level0-slowdown-writes-trigger" default:"20"` + // The maximum number of files at L0 that completely block write + Level0StopWritesTrigger int64 `json:"level0-stop-writes-trigger" default:"36"` + // The maximum number of bytes written into disk per compaction + MaxCompactionBytes string `json:"max-compaction-bytes" default:"2GB"` + // The default amplification multiple for each layer + MaxBytesForLevelMultiplier int64 `json:"max-bytes-for-level-multiplier" default:"10"` + // Enables or disables automatic compaction + DisableAutoCompactions bool `json:"disable-auto-compactions" default:"false"` + // The soft limit on the pending compaction bytes + SoftPendingCompactionBytesLimit string `json:"soft-pending-compaction-bytes-limit" default:"192GB"` + // The hard limit on the pending compaction bytes + HardPendingCompactionBytesLimit string `json:"hard-pending-compaction-bytes-limit" default:"1024GB"` + // The mode of processing blob files + Titan tikvDbCfgTitan `json:"titan"` +} +type tikvDbCfgTitan struct { + BlobRunMode string `json:"blob-run-mode" default:"normal"` +} + +type tikvStroageKeys struct { + // The size of shared block cache (supported since v4.0.3) + BlockCache tikvStorageBlockCache `json:"block-cache"` + // The number of threads in the Scheduler thread pool + SchedulerWorkerPoolSize int64 `json:"scheduler-worker-pool-size" default:"4"` +} + +type tikvStorageBlockCache struct { + Capacity string `json:"capacity" default:"IGNOREONDESTROY#[45% f the size of total system memory]"` +} + +type tikvSplitKeys struct { + // The threshold to execute load-base-split on a Region. If the QPS of read requests for a Region exceeds qps-threshold for a consecutive period of time, this Region should be + QpsThreshold string `json:"qps-threshold" default:"IGNOREONDESTROY#"` + // The threshold to execute load-base-split on a Region. If the traffic of read requests for a Region exceeds the byte-threshold for a consecutive period of time, this Region should be + ByteThreshold string `json:"byte-threshold" default:"IGNOREONDESTROY#"` + // The parameter of load-base-split, which ensures the load of the two split Regions is as balanced as possible. The smaller the value is, the more balanced the load is. But setting it too small might cause split failure. + SplitBalanceScore string `json:"split-balance-score" default:"IGNOREONDESTROY#"` + // The parameter of load-base- The smaller the value, the fewer cross-Region visits after Region + SplitContainedScore string `json:"split-contained-score" default:"IGNOREONDESTROY#"` +} +type tikvCdcKeys struct { + // The time interval at which Resolved TS is forwarded + MinTsInterval string `json:"min-ts-interval" default:"1s"` + // The upper limit of memory occupied by the TiCDC Old Value entries + OldValueCacheMemoryQuota string `json:"old-value-cache-memory-quota" default:"512MB"` + // The upper limit of memory occupied by TiCDC data change events + SinkMemoryQuota string `json:"sink-memory-quota" default:"512MB"` + // The upper limit on the speed of incremental scanning for historical data + IncrementalScanSpeedLimit string `json:"incremental-scan-speed-limit" default:"128MB"` + // The maximum number of concurrent incremental scanning tasks for historical data + IncrementalScanConcurrency int64 `json:"incremental-scan-concurrency" default:"6"` +} +type tikvReadpoolKeys struct { + Unifed tikvReadpoolUnified `json:"unifed"` +} + +type tikvReadpoolUnified struct { + //The maximum number of threads in the thread pool that uniformly processes read requests, which is the size of the UnifyReadPool thread pool + MaxThreadCount int64 `json:"max-thread-count" default:"IGNOREONDESTROY#[MAX(4, cpu_count * 0.8)]"` +} + +type tikvBackupKeys struct { + // The number of backup threads (supported since v4.0.3) + NumThreads int64 `json:"num-threads" default:"IGNOREONDESTROY#[MIN(CPU * 0.5, 8)]"` +} + +type tikvRaftstoreKeys struct { + // The number of Raft logs to be confirmed. If this number is exceeded, the Raft state machine slows down log sending. + RaftMaxInflightMsgs int64 `json:"raft-max-inflight-msgs" default:"256"` + // The time interval at which the polling task of deleting Raft logs is scheduled + RaftLogGcTickInterval string `json:"raft-log-gc-tick-interval" default:"3s"` + // The soft limit on the maximum allowable number of residual Raft logs + RaftLogGcThreshold int64 `json:"raft-log-gc-threshold" default:"50"` + // The hard limit on the allowable number of residual Raft logs + RaftLogGcCountLimit int64 `json:"raft-log-gc-count-limit" default:"IGNOREONDESTROY#the log number that can be accommodated in the 3/4 Region size (calculated as 1MB for each log)"` + // The hard limit on the allowable size of residual Raft logs + RaftLogGcSizeLimit int64 `json:"raft-log-gc-size-limit" default:"IGNOREONDESTROY#[region_size*3/4]"` + // The soft limit on the size of a single message packet that is allowed to be generated + RaftMaxSizePerMsg string `json:"raft-max-size-per-msg" default:"1MB"` + // The hard limit on the maximum size of a single Raft log + RaftEntryMaxSize string `json:"raft-entry-max-size" default:"8MB"` + // The maximum remaining time allowed for the log cache in memory + RaftEntryCacheLifeTime string `json:"raft-entry-cache-life-time" default:"30s"` + // The time interval at wh ich to check whether the Region split is needed + SplitRegionCheckTickInterval string `json:"split-region-check-tick-interval" default:"10s"` + // The maximum value by which the Region data is allowed to exceed before Region split + RegionSplitCheckDiff string `json:"region-split-check-diff" default:"IGNOREONDESTROY#[regions_size*1/16]"` + // The time interval at which to check whether it is necessary to manually trigger RocksDB compaction + RegionCompactCheckInterval string `json:"region-compact-check-interval" default:"5m"` + // The number of Regions checked at one time for each round of manual compaction + RegionCompactCheckStep int64 `json:"region-compact-check-step" default:"100"` + // The number of tombstones required to trigger RocksDB compaction + RegionCompactMinTombstones int64 `json:"region-compact-min-tombstones" default:"10000"` + // The proportion of tombstone required to trigger RocksDB compaction + RegionCompactTombstonesPercent int64 `json:"region-compact-tombstones-percent" default:"30"` + // The time interval at which a Region's heartbeat to PD is triggered + PdHeartbeatTickInterval string `json:"pd-heartbeat-tick-interval" default:"1m"` + // The time interval at which a store's heartbeat to PD is triggered + PdStoreHeartbeatTickInterval string `json:"pd-store-heartbeat-tick-interval" default:"10s"` + // The time interval at which the recycle of expired snapshot files is triggered + SnapMgrGcTickInterval string `json:"snap-mgr-gc-tick-interval" default:"1m"` + // The longest time for which a snapshot file is saved + SnapGcTimeout string `json:"snap-gc-timeout"` + // The time interval at which TiKV triggers a manual compaction for the Lock Column Family + LockCfCompactInterval string `json:"lock-cf-compact-interval" default:"10m"` + // The size at which TiKV triggers a manual compaction for the Lock Column Family + LockCfCompactBytesThreshold string `json:"lock-cf-compact-bytes-threshold" default:"256MB"` + // The maximum number of messages processed per batch + MessagesPerTick int64 `json:"messages-per-tick" default:"4096"` + // The longest inactive duration allowed for a peer + MaxPeerDownDuration string `json:"max-peer-down-duration" default:"10m"` + // The longest duration allowed for a peer to be without a leader. If this value is exceeded, the peer verifies with PD whether it has been deleted. + MaxLeaderMissingDuration string `json:"max-leader-missing-duration" default:"2h"` + // The normal duration allowed for a peer to be without a leader. If this value is exceeded, the peer is seen as abnormal and marked in metrics and logs. + AbnormalLeaderMissingDuration string `json:"abnormal-leader-missing-duration" default:"10m"` + // The time interval to check whether a peer is without a leader + PeerStaleStateCheckInterval string `json:"peer-stale-state-check-interval" default:"5m"` + // The time interval to check consistency (NOT recommended because it is not compatible with the garbage collection in TiDB) + ConsistencyCheckInterval string `json:"consistency-check-interval" default:"0s"` + // The longest trusted period of a Raft leader + RaftStoreMaxLeaderLease string `json:"raft-store-max-leader-lease" default:"9s"` + // The time interval for merge check + MergeCheckTickInterval string `json:"merge-check-tick-interval" default:"2s"` + // The time interval to check expired SST files + CleanupImportSstInterval string `json:"cleanup-import-sst-interval" default:"10m"` + // The maximum number of read requests processed in one batch + LocalReadBatchSize string `json:"local-read-batch-size" default:"1024"` + // The shortest wait duration before entering hibernation upon start. Within this duration, TiKV does not hibernate (not released). + HibernateTimeout int64 `json:"hibernate-timeout" default:"0"` + // The number of threads in the pool that flushes data to the disk, which is the size of the Apply thread pool + ApplyPoolSize int64 `json:"apply-pool-size" default:"2"` + // The number of threads in the pool that processes Raft, which is the size of the Raftstore thread pool + StorePoolSize int64 `json:"store-pool-size" default:"2"` + // Raft state machines process data write requests in batches by the BatchSystem. This configuration item specifies the maximum number of Raft state machines that can execute the requests in one batch. + ApplyMaxBatchSize int64 `json:"apply-max-batch-size" default:"256"` + // Raft state machines process requests for flushing logs into the disk in batches by the BatchSystem. This configuration item specifies the maximum number of Raft state machines that can process the requests in one batch. + StoreMaxBatchSize int64 `json:"store-max-batch-size" default:"IGNOREONDESTROY#If hibernate-regions is enabled, the default value is 256. If hibernate-regions is disabled, the default value is 1024."` +} + +type tikvCoprocessorKeys struct { + // Enables to split Region by table + SplitRegionOnTable bool `json:"split-region-on-table" default:"false"` + // The threshold of Region split in batches + BatchSplitLimit int64 `json:"batch-split-limit" default:"10"` + // The maximum size of a Region + RegionMaxSize string `json:"region-max-size" default:"IGNOREONDESTROY#[region-split-size/2*3]"` + // The size of the newly split Region + RegionSplitSize string `json:"region-split-size" default:"96MiB"` + // The maximum number of keys allowed in a Region + RegionMaxKeys string `json:"region-max-keys" default:"IGNOREONDESTROY#[region-split-keys/2*3]"` + // The number of keys in the newly split Region + RegionSplitKeys int64 `json:"region-split-keys" default:"960000"` +} + +type tikvPessimisticTxnKeys struct { + // The longest duration that a pessimistic transaction waits for the lock + WaitForLockTimeout string `json:"wait-for-lock-timeout" default:"1s"` + // The duration after which a pessimistic transaction is woken up + WakeUpDelayDuration string `json:"wake-up-delay-duration" default:"20ms"` + // Determines whether to enable the pipelined pessimistic locking process + Pipelined bool `json:"pipelined" default:"true"` + // Determines whether to enable the in-memory pessimistic lock + InMemory bool `json:"in-memory" default:"true"` +} diff --git a/mysql/resource_ti_config_variable.go b/mysql/resource_ti_config_variable.go new file mode 100644 index 000000000..85c31d3be --- /dev/null +++ b/mysql/resource_ti_config_variable.go @@ -0,0 +1,171 @@ +package mysql + +import ( + "database/sql" + "encoding/json" + "fmt" + "log" + "regexp" + "strings" + + "github.com/creasty/defaults" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation" + "github.com/tidwall/gjson" +) + +func resourceTiConfigVariable() *schema.Resource { + return &schema.Resource{ + Create: CreateOrUpdateConfigVariable, + Read: ReadConfigVariable, + Update: CreateOrUpdateConfigVariable, + Delete: DeleteConfigVariable, + Importer: &schema.ResourceImporter{ + StateContext: schema.ImportStatePassthroughContext, + }, + Schema: map[string]*schema.Schema{ + "name": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + }, + "value": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + }, + "type": { + Type: schema.TypeString, + Required: true, + ValidateFunc: validation.StringInSlice([]string{"pd", "tikv"}, true), + }, + "instance": { + Type: schema.TypeString, + Optional: true, + }, + }, + } +} + +func CreateOrUpdateConfigVariable(d *schema.ResourceData, meta interface{}) error { + db := meta.(*MySQLConfiguration).Db + + varName := d.Get("name").(string) + varValue := d.Get("value").(string) + varInstanceType := d.Get("type").(string) + varInstance := d.Get("instance").(string) + + var warnLevel, warnMessage string + var warnCode int = 0 + + configQuery := fmt.Sprintf("SET CONFIG %s %s=", varInstanceType, quoteIdentifier(varName)) + + if varInstance != "" { + configQuery = fmt.Sprintf("SET CONFIG \"%s\" %s=", varInstance, quoteIdentifier(varName)) + } + + if isNumeric(varValue) { + configQuery = fmt.Sprintf("%s %s", configQuery, varValue) + } else { + configQuery = fmt.Sprintf("%s'%s'", configQuery, varValue) + } + + log.Printf("[DEBUG] SQL: %s\n", configQuery) + + _, err := db.Exec(configQuery) + if err != nil { + return fmt.Errorf("error setting value: %s", err) + } + + db.QueryRow("SHOW WARNINGS").Scan(&warnLevel, &warnCode, &warnMessage) + + if warnCode != 0 { + return fmt.Errorf("error setting value: %s -> %s Error: %s", varName, varValue, warnMessage) + } + + newId := fmt.Sprintf("%s#%s", varInstanceType, varName) + if varInstance != "" { + newId = fmt.Sprintf("%s#%s#%s", varInstanceType, varName, varInstance) + } + + d.SetId(newId) + + return nil +} + +func ReadConfigVariable(d *schema.ResourceData, meta interface{}) error { + var resType, resInstance, resName, resValue string + + db := meta.(*MySQLConfiguration).Db + match, _ := regexp.MatchString("^(pd|tikv)#(.*)$", d.Id()) + if !match { + return fmt.Errorf("error parsing TiDB component (tikv or pd) type from ID. \n Acceptable format is ##") + } + + indexParts := strings.Split(d.Id(), "#") + splitedResType := indexParts[0] + splitedResName := indexParts[1] + + configQuery := fmt.Sprintf("SHOW CONFIG WHERE type = '%s' AND name = '%s'", splitedResType, splitedResName) + if len(indexParts) > 2 { + configQuery = configQuery + fmt.Sprintf(" AND instance = '%s'", (indexParts[2])) + } + + log.Printf("[DEBUG] SQL: %s\n", configQuery) + + err := db.QueryRow(configQuery).Scan(&resType, &resInstance, &resName, &resValue) + if err != nil && err != sql.ErrNoRows { + d.SetId("") + return fmt.Errorf("error during show config variables: %s", err) + } + + d.Set("name", resName) + d.Set("type", resType) + if len(indexParts) > 2 { + d.Set("instance", resInstance) + } + d.Set("value", resValue) + + return nil + +} + +func DeleteConfigVariable(d *schema.ResourceData, meta interface{}) error { + varName := d.Get("name").(string) + varInstanceType := d.Get("type").(string) + defCfg := &defaultConfig{} + var jsonCfg []byte + var err error + + if err := defaults.Set(defCfg); err != nil { + return fmt.Errorf("error during destroy config variables: %s", err) + } + + switch varInstanceType { + case "pd": + jsonCfg, err = json.MarshalIndent(&defCfg.Pd, "", " ") + case "tikv": + jsonCfg, err = json.MarshalIndent(&defCfg.TiKv, "", " ") + default: + return fmt.Errorf("error during destory config variables: %s is not allowed type", varInstanceType) + } + + if err != nil { + return fmt.Errorf("error during destroy config variables: %s", err) + } + + log.Printf("[DEBUG] JSON CFG: %s", jsonCfg) + defaultValue := gjson.Get(string(jsonCfg), varName) + log.Printf("[DEBUG]: DESTROY %s %s->%s\n", varInstanceType, varName, defaultValue) + match, _ := regexp.MatchString("^(IGNOREONDESTROY)#(.*)$", defaultValue.String()) + if match { + log.Printf("[WARN] Variable_name (%s) dont have default values; removing from state", d.Id()) + d.SetId("") + return nil + } + + d.Set("value", defaultValue.String()) + + return CreateOrUpdateConfigVariable(d, meta) + +} diff --git a/mysql/resource_ti_config_variable_test.go b/mysql/resource_ti_config_variable_test.go new file mode 100644 index 000000000..a21a4baf8 --- /dev/null +++ b/mysql/resource_ti_config_variable_test.go @@ -0,0 +1,208 @@ +package mysql + +import ( + "database/sql" + "fmt" + "regexp" + "testing" + + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" + "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" +) + +func TestPdConfigVar_basic(t *testing.T) { + varName := "log.level" + varValue := "warn" + varType := "pd" + varInstance := "" + resourceName := "mysql_ti_config.test" + + resource.Test(t, resource.TestCase{ + PreCheck: func() { + testAccPreCheck(t) + testAccPreCheckSkipNotTiDB(t) + }, + Providers: testAccProviders, + CheckDestroy: testAccConfigVarCheckDestroy(varName, varType), + Steps: []resource.TestStep{ + { + Config: testAccConfigVarConfig_basic(varName, varValue, varType), + Check: resource.ComposeTestCheckFunc( + testAccConfigVarExists(varName, varValue, varType), + resource.TestCheckResourceAttr(resourceName, "name", varName), + ), + }, + { + Config: testAccConfigVarConfig_withInstanceAndType(varName, varValue, varType, varInstance), + Check: resource.ComposeTestCheckFunc( + testAccConfigVarExists(varName, varValue, varType), + resource.TestCheckResourceAttr(resourceName, "name", varName), + ), + }, + { + Config: testAccConfigVarConfig_withInstanceAndType(varName, varValue, varType, getGetInstance(varType)), + Check: resource.ComposeTestCheckFunc( + testAccConfigVarExists(varName, varValue, varType), + resource.TestCheckResourceAttr(resourceName, "name", varName), + ), + }, + { + Config: testAccConfigVarConfig_withInstanceAndType(varName, varValue, varType, getGetInstance(varType)), + ExpectError: regexp.MustCompile("variable 'log.level' not found"), + Check: resource.ComposeTestCheckFunc( + testAccConfigVarExists(varName, varValue, "badType"), + resource.TestCheckResourceAttr(resourceName, "name", varName), + ), + }, + { + Config: testAccConfigVarConfig_withInstanceAndType("varName", varValue, varType, getGetInstance(varType)), + ExpectError: regexp.MustCompile(".*Error: bad request to:*"), + Check: resource.ComposeTestCheckFunc( + testAccConfigVarExists(varName, varValue, varType), + resource.TestCheckResourceAttr(resourceName, "name", varName), + ), + }, + { + Config: testAccConfigVarConfig_withInstanceAndType(varName, varValue, "varType", getGetInstance(varType)), + ExpectError: regexp.MustCompile(".*Error: expected type to be one of.*"), + Check: resource.ComposeTestCheckFunc( + testAccConfigVarExists(varName, varValue, varType), + resource.TestCheckResourceAttr(resourceName, "name", varName), + ), + }, + }, + }) +} + +func TestTiKvConfigVar_basic(t *testing.T) { + varName := "split.qps-threshold" + varValue := "1000" + varType := "tikv" + varInstance := "" + resourceName := "mysql_ti_config.test" + + resource.Test(t, resource.TestCase{ + PreCheck: func() { + testAccPreCheck(t) + testAccPreCheckSkipNotTiDB(t) + }, + Providers: testAccProviders, + CheckDestroy: testAccConfigVarCheckDestroy(varName, varType), + Steps: []resource.TestStep{ + { + Config: testAccConfigVarConfig_basic(varName, varValue, varType), + Check: resource.ComposeTestCheckFunc( + testAccConfigVarExists(varName, varValue, varType), + resource.TestCheckResourceAttr(resourceName, "name", varName), + ), + }, + { + Config: testAccConfigVarConfig_withInstanceAndType(varName, varValue, varType, varInstance), + Check: resource.ComposeTestCheckFunc( + testAccConfigVarExists(varName, varValue, varType), + resource.TestCheckResourceAttr(resourceName, "name", varName), + ), + }, + { + Config: testAccConfigVarConfig_withInstanceAndType(varName, "varValue", varType, varInstance), + ExpectError: regexp.MustCompile(".*Error: error setting value*"), + Check: resource.ComposeTestCheckFunc( + testAccConfigVarExists(varName, "varValue", varType), + resource.TestCheckResourceAttr(resourceName, "name", varName), + ), + }, + }, + }) +} + +func testAccConfigVarExists(varName string, varValue string, varType string) resource.TestCheckFunc { + return func(s *terraform.State) error { + db, err := connectToMySQL(testAccProvider.Meta().(*MySQLConfiguration)) + if err != nil { + return err + } + + resName, resValue, err := testAccGetConfigVar(varName, varType, db) + + if err != nil { + return err + } + + if resValue == varValue { + return nil + } + + return fmt.Errorf("variable '%s' not found. resName: %s, resValue: %s, err: %s", varName, resName, resValue, err) + } +} + +func getGetInstance(varType string) string { + var resInstanceType, resInstance, resName, resValue string + + db, err := connectToMySQL(testAccProvider.Meta().(*MySQLConfiguration)) + if err != nil { + return err.Error() + } + configQuery := "SHOW CONFIG WHERE type = ? AND name = 'log.level'" + + stmt, err := db.Prepare(configQuery) + + if err != nil { + return err.Error() + } + + err = stmt.QueryRow(varType).Scan(&resInstanceType, &resInstance, &resName, &resValue) + + if err != nil && err != sql.ErrNoRows { + return err.Error() + } + + return resInstance +} + +func testAccGetConfigVar(varName string, varType string, db *sql.DB) (string, string, error) { + var resType, resInstance, resName, resValue string + + configQuery := "SHOW CONFIG WHERE name = ? AND type = ?" + + stmt, err := db.Prepare(configQuery) + + if err != nil { + return "nil", "nil", err + } + + err = stmt.QueryRow(varName, varType).Scan(&resType, &resInstance, &resName, &resValue) + + if err != nil && err != sql.ErrNoRows { + return "nil", "nil", err + } + + return resName, resValue, nil +} + +func testAccConfigVarCheckDestroy(varName string, varType string) resource.TestCheckFunc { + return func(s *terraform.State) error { + return nil + } +} + +func testAccConfigVarConfig_basic(varName string, varValue string, varType string) string { + return fmt.Sprintf(` +resource "mysql_ti_config" "test" { + name = "%s" + value = "%s" + type = "%s" +} +`, varName, varValue, varType) +} + +func testAccConfigVarConfig_withInstanceAndType(varName string, varValue string, varType string, varInstance string) string { + return fmt.Sprintf(` +resource "mysql_ti_config" "test" { + name = "%s" + value = "%s" + type = "%s" + instance = "%s" +} +`, varName, varValue, varType, varInstance) +} diff --git a/mysql/utils.go b/mysql/utils.go index f5a46b26c..22e842c9a 100644 --- a/mysql/utils.go +++ b/mysql/utils.go @@ -3,8 +3,14 @@ package mysql import ( "crypto/sha256" "fmt" + "strconv" ) func hashSum(contents interface{}) string { return fmt.Sprintf("%x", sha256.Sum256([]byte(contents.(string)))) } + +func isNumeric(s string) bool { + _, err := strconv.ParseFloat(s, 64) + return err == nil +} diff --git a/vendor/github.com/creasty/defaults/.gitignore b/vendor/github.com/creasty/defaults/.gitignore new file mode 100644 index 000000000..e43b0f988 --- /dev/null +++ b/vendor/github.com/creasty/defaults/.gitignore @@ -0,0 +1 @@ +.DS_Store diff --git a/vendor/github.com/creasty/defaults/LICENSE b/vendor/github.com/creasty/defaults/LICENSE new file mode 100644 index 000000000..1483dd2d8 --- /dev/null +++ b/vendor/github.com/creasty/defaults/LICENSE @@ -0,0 +1,22 @@ +Copyright (c) 2017-present Yuki Iwanaga + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/creasty/defaults/Makefile b/vendor/github.com/creasty/defaults/Makefile new file mode 100644 index 000000000..bd42d46f4 --- /dev/null +++ b/vendor/github.com/creasty/defaults/Makefile @@ -0,0 +1,30 @@ +SHELL := /bin/bash -eu -o pipefail + +GO_TEST_FLAGS := -v + +PACKAGE_DIRS := $(shell go list ./... 2> /dev/null | grep -v /vendor/) +SRC_FILES := $(shell find . -name '*.go' -not -path './vendor/*') + + +# Tasks +#----------------------------------------------- +.PHONY: lint +lint: + @gofmt -e -d -s $(SRC_FILES) | awk '{ e = 1; print $0 } END { if (e) exit(1) }' + @echo $(SRC_FILES) | xargs -n1 golint -set_exit_status + @go vet $(PACKAGE_DIRS) + +.PHONY: test +test: lint + @go test $(GO_TEST_FLAGS) $(PACKAGE_DIRS) + +.PHONY: ci-test +ci-test: lint + @echo > coverage.txt + @for d in $(PACKAGE_DIRS); do \ + go test -coverprofile=profile.out -covermode=atomic -race -v $$d; \ + if [ -f profile.out ]; then \ + cat profile.out >> coverage.txt; \ + rm profile.out; \ + fi; \ + done diff --git a/vendor/github.com/creasty/defaults/README.md b/vendor/github.com/creasty/defaults/README.md new file mode 100644 index 000000000..de1ca3da7 --- /dev/null +++ b/vendor/github.com/creasty/defaults/README.md @@ -0,0 +1,73 @@ +defaults +======== + +[![CircleCI](https://circleci.com/gh/creasty/defaults/tree/master.svg?style=svg)](https://circleci.com/gh/creasty/defaults/tree/master) +[![codecov](https://codecov.io/gh/creasty/defaults/branch/master/graph/badge.svg)](https://codecov.io/gh/creasty/defaults) +[![GitHub release](https://img.shields.io/github/release/creasty/defaults.svg)](https://github.com/creasty/defaults/releases) +[![License](https://img.shields.io/github/license/creasty/defaults.svg)](./LICENSE) + +Initialize structs with default values + +- Supports almost all kind of types + - Scalar types + - `int/8/16/32/64`, `uint/8/16/32/64`, `float32/64` + - `uintptr`, `bool`, `string` + - Complex types + - `map`, `slice`, `struct` + - Aliased types + - `time.Duration` + - e.g., `type Enum string` + - Pointer types + - e.g., `*SampleStruct`, `*int` +- Recursively initializes fields in a struct +- Dynamically sets default values by [`defaults.Setter`](./setter.go) interface +- Preserves non-initial values from being reset with a default value + + +Usage +----- + +```go +type Gender string + +type Sample struct { + Name string `default:"John Smith"` + Age int `default:"27"` + Gender Gender `default:"m"` + + Slice []string `default:"[]"` + SliceByJSON []int `default:"[1, 2, 3]"` // Supports JSON + + Map map[string]int `default:"{}"` + MapByJSON map[string]int `default:"{\"foo\": 123}"` + MapOfStruct map[string]OtherStruct + MapOfPtrStruct map[string]*OtherStruct + MapOfStructWithTag map[string]OtherStruct `default:"{\"Key1\": {\"Foo\":123}}"` + + Struct OtherStruct `default:"{}"` + StructPtr *OtherStruct `default:"{\"Foo\": 123}"` + + NoTag OtherStruct // Recurses into a nested struct by default + OptOut OtherStruct `default:"-"` // Opt-out +} + +type OtherStruct struct { + Hello string `default:"world"` // Tags in a nested struct also work + Foo int `default:"-"` + Random int `default:"-"` +} + +// SetDefaults implements defaults.Setter interface +func (s *OtherStruct) SetDefaults() { + if defaults.CanUpdate(s.Random) { // Check if it's a zero value (recommended) + s.Random = rand.Int() // Set a dynamic value + } +} +``` + +```go +obj := &Sample{} +if err := defaults.Set(obj); err != nil { + panic(err) +} +``` diff --git a/vendor/github.com/creasty/defaults/defaults.go b/vendor/github.com/creasty/defaults/defaults.go new file mode 100644 index 000000000..3bb2c220a --- /dev/null +++ b/vendor/github.com/creasty/defaults/defaults.go @@ -0,0 +1,221 @@ +package defaults + +import ( + "encoding/json" + "errors" + "reflect" + "strconv" + "time" +) + +var ( + errInvalidType = errors.New("not a struct pointer") +) + +const ( + fieldName = "default" +) + +// Set initializes members in a struct referenced by a pointer. +// Maps and slices are initialized by `make` and other primitive types are set with default values. +// `ptr` should be a struct pointer +func Set(ptr interface{}) error { + if reflect.TypeOf(ptr).Kind() != reflect.Ptr { + return errInvalidType + } + + v := reflect.ValueOf(ptr).Elem() + t := v.Type() + + if t.Kind() != reflect.Struct { + return errInvalidType + } + + for i := 0; i < t.NumField(); i++ { + if defaultVal := t.Field(i).Tag.Get(fieldName); defaultVal != "-" { + if err := setField(v.Field(i), defaultVal); err != nil { + return err + } + } + } + callSetter(ptr) + return nil +} + +// MustSet function is a wrapper of Set function +// It will call Set and panic if err not equals nil. +func MustSet(ptr interface{}) { + if err := Set(ptr); err != nil { + panic(err) + } +} + +func setField(field reflect.Value, defaultVal string) error { + if !field.CanSet() { + return nil + } + + if !shouldInitializeField(field, defaultVal) { + return nil + } + + isInitial := isInitialValue(field) + if isInitial { + switch field.Kind() { + case reflect.Bool: + if val, err := strconv.ParseBool(defaultVal); err == nil { + field.Set(reflect.ValueOf(val).Convert(field.Type())) + } + case reflect.Int: + if val, err := strconv.ParseInt(defaultVal, 0, strconv.IntSize); err == nil { + field.Set(reflect.ValueOf(int(val)).Convert(field.Type())) + } + case reflect.Int8: + if val, err := strconv.ParseInt(defaultVal, 0, 8); err == nil { + field.Set(reflect.ValueOf(int8(val)).Convert(field.Type())) + } + case reflect.Int16: + if val, err := strconv.ParseInt(defaultVal, 0, 16); err == nil { + field.Set(reflect.ValueOf(int16(val)).Convert(field.Type())) + } + case reflect.Int32: + if val, err := strconv.ParseInt(defaultVal, 0, 32); err == nil { + field.Set(reflect.ValueOf(int32(val)).Convert(field.Type())) + } + case reflect.Int64: + if val, err := time.ParseDuration(defaultVal); err == nil { + field.Set(reflect.ValueOf(val).Convert(field.Type())) + } else if val, err := strconv.ParseInt(defaultVal, 0, 64); err == nil { + field.Set(reflect.ValueOf(val).Convert(field.Type())) + } + case reflect.Uint: + if val, err := strconv.ParseUint(defaultVal, 0, strconv.IntSize); err == nil { + field.Set(reflect.ValueOf(uint(val)).Convert(field.Type())) + } + case reflect.Uint8: + if val, err := strconv.ParseUint(defaultVal, 0, 8); err == nil { + field.Set(reflect.ValueOf(uint8(val)).Convert(field.Type())) + } + case reflect.Uint16: + if val, err := strconv.ParseUint(defaultVal, 0, 16); err == nil { + field.Set(reflect.ValueOf(uint16(val)).Convert(field.Type())) + } + case reflect.Uint32: + if val, err := strconv.ParseUint(defaultVal, 0, 32); err == nil { + field.Set(reflect.ValueOf(uint32(val)).Convert(field.Type())) + } + case reflect.Uint64: + if val, err := strconv.ParseUint(defaultVal, 0, 64); err == nil { + field.Set(reflect.ValueOf(val).Convert(field.Type())) + } + case reflect.Uintptr: + if val, err := strconv.ParseUint(defaultVal, 0, strconv.IntSize); err == nil { + field.Set(reflect.ValueOf(uintptr(val)).Convert(field.Type())) + } + case reflect.Float32: + if val, err := strconv.ParseFloat(defaultVal, 32); err == nil { + field.Set(reflect.ValueOf(float32(val)).Convert(field.Type())) + } + case reflect.Float64: + if val, err := strconv.ParseFloat(defaultVal, 64); err == nil { + field.Set(reflect.ValueOf(val).Convert(field.Type())) + } + case reflect.String: + field.Set(reflect.ValueOf(defaultVal).Convert(field.Type())) + + case reflect.Slice: + ref := reflect.New(field.Type()) + ref.Elem().Set(reflect.MakeSlice(field.Type(), 0, 0)) + if defaultVal != "" && defaultVal != "[]" { + if err := json.Unmarshal([]byte(defaultVal), ref.Interface()); err != nil { + return err + } + } + field.Set(ref.Elem().Convert(field.Type())) + case reflect.Map: + ref := reflect.New(field.Type()) + ref.Elem().Set(reflect.MakeMap(field.Type())) + if defaultVal != "" && defaultVal != "{}" { + if err := json.Unmarshal([]byte(defaultVal), ref.Interface()); err != nil { + return err + } + } + field.Set(ref.Elem().Convert(field.Type())) + case reflect.Struct: + if defaultVal != "" && defaultVal != "{}" { + if err := json.Unmarshal([]byte(defaultVal), field.Addr().Interface()); err != nil { + return err + } + } + case reflect.Ptr: + field.Set(reflect.New(field.Type().Elem())) + } + } + + switch field.Kind() { + case reflect.Ptr: + if isInitial || field.Elem().Kind() == reflect.Struct { + setField(field.Elem(), defaultVal) + callSetter(field.Interface()) + } + case reflect.Struct: + if err := Set(field.Addr().Interface()); err != nil { + return err + } + case reflect.Slice: + for j := 0; j < field.Len(); j++ { + if err := setField(field.Index(j), defaultVal); err != nil { + return err + } + } + case reflect.Map: + for _, e := range field.MapKeys() { + var v = field.MapIndex(e) + + switch v.Kind() { + case reflect.Ptr: + switch v.Elem().Kind() { + case reflect.Struct, reflect.Slice, reflect.Map: + if err := setField(v.Elem(), ""); err != nil { + return err + } + } + case reflect.Struct, reflect.Slice, reflect.Map: + ref := reflect.New(v.Type()) + ref.Elem().Set(v) + if err := setField(ref.Elem(), ""); err != nil { + return err + } + field.SetMapIndex(e, ref.Elem().Convert(v.Type())) + } + } + } + + return nil +} + +func isInitialValue(field reflect.Value) bool { + return reflect.DeepEqual(reflect.Zero(field.Type()).Interface(), field.Interface()) +} + +func shouldInitializeField(field reflect.Value, tag string) bool { + switch field.Kind() { + case reflect.Struct: + return true + case reflect.Ptr: + if !field.IsNil() && field.Elem().Kind() == reflect.Struct { + return true + } + case reflect.Slice: + return field.Len() > 0 || tag != "" + case reflect.Map: + return field.Len() > 0 || tag != "" + } + + return tag != "" +} + +// CanUpdate returns true when the given value is an initial value of its type +func CanUpdate(v interface{}) bool { + return isInitialValue(reflect.ValueOf(v)) +} diff --git a/vendor/github.com/creasty/defaults/setter.go b/vendor/github.com/creasty/defaults/setter.go new file mode 100644 index 000000000..1f64aa659 --- /dev/null +++ b/vendor/github.com/creasty/defaults/setter.go @@ -0,0 +1,12 @@ +package defaults + +// Setter is an interface for setting default values +type Setter interface { + SetDefaults() +} + +func callSetter(v interface{}) { + if ds, ok := v.(Setter); ok { + ds.SetDefaults() + } +} diff --git a/vendor/github.com/tidwall/gjson/LICENSE b/vendor/github.com/tidwall/gjson/LICENSE new file mode 100644 index 000000000..58f5819a4 --- /dev/null +++ b/vendor/github.com/tidwall/gjson/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2016 Josh Baker + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/tidwall/gjson/README.md b/vendor/github.com/tidwall/gjson/README.md new file mode 100644 index 000000000..00b4c9621 --- /dev/null +++ b/vendor/github.com/tidwall/gjson/README.md @@ -0,0 +1,497 @@ +

+GJSON +
+GoDoc +GJSON Playground +GJSON Syntax + +

+ +

get json values quickly

+ +GJSON is a Go package that provides a [fast](#performance) and [simple](#get-a-value) way to get values from a json document. +It has features such as [one line retrieval](#get-a-value), [dot notation paths](#path-syntax), [iteration](#iterate-through-an-object-or-array), and [parsing json lines](#json-lines). + +Also check out [SJSON](https://github.com/tidwall/sjson) for modifying json, and the [JJ](https://github.com/tidwall/jj) command line tool. + +This README is a quick overview of how to use GJSON, for more information check out [GJSON Syntax](SYNTAX.md). + +GJSON is also available for [Python](https://github.com/volans-/gjson-py) and [Rust](https://github.com/tidwall/gjson.rs) + +Getting Started +=============== + +## Installing + +To start using GJSON, install Go and run `go get`: + +```sh +$ go get -u github.com/tidwall/gjson +``` + +This will retrieve the library. + +## Get a value +Get searches json for the specified path. A path is in dot syntax, such as "name.last" or "age". When the value is found it's returned immediately. + +```go +package main + +import "github.com/tidwall/gjson" + +const json = `{"name":{"first":"Janet","last":"Prichard"},"age":47}` + +func main() { + value := gjson.Get(json, "name.last") + println(value.String()) +} +``` + +This will print: + +``` +Prichard +``` +*There's also the [GetMany](#get-multiple-values-at-once) function to get multiple values at once, and [GetBytes](#working-with-bytes) for working with JSON byte slices.* + +## Path Syntax + +Below is a quick overview of the path syntax, for more complete information please +check out [GJSON Syntax](SYNTAX.md). + +A path is a series of keys separated by a dot. +A key may contain special wildcard characters '\*' and '?'. +To access an array value use the index as the key. +To get the number of elements in an array or to access a child path, use the '#' character. +The dot and wildcard characters can be escaped with '\\'. + +```json +{ + "name": {"first": "Tom", "last": "Anderson"}, + "age":37, + "children": ["Sara","Alex","Jack"], + "fav.movie": "Deer Hunter", + "friends": [ + {"first": "Dale", "last": "Murphy", "age": 44, "nets": ["ig", "fb", "tw"]}, + {"first": "Roger", "last": "Craig", "age": 68, "nets": ["fb", "tw"]}, + {"first": "Jane", "last": "Murphy", "age": 47, "nets": ["ig", "tw"]} + ] +} +``` +``` +"name.last" >> "Anderson" +"age" >> 37 +"children" >> ["Sara","Alex","Jack"] +"children.#" >> 3 +"children.1" >> "Alex" +"child*.2" >> "Jack" +"c?ildren.0" >> "Sara" +"fav\.movie" >> "Deer Hunter" +"friends.#.first" >> ["Dale","Roger","Jane"] +"friends.1.last" >> "Craig" +``` + +You can also query an array for the first match by using `#(...)`, or find all +matches with `#(...)#`. Queries support the `==`, `!=`, `<`, `<=`, `>`, `>=` +comparison operators and the simple pattern matching `%` (like) and `!%` +(not like) operators. + +``` +friends.#(last=="Murphy").first >> "Dale" +friends.#(last=="Murphy")#.first >> ["Dale","Jane"] +friends.#(age>45)#.last >> ["Craig","Murphy"] +friends.#(first%"D*").last >> "Murphy" +friends.#(first!%"D*").last >> "Craig" +friends.#(nets.#(=="fb"))#.first >> ["Dale","Roger"] +``` + +*Please note that prior to v1.3.0, queries used the `#[...]` brackets. This was +changed in v1.3.0 as to avoid confusion with the new +[multipath](SYNTAX.md#multipaths) syntax. For backwards compatibility, +`#[...]` will continue to work until the next major release.* + +## Result Type + +GJSON supports the json types `string`, `number`, `bool`, and `null`. +Arrays and Objects are returned as their raw json types. + +The `Result` type holds one of these: + +``` +bool, for JSON booleans +float64, for JSON numbers +string, for JSON string literals +nil, for JSON null +``` + +To directly access the value: + +```go +result.Type // can be String, Number, True, False, Null, or JSON +result.Str // holds the string +result.Num // holds the float64 number +result.Raw // holds the raw json +result.Index // index of raw value in original json, zero means index unknown +result.Indexes // indexes of all the elements that match on a path containing the '#' query character. +``` + +There are a variety of handy functions that work on a result: + +```go +result.Exists() bool +result.Value() interface{} +result.Int() int64 +result.Uint() uint64 +result.Float() float64 +result.String() string +result.Bool() bool +result.Time() time.Time +result.Array() []gjson.Result +result.Map() map[string]gjson.Result +result.Get(path string) Result +result.ForEach(iterator func(key, value Result) bool) +result.Less(token Result, caseSensitive bool) bool +``` + +The `result.Value()` function returns an `interface{}` which requires type assertion and is one of the following Go types: + +```go +boolean >> bool +number >> float64 +string >> string +null >> nil +array >> []interface{} +object >> map[string]interface{} +``` + +The `result.Array()` function returns back an array of values. +If the result represents a non-existent value, then an empty array will be returned. +If the result is not a JSON array, the return value will be an array containing one result. + +### 64-bit integers + +The `result.Int()` and `result.Uint()` calls are capable of reading all 64 bits, allowing for large JSON integers. + +```go +result.Int() int64 // -9223372036854775808 to 9223372036854775807 +result.Uint() int64 // 0 to 18446744073709551615 +``` + +## Modifiers and path chaining + +New in version 1.2 is support for modifier functions and path chaining. + +A modifier is a path component that performs custom processing on the +json. + +Multiple paths can be "chained" together using the pipe character. +This is useful for getting results from a modified query. + +For example, using the built-in `@reverse` modifier on the above json document, +we'll get `children` array and reverse the order: + +``` +"children|@reverse" >> ["Jack","Alex","Sara"] +"children|@reverse|0" >> "Jack" +``` + +There are currently the following built-in modifiers: + +- `@reverse`: Reverse an array or the members of an object. +- `@ugly`: Remove all whitespace from a json document. +- `@pretty`: Make the json document more human readable. +- `@this`: Returns the current element. It can be used to retrieve the root element. +- `@valid`: Ensure the json document is valid. +- `@flatten`: Flattens an array. +- `@join`: Joins multiple objects into a single object. +- `@keys`: Returns an array of keys for an object. +- `@values`: Returns an array of values for an object. +- `@tostr`: Converts json to a string. Wraps a json string. +- `@fromstr`: Converts a string from json. Unwraps a json string. +- `@group`: Groups arrays of objects. See [e4fc67c](https://github.com/tidwall/gjson/commit/e4fc67c92aeebf2089fabc7872f010e340d105db). + +### Modifier arguments + +A modifier may accept an optional argument. The argument can be a valid JSON +document or just characters. + +For example, the `@pretty` modifier takes a json object as its argument. + +``` +@pretty:{"sortKeys":true} +``` + +Which makes the json pretty and orders all of its keys. + +```json +{ + "age":37, + "children": ["Sara","Alex","Jack"], + "fav.movie": "Deer Hunter", + "friends": [ + {"age": 44, "first": "Dale", "last": "Murphy"}, + {"age": 68, "first": "Roger", "last": "Craig"}, + {"age": 47, "first": "Jane", "last": "Murphy"} + ], + "name": {"first": "Tom", "last": "Anderson"} +} +``` + +*The full list of `@pretty` options are `sortKeys`, `indent`, `prefix`, and `width`. +Please see [Pretty Options](https://github.com/tidwall/pretty#customized-output) for more information.* + +### Custom modifiers + +You can also add custom modifiers. + +For example, here we create a modifier that makes the entire json document upper +or lower case. + +```go +gjson.AddModifier("case", func(json, arg string) string { + if arg == "upper" { + return strings.ToUpper(json) + } + if arg == "lower" { + return strings.ToLower(json) + } + return json +}) +``` + +``` +"children|@case:upper" >> ["SARA","ALEX","JACK"] +"children|@case:lower|@reverse" >> ["jack","alex","sara"] +``` + +## JSON Lines + +There's support for [JSON Lines](http://jsonlines.org/) using the `..` prefix, which treats a multilined document as an array. + +For example: + +``` +{"name": "Gilbert", "age": 61} +{"name": "Alexa", "age": 34} +{"name": "May", "age": 57} +{"name": "Deloise", "age": 44} +``` + +``` +..# >> 4 +..1 >> {"name": "Alexa", "age": 34} +..3 >> {"name": "Deloise", "age": 44} +..#.name >> ["Gilbert","Alexa","May","Deloise"] +..#(name="May").age >> 57 +``` + +The `ForEachLines` function will iterate through JSON lines. + +```go +gjson.ForEachLine(json, func(line gjson.Result) bool{ + println(line.String()) + return true +}) +``` + +## Get nested array values + +Suppose you want all the last names from the following json: + +```json +{ + "programmers": [ + { + "firstName": "Janet", + "lastName": "McLaughlin", + }, { + "firstName": "Elliotte", + "lastName": "Hunter", + }, { + "firstName": "Jason", + "lastName": "Harold", + } + ] +} +``` + +You would use the path "programmers.#.lastName" like such: + +```go +result := gjson.Get(json, "programmers.#.lastName") +for _, name := range result.Array() { + println(name.String()) +} +``` + +You can also query an object inside an array: + +```go +name := gjson.Get(json, `programmers.#(lastName="Hunter").firstName`) +println(name.String()) // prints "Elliotte" +``` + +## Iterate through an object or array + +The `ForEach` function allows for quickly iterating through an object or array. +The key and value are passed to the iterator function for objects. +Only the value is passed for arrays. +Returning `false` from an iterator will stop iteration. + +```go +result := gjson.Get(json, "programmers") +result.ForEach(func(key, value gjson.Result) bool { + println(value.String()) + return true // keep iterating +}) +``` + +## Simple Parse and Get + +There's a `Parse(json)` function that will do a simple parse, and `result.Get(path)` that will search a result. + +For example, all of these will return the same result: + +```go +gjson.Parse(json).Get("name").Get("last") +gjson.Get(json, "name").Get("last") +gjson.Get(json, "name.last") +``` + +## Check for the existence of a value + +Sometimes you just want to know if a value exists. + +```go +value := gjson.Get(json, "name.last") +if !value.Exists() { + println("no last name") +} else { + println(value.String()) +} + +// Or as one step +if gjson.Get(json, "name.last").Exists() { + println("has a last name") +} +``` + +## Validate JSON + +The `Get*` and `Parse*` functions expects that the json is well-formed. Bad json will not panic, but it may return back unexpected results. + +If you are consuming JSON from an unpredictable source then you may want to validate prior to using GJSON. + +```go +if !gjson.Valid(json) { + return errors.New("invalid json") +} +value := gjson.Get(json, "name.last") +``` + +## Unmarshal to a map + +To unmarshal to a `map[string]interface{}`: + +```go +m, ok := gjson.Parse(json).Value().(map[string]interface{}) +if !ok { + // not a map +} +``` + +## Working with Bytes + +If your JSON is contained in a `[]byte` slice, there's the [GetBytes](https://godoc.org/github.com/tidwall/gjson#GetBytes) function. This is preferred over `Get(string(data), path)`. + +```go +var json []byte = ... +result := gjson.GetBytes(json, path) +``` + +If you are using the `gjson.GetBytes(json, path)` function and you want to avoid converting `result.Raw` to a `[]byte`, then you can use this pattern: + +```go +var json []byte = ... +result := gjson.GetBytes(json, path) +var raw []byte +if result.Index > 0 { + raw = json[result.Index:result.Index+len(result.Raw)] +} else { + raw = []byte(result.Raw) +} +``` + +This is a best-effort no allocation sub slice of the original json. This method utilizes the `result.Index` field, which is the position of the raw data in the original json. It's possible that the value of `result.Index` equals zero, in which case the `result.Raw` is converted to a `[]byte`. + +## Get multiple values at once + +The `GetMany` function can be used to get multiple values at the same time. + +```go +results := gjson.GetMany(json, "name.first", "name.last", "age") +``` + +The return value is a `[]Result`, which will always contain exactly the same number of items as the input paths. + +## Performance + +Benchmarks of GJSON alongside [encoding/json](https://golang.org/pkg/encoding/json/), +[ffjson](https://github.com/pquerna/ffjson), +[EasyJSON](https://github.com/mailru/easyjson), +[jsonparser](https://github.com/buger/jsonparser), +and [json-iterator](https://github.com/json-iterator/go) + +``` +BenchmarkGJSONGet-16 11644512 311 ns/op 0 B/op 0 allocs/op +BenchmarkGJSONUnmarshalMap-16 1122678 3094 ns/op 1920 B/op 26 allocs/op +BenchmarkJSONUnmarshalMap-16 516681 6810 ns/op 2944 B/op 69 allocs/op +BenchmarkJSONUnmarshalStruct-16 697053 5400 ns/op 928 B/op 13 allocs/op +BenchmarkJSONDecoder-16 330450 10217 ns/op 3845 B/op 160 allocs/op +BenchmarkFFJSONLexer-16 1424979 2585 ns/op 880 B/op 8 allocs/op +BenchmarkEasyJSONLexer-16 3000000 729 ns/op 501 B/op 5 allocs/op +BenchmarkJSONParserGet-16 3000000 366 ns/op 21 B/op 0 allocs/op +BenchmarkJSONIterator-16 3000000 869 ns/op 693 B/op 14 allocs/op +``` + +JSON document used: + +```json +{ + "widget": { + "debug": "on", + "window": { + "title": "Sample Konfabulator Widget", + "name": "main_window", + "width": 500, + "height": 500 + }, + "image": { + "src": "Images/Sun.png", + "hOffset": 250, + "vOffset": 250, + "alignment": "center" + }, + "text": { + "data": "Click Here", + "size": 36, + "style": "bold", + "vOffset": 100, + "alignment": "center", + "onMouseUp": "sun1.opacity = (sun1.opacity / 100) * 90;" + } + } +} +``` + +Each operation was rotated through one of the following search paths: + +``` +widget.window.name +widget.image.hOffset +widget.text.onMouseUp +``` + +*These benchmarks were run on a MacBook Pro 16" 2.4 GHz Intel Core i9 using Go 1.17 and can be found [here](https://github.com/tidwall/gjson-benchmarks).* diff --git a/vendor/github.com/tidwall/gjson/SYNTAX.md b/vendor/github.com/tidwall/gjson/SYNTAX.md new file mode 100644 index 000000000..7a9b6a2d7 --- /dev/null +++ b/vendor/github.com/tidwall/gjson/SYNTAX.md @@ -0,0 +1,342 @@ +# GJSON Path Syntax + +A GJSON Path is a text string syntax that describes a search pattern for quickly retreiving values from a JSON payload. + +This document is designed to explain the structure of a GJSON Path through examples. + +- [Path structure](#path-structure) +- [Basic](#basic) +- [Wildcards](#wildcards) +- [Escape Character](#escape-character) +- [Arrays](#arrays) +- [Queries](#queries) +- [Dot vs Pipe](#dot-vs-pipe) +- [Modifiers](#modifiers) +- [Multipaths](#multipaths) +- [Literals](#literals) + +The definitive implemenation is [github.com/tidwall/gjson](https://github.com/tidwall/gjson). +Use the [GJSON Playground](https://gjson.dev) to experiment with the syntax online. + +## Path structure + +A GJSON Path is intended to be easily expressed as a series of components seperated by a `.` character. + +Along with `.` character, there are a few more that have special meaning, including `|`, `#`, `@`, `\`, `*`, `!`, and `?`. + +## Example + +Given this JSON + +```json +{ + "name": {"first": "Tom", "last": "Anderson"}, + "age":37, + "children": ["Sara","Alex","Jack"], + "fav.movie": "Deer Hunter", + "friends": [ + {"first": "Dale", "last": "Murphy", "age": 44, "nets": ["ig", "fb", "tw"]}, + {"first": "Roger", "last": "Craig", "age": 68, "nets": ["fb", "tw"]}, + {"first": "Jane", "last": "Murphy", "age": 47, "nets": ["ig", "tw"]} + ] +} +``` + +The following GJSON Paths evaluate to the accompanying values. + +### Basic + +In many cases you'll just want to retreive values by object name or array index. + +```go +name.last "Anderson" +name.first "Tom" +age 37 +children ["Sara","Alex","Jack"] +children.0 "Sara" +children.1 "Alex" +friends.1 {"first": "Roger", "last": "Craig", "age": 68} +friends.1.first "Roger" +``` + +### Wildcards + +A key may contain the special wildcard characters `*` and `?`. +The `*` will match on any zero+ characters, and `?` matches on any one character. + +```go +child*.2 "Jack" +c?ildren.0 "Sara" +``` + +### Escape character + +Special purpose characters, such as `.`, `*`, and `?` can be escaped with `\`. + +```go +fav\.movie "Deer Hunter" +``` + +You'll also need to make sure that the `\` character is correctly escaped when hardcoding a path in your source code. + +```go +// Go +val := gjson.Get(json, "fav\\.movie") // must escape the slash +val := gjson.Get(json, `fav\.movie`) // no need to escape the slash +``` + +```rust +// Rust +let val = gjson::get(json, "fav\\.movie") // must escape the slash +let val = gjson::get(json, r#"fav\.movie"#) // no need to escape the slash +``` + + +### Arrays + +The `#` character allows for digging into JSON Arrays. + +To get the length of an array you'll just use the `#` all by itself. + +```go +friends.# 3 +friends.#.age [44,68,47] +``` + +### Queries + +You can also query an array for the first match by using `#(...)`, or find all matches with `#(...)#`. +Queries support the `==`, `!=`, `<`, `<=`, `>`, `>=` comparison operators, +and the simple pattern matching `%` (like) and `!%` (not like) operators. + +```go +friends.#(last=="Murphy").first "Dale" +friends.#(last=="Murphy")#.first ["Dale","Jane"] +friends.#(age>45)#.last ["Craig","Murphy"] +friends.#(first%"D*").last "Murphy" +friends.#(first!%"D*").last "Craig" +``` + +To query for a non-object value in an array, you can forgo the string to the right of the operator. + +```go +children.#(!%"*a*") "Alex" +children.#(%"*a*")# ["Sara","Jack"] +``` + +Nested queries are allowed. + +```go +friends.#(nets.#(=="fb"))#.first >> ["Dale","Roger"] +``` + +*Please note that prior to v1.3.0, queries used the `#[...]` brackets. This was +changed in v1.3.0 as to avoid confusion with the new [multipath](#multipaths) +syntax. For backwards compatibility, `#[...]` will continue to work until the +next major release.* + +The `~` (tilde) operator will convert a value to a boolean before comparison. + +For example, using the following JSON: + +```json +{ + "vals": [ + { "a": 1, "b": true }, + { "a": 2, "b": true }, + { "a": 3, "b": false }, + { "a": 4, "b": "0" }, + { "a": 5, "b": 0 }, + { "a": 6, "b": "1" }, + { "a": 7, "b": 1 }, + { "a": 8, "b": "true" }, + { "a": 9, "b": false }, + { "a": 10, "b": null }, + { "a": 11 } + ] +} +``` + +You can now query for all true(ish) or false(ish) values: + +``` +vals.#(b==~true)#.a >> [1,2,6,7,8] +vals.#(b==~false)#.a >> [3,4,5,9,10,11] +``` + +The last value which was non-existent is treated as `false` + +### Dot vs Pipe + +The `.` is standard separator, but it's also possible to use a `|`. +In most cases they both end up returning the same results. +The cases where`|` differs from `.` is when it's used after the `#` for [Arrays](#arrays) and [Queries](#queries). + +Here are some examples + +```go +friends.0.first "Dale" +friends|0.first "Dale" +friends.0|first "Dale" +friends|0|first "Dale" +friends|# 3 +friends.# 3 +friends.#(last="Murphy")# [{"first": "Dale", "last": "Murphy", "age": 44},{"first": "Jane", "last": "Murphy", "age": 47}] +friends.#(last="Murphy")#.first ["Dale","Jane"] +friends.#(last="Murphy")#|first +friends.#(last="Murphy")#.0 [] +friends.#(last="Murphy")#|0 {"first": "Dale", "last": "Murphy", "age": 44} +friends.#(last="Murphy")#.# [] +friends.#(last="Murphy")#|# 2 +``` + +Let's break down a few of these. + +The path `friends.#(last="Murphy")#` all by itself results in + +```json +[{"first": "Dale", "last": "Murphy", "age": 44},{"first": "Jane", "last": "Murphy", "age": 47}] +``` + +The `.first` suffix will process the `first` path on each array element *before* returning the results. Which becomes + +```json +["Dale","Jane"] +``` + +But the `|first` suffix actually processes the `first` path *after* the previous result. +Since the previous result is an array, not an object, it's not possible to process +because `first` does not exist. + +Yet, `|0` suffix returns + +```json +{"first": "Dale", "last": "Murphy", "age": 44} +``` + +Because `0` is the first index of the previous result. + +### Modifiers + +A modifier is a path component that performs custom processing on the JSON. + +For example, using the built-in `@reverse` modifier on the above JSON payload will reverse the `children` array: + +```go +children.@reverse ["Jack","Alex","Sara"] +children.@reverse.0 "Jack" +``` + +There are currently the following built-in modifiers: + +- `@reverse`: Reverse an array or the members of an object. +- `@ugly`: Remove all whitespace from JSON. +- `@pretty`: Make the JSON more human readable. +- `@this`: Returns the current element. It can be used to retrieve the root element. +- `@valid`: Ensure the json document is valid. +- `@flatten`: Flattens an array. +- `@join`: Joins multiple objects into a single object. +- `@keys`: Returns an array of keys for an object. +- `@values`: Returns an array of values for an object. +- `@tostr`: Converts json to a string. Wraps a json string. +- `@fromstr`: Converts a string from json. Unwraps a json string. +- `@group`: Groups arrays of objects. See [e4fc67c](https://github.com/tidwall/gjson/commit/e4fc67c92aeebf2089fabc7872f010e340d105db). + +#### Modifier arguments + +A modifier may accept an optional argument. The argument can be a valid JSON payload or just characters. + +For example, the `@pretty` modifier takes a json object as its argument. + +``` +@pretty:{"sortKeys":true} +``` + +Which makes the json pretty and orders all of its keys. + +```json +{ + "age":37, + "children": ["Sara","Alex","Jack"], + "fav.movie": "Deer Hunter", + "friends": [ + {"age": 44, "first": "Dale", "last": "Murphy"}, + {"age": 68, "first": "Roger", "last": "Craig"}, + {"age": 47, "first": "Jane", "last": "Murphy"} + ], + "name": {"first": "Tom", "last": "Anderson"} +} +``` + +*The full list of `@pretty` options are `sortKeys`, `indent`, `prefix`, and `width`. +Please see [Pretty Options](https://github.com/tidwall/pretty#customized-output) for more information.* + +#### Custom modifiers + +You can also add custom modifiers. + +For example, here we create a modifier which makes the entire JSON payload upper or lower case. + +```go +gjson.AddModifier("case", func(json, arg string) string { + if arg == "upper" { + return strings.ToUpper(json) + } + if arg == "lower" { + return strings.ToLower(json) + } + return json +}) +"children.@case:upper" ["SARA","ALEX","JACK"] +"children.@case:lower.@reverse" ["jack","alex","sara"] +``` + +*Note: Custom modifiers are not yet available in the Rust version* + +### Multipaths + +Starting with v1.3.0, GJSON added the ability to join multiple paths together +to form new documents. Wrapping comma-separated paths between `[...]` or +`{...}` will result in a new array or object, respectively. + +For example, using the given multipath: + +``` +{name.first,age,"the_murphys":friends.#(last="Murphy")#.first} +``` + +Here we selected the first name, age, and the first name for friends with the +last name "Murphy". + +You'll notice that an optional key can be provided, in this case +"the_murphys", to force assign a key to a value. Otherwise, the name of the +actual field will be used, in this case "first". If a name cannot be +determined, then "_" is used. + +This results in + +```json +{"first":"Tom","age":37,"the_murphys":["Dale","Jane"]} +``` + +### Literals + +Starting with v1.12.0, GJSON added support of json literals, which provides a way for constructing static blocks of json. This is can be particularly useful when constructing a new json document using [multipaths](#multipaths). + +A json literal begins with the '!' declaration character. + +For example, using the given multipath: + +``` +{name.first,age,"company":!"Happysoft","employed":!true} +``` + +Here we selected the first name and age. Then add two new fields, "company" and "employed". + +This results in + +```json +{"first":"Tom","age":37,"company":"Happysoft","employed":true} +``` + +*See issue [#249](https://github.com/tidwall/gjson/issues/249) for additional context on JSON Literals.* diff --git a/vendor/github.com/tidwall/gjson/gjson.go b/vendor/github.com/tidwall/gjson/gjson.go new file mode 100644 index 000000000..9b0db4eb7 --- /dev/null +++ b/vendor/github.com/tidwall/gjson/gjson.go @@ -0,0 +1,3359 @@ +// Package gjson provides searching for json strings. +package gjson + +import ( + "strconv" + "strings" + "time" + "unicode/utf16" + "unicode/utf8" + "unsafe" + + "github.com/tidwall/match" + "github.com/tidwall/pretty" +) + +// Type is Result type +type Type int + +const ( + // Null is a null json value + Null Type = iota + // False is a json false boolean + False + // Number is json number + Number + // String is a json string + String + // True is a json true boolean + True + // JSON is a raw block of JSON + JSON +) + +// String returns a string representation of the type. +func (t Type) String() string { + switch t { + default: + return "" + case Null: + return "Null" + case False: + return "False" + case Number: + return "Number" + case String: + return "String" + case True: + return "True" + case JSON: + return "JSON" + } +} + +// Result represents a json value that is returned from Get(). +type Result struct { + // Type is the json type + Type Type + // Raw is the raw json + Raw string + // Str is the json string + Str string + // Num is the json number + Num float64 + // Index of raw value in original json, zero means index unknown + Index int + // Indexes of all the elements that match on a path containing the '#' + // query character. + Indexes []int +} + +// String returns a string representation of the value. +func (t Result) String() string { + switch t.Type { + default: + return "" + case False: + return "false" + case Number: + if len(t.Raw) == 0 { + // calculated result + return strconv.FormatFloat(t.Num, 'f', -1, 64) + } + var i int + if t.Raw[0] == '-' { + i++ + } + for ; i < len(t.Raw); i++ { + if t.Raw[i] < '0' || t.Raw[i] > '9' { + return strconv.FormatFloat(t.Num, 'f', -1, 64) + } + } + return t.Raw + case String: + return t.Str + case JSON: + return t.Raw + case True: + return "true" + } +} + +// Bool returns an boolean representation. +func (t Result) Bool() bool { + switch t.Type { + default: + return false + case True: + return true + case String: + b, _ := strconv.ParseBool(strings.ToLower(t.Str)) + return b + case Number: + return t.Num != 0 + } +} + +// Int returns an integer representation. +func (t Result) Int() int64 { + switch t.Type { + default: + return 0 + case True: + return 1 + case String: + n, _ := parseInt(t.Str) + return n + case Number: + // try to directly convert the float64 to int64 + i, ok := safeInt(t.Num) + if ok { + return i + } + // now try to parse the raw string + i, ok = parseInt(t.Raw) + if ok { + return i + } + // fallback to a standard conversion + return int64(t.Num) + } +} + +// Uint returns an unsigned integer representation. +func (t Result) Uint() uint64 { + switch t.Type { + default: + return 0 + case True: + return 1 + case String: + n, _ := parseUint(t.Str) + return n + case Number: + // try to directly convert the float64 to uint64 + i, ok := safeInt(t.Num) + if ok && i >= 0 { + return uint64(i) + } + // now try to parse the raw string + u, ok := parseUint(t.Raw) + if ok { + return u + } + // fallback to a standard conversion + return uint64(t.Num) + } +} + +// Float returns an float64 representation. +func (t Result) Float() float64 { + switch t.Type { + default: + return 0 + case True: + return 1 + case String: + n, _ := strconv.ParseFloat(t.Str, 64) + return n + case Number: + return t.Num + } +} + +// Time returns a time.Time representation. +func (t Result) Time() time.Time { + res, _ := time.Parse(time.RFC3339, t.String()) + return res +} + +// Array returns back an array of values. +// If the result represents a null value or is non-existent, then an empty +// array will be returned. +// If the result is not a JSON array, the return value will be an +// array containing one result. +func (t Result) Array() []Result { + if t.Type == Null { + return []Result{} + } + if !t.IsArray() { + return []Result{t} + } + r := t.arrayOrMap('[', false) + return r.a +} + +// IsObject returns true if the result value is a JSON object. +func (t Result) IsObject() bool { + return t.Type == JSON && len(t.Raw) > 0 && t.Raw[0] == '{' +} + +// IsArray returns true if the result value is a JSON array. +func (t Result) IsArray() bool { + return t.Type == JSON && len(t.Raw) > 0 && t.Raw[0] == '[' +} + +// IsBool returns true if the result value is a JSON boolean. +func (t Result) IsBool() bool { + return t.Type == True || t.Type == False +} + +// ForEach iterates through values. +// If the result represents a non-existent value, then no values will be +// iterated. If the result is an Object, the iterator will pass the key and +// value of each item. If the result is an Array, the iterator will only pass +// the value of each item. If the result is not a JSON array or object, the +// iterator will pass back one value equal to the result. +func (t Result) ForEach(iterator func(key, value Result) bool) { + if !t.Exists() { + return + } + if t.Type != JSON { + iterator(Result{}, t) + return + } + json := t.Raw + var obj bool + var i int + var key, value Result + for ; i < len(json); i++ { + if json[i] == '{' { + i++ + key.Type = String + obj = true + break + } else if json[i] == '[' { + i++ + key.Type = Number + key.Num = -1 + break + } + if json[i] > ' ' { + return + } + } + var str string + var vesc bool + var ok bool + var idx int + for ; i < len(json); i++ { + if obj { + if json[i] != '"' { + continue + } + s := i + i, str, vesc, ok = parseString(json, i+1) + if !ok { + return + } + if vesc { + key.Str = unescape(str[1 : len(str)-1]) + } else { + key.Str = str[1 : len(str)-1] + } + key.Raw = str + key.Index = s + t.Index + } else { + key.Num += 1 + } + for ; i < len(json); i++ { + if json[i] <= ' ' || json[i] == ',' || json[i] == ':' { + continue + } + break + } + s := i + i, value, ok = parseAny(json, i, true) + if !ok { + return + } + if t.Indexes != nil { + if idx < len(t.Indexes) { + value.Index = t.Indexes[idx] + } + } else { + value.Index = s + t.Index + } + if !iterator(key, value) { + return + } + idx++ + } +} + +// Map returns back a map of values. The result should be a JSON object. +// If the result is not a JSON object, the return value will be an empty map. +func (t Result) Map() map[string]Result { + if t.Type != JSON { + return map[string]Result{} + } + r := t.arrayOrMap('{', false) + return r.o +} + +// Get searches result for the specified path. +// The result should be a JSON array or object. +func (t Result) Get(path string) Result { + r := Get(t.Raw, path) + if r.Indexes != nil { + for i := 0; i < len(r.Indexes); i++ { + r.Indexes[i] += t.Index + } + } else { + r.Index += t.Index + } + return r +} + +type arrayOrMapResult struct { + a []Result + ai []interface{} + o map[string]Result + oi map[string]interface{} + vc byte +} + +func (t Result) arrayOrMap(vc byte, valueize bool) (r arrayOrMapResult) { + var json = t.Raw + var i int + var value Result + var count int + var key Result + if vc == 0 { + for ; i < len(json); i++ { + if json[i] == '{' || json[i] == '[' { + r.vc = json[i] + i++ + break + } + if json[i] > ' ' { + goto end + } + } + } else { + for ; i < len(json); i++ { + if json[i] == vc { + i++ + break + } + if json[i] > ' ' { + goto end + } + } + r.vc = vc + } + if r.vc == '{' { + if valueize { + r.oi = make(map[string]interface{}) + } else { + r.o = make(map[string]Result) + } + } else { + if valueize { + r.ai = make([]interface{}, 0) + } else { + r.a = make([]Result, 0) + } + } + for ; i < len(json); i++ { + if json[i] <= ' ' { + continue + } + // get next value + if json[i] == ']' || json[i] == '}' { + break + } + switch json[i] { + default: + if (json[i] >= '0' && json[i] <= '9') || json[i] == '-' { + value.Type = Number + value.Raw, value.Num = tonum(json[i:]) + value.Str = "" + } else { + continue + } + case '{', '[': + value.Type = JSON + value.Raw = squash(json[i:]) + value.Str, value.Num = "", 0 + case 'n': + value.Type = Null + value.Raw = tolit(json[i:]) + value.Str, value.Num = "", 0 + case 't': + value.Type = True + value.Raw = tolit(json[i:]) + value.Str, value.Num = "", 0 + case 'f': + value.Type = False + value.Raw = tolit(json[i:]) + value.Str, value.Num = "", 0 + case '"': + value.Type = String + value.Raw, value.Str = tostr(json[i:]) + value.Num = 0 + } + value.Index = i + t.Index + + i += len(value.Raw) - 1 + + if r.vc == '{' { + if count%2 == 0 { + key = value + } else { + if valueize { + if _, ok := r.oi[key.Str]; !ok { + r.oi[key.Str] = value.Value() + } + } else { + if _, ok := r.o[key.Str]; !ok { + r.o[key.Str] = value + } + } + } + count++ + } else { + if valueize { + r.ai = append(r.ai, value.Value()) + } else { + r.a = append(r.a, value) + } + } + } +end: + if t.Indexes != nil { + if len(t.Indexes) != len(r.a) { + for i := 0; i < len(r.a); i++ { + r.a[i].Index = 0 + } + } else { + for i := 0; i < len(r.a); i++ { + r.a[i].Index = t.Indexes[i] + } + } + } + return +} + +// Parse parses the json and returns a result. +// +// This function expects that the json is well-formed, and does not validate. +// Invalid json will not panic, but it may return back unexpected results. +// If you are consuming JSON from an unpredictable source then you may want to +// use the Valid function first. +func Parse(json string) Result { + var value Result + i := 0 + for ; i < len(json); i++ { + if json[i] == '{' || json[i] == '[' { + value.Type = JSON + value.Raw = json[i:] // just take the entire raw + break + } + if json[i] <= ' ' { + continue + } + switch json[i] { + case '+', '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + 'i', 'I', 'N': + value.Type = Number + value.Raw, value.Num = tonum(json[i:]) + case 'n': + if i+1 < len(json) && json[i+1] != 'u' { + // nan + value.Type = Number + value.Raw, value.Num = tonum(json[i:]) + } else { + // null + value.Type = Null + value.Raw = tolit(json[i:]) + } + case 't': + value.Type = True + value.Raw = tolit(json[i:]) + case 'f': + value.Type = False + value.Raw = tolit(json[i:]) + case '"': + value.Type = String + value.Raw, value.Str = tostr(json[i:]) + default: + return Result{} + } + break + } + if value.Exists() { + value.Index = i + } + return value +} + +// ParseBytes parses the json and returns a result. +// If working with bytes, this method preferred over Parse(string(data)) +func ParseBytes(json []byte) Result { + return Parse(string(json)) +} + +func squash(json string) string { + // expects that the lead character is a '[' or '{' or '(' or '"' + // squash the value, ignoring all nested arrays and objects. + var i, depth int + if json[0] != '"' { + i, depth = 1, 1 + } + for ; i < len(json); i++ { + if json[i] >= '"' && json[i] <= '}' { + switch json[i] { + case '"': + i++ + s2 := i + for ; i < len(json); i++ { + if json[i] > '\\' { + continue + } + if json[i] == '"' { + // look for an escaped slash + if json[i-1] == '\\' { + n := 0 + for j := i - 2; j > s2-1; j-- { + if json[j] != '\\' { + break + } + n++ + } + if n%2 == 0 { + continue + } + } + break + } + } + if depth == 0 { + if i >= len(json) { + return json + } + return json[:i+1] + } + case '{', '[', '(': + depth++ + case '}', ']', ')': + depth-- + if depth == 0 { + return json[:i+1] + } + } + } + } + return json +} + +func tonum(json string) (raw string, num float64) { + for i := 1; i < len(json); i++ { + // less than dash might have valid characters + if json[i] <= '-' { + if json[i] <= ' ' || json[i] == ',' { + // break on whitespace and comma + raw = json[:i] + num, _ = strconv.ParseFloat(raw, 64) + return + } + // could be a '+' or '-'. let's assume so. + } else if json[i] == ']' || json[i] == '}' { + // break on ']' or '}' + raw = json[:i] + num, _ = strconv.ParseFloat(raw, 64) + return + } + } + raw = json + num, _ = strconv.ParseFloat(raw, 64) + return +} + +func tolit(json string) (raw string) { + for i := 1; i < len(json); i++ { + if json[i] < 'a' || json[i] > 'z' { + return json[:i] + } + } + return json +} + +func tostr(json string) (raw string, str string) { + // expects that the lead character is a '"' + for i := 1; i < len(json); i++ { + if json[i] > '\\' { + continue + } + if json[i] == '"' { + return json[:i+1], json[1:i] + } + if json[i] == '\\' { + i++ + for ; i < len(json); i++ { + if json[i] > '\\' { + continue + } + if json[i] == '"' { + // look for an escaped slash + if json[i-1] == '\\' { + n := 0 + for j := i - 2; j > 0; j-- { + if json[j] != '\\' { + break + } + n++ + } + if n%2 == 0 { + continue + } + } + return json[:i+1], unescape(json[1:i]) + } + } + var ret string + if i+1 < len(json) { + ret = json[:i+1] + } else { + ret = json[:i] + } + return ret, unescape(json[1:i]) + } + } + return json, json[1:] +} + +// Exists returns true if value exists. +// +// if gjson.Get(json, "name.last").Exists(){ +// println("value exists") +// } +func (t Result) Exists() bool { + return t.Type != Null || len(t.Raw) != 0 +} + +// Value returns one of these types: +// +// bool, for JSON booleans +// float64, for JSON numbers +// Number, for JSON numbers +// string, for JSON string literals +// nil, for JSON null +// map[string]interface{}, for JSON objects +// []interface{}, for JSON arrays +// +func (t Result) Value() interface{} { + if t.Type == String { + return t.Str + } + switch t.Type { + default: + return nil + case False: + return false + case Number: + return t.Num + case JSON: + r := t.arrayOrMap(0, true) + if r.vc == '{' { + return r.oi + } else if r.vc == '[' { + return r.ai + } + return nil + case True: + return true + } +} + +func parseString(json string, i int) (int, string, bool, bool) { + var s = i + for ; i < len(json); i++ { + if json[i] > '\\' { + continue + } + if json[i] == '"' { + return i + 1, json[s-1 : i+1], false, true + } + if json[i] == '\\' { + i++ + for ; i < len(json); i++ { + if json[i] > '\\' { + continue + } + if json[i] == '"' { + // look for an escaped slash + if json[i-1] == '\\' { + n := 0 + for j := i - 2; j > 0; j-- { + if json[j] != '\\' { + break + } + n++ + } + if n%2 == 0 { + continue + } + } + return i + 1, json[s-1 : i+1], true, true + } + } + break + } + } + return i, json[s-1:], false, false +} + +func parseNumber(json string, i int) (int, string) { + var s = i + i++ + for ; i < len(json); i++ { + if json[i] <= ' ' || json[i] == ',' || json[i] == ']' || + json[i] == '}' { + return i, json[s:i] + } + } + return i, json[s:] +} + +func parseLiteral(json string, i int) (int, string) { + var s = i + i++ + for ; i < len(json); i++ { + if json[i] < 'a' || json[i] > 'z' { + return i, json[s:i] + } + } + return i, json[s:] +} + +type arrayPathResult struct { + part string + path string + pipe string + piped bool + more bool + alogok bool + arrch bool + alogkey string + query struct { + on bool + all bool + path string + op string + value string + } +} + +func parseArrayPath(path string) (r arrayPathResult) { + for i := 0; i < len(path); i++ { + if path[i] == '|' { + r.part = path[:i] + r.pipe = path[i+1:] + r.piped = true + return + } + if path[i] == '.' { + r.part = path[:i] + if !r.arrch && i < len(path)-1 && isDotPiperChar(path[i+1:]) { + r.pipe = path[i+1:] + r.piped = true + } else { + r.path = path[i+1:] + r.more = true + } + return + } + if path[i] == '#' { + r.arrch = true + if i == 0 && len(path) > 1 { + if path[1] == '.' { + r.alogok = true + r.alogkey = path[2:] + r.path = path[:1] + } else if path[1] == '[' || path[1] == '(' { + // query + r.query.on = true + qpath, op, value, _, fi, vesc, ok := + parseQuery(path[i:]) + if !ok { + // bad query, end now + break + } + if len(value) >= 2 && value[0] == '"' && + value[len(value)-1] == '"' { + value = value[1 : len(value)-1] + if vesc { + value = unescape(value) + } + } + r.query.path = qpath + r.query.op = op + r.query.value = value + + i = fi - 1 + if i+1 < len(path) && path[i+1] == '#' { + r.query.all = true + } + } + } + continue + } + } + r.part = path + r.path = "" + return +} + +// splitQuery takes a query and splits it into three parts: +// path, op, middle, and right. +// So for this query: +// #(first_name=="Murphy").last +// Becomes +// first_name # path +// =="Murphy" # middle +// .last # right +// Or, +// #(service_roles.#(=="one")).cap +// Becomes +// service_roles.#(=="one") # path +// # middle +// .cap # right +func parseQuery(query string) ( + path, op, value, remain string, i int, vesc, ok bool, +) { + if len(query) < 2 || query[0] != '#' || + (query[1] != '(' && query[1] != '[') { + return "", "", "", "", i, false, false + } + i = 2 + j := 0 // start of value part + depth := 1 + for ; i < len(query); i++ { + if depth == 1 && j == 0 { + switch query[i] { + case '!', '=', '<', '>', '%': + // start of the value part + j = i + continue + } + } + if query[i] == '\\' { + i++ + } else if query[i] == '[' || query[i] == '(' { + depth++ + } else if query[i] == ']' || query[i] == ')' { + depth-- + if depth == 0 { + break + } + } else if query[i] == '"' { + // inside selector string, balance quotes + i++ + for ; i < len(query); i++ { + if query[i] == '\\' { + vesc = true + i++ + } else if query[i] == '"' { + break + } + } + } + } + if depth > 0 { + return "", "", "", "", i, false, false + } + if j > 0 { + path = trim(query[2:j]) + value = trim(query[j:i]) + remain = query[i+1:] + // parse the compare op from the value + var opsz int + switch { + case len(value) == 1: + opsz = 1 + case value[0] == '!' && value[1] == '=': + opsz = 2 + case value[0] == '!' && value[1] == '%': + opsz = 2 + case value[0] == '<' && value[1] == '=': + opsz = 2 + case value[0] == '>' && value[1] == '=': + opsz = 2 + case value[0] == '=' && value[1] == '=': + value = value[1:] + opsz = 1 + case value[0] == '<': + opsz = 1 + case value[0] == '>': + opsz = 1 + case value[0] == '=': + opsz = 1 + case value[0] == '%': + opsz = 1 + } + op = value[:opsz] + value = trim(value[opsz:]) + } else { + path = trim(query[2:i]) + remain = query[i+1:] + } + return path, op, value, remain, i + 1, vesc, true +} + +func trim(s string) string { +left: + if len(s) > 0 && s[0] <= ' ' { + s = s[1:] + goto left + } +right: + if len(s) > 0 && s[len(s)-1] <= ' ' { + s = s[:len(s)-1] + goto right + } + return s +} + +// peek at the next byte and see if it's a '@', '[', or '{'. +func isDotPiperChar(s string) bool { + if DisableModifiers { + return false + } + c := s[0] + if c == '@' { + // check that the next component is *not* a modifier. + i := 1 + for ; i < len(s); i++ { + if s[i] == '.' || s[i] == '|' { + break + } + } + _, ok := modifiers[s[1:i]] + return ok + } + return c == '[' || c == '{' +} + +type objectPathResult struct { + part string + path string + pipe string + piped bool + wild bool + more bool +} + +func parseObjectPath(path string) (r objectPathResult) { + for i := 0; i < len(path); i++ { + if path[i] == '|' { + r.part = path[:i] + r.pipe = path[i+1:] + r.piped = true + return + } + if path[i] == '.' { + r.part = path[:i] + if i < len(path)-1 && isDotPiperChar(path[i+1:]) { + r.pipe = path[i+1:] + r.piped = true + } else { + r.path = path[i+1:] + r.more = true + } + return + } + if path[i] == '*' || path[i] == '?' { + r.wild = true + continue + } + if path[i] == '\\' { + // go into escape mode. this is a slower path that + // strips off the escape character from the part. + epart := []byte(path[:i]) + i++ + if i < len(path) { + epart = append(epart, path[i]) + i++ + for ; i < len(path); i++ { + if path[i] == '\\' { + i++ + if i < len(path) { + epart = append(epart, path[i]) + } + continue + } else if path[i] == '.' { + r.part = string(epart) + if i < len(path)-1 && isDotPiperChar(path[i+1:]) { + r.pipe = path[i+1:] + r.piped = true + } else { + r.path = path[i+1:] + } + r.more = true + return + } else if path[i] == '|' { + r.part = string(epart) + r.pipe = path[i+1:] + r.piped = true + return + } else if path[i] == '*' || path[i] == '?' { + r.wild = true + } + epart = append(epart, path[i]) + } + } + // append the last part + r.part = string(epart) + return + } + } + r.part = path + return +} + +func parseSquash(json string, i int) (int, string) { + // expects that the lead character is a '[' or '{' or '(' + // squash the value, ignoring all nested arrays and objects. + // the first '[' or '{' or '(' has already been read + s := i + i++ + depth := 1 + for ; i < len(json); i++ { + if json[i] >= '"' && json[i] <= '}' { + switch json[i] { + case '"': + i++ + s2 := i + for ; i < len(json); i++ { + if json[i] > '\\' { + continue + } + if json[i] == '"' { + // look for an escaped slash + if json[i-1] == '\\' { + n := 0 + for j := i - 2; j > s2-1; j-- { + if json[j] != '\\' { + break + } + n++ + } + if n%2 == 0 { + continue + } + } + break + } + } + case '{', '[', '(': + depth++ + case '}', ']', ')': + depth-- + if depth == 0 { + i++ + return i, json[s:i] + } + } + } + } + return i, json[s:] +} + +func parseObject(c *parseContext, i int, path string) (int, bool) { + var pmatch, kesc, vesc, ok, hit bool + var key, val string + rp := parseObjectPath(path) + if !rp.more && rp.piped { + c.pipe = rp.pipe + c.piped = true + } + for i < len(c.json) { + for ; i < len(c.json); i++ { + if c.json[i] == '"' { + // parse_key_string + // this is slightly different from getting s string value + // because we don't need the outer quotes. + i++ + var s = i + for ; i < len(c.json); i++ { + if c.json[i] > '\\' { + continue + } + if c.json[i] == '"' { + i, key, kesc, ok = i+1, c.json[s:i], false, true + goto parse_key_string_done + } + if c.json[i] == '\\' { + i++ + for ; i < len(c.json); i++ { + if c.json[i] > '\\' { + continue + } + if c.json[i] == '"' { + // look for an escaped slash + if c.json[i-1] == '\\' { + n := 0 + for j := i - 2; j > 0; j-- { + if c.json[j] != '\\' { + break + } + n++ + } + if n%2 == 0 { + continue + } + } + i, key, kesc, ok = i+1, c.json[s:i], true, true + goto parse_key_string_done + } + } + break + } + } + key, kesc, ok = c.json[s:], false, false + parse_key_string_done: + break + } + if c.json[i] == '}' { + return i + 1, false + } + } + if !ok { + return i, false + } + if rp.wild { + if kesc { + pmatch = matchLimit(unescape(key), rp.part) + } else { + pmatch = matchLimit(key, rp.part) + } + } else { + if kesc { + pmatch = rp.part == unescape(key) + } else { + pmatch = rp.part == key + } + } + hit = pmatch && !rp.more + for ; i < len(c.json); i++ { + var num bool + switch c.json[i] { + default: + continue + case '"': + i++ + i, val, vesc, ok = parseString(c.json, i) + if !ok { + return i, false + } + if hit { + if vesc { + c.value.Str = unescape(val[1 : len(val)-1]) + } else { + c.value.Str = val[1 : len(val)-1] + } + c.value.Raw = val + c.value.Type = String + return i, true + } + case '{': + if pmatch && !hit { + i, hit = parseObject(c, i+1, rp.path) + if hit { + return i, true + } + } else { + i, val = parseSquash(c.json, i) + if hit { + c.value.Raw = val + c.value.Type = JSON + return i, true + } + } + case '[': + if pmatch && !hit { + i, hit = parseArray(c, i+1, rp.path) + if hit { + return i, true + } + } else { + i, val = parseSquash(c.json, i) + if hit { + c.value.Raw = val + c.value.Type = JSON + return i, true + } + } + case 'n': + if i+1 < len(c.json) && c.json[i+1] != 'u' { + num = true + break + } + fallthrough + case 't', 'f': + vc := c.json[i] + i, val = parseLiteral(c.json, i) + if hit { + c.value.Raw = val + switch vc { + case 't': + c.value.Type = True + case 'f': + c.value.Type = False + } + return i, true + } + case '+', '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + 'i', 'I', 'N': + num = true + } + if num { + i, val = parseNumber(c.json, i) + if hit { + c.value.Raw = val + c.value.Type = Number + c.value.Num, _ = strconv.ParseFloat(val, 64) + return i, true + } + } + break + } + } + return i, false +} + +// matchLimit will limit the complexity of the match operation to avoid ReDos +// attacks from arbritary inputs. +// See the github.com/tidwall/match.MatchLimit function for more information. +func matchLimit(str, pattern string) bool { + matched, _ := match.MatchLimit(str, pattern, 10000) + return matched +} + +func queryMatches(rp *arrayPathResult, value Result) bool { + rpv := rp.query.value + if len(rpv) > 0 && rpv[0] == '~' { + // convert to bool + rpv = rpv[1:] + if value.Bool() { + value = Result{Type: True} + } else { + value = Result{Type: False} + } + } + if !value.Exists() { + return false + } + if rp.query.op == "" { + // the query is only looking for existence, such as: + // friends.#(name) + // which makes sure that the array "friends" has an element of + // "name" that exists + return true + } + switch value.Type { + case String: + switch rp.query.op { + case "=": + return value.Str == rpv + case "!=": + return value.Str != rpv + case "<": + return value.Str < rpv + case "<=": + return value.Str <= rpv + case ">": + return value.Str > rpv + case ">=": + return value.Str >= rpv + case "%": + return matchLimit(value.Str, rpv) + case "!%": + return !matchLimit(value.Str, rpv) + } + case Number: + rpvn, _ := strconv.ParseFloat(rpv, 64) + switch rp.query.op { + case "=": + return value.Num == rpvn + case "!=": + return value.Num != rpvn + case "<": + return value.Num < rpvn + case "<=": + return value.Num <= rpvn + case ">": + return value.Num > rpvn + case ">=": + return value.Num >= rpvn + } + case True: + switch rp.query.op { + case "=": + return rpv == "true" + case "!=": + return rpv != "true" + case ">": + return rpv == "false" + case ">=": + return true + } + case False: + switch rp.query.op { + case "=": + return rpv == "false" + case "!=": + return rpv != "false" + case "<": + return rpv == "true" + case "<=": + return true + } + } + return false +} +func parseArray(c *parseContext, i int, path string) (int, bool) { + var pmatch, vesc, ok, hit bool + var val string + var h int + var alog []int + var partidx int + var multires []byte + var queryIndexes []int + rp := parseArrayPath(path) + if !rp.arrch { + n, ok := parseUint(rp.part) + if !ok { + partidx = -1 + } else { + partidx = int(n) + } + } + if !rp.more && rp.piped { + c.pipe = rp.pipe + c.piped = true + } + + procQuery := func(qval Result) bool { + if rp.query.all { + if len(multires) == 0 { + multires = append(multires, '[') + } + } + var tmp parseContext + tmp.value = qval + fillIndex(c.json, &tmp) + parentIndex := tmp.value.Index + var res Result + if qval.Type == JSON { + res = qval.Get(rp.query.path) + } else { + if rp.query.path != "" { + return false + } + res = qval + } + if queryMatches(&rp, res) { + if rp.more { + left, right, ok := splitPossiblePipe(rp.path) + if ok { + rp.path = left + c.pipe = right + c.piped = true + } + res = qval.Get(rp.path) + } else { + res = qval + } + if rp.query.all { + raw := res.Raw + if len(raw) == 0 { + raw = res.String() + } + if raw != "" { + if len(multires) > 1 { + multires = append(multires, ',') + } + multires = append(multires, raw...) + queryIndexes = append(queryIndexes, res.Index+parentIndex) + } + } else { + c.value = res + return true + } + } + return false + } + for i < len(c.json)+1 { + if !rp.arrch { + pmatch = partidx == h + hit = pmatch && !rp.more + } + h++ + if rp.alogok { + alog = append(alog, i) + } + for ; ; i++ { + var ch byte + if i > len(c.json) { + break + } else if i == len(c.json) { + ch = ']' + } else { + ch = c.json[i] + } + var num bool + switch ch { + default: + continue + case '"': + i++ + i, val, vesc, ok = parseString(c.json, i) + if !ok { + return i, false + } + if rp.query.on { + var qval Result + if vesc { + qval.Str = unescape(val[1 : len(val)-1]) + } else { + qval.Str = val[1 : len(val)-1] + } + qval.Raw = val + qval.Type = String + if procQuery(qval) { + return i, true + } + } else if hit { + if rp.alogok { + break + } + if vesc { + c.value.Str = unescape(val[1 : len(val)-1]) + } else { + c.value.Str = val[1 : len(val)-1] + } + c.value.Raw = val + c.value.Type = String + return i, true + } + case '{': + if pmatch && !hit { + i, hit = parseObject(c, i+1, rp.path) + if hit { + if rp.alogok { + break + } + return i, true + } + } else { + i, val = parseSquash(c.json, i) + if rp.query.on { + if procQuery(Result{Raw: val, Type: JSON}) { + return i, true + } + } else if hit { + if rp.alogok { + break + } + c.value.Raw = val + c.value.Type = JSON + return i, true + } + } + case '[': + if pmatch && !hit { + i, hit = parseArray(c, i+1, rp.path) + if hit { + if rp.alogok { + break + } + return i, true + } + } else { + i, val = parseSquash(c.json, i) + if rp.query.on { + if procQuery(Result{Raw: val, Type: JSON}) { + return i, true + } + } else if hit { + if rp.alogok { + break + } + c.value.Raw = val + c.value.Type = JSON + return i, true + } + } + case 'n': + if i+1 < len(c.json) && c.json[i+1] != 'u' { + num = true + break + } + fallthrough + case 't', 'f': + vc := c.json[i] + i, val = parseLiteral(c.json, i) + if rp.query.on { + var qval Result + qval.Raw = val + switch vc { + case 't': + qval.Type = True + case 'f': + qval.Type = False + } + if procQuery(qval) { + return i, true + } + } else if hit { + if rp.alogok { + break + } + c.value.Raw = val + switch vc { + case 't': + c.value.Type = True + case 'f': + c.value.Type = False + } + return i, true + } + case '+', '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + 'i', 'I', 'N': + num = true + case ']': + if rp.arrch && rp.part == "#" { + if rp.alogok { + left, right, ok := splitPossiblePipe(rp.alogkey) + if ok { + rp.alogkey = left + c.pipe = right + c.piped = true + } + var indexes = make([]int, 0, 64) + var jsons = make([]byte, 0, 64) + jsons = append(jsons, '[') + for j, k := 0, 0; j < len(alog); j++ { + idx := alog[j] + for idx < len(c.json) { + switch c.json[idx] { + case ' ', '\t', '\r', '\n': + idx++ + continue + } + break + } + if idx < len(c.json) && c.json[idx] != ']' { + _, res, ok := parseAny(c.json, idx, true) + if ok { + res := res.Get(rp.alogkey) + if res.Exists() { + if k > 0 { + jsons = append(jsons, ',') + } + raw := res.Raw + if len(raw) == 0 { + raw = res.String() + } + jsons = append(jsons, []byte(raw)...) + indexes = append(indexes, res.Index) + k++ + } + } + } + } + jsons = append(jsons, ']') + c.value.Type = JSON + c.value.Raw = string(jsons) + c.value.Indexes = indexes + return i + 1, true + } + if rp.alogok { + break + } + + c.value.Type = Number + c.value.Num = float64(h - 1) + c.value.Raw = strconv.Itoa(h - 1) + c.calcd = true + return i + 1, true + } + if !c.value.Exists() { + if len(multires) > 0 { + c.value = Result{ + Raw: string(append(multires, ']')), + Type: JSON, + Indexes: queryIndexes, + } + } else if rp.query.all { + c.value = Result{ + Raw: "[]", + Type: JSON, + } + } + } + return i + 1, false + } + if num { + i, val = parseNumber(c.json, i) + if rp.query.on { + var qval Result + qval.Raw = val + qval.Type = Number + qval.Num, _ = strconv.ParseFloat(val, 64) + if procQuery(qval) { + return i, true + } + } else if hit { + if rp.alogok { + break + } + c.value.Raw = val + c.value.Type = Number + c.value.Num, _ = strconv.ParseFloat(val, 64) + return i, true + } + } + break + } + } + return i, false +} + +func splitPossiblePipe(path string) (left, right string, ok bool) { + // take a quick peek for the pipe character. If found we'll split the piped + // part of the path into the c.pipe field and shorten the rp. + var possible bool + for i := 0; i < len(path); i++ { + if path[i] == '|' { + possible = true + break + } + } + if !possible { + return + } + + if len(path) > 0 && path[0] == '{' { + squashed := squash(path[1:]) + if len(squashed) < len(path)-1 { + squashed = path[:len(squashed)+1] + remain := path[len(squashed):] + if remain[0] == '|' { + return squashed, remain[1:], true + } + } + return + } + + // split the left and right side of the path with the pipe character as + // the delimiter. This is a little tricky because we'll need to basically + // parse the entire path. + for i := 0; i < len(path); i++ { + if path[i] == '\\' { + i++ + } else if path[i] == '.' { + if i == len(path)-1 { + return + } + if path[i+1] == '#' { + i += 2 + if i == len(path) { + return + } + if path[i] == '[' || path[i] == '(' { + var start, end byte + if path[i] == '[' { + start, end = '[', ']' + } else { + start, end = '(', ')' + } + // inside selector, balance brackets + i++ + depth := 1 + for ; i < len(path); i++ { + if path[i] == '\\' { + i++ + } else if path[i] == start { + depth++ + } else if path[i] == end { + depth-- + if depth == 0 { + break + } + } else if path[i] == '"' { + // inside selector string, balance quotes + i++ + for ; i < len(path); i++ { + if path[i] == '\\' { + i++ + } else if path[i] == '"' { + break + } + } + } + } + } + } + } else if path[i] == '|' { + return path[:i], path[i+1:], true + } + } + return +} + +// ForEachLine iterates through lines of JSON as specified by the JSON Lines +// format (http://jsonlines.org/). +// Each line is returned as a GJSON Result. +func ForEachLine(json string, iterator func(line Result) bool) { + var res Result + var i int + for { + i, res, _ = parseAny(json, i, true) + if !res.Exists() { + break + } + if !iterator(res) { + return + } + } +} + +type subSelector struct { + name string + path string +} + +// parseSubSelectors returns the subselectors belonging to a '[path1,path2]' or +// '{"field1":path1,"field2":path2}' type subSelection. It's expected that the +// first character in path is either '[' or '{', and has already been checked +// prior to calling this function. +func parseSubSelectors(path string) (sels []subSelector, out string, ok bool) { + modifier := 0 + depth := 1 + colon := 0 + start := 1 + i := 1 + pushSel := func() { + var sel subSelector + if colon == 0 { + sel.path = path[start:i] + } else { + sel.name = path[start:colon] + sel.path = path[colon+1 : i] + } + sels = append(sels, sel) + colon = 0 + modifier = 0 + start = i + 1 + } + for ; i < len(path); i++ { + switch path[i] { + case '\\': + i++ + case '@': + if modifier == 0 && i > 0 && (path[i-1] == '.' || path[i-1] == '|') { + modifier = i + } + case ':': + if modifier == 0 && colon == 0 && depth == 1 { + colon = i + } + case ',': + if depth == 1 { + pushSel() + } + case '"': + i++ + loop: + for ; i < len(path); i++ { + switch path[i] { + case '\\': + i++ + case '"': + break loop + } + } + case '[', '(', '{': + depth++ + case ']', ')', '}': + depth-- + if depth == 0 { + pushSel() + path = path[i+1:] + return sels, path, true + } + } + } + return +} + +// nameOfLast returns the name of the last component +func nameOfLast(path string) string { + for i := len(path) - 1; i >= 0; i-- { + if path[i] == '|' || path[i] == '.' { + if i > 0 { + if path[i-1] == '\\' { + continue + } + } + return path[i+1:] + } + } + return path +} + +func isSimpleName(component string) bool { + for i := 0; i < len(component); i++ { + if component[i] < ' ' { + return false + } + switch component[i] { + case '[', ']', '{', '}', '(', ')', '#', '|', '!': + return false + } + } + return true +} + +var hexchars = [...]byte{ + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + 'a', 'b', 'c', 'd', 'e', 'f', +} + +func appendHex16(dst []byte, x uint16) []byte { + return append(dst, + hexchars[x>>12&0xF], hexchars[x>>8&0xF], + hexchars[x>>4&0xF], hexchars[x>>0&0xF], + ) +} + +// AppendJSONString is a convenience function that converts the provided string +// to a valid JSON string and appends it to dst. +func AppendJSONString(dst []byte, s string) []byte { + dst = append(dst, make([]byte, len(s)+2)...) + dst = append(dst[:len(dst)-len(s)-2], '"') + for i := 0; i < len(s); i++ { + if s[i] < ' ' { + dst = append(dst, '\\') + switch s[i] { + case '\n': + dst = append(dst, 'n') + case '\r': + dst = append(dst, 'r') + case '\t': + dst = append(dst, 't') + default: + dst = append(dst, 'u') + dst = appendHex16(dst, uint16(s[i])) + } + } else if s[i] == '>' || s[i] == '<' || s[i] == '&' { + dst = append(dst, '\\', 'u') + dst = appendHex16(dst, uint16(s[i])) + } else if s[i] == '\\' { + dst = append(dst, '\\', '\\') + } else if s[i] == '"' { + dst = append(dst, '\\', '"') + } else if s[i] > 127 { + // read utf8 character + r, n := utf8.DecodeRuneInString(s[i:]) + if n == 0 { + break + } + if r == utf8.RuneError && n == 1 { + dst = append(dst, `\ufffd`...) + } else if r == '\u2028' || r == '\u2029' { + dst = append(dst, `\u202`...) + dst = append(dst, hexchars[r&0xF]) + } else { + dst = append(dst, s[i:i+n]...) + } + i = i + n - 1 + } else { + dst = append(dst, s[i]) + } + } + return append(dst, '"') +} + +type parseContext struct { + json string + value Result + pipe string + piped bool + calcd bool + lines bool +} + +// Get searches json for the specified path. +// A path is in dot syntax, such as "name.last" or "age". +// When the value is found it's returned immediately. +// +// A path is a series of keys separated by a dot. +// A key may contain special wildcard characters '*' and '?'. +// To access an array value use the index as the key. +// To get the number of elements in an array or to access a child path, use +// the '#' character. +// The dot and wildcard character can be escaped with '\'. +// +// { +// "name": {"first": "Tom", "last": "Anderson"}, +// "age":37, +// "children": ["Sara","Alex","Jack"], +// "friends": [ +// {"first": "James", "last": "Murphy"}, +// {"first": "Roger", "last": "Craig"} +// ] +// } +// "name.last" >> "Anderson" +// "age" >> 37 +// "children" >> ["Sara","Alex","Jack"] +// "children.#" >> 3 +// "children.1" >> "Alex" +// "child*.2" >> "Jack" +// "c?ildren.0" >> "Sara" +// "friends.#.first" >> ["James","Roger"] +// +// This function expects that the json is well-formed, and does not validate. +// Invalid json will not panic, but it may return back unexpected results. +// If you are consuming JSON from an unpredictable source then you may want to +// use the Valid function first. +func Get(json, path string) Result { + if len(path) > 1 { + if (path[0] == '@' && !DisableModifiers) || path[0] == '!' { + // possible modifier + var ok bool + var npath string + var rjson string + if path[0] == '@' && !DisableModifiers { + npath, rjson, ok = execModifier(json, path) + } else if path[0] == '!' { + npath, rjson, ok = execStatic(json, path) + } + if ok { + path = npath + if len(path) > 0 && (path[0] == '|' || path[0] == '.') { + res := Get(rjson, path[1:]) + res.Index = 0 + res.Indexes = nil + return res + } + return Parse(rjson) + } + } + if path[0] == '[' || path[0] == '{' { + // using a subselector path + kind := path[0] + var ok bool + var subs []subSelector + subs, path, ok = parseSubSelectors(path) + if ok { + if len(path) == 0 || (path[0] == '|' || path[0] == '.') { + var b []byte + b = append(b, kind) + var i int + for _, sub := range subs { + res := Get(json, sub.path) + if res.Exists() { + if i > 0 { + b = append(b, ',') + } + if kind == '{' { + if len(sub.name) > 0 { + if sub.name[0] == '"' && Valid(sub.name) { + b = append(b, sub.name...) + } else { + b = AppendJSONString(b, sub.name) + } + } else { + last := nameOfLast(sub.path) + if isSimpleName(last) { + b = AppendJSONString(b, last) + } else { + b = AppendJSONString(b, "_") + } + } + b = append(b, ':') + } + var raw string + if len(res.Raw) == 0 { + raw = res.String() + if len(raw) == 0 { + raw = "null" + } + } else { + raw = res.Raw + } + b = append(b, raw...) + i++ + } + } + b = append(b, kind+2) + var res Result + res.Raw = string(b) + res.Type = JSON + if len(path) > 0 { + res = res.Get(path[1:]) + } + res.Index = 0 + return res + } + } + } + } + var i int + var c = &parseContext{json: json} + if len(path) >= 2 && path[0] == '.' && path[1] == '.' { + c.lines = true + parseArray(c, 0, path[2:]) + } else { + for ; i < len(c.json); i++ { + if c.json[i] == '{' { + i++ + parseObject(c, i, path) + break + } + if c.json[i] == '[' { + i++ + parseArray(c, i, path) + break + } + } + } + if c.piped { + res := c.value.Get(c.pipe) + res.Index = 0 + return res + } + fillIndex(json, c) + return c.value +} + +// GetBytes searches json for the specified path. +// If working with bytes, this method preferred over Get(string(data), path) +func GetBytes(json []byte, path string) Result { + return getBytes(json, path) +} + +// runeit returns the rune from the the \uXXXX +func runeit(json string) rune { + n, _ := strconv.ParseUint(json[:4], 16, 64) + return rune(n) +} + +// unescape unescapes a string +func unescape(json string) string { + var str = make([]byte, 0, len(json)) + for i := 0; i < len(json); i++ { + switch { + default: + str = append(str, json[i]) + case json[i] < ' ': + return string(str) + case json[i] == '\\': + i++ + if i >= len(json) { + return string(str) + } + switch json[i] { + default: + return string(str) + case '\\': + str = append(str, '\\') + case '/': + str = append(str, '/') + case 'b': + str = append(str, '\b') + case 'f': + str = append(str, '\f') + case 'n': + str = append(str, '\n') + case 'r': + str = append(str, '\r') + case 't': + str = append(str, '\t') + case '"': + str = append(str, '"') + case 'u': + if i+5 > len(json) { + return string(str) + } + r := runeit(json[i+1:]) + i += 5 + if utf16.IsSurrogate(r) { + // need another code + if len(json[i:]) >= 6 && json[i] == '\\' && + json[i+1] == 'u' { + // we expect it to be correct so just consume it + r = utf16.DecodeRune(r, runeit(json[i+2:])) + i += 6 + } + } + // provide enough space to encode the largest utf8 possible + str = append(str, 0, 0, 0, 0, 0, 0, 0, 0) + n := utf8.EncodeRune(str[len(str)-8:], r) + str = str[:len(str)-8+n] + i-- // backtrack index by one + } + } + } + return string(str) +} + +// Less return true if a token is less than another token. +// The caseSensitive paramater is used when the tokens are Strings. +// The order when comparing two different type is: +// +// Null < False < Number < String < True < JSON +// +func (t Result) Less(token Result, caseSensitive bool) bool { + if t.Type < token.Type { + return true + } + if t.Type > token.Type { + return false + } + if t.Type == String { + if caseSensitive { + return t.Str < token.Str + } + return stringLessInsensitive(t.Str, token.Str) + } + if t.Type == Number { + return t.Num < token.Num + } + return t.Raw < token.Raw +} + +func stringLessInsensitive(a, b string) bool { + for i := 0; i < len(a) && i < len(b); i++ { + if a[i] >= 'A' && a[i] <= 'Z' { + if b[i] >= 'A' && b[i] <= 'Z' { + // both are uppercase, do nothing + if a[i] < b[i] { + return true + } else if a[i] > b[i] { + return false + } + } else { + // a is uppercase, convert a to lowercase + if a[i]+32 < b[i] { + return true + } else if a[i]+32 > b[i] { + return false + } + } + } else if b[i] >= 'A' && b[i] <= 'Z' { + // b is uppercase, convert b to lowercase + if a[i] < b[i]+32 { + return true + } else if a[i] > b[i]+32 { + return false + } + } else { + // neither are uppercase + if a[i] < b[i] { + return true + } else if a[i] > b[i] { + return false + } + } + } + return len(a) < len(b) +} + +// parseAny parses the next value from a json string. +// A Result is returned when the hit param is set. +// The return values are (i int, res Result, ok bool) +func parseAny(json string, i int, hit bool) (int, Result, bool) { + var res Result + var val string + for ; i < len(json); i++ { + if json[i] == '{' || json[i] == '[' { + i, val = parseSquash(json, i) + if hit { + res.Raw = val + res.Type = JSON + } + var tmp parseContext + tmp.value = res + fillIndex(json, &tmp) + return i, tmp.value, true + } + if json[i] <= ' ' { + continue + } + var num bool + switch json[i] { + case '"': + i++ + var vesc bool + var ok bool + i, val, vesc, ok = parseString(json, i) + if !ok { + return i, res, false + } + if hit { + res.Type = String + res.Raw = val + if vesc { + res.Str = unescape(val[1 : len(val)-1]) + } else { + res.Str = val[1 : len(val)-1] + } + } + return i, res, true + case 'n': + if i+1 < len(json) && json[i+1] != 'u' { + num = true + break + } + fallthrough + case 't', 'f': + vc := json[i] + i, val = parseLiteral(json, i) + if hit { + res.Raw = val + switch vc { + case 't': + res.Type = True + case 'f': + res.Type = False + } + return i, res, true + } + case '+', '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + 'i', 'I', 'N': + num = true + } + if num { + i, val = parseNumber(json, i) + if hit { + res.Raw = val + res.Type = Number + res.Num, _ = strconv.ParseFloat(val, 64) + } + return i, res, true + } + + } + return i, res, false +} + +// GetMany searches json for the multiple paths. +// The return value is a Result array where the number of items +// will be equal to the number of input paths. +func GetMany(json string, path ...string) []Result { + res := make([]Result, len(path)) + for i, path := range path { + res[i] = Get(json, path) + } + return res +} + +// GetManyBytes searches json for the multiple paths. +// The return value is a Result array where the number of items +// will be equal to the number of input paths. +func GetManyBytes(json []byte, path ...string) []Result { + res := make([]Result, len(path)) + for i, path := range path { + res[i] = GetBytes(json, path) + } + return res +} + +func validpayload(data []byte, i int) (outi int, ok bool) { + for ; i < len(data); i++ { + switch data[i] { + default: + i, ok = validany(data, i) + if !ok { + return i, false + } + for ; i < len(data); i++ { + switch data[i] { + default: + return i, false + case ' ', '\t', '\n', '\r': + continue + } + } + return i, true + case ' ', '\t', '\n', '\r': + continue + } + } + return i, false +} +func validany(data []byte, i int) (outi int, ok bool) { + for ; i < len(data); i++ { + switch data[i] { + default: + return i, false + case ' ', '\t', '\n', '\r': + continue + case '{': + return validobject(data, i+1) + case '[': + return validarray(data, i+1) + case '"': + return validstring(data, i+1) + case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + return validnumber(data, i+1) + case 't': + return validtrue(data, i+1) + case 'f': + return validfalse(data, i+1) + case 'n': + return validnull(data, i+1) + } + } + return i, false +} +func validobject(data []byte, i int) (outi int, ok bool) { + for ; i < len(data); i++ { + switch data[i] { + default: + return i, false + case ' ', '\t', '\n', '\r': + continue + case '}': + return i + 1, true + case '"': + key: + if i, ok = validstring(data, i+1); !ok { + return i, false + } + if i, ok = validcolon(data, i); !ok { + return i, false + } + if i, ok = validany(data, i); !ok { + return i, false + } + if i, ok = validcomma(data, i, '}'); !ok { + return i, false + } + if data[i] == '}' { + return i + 1, true + } + i++ + for ; i < len(data); i++ { + switch data[i] { + default: + return i, false + case ' ', '\t', '\n', '\r': + continue + case '"': + goto key + } + } + return i, false + } + } + return i, false +} +func validcolon(data []byte, i int) (outi int, ok bool) { + for ; i < len(data); i++ { + switch data[i] { + default: + return i, false + case ' ', '\t', '\n', '\r': + continue + case ':': + return i + 1, true + } + } + return i, false +} +func validcomma(data []byte, i int, end byte) (outi int, ok bool) { + for ; i < len(data); i++ { + switch data[i] { + default: + return i, false + case ' ', '\t', '\n', '\r': + continue + case ',': + return i, true + case end: + return i, true + } + } + return i, false +} +func validarray(data []byte, i int) (outi int, ok bool) { + for ; i < len(data); i++ { + switch data[i] { + default: + for ; i < len(data); i++ { + if i, ok = validany(data, i); !ok { + return i, false + } + if i, ok = validcomma(data, i, ']'); !ok { + return i, false + } + if data[i] == ']' { + return i + 1, true + } + } + case ' ', '\t', '\n', '\r': + continue + case ']': + return i + 1, true + } + } + return i, false +} +func validstring(data []byte, i int) (outi int, ok bool) { + for ; i < len(data); i++ { + if data[i] < ' ' { + return i, false + } else if data[i] == '\\' { + i++ + if i == len(data) { + return i, false + } + switch data[i] { + default: + return i, false + case '"', '\\', '/', 'b', 'f', 'n', 'r', 't': + case 'u': + for j := 0; j < 4; j++ { + i++ + if i >= len(data) { + return i, false + } + if !((data[i] >= '0' && data[i] <= '9') || + (data[i] >= 'a' && data[i] <= 'f') || + (data[i] >= 'A' && data[i] <= 'F')) { + return i, false + } + } + } + } else if data[i] == '"' { + return i + 1, true + } + } + return i, false +} +func validnumber(data []byte, i int) (outi int, ok bool) { + i-- + // sign + if data[i] == '-' { + i++ + if i == len(data) { + return i, false + } + if data[i] < '0' || data[i] > '9' { + return i, false + } + } + // int + if i == len(data) { + return i, false + } + if data[i] == '0' { + i++ + } else { + for ; i < len(data); i++ { + if data[i] >= '0' && data[i] <= '9' { + continue + } + break + } + } + // frac + if i == len(data) { + return i, true + } + if data[i] == '.' { + i++ + if i == len(data) { + return i, false + } + if data[i] < '0' || data[i] > '9' { + return i, false + } + i++ + for ; i < len(data); i++ { + if data[i] >= '0' && data[i] <= '9' { + continue + } + break + } + } + // exp + if i == len(data) { + return i, true + } + if data[i] == 'e' || data[i] == 'E' { + i++ + if i == len(data) { + return i, false + } + if data[i] == '+' || data[i] == '-' { + i++ + } + if i == len(data) { + return i, false + } + if data[i] < '0' || data[i] > '9' { + return i, false + } + i++ + for ; i < len(data); i++ { + if data[i] >= '0' && data[i] <= '9' { + continue + } + break + } + } + return i, true +} + +func validtrue(data []byte, i int) (outi int, ok bool) { + if i+3 <= len(data) && data[i] == 'r' && data[i+1] == 'u' && + data[i+2] == 'e' { + return i + 3, true + } + return i, false +} +func validfalse(data []byte, i int) (outi int, ok bool) { + if i+4 <= len(data) && data[i] == 'a' && data[i+1] == 'l' && + data[i+2] == 's' && data[i+3] == 'e' { + return i + 4, true + } + return i, false +} +func validnull(data []byte, i int) (outi int, ok bool) { + if i+3 <= len(data) && data[i] == 'u' && data[i+1] == 'l' && + data[i+2] == 'l' { + return i + 3, true + } + return i, false +} + +// Valid returns true if the input is valid json. +// +// if !gjson.Valid(json) { +// return errors.New("invalid json") +// } +// value := gjson.Get(json, "name.last") +// +func Valid(json string) bool { + _, ok := validpayload(stringBytes(json), 0) + return ok +} + +// ValidBytes returns true if the input is valid json. +// +// if !gjson.Valid(json) { +// return errors.New("invalid json") +// } +// value := gjson.Get(json, "name.last") +// +// If working with bytes, this method preferred over ValidBytes(string(data)) +// +func ValidBytes(json []byte) bool { + _, ok := validpayload(json, 0) + return ok +} + +func parseUint(s string) (n uint64, ok bool) { + var i int + if i == len(s) { + return 0, false + } + for ; i < len(s); i++ { + if s[i] >= '0' && s[i] <= '9' { + n = n*10 + uint64(s[i]-'0') + } else { + return 0, false + } + } + return n, true +} + +func parseInt(s string) (n int64, ok bool) { + var i int + var sign bool + if len(s) > 0 && s[0] == '-' { + sign = true + i++ + } + if i == len(s) { + return 0, false + } + for ; i < len(s); i++ { + if s[i] >= '0' && s[i] <= '9' { + n = n*10 + int64(s[i]-'0') + } else { + return 0, false + } + } + if sign { + return n * -1, true + } + return n, true +} + +// safeInt validates a given JSON number +// ensures it lies within the minimum and maximum representable JSON numbers +func safeInt(f float64) (n int64, ok bool) { + // https://tc39.es/ecma262/#sec-number.min_safe_integer + // https://tc39.es/ecma262/#sec-number.max_safe_integer + if f < -9007199254740991 || f > 9007199254740991 { + return 0, false + } + return int64(f), true +} + +// execStatic parses the path to find a static value. +// The input expects that the path already starts with a '!' +func execStatic(json, path string) (pathOut, res string, ok bool) { + name := path[1:] + if len(name) > 0 { + switch name[0] { + case '{', '[', '"', '+', '-', '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9': + _, res = parseSquash(name, 0) + pathOut = name[len(res):] + return pathOut, res, true + } + } + for i := 1; i < len(path); i++ { + if path[i] == '|' { + pathOut = path[i:] + name = path[1:i] + break + } + if path[i] == '.' { + pathOut = path[i:] + name = path[1:i] + break + } + } + switch strings.ToLower(name) { + case "true", "false", "null", "nan", "inf": + return pathOut, name, true + } + return pathOut, res, false +} + +// execModifier parses the path to find a matching modifier function. +// The input expects that the path already starts with a '@' +func execModifier(json, path string) (pathOut, res string, ok bool) { + name := path[1:] + var hasArgs bool + for i := 1; i < len(path); i++ { + if path[i] == ':' { + pathOut = path[i+1:] + name = path[1:i] + hasArgs = len(pathOut) > 0 + break + } + if path[i] == '|' { + pathOut = path[i:] + name = path[1:i] + break + } + if path[i] == '.' { + pathOut = path[i:] + name = path[1:i] + break + } + } + if fn, ok := modifiers[name]; ok { + var args string + if hasArgs { + var parsedArgs bool + switch pathOut[0] { + case '{', '[', '"': + res := Parse(pathOut) + if res.Exists() { + args = squash(pathOut) + pathOut = pathOut[len(args):] + parsedArgs = true + } + } + if !parsedArgs { + idx := strings.IndexByte(pathOut, '|') + if idx == -1 { + args = pathOut + pathOut = "" + } else { + args = pathOut[:idx] + pathOut = pathOut[idx:] + } + } + } + return pathOut, fn(json, args), true + } + return pathOut, res, false +} + +// unwrap removes the '[]' or '{}' characters around json +func unwrap(json string) string { + json = trim(json) + if len(json) >= 2 && (json[0] == '[' || json[0] == '{') { + json = json[1 : len(json)-1] + } + return json +} + +// DisableModifiers will disable the modifier syntax +var DisableModifiers = false + +var modifiers = map[string]func(json, arg string) string{ + "pretty": modPretty, + "ugly": modUgly, + "reverse": modReverse, + "this": modThis, + "flatten": modFlatten, + "join": modJoin, + "valid": modValid, + "keys": modKeys, + "values": modValues, + "tostr": modToStr, + "fromstr": modFromStr, + "group": modGroup, +} + +// AddModifier binds a custom modifier command to the GJSON syntax. +// This operation is not thread safe and should be executed prior to +// using all other gjson function. +func AddModifier(name string, fn func(json, arg string) string) { + modifiers[name] = fn +} + +// ModifierExists returns true when the specified modifier exists. +func ModifierExists(name string, fn func(json, arg string) string) bool { + _, ok := modifiers[name] + return ok +} + +// cleanWS remove any non-whitespace from string +func cleanWS(s string) string { + for i := 0; i < len(s); i++ { + switch s[i] { + case ' ', '\t', '\n', '\r': + continue + default: + var s2 []byte + for i := 0; i < len(s); i++ { + switch s[i] { + case ' ', '\t', '\n', '\r': + s2 = append(s2, s[i]) + } + } + return string(s2) + } + } + return s +} + +// @pretty modifier makes the json look nice. +func modPretty(json, arg string) string { + if len(arg) > 0 { + opts := *pretty.DefaultOptions + Parse(arg).ForEach(func(key, value Result) bool { + switch key.String() { + case "sortKeys": + opts.SortKeys = value.Bool() + case "indent": + opts.Indent = cleanWS(value.String()) + case "prefix": + opts.Prefix = cleanWS(value.String()) + case "width": + opts.Width = int(value.Int()) + } + return true + }) + return bytesString(pretty.PrettyOptions(stringBytes(json), &opts)) + } + return bytesString(pretty.Pretty(stringBytes(json))) +} + +// @this returns the current element. Can be used to retrieve the root element. +func modThis(json, arg string) string { + return json +} + +// @ugly modifier removes all whitespace. +func modUgly(json, arg string) string { + return bytesString(pretty.Ugly(stringBytes(json))) +} + +// @reverse reverses array elements or root object members. +func modReverse(json, arg string) string { + res := Parse(json) + if res.IsArray() { + var values []Result + res.ForEach(func(_, value Result) bool { + values = append(values, value) + return true + }) + out := make([]byte, 0, len(json)) + out = append(out, '[') + for i, j := len(values)-1, 0; i >= 0; i, j = i-1, j+1 { + if j > 0 { + out = append(out, ',') + } + out = append(out, values[i].Raw...) + } + out = append(out, ']') + return bytesString(out) + } + if res.IsObject() { + var keyValues []Result + res.ForEach(func(key, value Result) bool { + keyValues = append(keyValues, key, value) + return true + }) + out := make([]byte, 0, len(json)) + out = append(out, '{') + for i, j := len(keyValues)-2, 0; i >= 0; i, j = i-2, j+1 { + if j > 0 { + out = append(out, ',') + } + out = append(out, keyValues[i+0].Raw...) + out = append(out, ':') + out = append(out, keyValues[i+1].Raw...) + } + out = append(out, '}') + return bytesString(out) + } + return json +} + +// @flatten an array with child arrays. +// [1,[2],[3,4],[5,[6,7]]] -> [1,2,3,4,5,[6,7]] +// The {"deep":true} arg can be provide for deep flattening. +// [1,[2],[3,4],[5,[6,7]]] -> [1,2,3,4,5,6,7] +// The original json is returned when the json is not an array. +func modFlatten(json, arg string) string { + res := Parse(json) + if !res.IsArray() { + return json + } + var deep bool + if arg != "" { + Parse(arg).ForEach(func(key, value Result) bool { + if key.String() == "deep" { + deep = value.Bool() + } + return true + }) + } + var out []byte + out = append(out, '[') + var idx int + res.ForEach(func(_, value Result) bool { + var raw string + if value.IsArray() { + if deep { + raw = unwrap(modFlatten(value.Raw, arg)) + } else { + raw = unwrap(value.Raw) + } + } else { + raw = value.Raw + } + raw = strings.TrimSpace(raw) + if len(raw) > 0 { + if idx > 0 { + out = append(out, ',') + } + out = append(out, raw...) + idx++ + } + return true + }) + out = append(out, ']') + return bytesString(out) +} + +// @keys extracts the keys from an object. +// {"first":"Tom","last":"Smith"} -> ["first","last"] +func modKeys(json, arg string) string { + v := Parse(json) + if !v.Exists() { + return "[]" + } + obj := v.IsObject() + var out strings.Builder + out.WriteByte('[') + var i int + v.ForEach(func(key, _ Result) bool { + if i > 0 { + out.WriteByte(',') + } + if obj { + out.WriteString(key.Raw) + } else { + out.WriteString("null") + } + i++ + return true + }) + out.WriteByte(']') + return out.String() +} + +// @values extracts the values from an object. +// {"first":"Tom","last":"Smith"} -> ["Tom","Smith"] +func modValues(json, arg string) string { + v := Parse(json) + if !v.Exists() { + return "[]" + } + if v.IsArray() { + return json + } + var out strings.Builder + out.WriteByte('[') + var i int + v.ForEach(func(_, value Result) bool { + if i > 0 { + out.WriteByte(',') + } + out.WriteString(value.Raw) + i++ + return true + }) + out.WriteByte(']') + return out.String() +} + +// @join multiple objects into a single object. +// [{"first":"Tom"},{"last":"Smith"}] -> {"first","Tom","last":"Smith"} +// The arg can be "true" to specify that duplicate keys should be preserved. +// [{"first":"Tom","age":37},{"age":41}] -> {"first","Tom","age":37,"age":41} +// Without preserved keys: +// [{"first":"Tom","age":37},{"age":41}] -> {"first","Tom","age":41} +// The original json is returned when the json is not an object. +func modJoin(json, arg string) string { + res := Parse(json) + if !res.IsArray() { + return json + } + var preserve bool + if arg != "" { + Parse(arg).ForEach(func(key, value Result) bool { + if key.String() == "preserve" { + preserve = value.Bool() + } + return true + }) + } + var out []byte + out = append(out, '{') + if preserve { + // Preserve duplicate keys. + var idx int + res.ForEach(func(_, value Result) bool { + if !value.IsObject() { + return true + } + if idx > 0 { + out = append(out, ',') + } + out = append(out, unwrap(value.Raw)...) + idx++ + return true + }) + } else { + // Deduplicate keys and generate an object with stable ordering. + var keys []Result + kvals := make(map[string]Result) + res.ForEach(func(_, value Result) bool { + if !value.IsObject() { + return true + } + value.ForEach(func(key, value Result) bool { + k := key.String() + if _, ok := kvals[k]; !ok { + keys = append(keys, key) + } + kvals[k] = value + return true + }) + return true + }) + for i := 0; i < len(keys); i++ { + if i > 0 { + out = append(out, ',') + } + out = append(out, keys[i].Raw...) + out = append(out, ':') + out = append(out, kvals[keys[i].String()].Raw...) + } + } + out = append(out, '}') + return bytesString(out) +} + +// @valid ensures that the json is valid before moving on. An empty string is +// returned when the json is not valid, otherwise it returns the original json. +func modValid(json, arg string) string { + if !Valid(json) { + return "" + } + return json +} + +// @fromstr converts a string to json +// "{\"id\":1023,\"name\":\"alert\"}" -> {"id":1023,"name":"alert"} +func modFromStr(json, arg string) string { + if !Valid(json) { + return "" + } + return Parse(json).String() +} + +// @tostr converts a string to json +// {"id":1023,"name":"alert"} -> "{\"id\":1023,\"name\":\"alert\"}" +func modToStr(str, arg string) string { + return string(AppendJSONString(nil, str)) +} + +func modGroup(json, arg string) string { + res := Parse(json) + if !res.IsObject() { + return "" + } + var all [][]byte + res.ForEach(func(key, value Result) bool { + if !value.IsArray() { + return true + } + var idx int + value.ForEach(func(_, value Result) bool { + if idx == len(all) { + all = append(all, []byte{}) + } + all[idx] = append(all[idx], ("," + key.Raw + ":" + value.Raw)...) + idx++ + return true + }) + return true + }) + var data []byte + data = append(data, '[') + for i, item := range all { + if i > 0 { + data = append(data, ',') + } + data = append(data, '{') + data = append(data, item[1:]...) + data = append(data, '}') + } + data = append(data, ']') + return string(data) +} + +// stringHeader instead of reflect.StringHeader +type stringHeader struct { + data unsafe.Pointer + len int +} + +// sliceHeader instead of reflect.SliceHeader +type sliceHeader struct { + data unsafe.Pointer + len int + cap int +} + +// getBytes casts the input json bytes to a string and safely returns the +// results as uniquely allocated data. This operation is intended to minimize +// copies and allocations for the large json string->[]byte. +func getBytes(json []byte, path string) Result { + var result Result + if json != nil { + // unsafe cast to string + result = Get(*(*string)(unsafe.Pointer(&json)), path) + // safely get the string headers + rawhi := *(*stringHeader)(unsafe.Pointer(&result.Raw)) + strhi := *(*stringHeader)(unsafe.Pointer(&result.Str)) + // create byte slice headers + rawh := sliceHeader{data: rawhi.data, len: rawhi.len, cap: rawhi.len} + strh := sliceHeader{data: strhi.data, len: strhi.len, cap: rawhi.len} + if strh.data == nil { + // str is nil + if rawh.data == nil { + // raw is nil + result.Raw = "" + } else { + // raw has data, safely copy the slice header to a string + result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh))) + } + result.Str = "" + } else if rawh.data == nil { + // raw is nil + result.Raw = "" + // str has data, safely copy the slice header to a string + result.Str = string(*(*[]byte)(unsafe.Pointer(&strh))) + } else if uintptr(strh.data) >= uintptr(rawh.data) && + uintptr(strh.data)+uintptr(strh.len) <= + uintptr(rawh.data)+uintptr(rawh.len) { + // Str is a substring of Raw. + start := uintptr(strh.data) - uintptr(rawh.data) + // safely copy the raw slice header + result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh))) + // substring the raw + result.Str = result.Raw[start : start+uintptr(strh.len)] + } else { + // safely copy both the raw and str slice headers to strings + result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh))) + result.Str = string(*(*[]byte)(unsafe.Pointer(&strh))) + } + } + return result +} + +// fillIndex finds the position of Raw data and assigns it to the Index field +// of the resulting value. If the position cannot be found then Index zero is +// used instead. +func fillIndex(json string, c *parseContext) { + if len(c.value.Raw) > 0 && !c.calcd { + jhdr := *(*stringHeader)(unsafe.Pointer(&json)) + rhdr := *(*stringHeader)(unsafe.Pointer(&(c.value.Raw))) + c.value.Index = int(uintptr(rhdr.data) - uintptr(jhdr.data)) + if c.value.Index < 0 || c.value.Index >= len(json) { + c.value.Index = 0 + } + } +} + +func stringBytes(s string) []byte { + return *(*[]byte)(unsafe.Pointer(&sliceHeader{ + data: (*stringHeader)(unsafe.Pointer(&s)).data, + len: len(s), + cap: len(s), + })) +} + +func bytesString(b []byte) string { + return *(*string)(unsafe.Pointer(&b)) +} + +func revSquash(json string) string { + // reverse squash + // expects that the tail character is a ']' or '}' or ')' or '"' + // squash the value, ignoring all nested arrays and objects. + i := len(json) - 1 + var depth int + if json[i] != '"' { + depth++ + } + if json[i] == '}' || json[i] == ']' || json[i] == ')' { + i-- + } + for ; i >= 0; i-- { + switch json[i] { + case '"': + i-- + for ; i >= 0; i-- { + if json[i] == '"' { + esc := 0 + for i > 0 && json[i-1] == '\\' { + i-- + esc++ + } + if esc%2 == 1 { + continue + } + i += esc + break + } + } + if depth == 0 { + if i < 0 { + i = 0 + } + return json[i:] + } + case '}', ']', ')': + depth++ + case '{', '[', '(': + depth-- + if depth == 0 { + return json[i:] + } + } + } + return json +} + +// Paths returns the original GJSON paths for a Result where the Result came +// from a simple query path that returns an array, like: +// +// gjson.Get(json, "friends.#.first") +// +// The returned value will be in the form of a JSON array: +// +// ["friends.0.first","friends.1.first","friends.2.first"] +// +// The param 'json' must be the original JSON used when calling Get. +// +// Returns an empty string if the paths cannot be determined, which can happen +// when the Result came from a path that contained a multipath, modifier, +// or a nested query. +func (t Result) Paths(json string) []string { + if t.Indexes == nil { + return nil + } + paths := make([]string, 0, len(t.Indexes)) + t.ForEach(func(_, value Result) bool { + paths = append(paths, value.Path(json)) + return true + }) + if len(paths) != len(t.Indexes) { + return nil + } + return paths +} + +// Path returns the original GJSON path for a Result where the Result came +// from a simple path that returns a single value, like: +// +// gjson.Get(json, "friends.#(last=Murphy)") +// +// The returned value will be in the form of a JSON string: +// +// "friends.0" +// +// The param 'json' must be the original JSON used when calling Get. +// +// Returns an empty string if the paths cannot be determined, which can happen +// when the Result came from a path that contained a multipath, modifier, +// or a nested query. +func (t Result) Path(json string) string { + var path []byte + var comps []string // raw components + i := t.Index - 1 + if t.Index+len(t.Raw) > len(json) { + // JSON cannot safely contain Result. + goto fail + } + if !strings.HasPrefix(json[t.Index:], t.Raw) { + // Result is not at the JSON index as exepcted. + goto fail + } + for ; i >= 0; i-- { + if json[i] <= ' ' { + continue + } + if json[i] == ':' { + // inside of object, get the key + for ; i >= 0; i-- { + if json[i] != '"' { + continue + } + break + } + raw := revSquash(json[:i+1]) + i = i - len(raw) + comps = append(comps, raw) + // key gotten, now squash the rest + raw = revSquash(json[:i+1]) + i = i - len(raw) + i++ // increment the index for next loop step + } else if json[i] == '{' { + // Encountered an open object. The original result was probably an + // object key. + goto fail + } else if json[i] == ',' || json[i] == '[' { + // inside of an array, count the position + var arrIdx int + if json[i] == ',' { + arrIdx++ + i-- + } + for ; i >= 0; i-- { + if json[i] == ':' { + // Encountered an unexpected colon. The original result was + // probably an object key. + goto fail + } else if json[i] == ',' { + arrIdx++ + } else if json[i] == '[' { + comps = append(comps, strconv.Itoa(arrIdx)) + break + } else if json[i] == ']' || json[i] == '}' || json[i] == '"' { + raw := revSquash(json[:i+1]) + i = i - len(raw) + 1 + } + } + } + } + if len(comps) == 0 { + if DisableModifiers { + goto fail + } + return "@this" + } + for i := len(comps) - 1; i >= 0; i-- { + rcomp := Parse(comps[i]) + if !rcomp.Exists() { + goto fail + } + comp := escapeComp(rcomp.String()) + path = append(path, '.') + path = append(path, comp...) + } + if len(path) > 0 { + path = path[1:] + } + return string(path) +fail: + return "" +} + +// isSafePathKeyChar returns true if the input character is safe for not +// needing escaping. +func isSafePathKeyChar(c byte) bool { + return c <= ' ' || c > '~' || c == '_' || c == '-' || c == ':' || + (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') +} + +// escapeComp escaped a path compontent, making it safe for generating a +// path for later use. +func escapeComp(comp string) string { + for i := 0; i < len(comp); i++ { + if !isSafePathKeyChar(comp[i]) { + ncomp := []byte(comp[:i]) + for ; i < len(comp); i++ { + if !isSafePathKeyChar(comp[i]) { + ncomp = append(ncomp, '\\') + } + ncomp = append(ncomp, comp[i]) + } + return string(ncomp) + } + } + return comp +} diff --git a/vendor/github.com/tidwall/gjson/logo.png b/vendor/github.com/tidwall/gjson/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..17a8bbe9d651e5d79cbd59a3645a152443440ad6 GIT binary patch literal 15936 zcmdseRa9I})9xU_0znfrFi3EBcXtB8-Q8tyw?MEE+&u)>;O-g-!QF!n?wrm0ecyln zi+`Pqb92@@b204MboZ|AuIj4isoIgsic)AO1SlX72u(&>{38ei4_w2jBEbT8g#0}> zfj{`J5}K~64(6^NM$Tp+5mN_aGq8-Ek%ieuGb2+^ry(I^6HUo1oax$2(u&{u+covw&X$WR|Y3j=W4v9v&Wy9&C&b&K688JUl#1%&bhTtPDU41{Y6zS0f06 zy$kt&Mi4i1F>$tXbhUD@2fvJHWbEMPDnJHE`mZV2IVvdp*TD8J|6V9y$(SHUj!Z0! z%uIH6FZ24RwTtUVv;Qr||Jd3^&C}70>7$v8gPXGnupj2+|LF{@-T(PPFAV`{c$J*3 zfK4&76?ZUkvoo`Il@S*p1OCHkYGukR$|buuylm3H z<}7aJY~^ldD(UQC2mWW3d9D5jDR|g;*tw0_nHkJkjZ7HWS$Vh_jJR2u8Q6HtIgQO& zSd4j$Sjqm~-}Jw&-oLaIxC6|@@jn9bnzJ+W7@M$}GZ-0hn=-JlaPctk7_+f5@NjXm zahsTP8Jln!kud=xGQB9ye^aFY+yb=o6ILj1BiHWiOsHjwSN(6-S#&IQQX(?!~oRWla6k zvj3m#lk>9cCv!wc6L6R*c~`T5NT-O!e%&k`LRLWCk5V)xyH@qD}#ba7*ern zD^KwwL26x@_H9n^_xJEwj%`LF z2_M&3l{$q7;~URr@StEw0z3u!#j3ZdE5%$S1w0iM^K2@_(3KW1T0xFBPH3#1nB2_f zvBMJo!z+(+E_6G$0(R-JL<$OXHd^e=iug&_x%->yfVBZnFV3sIHbvZbAFwXaW7AoKsJ9QG z>$+mQV_~P4&z8jBcSvrf!^$eZC-BSslJQJOm4Sir>t!xQeA0}meB*bk78Ll6JYotw zFN_tm1+Ok3dz)A2wN_Xinfj#Ee?*6SsMzd_g;n4n@*KNU9c5|uUDdVb5;09jU=BA8 z7Ip^CUs>Q(aGI&ybtyH$7K|L2U(LdFslcD0RJYaHH%xx{O^TZmsdabO$yfZSTX|)r z=-oR$toP)*A6N)!Ci&fYk#B|NMo}VC;kw_(;DJ=HK8MnXxCRjx zm+Ele5%M~`%RXZFhRaja zofW}k`izaw_BnN>(4NODngeC2ev#$a0Za@}FunO>tS9?CYq6*J5?4v-!SCmNEg}($ zsiIfoly7`UXRh?F`!)YcJa>S4yLz&L#;kL&6#~9Ni+?|0z}yLl+KUVh*{UN zQL82>H7J0_BwzsZbE2-owg@HT^}ncXoZ;dh4@Ag(^&lQPK)^E=LOJs3m1N!Uj9F54V7Fy*9;Hf!S| z2Vtfjj8{AZ6|UJ&-*wZR;=h8&K-WF?$U44F^rNysF*k#YfwM3ww(UIiz!K$Vl6g^; zpZSmDI41>YtUMi>*8?muaBUxB;C6#-g+)6l$2v@q$uZDbJ6wES8#l*s2D<1?VzXJ$ zNn3AE*NNnAtmKenlM+7=mM9>ZV6zb+`lI$2@hpIeP1DdcS*Cvz5A~9(XQ5ee8Zy?1 zV$H!Cd=InD(OPcd;^t`I|2d8dNC%ws6z&4#gegDv>rH+oz!8Nz>NP}eD-R;bVvA0S z5fJb?Ou@|fK(P*e**ICmfISbcs}Y$fZuREW@ZFBDNYmhXW7PA6V7+}jLHzU1y=p!n z^hyRvQ|hIqYYoin6oO1NuT}m&C#3Y93YZnNA2Tz$8cr96%FkEFIxLhO1c8xa?YS>1 zYfbRvnrv%W@wwZlMg$41zv!F1Uthy~PJ0n;XM;%WG#G1Z(D~^_heW34*YaC}4fwaY zQ_|5S2@Q6+L&grf$wpF2KXm2n`%skl-*5HsEQC3gz~7nJ8i!$efQc!-p`}FGdT|bp zjc+K291ok>nAU4-{|as^#|^q`l>3ommlA=!Yk*~f4lIlN?BhIKO<)tnThs?ySx%(Q zZ{BqMNZOg7QO=}w219Yn;z5ayrjclN12jx{->DqdA?>)@B*M55Z6*>9uOVG^g4+gB$$ z&!XEFAyF0vK@Hi45CHqg@K*IEb;v3e@Qc{QP%M+qSM^o+6flzV`6>&uh)6U4UOl>Y zUko)LSj?`l;=xRs21!m{!rvOi>)at<`A6r#TDg{HRsnKvk6eoC&S-x#Mq{7vf>j;i zpoU-IgJbwqGu*^D+T5dUG0a_I-j;HEIj*%e?#p&>hhO%ho;0*k&ga~7^2w21Ks!^N zJ88HQ;e-ECm0XE^;oQz-+JymauGTlg2o=jQD#D!&eneT>3#Y{`@=}~!gct3epErdK zU`nI?9GgnOgA%X>7A8>kKcDjv8zn1;!d*!BC&n1*8?K}4xYS*8J-uGJt;RUTVXir%{}C0=q5B02D<=xOiYxcn?4l1}Em4lV07IMeqs8t@G*e)leU z^LZ&bK;v9xiiAiYvcgAIY^ z{hsN<54t}2vf2`*A~p<)503a+JOop=cS87$R%0Y9b}n%u{HMx2QvUVcnksc$0lfh| zO+0575+Z)~?&@C8M99dsa?f$2%sd?hq|Ds&;G=9fK537ECE{4uO>1(q_^&|Vlh_Gr>&p4j#CTe1bR2DJ_0}1+G9&6IH~WZ{A=YBXE4q6-nx9bn^kCGD zKq`a|i1oH?5f(Dp2PUamK}6jL3(BY6_{%!xtDA#drfkJ=#M6{`HRjVF)bpUQq&Ef6 zyRCSNEr>MF%a_m1u9p-Py~pK8D7c5rSo^`&g9y&-@)_~;#h*R~7VEJ@5*>Y+Ig5{* z{KaPMCB3%XAcWYoc^46OK;ZSTx9HGO-}GxhBWm@2u4qc+2r6b_aBFo$7yfX_H3CEQ zaAx(~OTHSE^w9jPe|Db0^FFGfRAv|V-o14(YFFLKu2^KV)OYV$MilR}0%8L8mXrX# z?aXNsv{y&t%MTik%-A};(yQA&Eexy&Tel+t@?5=F>qE*mRdi^0cko1yYtB`K{ocK2 ze?^g1EeKeXFLrFjvhUiy$_GM26OFkX+qLLsXfuf_ zAiE6m$DGN>RDMk}<35n$|sHc(gYzBw5(l8USpk56G_tt5v zT2RPk>)0{R5nwf2>%09ZBb^t1O4UtbrXz%J=bQ7Aj>{^+T`g!YD&6-HAA&h~?oH{F z-*(;OvA=)%wE6M!TY|;d-~Kce;ebbI&U$_a!QI&6cp@;QC-Ynh;FOeB9wl}vAGWi_ zI6Eb)y%&;;oN5BIUcqEK(H~zuZ*l#_kGGu+l9nrvvTET;x{?m&Pz z)6(6l?`Nf+N{at87srvs@R$3YNC|1B5vn8BpnndfCUEtywUobq0}Eh3Y)W$a((OMt zeVTMUi_J2y7^X9|!b=N>O53xrkSYtG=HIjP+ixUv0p0}H|0JbRs^%hA!Jwk}>L&MM z80l=$8*p?p5nqzB2flUC+qS5co0cA{{~DgUCuuOfxzg5oVle|X6R|&z9HyjTBz7=W z6(hQ3!ou1{BT@o3I#0`zwh{7Z9D2aZ8IS0|LDbPB~{a9 zxp|-)J|}zIePx$CI@kZ{&VL_`9mP8HDU)6(_8QG5JF(05;fr7V+K5bA1Etfxgh#5W>6P@@ucg&Jzly%#@3VwSKV{@&j#^M( zTlz^^ZmfJ%6rHQ3%yEH?9(L!Qb(!Do<{dt5FB1GvxNV4LI{Pjr0R!MUrMb5A4pX>C zAxvpRy)idQ1E-@e{fUAX=xnu;V!H}&V4CM$%=yv*v=$M;MOYIcj@A-jG z?Rpip=$Wr@_3CPVC#tvr@>g2)Q4aUQ5cATrcEjnW2##{myh=>MLI$B&y6jSWY!swkxM01Od*SEg zL*Mh~ni#4$)U3y+i&vQhK|9qf!@Ie%77J-pfjx;IqN%|*VX(lO%{8>t0l{p^gB)j@ zJH57Z00N>gy2Swd*!!iFW0?|!a>O(rOZL2g&I>ppw~Od1Z5(JPKd=Q&^ICR7OvUvu z?=;QVzQ(dwYT=aA*Bg?3&P}yAThCa2|K}GvR-m-L*_!MseF=o;M zOc|!hnA_@||7VYZk>zZ=UNBMA*Vtu^yo#aiaXg2+Bh3A_ z*k(u*3VX`)Fpa0tfiD<{pqWjZHrb&h8UI8iP-1;S@Ctq-P$n^!{&NzMp(LvzzPbW5 z&{drM`nf{0E^A*AS;QZ4;qFHOENld#)$;q_vGmq_=O^r}ovg!Le`|uzGHczAnUBcG z`*F2&g&n(N8{#Biba+rq`+AT}7E##C*+mWAHcl~e8IF*BlD}ptBDDp7inFHgCu%CG zMQGj9<9umYwTdv9IK}7?I-Az9B4JJeXGNp&Y~9dg6EMByhM;=CJgzMOHPo|HyPwEF z>AJ~_8b(2SFF>9)iwy@AzPGmoydb?M{G9dKy6!c@N}i|Lt*7q!;Msy>FWlp16n&Oo zmA=5&{-W||a^yg79H=mpEMh=;B08fpy9ZhoVvt6ohvYk&9Dko*huy|_Vcj-#n-^%Y zs(?&;92eWq(+`)_??I^-a?v+g0KQKt*?l$Eba3dO7`iLIHz8VPfP?g+cWvw%=+?0$ zw8JC0o-q5rI(C{(+>M|Yd?#NqbO%~Z8MLGynQ3(_s2no9^BqZSUb>Vsq3+-&Z~pLd z_+I&t*zl@gEawwB70Gx!2LBP^FYe$`QL1{2+{*4>GM~87Lx9~C-tG~*?hRhhc=M|v zKz`3aEk0yz8Txxs?~Urdp+R0uG5?!T)2G;O-?IC#fm9y}er2kwt<)7;#|LG5_jh)$ z2Sc<@EHgtod#3K*7;KqZ%_)YTiJL+t8o!4XIkwUN@t`E$Yq7=AK*!U9H?3HzBPCK4 z1;{FwV^;P{uD18>%3tX3j%6R_MIMv&@f1V4k%w(DWcZccnI;9GVC^$jwAX_dysW1j zM`z~{y%DsP60N1h*GBx4LRqvFn|B3AG`4>4XHx@(`6tg$H8rQaVQ_b9%`q`{{4}r2 z=OP;`SSJVDA#p82i#aqb_VL+eJ|9?gaG4go0A~hZ1+MA=FDhbXUeMCt&#CDcTTP6} zxtZStJ{EiC|G-Fk6MeJpZ%|$9Rn@xfF(WyQBgyfy#vHNweKWz*P#EpEYl?H6KHpMe zvf?*tGh4Og7>Gv?^om*Qpu)Eyho@?PDF~@Ea^wN^&DqhE;aD3KU>Xh(01Gsi_yYOZ*mM@2DpdkOw<+vQpws53fo{ z&wajB`u(vzs_9!4?h&i8?CuR2u+G84+26IV*Kn`^r>fu|-veN(1Zn=_`3YN(XcZ|e1tk=THcWJbl zsYeS(c=;b95uXNv-?n*7v=S{~uMPFRje+fM2G~eIo<0k{7KhLSksr739qkp}2P;MF z#RHd6+;5fK#zzu#)US{5c4HuGKPuG*;1FXZM{@sop4IH+(rfX}ZIm^whc{X<>-WUx=Qlw6^^~L>&&#_wbjA%NM-BO- ziWfJ(=D?Q^MhL3 zhf&VlDz1)-0Zchw2k$O-icKDM^}Uif#Kk9m;R`tpuwOp5cG?~QR}6ZxbSN_q>1prP zakr4;NOzZ;Y^{A;uGpF=CV3`sPEq~4a=}w;)_$M_5ewbN6EZm-FnM6t)+yo$1Xzo8 zIpW+8HNz7 zlkMVkuwUHCF6C}Aw-E@>xDL3Tfbn_4oq*Rv>wJw&{jvDbvdHt}yMJ0Gb!csez+k{> zteQ9MH2^!E*)-B03O|-kOhidM>4!7jddvni6e+3Rhgmz{jFUl94}&j}0ME)Fp%mZ0 zXlE;DA$P!z`Ey7={LOdHkNp(zqOd+a5rWUY-&`B1w6#jF;9ooo#D?cdNf9r=4ZwBl zOS1FBam;Twsy90oaUIT{CrE+w=xr2`$no9-7C=scPgFEzAOBOZ<*$GeQ77bZwMSg@@7fXHj)VSZO3m{PYxnm7QPW=ydOs z{#chl`}vB`$gV*CX!a!p0D-WAVE#D;v=)%6LUt~fUM3C-2r&%X-h~3n92m=ymRYwXeipL(xBw46A4m}b zz`ipJfSi^h{{;9`Am@=3^@?4(vH)Plzr`-h3~f>NWL@{d&CLaZiG|xLcAC$!-`noB z4&_SEZ1pq&)UusTnZ1RoHFypZcN}#Eq#vv|MOIILF6sZa{{;$X363&PR_E3MC1@C^ zx;^*2YnYQiXAqu1qRWO2EB%sab?nau7YG4niID=(CE2JAH3TMJ{gmd4t!{42oMXA5 z=pHOdBnf^{06X{BZ;N29axO0saaETG~y-q@bv})p+406EHxk$8IMo93knK5R6iV?^+?4JEa=@BB{Wr;7$09%)WWpwdSJpc&gy?Xb#y=!13d4 znM6rJyf+X%lZo?o%SI9DF_A11@m+d`!3G!<29W9k_{MPczg;h)nTL@kmCpN_655)+ zV4#=PKa9~L!Z2&Ah&=5fCU1*LOJ_x$W27L$LOfiib%hP1*Z~zUp#+jQq7DbI-U%rP zk^T9Tl?*Eoupe)4-y&KT~+O##fDEImn4y>NutF;IVJ3=m*! z*rEW%TlVNiM?a>>U$u_$JuA>L6CZFXb!s@yP&g!R3gv~Bp}AgpK2=m0jV8VvIZv2C zmd^lRyVaiKaS$v6oYCt@$k#%)MsE3;z}yQ2L<8`{hpSucn4z6B`)k9mk!I(}zCa8Y zec8sli-a|~e-I8yodRg%#-Fe zUUq-YRYyjqA8u02I*uj{d;W>=)zk|nH^OPBH+T%!|4ND7zpZ*;!cs{XDZAc=)TpYy zygF?tJ_g{6^&8==X_&x=`hHuSa{O~;8JsY}g+Ry)97Lmf`m;+MhVx9uQw2LJB)YCN z@x%=aA>jGW@-K6kNsyeSxy?!*B;mpG3B8IhiJ`aL(iYA4>ejWW`Ba}KuFQ=qb|9UD zd1@S)Q6&PF^2(HexS@I93W4L)kM;M0t02}wjNHH_W_t5)TVGK6pzr>0Q^W#T7H~ar z!`O{mW@83Sl9!-P1c_I-0Gd=n(Al2Ah)MotJ8AbzM;HOuxA3PI2G5Xa4-KFnd(R<& zl~(O&-Jsc|!_LVQcPi|XD}`wzR1Xn(cG2(D_N3D~k*V(8k4i+i9K+ulxOuCy!l zG3U%zV9j+$hqsRH4K8qiX)Y%5kpNKe-@i7c^ly-X9EHpqEaj*3o>oZwj-~iM`20S0 zT+X3*f&BF+O!NkR{xf~GM<9^RlYs$K*d}c0boMm+LsFzfZDJ%7->?En1@nR69#5W9 zha7r+_<*>ozO^~mkB|Jz*z!F@=x~3DwK8pu~`vT=$gd#s-trjR06bA&FSRn zXosx!+Ar!=CCEZdm*mgGVygGr=a;bPp3`1$ojoLd5*z)h`E`i{b~Tk6I_M{*hrhdq z%km?-j{K^V=N^-HEYrs)!oparg}x|u^fEHypM^&WQ?T7d$N;k6=t`n8b5A;R7+fd< zweEqw_&I7h2Q`^|Nhl+xehysMZI!+Zv%IfOO*?!yw98BGB}51~t1J8V4~T$`Y~Rpg zC&EB-_kDWzSVdXeG7o$eVQN}aM@*<;RL@^rCgVF8kWxLDx|}q#y}Sgg z??KODbbqBH3$u%uWAZp`x1p_9+vh$RDq>q8DArehLiv#? zS7p(k_klZ2h^UNb09!*yqk7YKmjb|*xuZ{kcYnK_eZDQDgCKrv@~G^rCEkhh1KQ8t zT~&5d1+9jn{HLO`4hJ}XbT4W*8iQ!OH&R^30MdVf>goXR<52$G7-HwbKISxdG2?gX zUWamaU{hM>`{N6CBF`m|UO7f;#%00^HUQjv90sS3{=D}p=qL&|w9G5VQ=hdE?qpB* z8CVyn;Vhr1QP!CQcS~LOSrN!b4hgV%PU9lhY4qrPo;TID@g4v|;8auwD2G~SY|8TM zRd4Zg*tNyl*2c@A_Z{0+cYF_Pnwm+58d;%$qhs2q2XkwXMBxG|t)dcF{495+0LXAt z%=xkC)NYk>2z6<8b(io?P7{c7z!dMrujH=okhn5qE$`KV2r?QH33y;KVw{~_a|@d1 zgc0Z`b1>*sRl_Co)qBcExZwdxbAVy&napFK=rMOLMPFe2WYbbO%19sPa*w$o;yi)Q z){i?j*$ujpB;L{(m!Q9Sdnfg#tTM-#YfR{CBw}t`;qUq-!x&bmVfcN}-Po_I4}X}_ z^t5y1VNMmk9z)``Ic6iQsFY7DwU8AJM3TN>l5#qSx#b5zyk}gA%k0LtM)YSep6s(K^!|HPUZnNmZx2o z56)-Om(G2o{0FL>zz=R>3h!{P^*+b!qaH#4@l(IQkuiEnE!ScPnFiaVAO&1gyai|l zAdJrT+kK3Se%<(USIy}n=jk}2Ht2IX#e*edEZ~-uUSJSqWC~k*1T5@YJrKyPnw|X< z+W3-Y{WKrQdHr~R3#5&-TZ3zM;w6TqCbhn|Rj2*Z?S*6e-R?XXaRSs|oId=TVXLu{ zj|URNRPe7+kR)4)8|nkVLsFM7<=Vmf~HeV0cp1|XXS5L@wjNj5yKie+KA z%09g+z^n`}M7P_vg>2)FYXMz=YLgchlQ~%ARY;7G(ys;3HB-ed^wLOTkupT@Nfk&o zqI^Bl$76XEwk1?h=qO6mkigTz!Fhq&Y~rX|9n!A;SBB16W?nFjd=|4(?*6`uT>Om4 zq!ivu+WiqM6jPn_bYNMC+&k!sajqnXW6Bh6mHZwrOTOLYAspD>VRI&w0>>Ym%QSk7jV!gm~4K#@3Ekv@C& z1ZW^9I}1pIxbnRqUm@~9k$hFsgO$SclmV)dsY(@axX-x;K@08<7z*M;!)D+)GnB-h_pX#E4&BdOm}hnZK05O34%JkmnSPvltCp%pfh1YkKaB; z3r{eK1y}F0zteZ!VbKxN(mGE+RBO2}il&HvJsmW5)9f&i7~9pNsoJ;9UuU{fdVZPdaTpl3xn;a*6FGldFooO)ID5!f~o{A(@Q|Z#mgd97e)$p~ZNmCyO%SVP)0qcao?}@xnBznLG_-jRz&F1t?k^&(O-9@Xe66ob6E_pADB_QBgqLn-gy;LfXQBSJbXm4Qa z)@0fUbjL-DMg$ghS|B?CBkpf7|ESPv)cUf`>w<=*s{H62A8V`IiK$3LELwT3jzOb8 zl!S`+Tx&GX2;@<}_wVf9BLV{P^(7He508+NcJ@gXb$|bH_KiMExuqD$AfHUdt4cDv znhtuostMM!A9W3zyP;)%a8-~&aywa>^w*%F9kkeh*?k3sl)5}_ql5OAK9{{G;i=&{ zUtHfQwYq)Mv2b#<<*l>m4R5ck-D}D4z}ASJ5n|#5oacfBD7c8Ej7x(YE2s{iT$}G8 zwe%kv{OB)T?3TF&{@pgAVDqA%#pj(8sb{K9MGKWIs8|2FC9!(el>JYu3twNt=QY>E zEfe)z$ve+s#4HImOHv>x>;K~XCmW++46hoJ3_n6i`N8IeHpqE;laOh)fhBpsdn`Zm z=#=<=nmx{M{f?01b_7r#Sw*A=7N%8-I`SyqirAZn#cfO6?4!h|ub2b6?Zr>?Gx+w7 z<{S4WRHS>hqD7{`zLA&m^E6s)FM4@tcOl;dBnnQWimCjB_^)p8k672RtIbfAuFEw6 zgYuq`4j>h|oy0tO_clJuOF6%_;JJXGg4^VC^xm%7&7Hy10VSjE=3!N?H7J&iqdP}I zNOMAsr-?az+$AOgF>TROKSqm>+JU2`&iM{Ko72ZNnN>6ZNGPvfr7rwp!%~yBJSI?n zB3R6?SX`*e^MKP=??Xlg7}vTlQ%RIYvO<*`dtIXjAi@O&*XGS5Uh^+>(R3{UOg9!# zeFU6}>N+X$p5K_^4*d4ma(tvz--++&_^^mulRJJ-8!^^{Pd@#l&?DVnNZmyNE?sJo z1v4i@CO?oX!Dg1Lqv3TWiOw2-FS)k+Ljd}Ilpuew^DX&cs7!n~Bu5*_ zbNkTWkOG>zvpx~^d~foOLY<`#J9-4ss8Adupv(u66p=MohfRAYxhNA>HSU|gUFPg!w!n|-*n&2dvMVX3i(q* z9avlqEO5CvT+~(A1xN=y58K~En@L`vMnKT5ufo5MXk%ca7vW2=31I*Km~>+KC%76EmzQ{Epl*zpBy66^=k| zgxepQc7MxTEKkj~;bY@6Gd!n`K!v>&``bhrGUsu ziUsg|6^14}8h@IKz!**2yt0@~gdZ8TQ6A456}yw%F8pOXchJfG;IsFRiK@Z=(_=hNf1EVaQxVt%sqi_6s+|aA2Hs^Z1^jkF} z)1c0f3(n&QqBF4M>5l>N+2Cviy}C>Nvp)=5PwiM9D`yM8h&eBJ^iH_7m4)X9Q%5MW z!_c~><|XVWOt+Ve`j5xtKb#eCe342+D}=)aV%}@?*l_nK;*CILu4J}tqmJH`|Q{N79k>Ago$ zw+kbnqFbA;2Yd~D;IrHI?7OkQTl}|AhyXHl!kF?*%64_xllGp7XvzmFLZx)x^DQ#( z8yw9_Kg)1_V%`v*<7w+#aJ1^1V0 zi}6B_vKX`{IOQ9xZOX1JvESZ!V(|c{SDSNfEs(d|CL$lQ?eu!R|NTpI?eS+Kf7vNl z%!IbKTC}f)@d@}t?+klTHUjTrMxbJXOeb(+XezwY*G|WOOma4Pdp;kk0Fl~k&%h$# zkNqIcqQ*;D3DT7}0g$zkqn+g9IR#X2V^fthIf z6xut?2mox2Q$mF~UZ_v*={bqjQ*QF8smcNS^TPHE8vGd1^gz*sl^2S)?%;8oMxCX* z0SKz`wGxRnWD$dRPEJ3QEZIK37gU^>f=z&GmJiDQqp92G7HAK&!?*nn%c^fgy_Rc6 z2i%y;23Y}W_x9$Q$t)tiX=};p&i0?ga{!QL{KM0y9!)RXqLg$ix6$n3zDrBF#amXMJ8=viyxAutbQYXsIV&Y%NuDPMK&4 z49)cj0zMhnla%IgMc=|Vc20Hv6p#_`b$l?h_yQC$DIz&u_>1yOOYIZ}$7ErKy`9Iw zO&^zH_d%1a{z{drpduoOO9>4K0hs_Wqej8OE3;~Jz;l|XDy($&>P*2><8H_*uLF

Bg_)L$48&o}ack zt!HAQladuqH6nxcdZjxF6vTK*7%AV0IOX}JZ^u_OkMumMel#gIzv1VJUb|?$j71?Q zE%B-a&vFk0&Z20HLn$L5hH9dQ4Ef5bKHOX_RfA5_`gJvR%bW4}0DCGg0Va&YC>upU z1&8GC>xLY}LkxejF;yq2sOY~5Cqo4UPpDVfIk~O=P(Gb%k`up^XsPRv9)fXL9y}y5 z_nYE{jUJUly7pJ?$r_eiajL0B@N?t6efF1pS9lk2UGDMWIj>@eS3OH*kme)>qMs4c zXm_q!8hu`I8Dhv5J1?UvlYLJ*X6D(~=y^sb&mT@}>JxIp-$sTvzSdP`p2 zJSTpI%GX_eG4+h<`8LXOC{XRWB^Zd0<_&>b2eoF`!(!~j1?f~Xrey`D;63!OSo@1j zhVkNITfkbp3}EkdgcmMq1ZjCrPrG8@X$!y3nZFDG<8?k82gdmVJ<<=koN(R0uQJ&q&;Hm9we)+ z>EiW_<2L!S=W{bXQR{t-Xk&^=i3!xJ1GH*$ft^*`kFY2U5$7}?R(xpv1OE!wXhM-U z{|-m&g!{Wci5J~AR7xhAcz2%T1X}X;Lm5kqn$n>CdZG%!jlgyEgu}8cSE=Tna>KaW zE{i{1$MRasX61aWzh$Kz+gg_XXfmLI)gJjN8%-etjvKrZa0TIVu@FNzi_DBbgG}?X zc%`<^VRHLdaERnl?k6tkmy~V-6wDDz--2n{wg8GA9pEX|*>WDA)uFS%@&MN%-fruuv;9cz5hkb^M5ny{e|a>hsQrwX-D^ zNBO*f*^1b~?2NHQ2g~@Hiq$_V;Z8|Q@sz(OJ-_;@xOeF3^57a-&;bjXTR0^(9&O1`Qs5mDXKaZ{gYh~^a4>N0el@Q8ji@;G0VwtnJC=gh zkvhS5%hY0~-FeS=8*`P?hYjBB(TJ(h3vvLJ;q{z?7Jf~hQZOlh z8(M!wZ6}#fur(A{mkQvWp^(z{kJ9Qcf&hRH6HK!c5b1?W28wck^aQ{Ju^0{PD&(m< zLbu{`I17`!l>tz)uDi~j;Q*7#M!X;*rP#E#K`0MHVO=C_Y>Kpq3I%tz)KS21z(70w zu+S+efLBlwQ<3knLW3-aM&dW%uuv1r?kQqDWsn|wHc35K!jBl%t~9oLn(8w!(iF2Y zj)Ca3zC|-dT69O*c44Drzb zTWRcGB4pjuzq;tNLKJ7H7&3CTH`^5&fs^&Csh+4TdNT~n>o18P123FK1<6Ap7>9OE zH?~wjjt8TicxY-E;Ld@{^P%j9BpeC^HJlw8iW2I<*Vc#1VH5Y&k@$T23>riN5a^fE z1xtK3We+L~pTm`C@mKj&g@flbKtRjuc?i2+3nW^d516YT2Qx*|&}0%IIZ{BK)n@-A z7W$f>T8g%2qR?15Ezb7#Al1jSng=Um;<;^4CZ)UF4*B#s16}WxXDR&e;IFDemyJBP zQD>pCTMx2W)Iwg$ckc4ElkYeRp?hVt(&_O^6u-=WO4y!T>T70vx)M)1f17nmsqw{$ zZhYP=C<2j}m^Lg-ex&8RKf~0WrfrL!~NLQ^Xk?)No>Z1xiNn~MBa)Y#@)-369lm6 zDH#U3Pv&kw{mt3+_t3uW8=*{`^%?Rl0~ff@XZrjZDZCjasI5PtLP6)zBrRO$pYlfV z4DCdS&(8)uR*23AuP?jhTi@#6yL=G*b~7!5d3?2d_2DzzR3V+SVCZe@ULo|*Qz`CV z*j}+=>D|^uy$<{s0tgCLYba;t*4jWsj}CfRI?T4cS90us!fy8Lz;(qp7V}H&`+loKEpPZr6wt`2EqWMWj z6-EjjNWU@D@IQdNmlvT5>qUq{`9m maxcomp*len(str). +func MatchLimit(str, pattern string, maxcomp int) (matched, stopped bool) { + if pattern == "*" { + return true, false + } + counter := 0 + r := match(str, pattern, len(str), &counter, maxcomp) + if r == rStop { + return false, true + } + return r == rMatch, false +} + +type result int + +const ( + rNoMatch result = iota + rMatch + rStop +) + +func match(str, pat string, slen int, counter *int, maxcomp int) result { + // check complexity limit + if maxcomp > -1 { + if *counter > slen*maxcomp { + return rStop + } + *counter++ + } + + for len(pat) > 0 { + var wild bool + pc, ps := rune(pat[0]), 1 + if pc > 0x7f { + pc, ps = utf8.DecodeRuneInString(pat) + } + var sc rune + var ss int + if len(str) > 0 { + sc, ss = rune(str[0]), 1 + if sc > 0x7f { + sc, ss = utf8.DecodeRuneInString(str) + } + } + switch pc { + case '?': + if ss == 0 { + return rNoMatch + } + case '*': + // Ignore repeating stars. + for len(pat) > 1 && pat[1] == '*' { + pat = pat[1:] + } + + // If this star is the last character then it must be a match. + if len(pat) == 1 { + return rMatch + } + + // Match and trim any non-wildcard suffix characters. + var ok bool + str, pat, ok = matchTrimSuffix(str, pat) + if !ok { + return rNoMatch + } + + // Check for single star again. + if len(pat) == 1 { + return rMatch + } + + // Perform recursive wildcard search. + r := match(str, pat[1:], slen, counter, maxcomp) + if r != rNoMatch { + return r + } + if len(str) == 0 { + return rNoMatch + } + wild = true + default: + if ss == 0 { + return rNoMatch + } + if pc == '\\' { + pat = pat[ps:] + pc, ps = utf8.DecodeRuneInString(pat) + if ps == 0 { + return rNoMatch + } + } + if sc != pc { + return rNoMatch + } + } + str = str[ss:] + if !wild { + pat = pat[ps:] + } + } + if len(str) == 0 { + return rMatch + } + return rNoMatch +} + +// matchTrimSuffix matches and trims any non-wildcard suffix characters. +// Returns the trimed string and pattern. +// +// This is called because the pattern contains extra data after the wildcard +// star. Here we compare any suffix characters in the pattern to the suffix of +// the target string. Basically a reverse match that stops when a wildcard +// character is reached. This is a little trickier than a forward match because +// we need to evaluate an escaped character in reverse. +// +// Any matched characters will be trimmed from both the target +// string and the pattern. +func matchTrimSuffix(str, pat string) (string, string, bool) { + // It's expected that the pattern has at least two bytes and the first byte + // is a wildcard star '*' + match := true + for len(str) > 0 && len(pat) > 1 { + pc, ps := utf8.DecodeLastRuneInString(pat) + var esc bool + for i := 0; ; i++ { + if pat[len(pat)-ps-i-1] != '\\' { + if i&1 == 1 { + esc = true + ps++ + } + break + } + } + if pc == '*' && !esc { + match = true + break + } + sc, ss := utf8.DecodeLastRuneInString(str) + if !((pc == '?' && !esc) || pc == sc) { + match = false + break + } + str = str[:len(str)-ss] + pat = pat[:len(pat)-ps] + } + return str, pat, match +} + +var maxRuneBytes = [...]byte{244, 143, 191, 191} + +// Allowable parses the pattern and determines the minimum and maximum allowable +// values that the pattern can represent. +// When the max cannot be determined, 'true' will be returned +// for infinite. +func Allowable(pattern string) (min, max string) { + if pattern == "" || pattern[0] == '*' { + return "", "" + } + + minb := make([]byte, 0, len(pattern)) + maxb := make([]byte, 0, len(pattern)) + var wild bool + for i := 0; i < len(pattern); i++ { + if pattern[i] == '*' { + wild = true + break + } + if pattern[i] == '?' { + minb = append(minb, 0) + maxb = append(maxb, maxRuneBytes[:]...) + } else { + minb = append(minb, pattern[i]) + maxb = append(maxb, pattern[i]) + } + } + if wild { + r, n := utf8.DecodeLastRune(maxb) + if r != utf8.RuneError { + if r < utf8.MaxRune { + r++ + if r > 0x7f { + b := make([]byte, 4) + nn := utf8.EncodeRune(b, r) + maxb = append(maxb[:len(maxb)-n], b[:nn]...) + } else { + maxb = append(maxb[:len(maxb)-n], byte(r)) + } + } + } + } + return string(minb), string(maxb) +} + +// IsPattern returns true if the string is a pattern. +func IsPattern(str string) bool { + for i := 0; i < len(str); i++ { + if str[i] == '*' || str[i] == '?' { + return true + } + } + return false +} diff --git a/vendor/github.com/tidwall/pretty/LICENSE b/vendor/github.com/tidwall/pretty/LICENSE new file mode 100644 index 000000000..993b83f23 --- /dev/null +++ b/vendor/github.com/tidwall/pretty/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2017 Josh Baker + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/tidwall/pretty/README.md b/vendor/github.com/tidwall/pretty/README.md new file mode 100644 index 000000000..d3be5e54e --- /dev/null +++ b/vendor/github.com/tidwall/pretty/README.md @@ -0,0 +1,122 @@ +# Pretty + +[![GoDoc](https://img.shields.io/badge/api-reference-blue.svg?style=flat-square)](https://pkg.go.dev/github.com/tidwall/pretty) + +Pretty is a Go package that provides [fast](#performance) methods for formatting JSON for human readability, or to compact JSON for smaller payloads. + +Getting Started +=============== + +## Installing + +To start using Pretty, install Go and run `go get`: + +```sh +$ go get -u github.com/tidwall/pretty +``` + +This will retrieve the library. + +## Pretty + +Using this example: + +```json +{"name": {"first":"Tom","last":"Anderson"}, "age":37, +"children": ["Sara","Alex","Jack"], +"fav.movie": "Deer Hunter", "friends": [ + {"first": "Janet", "last": "Murphy", "age": 44} + ]} +``` + +The following code: +```go +result = pretty.Pretty(example) +``` + +Will format the json to: + +```json +{ + "name": { + "first": "Tom", + "last": "Anderson" + }, + "age": 37, + "children": ["Sara", "Alex", "Jack"], + "fav.movie": "Deer Hunter", + "friends": [ + { + "first": "Janet", + "last": "Murphy", + "age": 44 + } + ] +} +``` + +## Color + +Color will colorize the json for outputing to the screen. + +```json +result = pretty.Color(json, nil) +``` + +Will add color to the result for printing to the terminal. +The second param is used for a customizing the style, and passing nil will use the default `pretty.TerminalStyle`. + +## Ugly + +The following code: +```go +result = pretty.Ugly(example) +``` + +Will format the json to: + +```json +{"name":{"first":"Tom","last":"Anderson"},"age":37,"children":["Sara","Alex","Jack"],"fav.movie":"Deer Hunter","friends":[{"first":"Janet","last":"Murphy","age":44}]}``` +``` + +## Customized output + +There's a `PrettyOptions(json, opts)` function which allows for customizing the output with the following options: + +```go +type Options struct { + // Width is an max column width for single line arrays + // Default is 80 + Width int + // Prefix is a prefix for all lines + // Default is an empty string + Prefix string + // Indent is the nested indentation + // Default is two spaces + Indent string + // SortKeys will sort the keys alphabetically + // Default is false + SortKeys bool +} +``` +## Performance + +Benchmarks of Pretty alongside the builtin `encoding/json` Indent/Compact methods. +``` +BenchmarkPretty-16 1000000 1034 ns/op 720 B/op 2 allocs/op +BenchmarkPrettySortKeys-16 586797 1983 ns/op 2848 B/op 14 allocs/op +BenchmarkUgly-16 4652365 254 ns/op 240 B/op 1 allocs/op +BenchmarkUglyInPlace-16 6481233 183 ns/op 0 B/op 0 allocs/op +BenchmarkJSONIndent-16 450654 2687 ns/op 1221 B/op 0 allocs/op +BenchmarkJSONCompact-16 685111 1699 ns/op 442 B/op 0 allocs/op +``` + +*These benchmarks were run on a MacBook Pro 2.4 GHz 8-Core Intel Core i9.* + +## Contact +Josh Baker [@tidwall](http://twitter.com/tidwall) + +## License + +Pretty source code is available under the MIT [License](/LICENSE). + diff --git a/vendor/github.com/tidwall/pretty/pretty.go b/vendor/github.com/tidwall/pretty/pretty.go new file mode 100644 index 000000000..f3f756aad --- /dev/null +++ b/vendor/github.com/tidwall/pretty/pretty.go @@ -0,0 +1,674 @@ +package pretty + +import ( + "bytes" + "encoding/json" + "sort" + "strconv" +) + +// Options is Pretty options +type Options struct { + // Width is an max column width for single line arrays + // Default is 80 + Width int + // Prefix is a prefix for all lines + // Default is an empty string + Prefix string + // Indent is the nested indentation + // Default is two spaces + Indent string + // SortKeys will sort the keys alphabetically + // Default is false + SortKeys bool +} + +// DefaultOptions is the default options for pretty formats. +var DefaultOptions = &Options{Width: 80, Prefix: "", Indent: " ", SortKeys: false} + +// Pretty converts the input json into a more human readable format where each +// element is on it's own line with clear indentation. +func Pretty(json []byte) []byte { return PrettyOptions(json, nil) } + +// PrettyOptions is like Pretty but with customized options. +func PrettyOptions(json []byte, opts *Options) []byte { + if opts == nil { + opts = DefaultOptions + } + buf := make([]byte, 0, len(json)) + if len(opts.Prefix) != 0 { + buf = append(buf, opts.Prefix...) + } + buf, _, _, _ = appendPrettyAny(buf, json, 0, true, + opts.Width, opts.Prefix, opts.Indent, opts.SortKeys, + 0, 0, -1) + if len(buf) > 0 { + buf = append(buf, '\n') + } + return buf +} + +// Ugly removes insignificant space characters from the input json byte slice +// and returns the compacted result. +func Ugly(json []byte) []byte { + buf := make([]byte, 0, len(json)) + return ugly(buf, json) +} + +// UglyInPlace removes insignificant space characters from the input json +// byte slice and returns the compacted result. This method reuses the +// input json buffer to avoid allocations. Do not use the original bytes +// slice upon return. +func UglyInPlace(json []byte) []byte { return ugly(json, json) } + +func ugly(dst, src []byte) []byte { + dst = dst[:0] + for i := 0; i < len(src); i++ { + if src[i] > ' ' { + dst = append(dst, src[i]) + if src[i] == '"' { + for i = i + 1; i < len(src); i++ { + dst = append(dst, src[i]) + if src[i] == '"' { + j := i - 1 + for ; ; j-- { + if src[j] != '\\' { + break + } + } + if (j-i)%2 != 0 { + break + } + } + } + } + } + } + return dst +} + +func isNaNOrInf(src []byte) bool { + return src[0] == 'i' || //Inf + src[0] == 'I' || // inf + src[0] == '+' || // +Inf + src[0] == 'N' || // Nan + (src[0] == 'n' && len(src) > 1 && src[1] != 'u') // nan +} + +func appendPrettyAny(buf, json []byte, i int, pretty bool, width int, prefix, indent string, sortkeys bool, tabs, nl, max int) ([]byte, int, int, bool) { + for ; i < len(json); i++ { + if json[i] <= ' ' { + continue + } + if json[i] == '"' { + return appendPrettyString(buf, json, i, nl) + } + + if (json[i] >= '0' && json[i] <= '9') || json[i] == '-' || isNaNOrInf(json[i:]) { + return appendPrettyNumber(buf, json, i, nl) + } + if json[i] == '{' { + return appendPrettyObject(buf, json, i, '{', '}', pretty, width, prefix, indent, sortkeys, tabs, nl, max) + } + if json[i] == '[' { + return appendPrettyObject(buf, json, i, '[', ']', pretty, width, prefix, indent, sortkeys, tabs, nl, max) + } + switch json[i] { + case 't': + return append(buf, 't', 'r', 'u', 'e'), i + 4, nl, true + case 'f': + return append(buf, 'f', 'a', 'l', 's', 'e'), i + 5, nl, true + case 'n': + return append(buf, 'n', 'u', 'l', 'l'), i + 4, nl, true + } + } + return buf, i, nl, true +} + +type pair struct { + kstart, kend int + vstart, vend int +} + +type byKeyVal struct { + sorted bool + json []byte + buf []byte + pairs []pair +} + +func (arr *byKeyVal) Len() int { + return len(arr.pairs) +} +func (arr *byKeyVal) Less(i, j int) bool { + if arr.isLess(i, j, byKey) { + return true + } + if arr.isLess(j, i, byKey) { + return false + } + return arr.isLess(i, j, byVal) +} +func (arr *byKeyVal) Swap(i, j int) { + arr.pairs[i], arr.pairs[j] = arr.pairs[j], arr.pairs[i] + arr.sorted = true +} + +type byKind int + +const ( + byKey byKind = 0 + byVal byKind = 1 +) + +type jtype int + +const ( + jnull jtype = iota + jfalse + jnumber + jstring + jtrue + jjson +) + +func getjtype(v []byte) jtype { + if len(v) == 0 { + return jnull + } + switch v[0] { + case '"': + return jstring + case 'f': + return jfalse + case 't': + return jtrue + case 'n': + return jnull + case '[', '{': + return jjson + default: + return jnumber + } +} + +func (arr *byKeyVal) isLess(i, j int, kind byKind) bool { + k1 := arr.json[arr.pairs[i].kstart:arr.pairs[i].kend] + k2 := arr.json[arr.pairs[j].kstart:arr.pairs[j].kend] + var v1, v2 []byte + if kind == byKey { + v1 = k1 + v2 = k2 + } else { + v1 = bytes.TrimSpace(arr.buf[arr.pairs[i].vstart:arr.pairs[i].vend]) + v2 = bytes.TrimSpace(arr.buf[arr.pairs[j].vstart:arr.pairs[j].vend]) + if len(v1) >= len(k1)+1 { + v1 = bytes.TrimSpace(v1[len(k1)+1:]) + } + if len(v2) >= len(k2)+1 { + v2 = bytes.TrimSpace(v2[len(k2)+1:]) + } + } + t1 := getjtype(v1) + t2 := getjtype(v2) + if t1 < t2 { + return true + } + if t1 > t2 { + return false + } + if t1 == jstring { + s1 := parsestr(v1) + s2 := parsestr(v2) + return string(s1) < string(s2) + } + if t1 == jnumber { + n1, _ := strconv.ParseFloat(string(v1), 64) + n2, _ := strconv.ParseFloat(string(v2), 64) + return n1 < n2 + } + return string(v1) < string(v2) + +} + +func parsestr(s []byte) []byte { + for i := 1; i < len(s); i++ { + if s[i] == '\\' { + var str string + json.Unmarshal(s, &str) + return []byte(str) + } + if s[i] == '"' { + return s[1:i] + } + } + return nil +} + +func appendPrettyObject(buf, json []byte, i int, open, close byte, pretty bool, width int, prefix, indent string, sortkeys bool, tabs, nl, max int) ([]byte, int, int, bool) { + var ok bool + if width > 0 { + if pretty && open == '[' && max == -1 { + // here we try to create a single line array + max := width - (len(buf) - nl) + if max > 3 { + s1, s2 := len(buf), i + buf, i, _, ok = appendPrettyObject(buf, json, i, '[', ']', false, width, prefix, "", sortkeys, 0, 0, max) + if ok && len(buf)-s1 <= max { + return buf, i, nl, true + } + buf = buf[:s1] + i = s2 + } + } else if max != -1 && open == '{' { + return buf, i, nl, false + } + } + buf = append(buf, open) + i++ + var pairs []pair + if open == '{' && sortkeys { + pairs = make([]pair, 0, 8) + } + var n int + for ; i < len(json); i++ { + if json[i] <= ' ' { + continue + } + if json[i] == close { + if pretty { + if open == '{' && sortkeys { + buf = sortPairs(json, buf, pairs) + } + if n > 0 { + nl = len(buf) + if buf[nl-1] == ' ' { + buf[nl-1] = '\n' + } else { + buf = append(buf, '\n') + } + } + if buf[len(buf)-1] != open { + buf = appendTabs(buf, prefix, indent, tabs) + } + } + buf = append(buf, close) + return buf, i + 1, nl, open != '{' + } + if open == '[' || json[i] == '"' { + if n > 0 { + buf = append(buf, ',') + if width != -1 && open == '[' { + buf = append(buf, ' ') + } + } + var p pair + if pretty { + nl = len(buf) + if buf[nl-1] == ' ' { + buf[nl-1] = '\n' + } else { + buf = append(buf, '\n') + } + if open == '{' && sortkeys { + p.kstart = i + p.vstart = len(buf) + } + buf = appendTabs(buf, prefix, indent, tabs+1) + } + if open == '{' { + buf, i, nl, _ = appendPrettyString(buf, json, i, nl) + if sortkeys { + p.kend = i + } + buf = append(buf, ':') + if pretty { + buf = append(buf, ' ') + } + } + buf, i, nl, ok = appendPrettyAny(buf, json, i, pretty, width, prefix, indent, sortkeys, tabs+1, nl, max) + if max != -1 && !ok { + return buf, i, nl, false + } + if pretty && open == '{' && sortkeys { + p.vend = len(buf) + if p.kstart > p.kend || p.vstart > p.vend { + // bad data. disable sorting + sortkeys = false + } else { + pairs = append(pairs, p) + } + } + i-- + n++ + } + } + return buf, i, nl, open != '{' +} +func sortPairs(json, buf []byte, pairs []pair) []byte { + if len(pairs) == 0 { + return buf + } + vstart := pairs[0].vstart + vend := pairs[len(pairs)-1].vend + arr := byKeyVal{false, json, buf, pairs} + sort.Stable(&arr) + if !arr.sorted { + return buf + } + nbuf := make([]byte, 0, vend-vstart) + for i, p := range pairs { + nbuf = append(nbuf, buf[p.vstart:p.vend]...) + if i < len(pairs)-1 { + nbuf = append(nbuf, ',') + nbuf = append(nbuf, '\n') + } + } + return append(buf[:vstart], nbuf...) +} + +func appendPrettyString(buf, json []byte, i, nl int) ([]byte, int, int, bool) { + s := i + i++ + for ; i < len(json); i++ { + if json[i] == '"' { + var sc int + for j := i - 1; j > s; j-- { + if json[j] == '\\' { + sc++ + } else { + break + } + } + if sc%2 == 1 { + continue + } + i++ + break + } + } + return append(buf, json[s:i]...), i, nl, true +} + +func appendPrettyNumber(buf, json []byte, i, nl int) ([]byte, int, int, bool) { + s := i + i++ + for ; i < len(json); i++ { + if json[i] <= ' ' || json[i] == ',' || json[i] == ':' || json[i] == ']' || json[i] == '}' { + break + } + } + return append(buf, json[s:i]...), i, nl, true +} + +func appendTabs(buf []byte, prefix, indent string, tabs int) []byte { + if len(prefix) != 0 { + buf = append(buf, prefix...) + } + if len(indent) == 2 && indent[0] == ' ' && indent[1] == ' ' { + for i := 0; i < tabs; i++ { + buf = append(buf, ' ', ' ') + } + } else { + for i := 0; i < tabs; i++ { + buf = append(buf, indent...) + } + } + return buf +} + +// Style is the color style +type Style struct { + Key, String, Number [2]string + True, False, Null [2]string + Escape [2]string + Append func(dst []byte, c byte) []byte +} + +func hexp(p byte) byte { + switch { + case p < 10: + return p + '0' + default: + return (p - 10) + 'a' + } +} + +// TerminalStyle is for terminals +var TerminalStyle *Style + +func init() { + TerminalStyle = &Style{ + Key: [2]string{"\x1B[94m", "\x1B[0m"}, + String: [2]string{"\x1B[92m", "\x1B[0m"}, + Number: [2]string{"\x1B[93m", "\x1B[0m"}, + True: [2]string{"\x1B[96m", "\x1B[0m"}, + False: [2]string{"\x1B[96m", "\x1B[0m"}, + Null: [2]string{"\x1B[91m", "\x1B[0m"}, + Escape: [2]string{"\x1B[35m", "\x1B[0m"}, + Append: func(dst []byte, c byte) []byte { + if c < ' ' && (c != '\r' && c != '\n' && c != '\t' && c != '\v') { + dst = append(dst, "\\u00"...) + dst = append(dst, hexp((c>>4)&0xF)) + return append(dst, hexp((c)&0xF)) + } + return append(dst, c) + }, + } +} + +// Color will colorize the json. The style parma is used for customizing +// the colors. Passing nil to the style param will use the default +// TerminalStyle. +func Color(src []byte, style *Style) []byte { + if style == nil { + style = TerminalStyle + } + apnd := style.Append + if apnd == nil { + apnd = func(dst []byte, c byte) []byte { + return append(dst, c) + } + } + type stackt struct { + kind byte + key bool + } + var dst []byte + var stack []stackt + for i := 0; i < len(src); i++ { + if src[i] == '"' { + key := len(stack) > 0 && stack[len(stack)-1].key + if key { + dst = append(dst, style.Key[0]...) + } else { + dst = append(dst, style.String[0]...) + } + dst = apnd(dst, '"') + esc := false + uesc := 0 + for i = i + 1; i < len(src); i++ { + if src[i] == '\\' { + if key { + dst = append(dst, style.Key[1]...) + } else { + dst = append(dst, style.String[1]...) + } + dst = append(dst, style.Escape[0]...) + dst = apnd(dst, src[i]) + esc = true + if i+1 < len(src) && src[i+1] == 'u' { + uesc = 5 + } else { + uesc = 1 + } + } else if esc { + dst = apnd(dst, src[i]) + if uesc == 1 { + esc = false + dst = append(dst, style.Escape[1]...) + if key { + dst = append(dst, style.Key[0]...) + } else { + dst = append(dst, style.String[0]...) + } + } else { + uesc-- + } + } else { + dst = apnd(dst, src[i]) + } + if src[i] == '"' { + j := i - 1 + for ; ; j-- { + if src[j] != '\\' { + break + } + } + if (j-i)%2 != 0 { + break + } + } + } + if esc { + dst = append(dst, style.Escape[1]...) + } else if key { + dst = append(dst, style.Key[1]...) + } else { + dst = append(dst, style.String[1]...) + } + } else if src[i] == '{' || src[i] == '[' { + stack = append(stack, stackt{src[i], src[i] == '{'}) + dst = apnd(dst, src[i]) + } else if (src[i] == '}' || src[i] == ']') && len(stack) > 0 { + stack = stack[:len(stack)-1] + dst = apnd(dst, src[i]) + } else if (src[i] == ':' || src[i] == ',') && len(stack) > 0 && stack[len(stack)-1].kind == '{' { + stack[len(stack)-1].key = !stack[len(stack)-1].key + dst = apnd(dst, src[i]) + } else { + var kind byte + if (src[i] >= '0' && src[i] <= '9') || src[i] == '-' || isNaNOrInf(src[i:]) { + kind = '0' + dst = append(dst, style.Number[0]...) + } else if src[i] == 't' { + kind = 't' + dst = append(dst, style.True[0]...) + } else if src[i] == 'f' { + kind = 'f' + dst = append(dst, style.False[0]...) + } else if src[i] == 'n' { + kind = 'n' + dst = append(dst, style.Null[0]...) + } else { + dst = apnd(dst, src[i]) + } + if kind != 0 { + for ; i < len(src); i++ { + if src[i] <= ' ' || src[i] == ',' || src[i] == ':' || src[i] == ']' || src[i] == '}' { + i-- + break + } + dst = apnd(dst, src[i]) + } + if kind == '0' { + dst = append(dst, style.Number[1]...) + } else if kind == 't' { + dst = append(dst, style.True[1]...) + } else if kind == 'f' { + dst = append(dst, style.False[1]...) + } else if kind == 'n' { + dst = append(dst, style.Null[1]...) + } + } + } + } + return dst +} + +// Spec strips out comments and trailing commas and convert the input to a +// valid JSON per the official spec: https://tools.ietf.org/html/rfc8259 +// +// The resulting JSON will always be the same length as the input and it will +// include all of the same line breaks at matching offsets. This is to ensure +// the result can be later processed by a external parser and that that +// parser will report messages or errors with the correct offsets. +func Spec(src []byte) []byte { + return spec(src, nil) +} + +// SpecInPlace is the same as Spec, but this method reuses the input json +// buffer to avoid allocations. Do not use the original bytes slice upon return. +func SpecInPlace(src []byte) []byte { + return spec(src, src) +} + +func spec(src, dst []byte) []byte { + dst = dst[:0] + for i := 0; i < len(src); i++ { + if src[i] == '/' { + if i < len(src)-1 { + if src[i+1] == '/' { + dst = append(dst, ' ', ' ') + i += 2 + for ; i < len(src); i++ { + if src[i] == '\n' { + dst = append(dst, '\n') + break + } else if src[i] == '\t' || src[i] == '\r' { + dst = append(dst, src[i]) + } else { + dst = append(dst, ' ') + } + } + continue + } + if src[i+1] == '*' { + dst = append(dst, ' ', ' ') + i += 2 + for ; i < len(src)-1; i++ { + if src[i] == '*' && src[i+1] == '/' { + dst = append(dst, ' ', ' ') + i++ + break + } else if src[i] == '\n' || src[i] == '\t' || + src[i] == '\r' { + dst = append(dst, src[i]) + } else { + dst = append(dst, ' ') + } + } + continue + } + } + } + dst = append(dst, src[i]) + if src[i] == '"' { + for i = i + 1; i < len(src); i++ { + dst = append(dst, src[i]) + if src[i] == '"' { + j := i - 1 + for ; ; j-- { + if src[j] != '\\' { + break + } + } + if (j-i)%2 != 0 { + break + } + } + } + } else if src[i] == '}' || src[i] == ']' { + for j := len(dst) - 2; j >= 0; j-- { + if dst[j] <= ' ' { + continue + } + if dst[j] == ',' { + dst[j] = ' ' + } + break + } + } + } + return dst +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 1f35db147..9b3319347 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -4,6 +4,9 @@ github.com/agext/levenshtein # github.com/apparentlymart/go-textseg/v13 v13.0.0 ## explicit; go 1.16 github.com/apparentlymart/go-textseg/v13/textseg +# github.com/creasty/defaults v1.6.0 +## explicit; go 1.14 +github.com/creasty/defaults # github.com/davecgh/go-spew v1.1.1 ## explicit github.com/davecgh/go-spew/spew @@ -169,6 +172,15 @@ github.com/mitchellh/reflectwalk # github.com/oklog/run v1.1.0 ## explicit; go 1.13 github.com/oklog/run +# github.com/tidwall/gjson v1.14.2 +## explicit; go 1.12 +github.com/tidwall/gjson +# github.com/tidwall/match v1.1.1 +## explicit; go 1.15 +github.com/tidwall/match +# github.com/tidwall/pretty v1.2.0 +## explicit; go 1.16 +github.com/tidwall/pretty # github.com/vmihailenco/msgpack v4.0.4+incompatible ## explicit github.com/vmihailenco/msgpack