diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cf1d3bff..41cc4526 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,6 +27,7 @@ jobs: args: release --clean --config .goreleaser.yml env: GITHUB_TOKEN: ${{ secrets.GORELEASER_TOKEN }} + dream: strategy: matrix: @@ -56,3 +57,33 @@ jobs: args: release --config tools/dream/${{ matrix.config }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + taucorder: + strategy: + matrix: + include: + - os: ubuntu-latest + config: '.goreleaser.linux.yml' + - os: macos-latest + config: '.goreleaser.darwin.yml' + - os: windows-latest + config: '.goreleaser.windows.yml' + runs-on: ${{ matrix.os }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: '1.22.0' + + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@v4 + with: + version: latest + args: release --config tools/taucorder/${{ matrix.config }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/README.md b/README.md index 8e6eff25..e1ecf073 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ As a result, `tau` has no API calls to create a serverless function, for example ### Versioning Git being core to `tau` means that nodes in the cloud do tune to a specific branch, by default main or master. Among what it enables is an easy way to set up development environments, for example. -A specific use case is local development in which case [dreamland](https://github.com/taubyte/dreamland) nodes can also be tuned to the current branch. +A specific use case is local development in which case [dreamland](https://github.com/taubyte/tau/clients/http/dream) nodes can also be tuned to the current branch. In addition to the nodes being on a branch, the application registry, managed by the 'tns' protocol, uses commit ids to version entries, allowing nodes serving the assets to detect new versions, or a roll-back for that matter. @@ -129,7 +129,7 @@ If you're looking to create E2E tests for projects hosted on `tau`, you can use ## Running a Local Cloud -While you can't practically run `tau` on your local machine, you can do so using [dreamland](https://github.com/taubyte/dreamland), which is a CLI wrapper around `libdream`. It creates local cloud environments mirroring production settings. Unlike `tau`, it offers an API for real-time configuration and testing. +While you can't practically run `tau` on your local machine, you can do so using [dreamland](https://github.com/taubyte/tau/clients/http/dream), which is a CLI wrapper around `libdream`. It creates local cloud environments mirroring production settings. Unlike `tau`, it offers an API for real-time configuration and testing. ## Extending Tau diff --git a/clients/http/dream/universe_test.go b/clients/http/dream/universe_test.go index ba1680e8..c6910c60 100644 --- a/clients/http/dream/universe_test.go +++ b/clients/http/dream/universe_test.go @@ -10,6 +10,7 @@ import ( "github.com/taubyte/tau/dream" "github.com/taubyte/tau/dream/api" _ "github.com/taubyte/tau/services/auth" + "gotest.tools/v3/assert" _ "github.com/taubyte/tau/services/hoarder" _ "github.com/taubyte/tau/services/monkey" @@ -26,10 +27,7 @@ func TestRoutes(t *testing.T) { univerName := "dreamland-http" // start multiverse err := api.BigBang() - if err != nil { - t.Errorf("Failed big bang with error: %v", err) - return - } + assert.NilError(t, err) u := dream.New(dream.UniverseConfig{Name: univerName}) defer u.Stop() @@ -53,20 +51,21 @@ func TestRoutes(t *testing.T) { }, }, }) - if err != nil { - t.Error(err) - return - } + assert.NilError(t, err) ctx := context.Background() time.Sleep(2 * time.Second) client, err := New(ctx, URL("http://localhost:1421"), Timeout(60*time.Second)) - if err != nil { - t.Errorf("Failed creating http client error: %v", err) - return - } + assert.NilError(t, err) + + univs, err := client.Universes() + assert.NilError(t, err) + + assert.DeepEqual(t, univs[univerName].SwarmKey, u.SwarmKey()) + + assert.Equal(t, univs[univerName].NodeCount, 8) universe := client.Universe(univerName) diff --git a/clients/http/dream/universes.go b/clients/http/dream/universes.go new file mode 100644 index 00000000..759457fc --- /dev/null +++ b/clients/http/dream/universes.go @@ -0,0 +1,18 @@ +package http + +type UniverseInfo struct { + SwarmKey []byte `json:"swarm"` + NodeCount int `json:"node-count"` +} + +type MultiverseInfo map[string]UniverseInfo + +func (c *Client) Universes() (MultiverseInfo, error) { + resp := make(MultiverseInfo) + err := c.get("/universes", &resp) + if err != nil { + return nil, err + } + + return resp, nil +} diff --git a/dream/api/http_routes.go b/dream/api/http_routes.go index 4b046fcc..23f8654a 100644 --- a/dream/api/http_routes.go +++ b/dream/api/http_routes.go @@ -6,6 +6,7 @@ func (srv *multiverseService) setUpHttpRoutes() httpIface.Service { srv.corsHttp() srv.statusHttp() + srv.universesHttp() srv.lesMiesrablesHttp() srv.fixtureHttp() srv.idHttp() diff --git a/dream/api/universes_http.go b/dream/api/universes_http.go new file mode 100644 index 00000000..b50fa020 --- /dev/null +++ b/dream/api/universes_http.go @@ -0,0 +1,14 @@ +package api + +import ( + httpIface "github.com/taubyte/http" +) + +func (srv *multiverseService) universesHttp() { + srv.rest.GET(&httpIface.RouteDefinition{ + Path: "/universes", + Handler: func(ctx httpIface.Context) (interface{}, error) { + return srv.Universes(), nil + }, + }) +} diff --git a/dream/multiverse.go b/dream/multiverse.go index 01954adc..089a2cb1 100644 --- a/dream/multiverse.go +++ b/dream/multiverse.go @@ -69,3 +69,16 @@ func (m *Multiverse) Status() interface{} { } return status } + +func (m *Multiverse) Universes() interface{} { + status := make(map[string]interface{}) + for _, u := range universes { + u.lock.RLock() + status[u.name] = map[string]interface{}{ + "swarm": u.swarmKey, + "node-count": len(u.all), + } + u.lock.RUnlock() + } + return status +} diff --git a/dream/universe.go b/dream/universe.go index ad0d46ff..63da0d2e 100644 --- a/dream/universe.go +++ b/dream/universe.go @@ -324,3 +324,7 @@ func (u *Universe) Cleanup() { func (u *Universe) Id() string { return u.id } + +func (u *Universe) SwarmKey() []byte { + return u.swarmKey +} diff --git a/go.mod b/go.mod index 9d3d139a..71a70658 100644 --- a/go.mod +++ b/go.mod @@ -50,7 +50,7 @@ require ( golang.org/x/exp v0.0.0-20240213143201-ec583247a57a golang.org/x/net v0.21.0 golang.org/x/oauth2 v0.16.0 - golang.org/x/sys v0.17.0 + golang.org/x/sys v0.19.0 golang.org/x/term v0.17.0 gopkg.in/go-playground/webhooks.v5 v5.17.0 gopkg.in/yaml.v3 v3.0.1 @@ -59,12 +59,16 @@ require ( ) require ( + github.com/c-bata/go-prompt v0.2.6 + github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be github.com/ethereum/go-ethereum v1.12.0 github.com/fsnotify/fsnotify v1.6.0 + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/hashicorp/go-plugin v1.4.10 github.com/ipfs/go-ds-pebble v0.3.1 github.com/multiformats/go-multihash v0.2.3 github.com/tetratelabs/wazero v1.6.0 + github.com/vbauerster/mpb/v8 v8.7.3 google.golang.org/grpc v1.60.1 google.golang.org/protobuf v1.32.0 ) @@ -78,6 +82,8 @@ require ( github.com/Jorropo/jsync v1.0.1 // indirect github.com/Microsoft/go-winio v0.5.2 // indirect github.com/ProtonMail/go-crypto v0.0.0-20220407094043-a94812496cf5 // indirect + github.com/VividCortex/ewma v1.2.0 // indirect + github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/acomagu/bufpipe v1.0.3 // indirect github.com/andybalholm/brotli v1.0.4 // indirect github.com/benbjohnson/clock v1.3.5 // indirect @@ -184,6 +190,7 @@ require ( github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/mattn/go-tty v0.0.3 // indirect github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect @@ -204,6 +211,7 @@ require ( github.com/opencontainers/runtime-spec v1.2.0 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect + github.com/pkg/term v1.2.0-beta.2 // indirect github.com/polydawn/refmt v0.89.0 // indirect github.com/prometheus/client_golang v1.18.0 // indirect github.com/prometheus/client_model v0.6.0 // indirect @@ -213,7 +221,7 @@ require ( github.com/quic-go/quic-go v0.41.0 // indirect github.com/quic-go/webtransport-go v0.6.0 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect - github.com/rivo/uniseg v0.4.4 // indirect + github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.11.0 // indirect github.com/rs/cors v1.8.3 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect diff --git a/go.sum b/go.sum index 3325d441..c27ad243 100644 --- a/go.sum +++ b/go.sum @@ -82,6 +82,10 @@ github.com/ProtonMail/go-crypto v0.0.0-20220407094043-a94812496cf5 h1:cSHEbLj0GZ github.com/ProtonMail/go-crypto v0.0.0-20220407094043-a94812496cf5/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= github.com/VictoriaMetrics/fastcache v1.6.0 h1:C/3Oi3EiBCqufydp1neRZkqcwmEiuRT9c3fqvvgKm5o= github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw= +github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= +github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk= github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 h1:ez/4by2iGztzR4L0zgAOR8lTQK9VlyBVVd7G4omaOQs= @@ -108,6 +112,8 @@ github.com/btcsuite/btcd/btcec/v2 v2.3.2/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= +github.com/c-bata/go-prompt v0.2.6 h1:POP+nrHE+DfLYx370bedwNhsqmpCUynWPxuHi0C5vZI= +github.com/c-bata/go-prompt v0.2.6/go.mod h1:/LMAke8wD2FsNu9EXNdHxNLbd9MedkPnCdfpU9wwHfY= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -139,6 +145,8 @@ github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwP github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= +github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be h1:J5BL2kskAlV9ckgEsNQXscjIaLiOYiZ75d4e94E6dcQ= +github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be/go.mod h1:mk5IQ+Y0ZeO87b858TlA645sVcEcbiX6YqP98kt+7+w= github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= @@ -346,6 +354,8 @@ github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5 h1:E/LAvt58di64hlYjx7AsNS6C/ysHWYo+2qPCZKTQhRo= github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -556,17 +566,23 @@ github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-tty v0.0.3 h1:5OfyWorkyO7xP52Mq7tB36ajHDG5OHrmBGIS/DtakQI= +github.com/mattn/go-tty v0.0.3/go.mod h1:ihxohKRERHTVzN+aSVRwACLCeqIoZAWpoICkkvrWyR0= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= github.com/miekg/dns v1.1.25/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= @@ -662,6 +678,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.6.0/go.mod h1:qBsxPvzyUincmltOk6iyRVxHYg4adc0OFOv72ZdLa18= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= +github.com/pkg/term v1.2.0-beta.2 h1:L3y/h2jkuBVFdWiJvNfYfKmzcCnILw7mJWm2JQuMppw= +github.com/pkg/term v1.2.0-beta.2/go.mod h1:E25nymQcrSllhX42Ok8MRm1+hyBdHY0dCeiKZ9jpNGw= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/polydawn/refmt v0.89.0 h1:ADJTApkvkeBZsN0tBTx8QjpD9JkmxbKp0cxfr9qszm4= @@ -697,8 +715,8 @@ github.com/quic-go/webtransport-go v0.6.0/go.mod h1:9KjU4AEBqEQidGHNDkZrb8CAa1ab github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtBsk= github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= -github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= @@ -817,6 +835,8 @@ github.com/urfave/negroni v1.0.0 h1:kIimOitoypq34K7TG7DUaJ9kq/N4Ofuwi1sjz0KipXc= github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/valyala/gozstd v1.11.0 h1:VV6qQFt+4sBBj9OJ7eKVvsFAMy59Urcs9Lgd+o5FOw0= github.com/valyala/gozstd v1.11.0/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ= +github.com/vbauerster/mpb/v8 v8.7.3 h1:n/mKPBav4FFWp5fH4U0lPpXfiOmCEgl5Yx/NM3tKJA0= +github.com/vbauerster/mpb/v8 v8.7.3/go.mod h1:9nFlNpDGVoTmQ4QvNjSLtwLmAFjwmq0XaAF26toHGNM= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= github.com/warpfork/go-testmark v0.12.1 h1:rMgCpJfwy1sJ50x0M0NgyphxYYPMOODIJHhsXyEHU0s= @@ -1055,9 +1075,11 @@ golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1073,6 +1095,8 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200918174421-af09f7315aff/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1100,8 +1124,8 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/tools/taucorder/.goreleaser.darwin.yml b/tools/taucorder/.goreleaser.darwin.yml new file mode 100644 index 00000000..6f96540f --- /dev/null +++ b/tools/taucorder/.goreleaser.darwin.yml @@ -0,0 +1,43 @@ +project_name: taucorder +release: + # Repo in which the release will be created. + github: + owner: taubyte + name: tau + + # Control the draft and prerelease settings + draft: false + replace_existing_draft: false + prerelease: auto + + # Release naming and SCM interaction + name_template: "" + disable: false + skip_upload: false + +builds: + - id: "darwin" + main: "./tools/taucorder/main.go" + binary: "taucorder" + goos: + - darwin + goarch: + - arm64 + - amd64 + env: + - CGO_ENABLED=1 + flags: + - -trimpath + tags: + - darwin + - odo + +changelog: + disable: true + +checksum: + disable: true + +archives: + - files: + - none* diff --git a/tools/taucorder/.goreleaser.linux.yml b/tools/taucorder/.goreleaser.linux.yml new file mode 100644 index 00000000..dbc8c884 --- /dev/null +++ b/tools/taucorder/.goreleaser.linux.yml @@ -0,0 +1,39 @@ +project_name: taucorder +release: + # Repo in which the release will be created. + github: + owner: taubyte + name: tau + + # Control the draft and prerelease settings + draft: false + replace_existing_draft: false + prerelease: auto + + # Release naming and SCM interaction + name_template: "" + disable: false + skip_upload: false + +builds: + - id: "linux" + main: "./tools/taucorder/main.go" + binary: "taucorder" + goos: + - linux + goarch: + - amd64 + - arm64 + env: + - CGO_ENABLED=0 + - GOAMD64=v2 + +changelog: + disable: true + +checksum: + disable: true + +archives: + - files: + - none* diff --git a/tools/taucorder/.goreleaser.windows.yml b/tools/taucorder/.goreleaser.windows.yml new file mode 100644 index 00000000..817fc0c7 --- /dev/null +++ b/tools/taucorder/.goreleaser.windows.yml @@ -0,0 +1,40 @@ +project_name: taucorder +release: + # Repo in which the release will be created. + github: + owner: taubyte + name: tau + + # Control the draft and prerelease settings + draft: false + replace_existing_draft: false + prerelease: auto + + # Release naming and SCM interaction + name_template: "" + disable: false + skip_upload: false + +builds: + - id: "windows" + main: "./tools/taucorder/main.go" + binary: "taucorder" + goos: + - windows + goarch: + - amd64 + env: + - CGO_ENABLED=0 + - GOAMD64=v2 + tags: + - windows + +changelog: + disable: true + +checksum: + disable: true + +archives: + - files: + - none* diff --git a/tools/taucorder/README.md b/tools/taucorder/README.md new file mode 100644 index 00000000..82e049b3 --- /dev/null +++ b/tools/taucorder/README.md @@ -0,0 +1 @@ +go run . use --port 4201 --key path/to/swarm.key fqdn \ No newline at end of file diff --git a/tools/taucorder/cli.go b/tools/taucorder/cli.go new file mode 100644 index 00000000..cdf7f9ed --- /dev/null +++ b/tools/taucorder/cli.go @@ -0,0 +1,470 @@ +package main + +import ( + "context" + "encoding/base64" + "errors" + "fmt" + "net" + "os" + "strings" + "time" + + "golang.org/x/exp/maps" + + dreamApi "github.com/taubyte/tau/clients/http/dream" + "github.com/vbauerster/mpb/v8" + "github.com/vbauerster/mpb/v8/decor" + + "github.com/libp2p/go-libp2p/core/crypto" + peer "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/pnet" + "github.com/multiformats/go-multiaddr" + "github.com/taubyte/tau/tools/taucorder/common" + "github.com/taubyte/tau/tools/taucorder/helpers/p2p" + "github.com/urfave/cli/v2" + + "github.com/libp2p/go-libp2p/core/sec" + "github.com/libp2p/go-libp2p/p2p/net/swarm" + + "github.com/miekg/dns" + + "github.com/taubyte/p2p/keypair" + + commonSpecs "github.com/taubyte/tau/pkg/specs/common" +) + +func getDNSRecords(domain string, dnsServer string) ([]net.IP, error) { + var ips []net.IP + + c := dns.Client{} + m := dns.Msg{} + m.SetQuestion(dns.Fqdn(domain), dns.TypeA) + r, _, err := c.Exchange(&m, dnsServer+":53") + if err != nil { + return nil, err + } + + for _, ans := range r.Answer { + if aRecord, ok := ans.(*dns.A); ok { + ips = append(ips, aRecord.A) + } + } + + return ips, nil +} + +func generateNodeKeyAndID(pkey string) (string, string, error) { + var ( + key crypto.PrivKey + keyData []byte + err error + ) + if pkey == "" { + key = keypair.New() + keyData, err = crypto.MarshalPrivateKey(key) + if err != nil { + return "", "", fmt.Errorf("marshal private key failed with %w", err) + } + } else { + keyData, err = base64.StdEncoding.DecodeString(pkey) + if err != nil { + return "", "", fmt.Errorf("decode private key failed with %w", err) + } + + key, err = crypto.UnmarshalPrivateKey(keyData) + if err != nil { + return "", "", fmt.Errorf("read private key failed with %w", err) + } + } + + id, err := peer.IDFromPublicKey(key.GetPublic()) + if err != nil { + return "", "", fmt.Errorf("id from private key failed with %w", err) + } + + return id.String(), base64.StdEncoding.EncodeToString(keyData), nil +} + +func AsErrPeerIDMismatch(err error) *sec.ErrPeerIDMismatch { + var dialerr *swarm.DialError + if !errors.As(err, &dialerr) { + return nil + } + + var mis sec.ErrPeerIDMismatch + for _, te := range dialerr.DialErrors { + if errors.As(te.Cause, &mis) { + return &mis + } + } + + return nil +} + +func deleteEmpty(s []string) []string { + if len(s) == 0 { + return nil + } + + r := make([]string, 0, len(s)) + for _, str := range s { + if str != "" { + r = append(r, str) + } + } + return r +} + +var ( + expectedKeyLength = 6 +) + +var ( + frames = []string{"∙∙∙", "●∙∙", "∙●∙", "∙∙●", "∙∙∙"} +) + +func formatSwarmKey(key []byte) (pnet.PSK, error) { + _key := strings.Split(string(key), "/") + _key = deleteEmpty(_key) + + if len(_key) != expectedKeyLength { + return nil, errors.New("swarm key is not correctly formatted") + } + + format := fmt.Sprintf(`/%s/%s/%s/%s/ +/%s/ +%s`, _key[0], _key[1], _key[2], _key[3], _key[4], _key[5]) + + return []byte(format), nil +} + +func newCLI() *(cli.App) { + app := &cli.App{ + UseShortOptionHandling: true, + EnableBashCompletion: true, + Action: func(ctx *cli.Context) error { return nil }, + } + defineCLI(app) + return app +} + +func ParseCommandLine() error { + return newCLI().RunContext(common.GlobalContext, os.Args) +} + +func defineCLI(app *cli.App) { + app.Commands = []*cli.Command{ + { + Name: "dream", + Aliases: []string{"local"}, + Usage: "Run using local dreamland", + Subcommands: []*cli.Command{ + { + Name: "with", + Aliases: []string{"use"}, + Action: func(c *cli.Context) error { + universe := c.Args().First() + if universe == "" { + return errors.New("provide the name of universe to connect to") + } + + client, err := dreamApi.New( + common.GlobalContext, + dreamApi.Unsecure(), + dreamApi.URL("http://127.0.0.1:1421"), + ) + if err != nil { + return fmt.Errorf("failed creating dreamland http client with error: %v", err) + } + + stats, err := client.Status() + if err != nil { + return fmt.Errorf("failed client status with error: %v", err) + } + + info, err := client.Universes() + if err != nil { + return fmt.Errorf("failed client status with error: %v", err) + } + + if _, ok := stats[universe]; !ok { + return fmt.Errorf("universe %s does not exist", universe) + } + + if _, ok := info[universe]; !ok { + return fmt.Errorf("failed to fetch info for universe %s", universe) + } + + // List for bootstrapping + _nodes := make([]peer.AddrInfo, 0, len(stats[universe].Nodes)) + + for id, addr := range stats[universe].Nodes { + node_addrs := make([]multiaddr.Multiaddr, 0) + for _, _addr := range addr { + node_addrs = append(node_addrs, multiaddr.StringCast(_addr)) + } + _pid, err := peer.Decode(id) + if err != nil { + return fmt.Errorf("failed peer id decode with error: %v", err) + } + node := peer.AddrInfo{ID: _pid, Addrs: node_addrs} + _nodes = append(_nodes, node) + } + + node, err = p2p.New(common.GlobalContext, _nodes, info[universe].SwarmKey) + if err != nil { + return fmt.Errorf("failed new with bootstrap list with error: %v", err) + } + return nil + }, + }, + { + Name: "list", + Aliases: []string{"l"}, + Action: func(c *cli.Context) error { + client, err := dreamApi.New( + common.GlobalContext, + dreamApi.Unsecure(), + dreamApi.URL("http://127.0.0.1:1421"), + ) + if err != nil { + return fmt.Errorf("failed creating dreamland http client with error: %v", err) + } + + stats, err := client.Status() + if err != nil { + return fmt.Errorf("failed client status with error: %v", err) + } + + for _, universe := range maps.Keys(stats) { + fmt.Println(universe) + } + + return nil + }, + }, + }, + }, + { + Name: "use", + Aliases: []string{"u"}, + Usage: "use a remote cloud", + Flags: []cli.Flag{ + &cli.Uint64SliceFlag{ + Name: "port", + }, + &cli.StringFlag{ + Name: "swarm-key", + Aliases: []string{"swarm", "key"}, + }, + }, + // Action: func(c *cli.Context) error { + // fqdn := c.Args().First() + // if fqdn == "" { + // return errors.New("provide the fqdn of cloud to connect to") + // } + + // swarmKey, err := os.ReadFile(c.String("swarm-key")) + // if err != nil { + // return fmt.Errorf("failed to open swarm file `%s` with %w", c.String("swarm-key"), err) + // } + + // swarmKey, err = formatSwarmKey(swarmKey) + // if err != nil { + // return fmt.Errorf("failed to format swarm key with %w", err) + // } + + // // spinner here + // ips, err := getDNSRecords("seer.tau."+fqdn, "8.8.8.8") + // if err != nil { + // return fmt.Errorf("failed to find seer nodes with %w", err) + // } + + // tmpCtx, tmpCtxC := context.WithCancel(common.GlobalContext) + // node, err = p2p.New(tmpCtx, nil, swarmKey) + // if err != nil { + // tmpCtxC() + // return fmt.Errorf("creating temporary node failed with %w", err) + // } + + // // progress bar group like mpb.New(mpb.WithWidth(60),mpb.WithRefreshRate(300*time.Millisecond)) + // peers := make([]peer.AddrInfo, 0, len(ips)) + // for _, ip := range ips { + // /* + // p.AddBar spinner + // make this for loop start go routines so we do up to count=4 in parallell + // */ + // pid, _, _ := generateNodeKeyAndID("") + // for _, port := range c.Uint64Slice("port") { + // peerAddrStr := fmt.Sprintf("/ip4/%s/tcp/%d/p2p/%s", ip.String(), port, pid) + // ma, err := multiaddr.NewMultiaddr(peerAddrStr) + // if err != nil { + // continue + // } + + // addrInfo, err := peer.AddrInfoFromP2pAddr(ma) + // if err != nil { + // continue + // } + + // err = node.Peer().Connect(context.Background(), *addrInfo) + // secerr := AsErrPeerIDMismatch(err) + // if secerr == nil { + // continue + // } + + // addrInfo.ID = secerr.Actual + + // fmt.Printf("Found %s\n", ip.String()) + // peers = append(peers, *addrInfo) + // break + // } + // } + + // node.Close() + // tmpCtxC() + + // node, err = p2p.New(common.GlobalContext, peers, swarmKey) + // if err != nil { + // return fmt.Errorf("failed new with bootstrap list with error: %v", err) + // } + + // node.WaitForSwarm(10 * time.Second) + + // return nil + // }, + Action: func(c *cli.Context) error { + fqdn := c.Args().First() + if fqdn == "" { + return errors.New("provide the fqdn of cloud to connect to") + } + + swarmKey, err := os.ReadFile(c.String("swarm-key")) + if err != nil { + return fmt.Errorf("failed to open swarm file `%s` with %w", c.String("swarm-key"), err) + } + + swarmKey, err = formatSwarmKey(swarmKey) + if err != nil { + return fmt.Errorf("failed to format swarm key with %w", err) + } + + // Progress bar setup + progress := mpb.New(mpb.WithWidth(60), mpb.WithRefreshRate(300*time.Millisecond)) + name := "Fetching DNS records" + dnsBar, _ := progress.Add(1, + mpb.SpinnerStyle(frames...).Build(), + mpb.BarRemoveOnComplete(), + mpb.BarFillerTrim(), + mpb.PrependDecorators( + decor.Name(name), + //decor.OnComplete(decor.Name(name), ""), + ), + ) + + ips := make(map[string]net.IP) + for _, pr := range commonSpecs.Services { + _ips, _ := getDNSRecords(pr+".tau."+fqdn, "8.8.8.8") + for _, ip := range _ips { + ips[ip.String()] = ip + } + } + //ips, err := getDNSRecords("seer.tau."+fqdn, "8.8.8.8") + dnsBar.Increment() + dnsBar.Wait() + + time.Sleep(time.Second) + + if len(ips) == 0 { + return errors.New("no peer were found") + } + + // if err != nil { + // return fmt.Errorf("failed to find seer nodes with %w", err) + // } + + tmpCtx, tmpCtxC := context.WithCancel(context.Background()) + node, err = p2p.New(tmpCtx, nil, swarmKey) + if err != nil { + tmpCtxC() + return fmt.Errorf("creating temporary node failed with %w", err) + } + + // Progress bar setup for connecting to peers + total := len(ips) + connectBar := progress.AddBar(int64(total), + mpb.PrependDecorators( + decor.Name("Discovering peers: "), + decor.CountersNoUnit("%d / %d"), + ), + mpb.AppendDecorators(decor.Percentage()), + mpb.BarRemoveOnComplete(), + ) + + peers := make([]peer.AddrInfo, 0, len(ips)) + sem := make(chan struct{}, 4) // limit to 4 concurrent goroutines + results := make(chan *peer.AddrInfo, len(ips)) + + for _, ip := range ips { + sem <- struct{}{} + go func(ip net.IP) { + defer func() { <-sem }() + defer connectBar.Increment() + + pid, _, _ := generateNodeKeyAndID("") + for _, port := range c.Uint64Slice("port") { + peerAddrStr := fmt.Sprintf("/ip4/%s/tcp/%d/p2p/%s", ip.String(), port, pid) + ma, err := multiaddr.NewMultiaddr(peerAddrStr) + if err != nil { + continue + } + + addrInfo, err := peer.AddrInfoFromP2pAddr(ma) + if err != nil { + continue + } + + err = node.Peer().Connect(context.Background(), *addrInfo) + secerr := AsErrPeerIDMismatch(err) + if secerr == nil { + continue + } + + addrInfo.ID = secerr.Actual + + //fmt.Printf("Found %s\n", ip.) + results <- addrInfo + } + }(ip) + } + + // Wait for all goroutines to finish + go func() { + for i := 0; i < total; i++ { + result := <-results + if result != nil { + peers = append(peers, *result) + } + } + close(results) + }() + progress.Wait() + + time.Sleep(time.Second) + + node.Close() + tmpCtxC() + + node, err = p2p.New(context.Background(), peers, swarmKey) + if err != nil { + return fmt.Errorf("failed new with bootstrap list with error: %v", err) + } + + node.WaitForSwarm(10 * time.Second) + + return nil + }, + }, + } +} diff --git a/tools/taucorder/common/signals.go b/tools/taucorder/common/signals.go new file mode 100644 index 00000000..a7822fa7 --- /dev/null +++ b/tools/taucorder/common/signals.go @@ -0,0 +1,26 @@ +package common + +import ( + "context" + "os" +) + +var ( + GlobalContext context.Context + GlobalContextCancel context.CancelFunc +) + +func init() { + GlobalContext, GlobalContextCancel = context.WithCancel(context.Background()) + + go func() { + select { + case <-GlobalContext.Done(): + os.Exit(3) + } + }() +} + +func Exit() { + GlobalContextCancel() +} diff --git a/tools/taucorder/helpers/json.go b/tools/taucorder/helpers/json.go new file mode 100644 index 00000000..12811970 --- /dev/null +++ b/tools/taucorder/helpers/json.go @@ -0,0 +1,58 @@ +package helpers + +import ( + "encoding/json" + + "github.com/jedib0t/go-pretty/v6/table" +) + +// https://github.com/go-testfixtures/testfixtures/blob/master/json.go +func appendMapInterface(t table.Writer, iface interface{}, level int, key string) { + switch iface.(type) { + case map[interface{}]interface{}: + for _key, _iface := range iface.(map[interface{}]interface{}) { + t.AppendRow([]interface{}{_key}) + appendMapInterface(t, _iface, level+1, _key.(string)) + } + case interface{}: + empty := make([]interface{}, level) + for idx := range empty { + empty[idx] = "" + if idx == level { + break + } + } + empty = append(empty, key) + empty = append(empty, iface) + t.AppendRow(empty) + t.AppendSeparator() + } +} + +type jsonArray []interface{} +type jsonMap map[string]interface{} + +func (m jsonMap) Value() (interface{}, error) { + return json.Marshal(m) +} + +// Go refuses to convert map[interface{}]interface{} to JSON because JSON only support string keys +// So it's necessary to recursively convert all map[interface]interface{} to map[string]interface{} +func recursiveToJSON(v interface{}) (r interface{}) { + switch v := v.(type) { + case []interface{}: + for i, e := range v { + v[i] = recursiveToJSON(e) + } + r = jsonArray(v) + case map[interface{}]interface{}: + newMap := make(map[string]interface{}, len(v)) + for k, e := range v { + newMap[k.(string)] = recursiveToJSON(e) + } + r = jsonMap(newMap) + default: + r = v + } + return +} diff --git a/tools/taucorder/helpers/list.go b/tools/taucorder/helpers/list.go new file mode 100644 index 00000000..5dfe3d4d --- /dev/null +++ b/tools/taucorder/helpers/list.go @@ -0,0 +1,54 @@ +package helpers + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "os" + + "github.com/jedib0t/go-pretty/v6/table" +) + +func CreateTableIds(ids []string, title string) { + t := table.NewWriter() + t.SetOutputMirror(os.Stdout) + t.AppendHeader(table.Row{title}) + t.SetStyle(table.StyleLight) + t.SetColumnConfigs([]table.ColumnConfig{ + { + Number: 1, + AutoMerge: true, + }, + }) + for _, _ids := range ids { + if ids == nil { + t.AppendRows([]table.Row{{"--"}}, + table.RowConfig{}) + } else { + t.AppendRows([]table.Row{{_ids}}, + table.RowConfig{}) + } + t.AppendSeparator() + } + t.Render() +} + +func CreateTableInterface(title string, iface interface{}) error { + js := recursiveToJSON(iface) + marshalled, err := json.Marshal(js) + if err != nil { + return fmt.Errorf("Failed marshall with %v", err) + } + + var buf bytes.Buffer + err = json.Indent(&buf, marshalled, "", " ") + if err != nil { + return fmt.Errorf("Failed json indent with %v", err) + } + + _, err = io.Copy(os.Stdout, &buf) + fmt.Println() + + return err +} diff --git a/tools/taucorder/helpers/p2p/p2p.go b/tools/taucorder/helpers/p2p/p2p.go new file mode 100644 index 00000000..48e1733a --- /dev/null +++ b/tools/taucorder/helpers/p2p/p2p.go @@ -0,0 +1,23 @@ +package p2p + +import ( + "context" + "fmt" + + corePeer "github.com/libp2p/go-libp2p/core/peer" + "github.com/taubyte/p2p/keypair" + "github.com/taubyte/p2p/peer" +) + +func New(ctx context.Context, nodes []corePeer.AddrInfo, swarmKey []byte) (peer.Node, error) { + return peer.NewClientNode( + ctx, + nil, + keypair.NewRaw(), + swarmKey, + []string{fmt.Sprintf("/ip4/0.0.0.0/tcp/%d", 11111)}, + nil, + true, + nodes, + ) +} diff --git a/tools/taucorder/main.go b/tools/taucorder/main.go new file mode 100644 index 00000000..72a72562 --- /dev/null +++ b/tools/taucorder/main.go @@ -0,0 +1,50 @@ +package main + +import ( + "fmt" + + "github.com/common-nighthawk/go-figure" + "github.com/taubyte/p2p/peer" + "github.com/taubyte/tau/tools/taucorder/common" + "github.com/taubyte/tau/tools/taucorder/prompt" +) + +var node peer.Node + +func main() { + err := ParseCommandLine() + if err != nil { + fmt.Println("Parsing command line failed with error:", err) + return + } + + if node == nil { + fmt.Println() + fmt.Println("You need to select a cloud") + return + } + + banner() + p, err := prompt.New(common.GlobalContext) + if err != nil { + fmt.Println("Prompt new failed with error:", err) + return + } + + err = p.Run(node) + if err != nil { + fmt.Println("Running prompt failed with error:", err) + return + } +} + +func banner() { + myFigure := figure.NewColorFigure( + "TAUCORDER", + "speed", + "green", + true, + ) + myFigure.Print() + fmt.Println() +} diff --git a/tools/taucorder/prompt/acme.go b/tools/taucorder/prompt/acme.go new file mode 100644 index 00000000..51b1e5f3 --- /dev/null +++ b/tools/taucorder/prompt/acme.go @@ -0,0 +1,99 @@ +package prompt + +import ( + "errors" + "fmt" + "os" + + goPrompt "github.com/c-bata/go-prompt" +) + +var acmeTree = &tctree{ + leafs: []*leaf{ + exitLeaf, + { + validator: stringValidator("injectStaticCert"), + ret: []goPrompt.Suggest{ + { + Text: "injectStaticCert", + Description: "inject a certificate into auth. Ex: inject domainName certFile", + }, + }, + handler: injectStaticCert, + }, + { + validator: stringValidator("getCert"), + ret: []goPrompt.Suggest{ + { + Text: "getCert", + Description: "gets the certificate file for a domain in /acme", + }, + }, + handler: getCertificate, + }, + { + validator: stringValidator("getStaticCert"), + ret: []goPrompt.Suggest{ + { + Text: "getStaticCert", + Description: "gets the certificate file for a domain in /static", + }, + }, + handler: getStaticCertificate, + }, + }, +} + +func injectStaticCert(p Prompt, args []string) error { + if len(args) != 3 { + fmt.Println("Must provide a domain and certificate file") + return errors.New("must provide an domain and certificate file") + } + + fileBytes, err := os.ReadFile(args[2]) + if err != nil { + fmt.Println("Failed reading file with ", err) + return fmt.Errorf("failed reading file with %v", err) + } + + err = p.TaubyteAuthClient().InjectStaticCertificate(args[1], fileBytes) + if err != nil { + fmt.Printf("Failed injecting certificate for %s with %v\n", args[1], err) + return fmt.Errorf(" Failed injecting certificate for %s with %v", args[1], err) + } + + fmt.Printf("Successfully injected certificate for %s\n", args[1]) + return nil +} + +func getCertificate(p Prompt, args []string) error { + if len(args) != 2 { + fmt.Println("Must provide an domain") + return errors.New("must provide an domain and key file") + } + + _pem, err := p.TaubyteAuthClient().GetCertificate(args[1]) + if err != nil { + fmt.Printf("Failed getting certificate for %s with %v\n", args[1], err) + return fmt.Errorf(" Failed getting certificate for %s with %v", args[1], err) + } + + fmt.Println(string(_pem)) + return nil +} + +func getStaticCertificate(p Prompt, args []string) error { + if len(args) != 2 { + fmt.Println("Must provide an domain") + return errors.New("must provide an domain and key file") + } + + cert, err := p.TaubyteAuthClient().GetStaticCertificate(args[1]) + if err != nil { + fmt.Printf("Failed getting certificate for %s with %v\n", args[1], err) + return fmt.Errorf(" Failed getting certificate for %s with %v", args[1], err) + } + + fmt.Printf("\nCertificate:\n%#v", cert) + return nil +} diff --git a/tools/taucorder/prompt/auth.go b/tools/taucorder/prompt/auth.go new file mode 100644 index 00000000..dead6ccc --- /dev/null +++ b/tools/taucorder/prompt/auth.go @@ -0,0 +1,75 @@ +package prompt + +import ( + goPrompt "github.com/c-bata/go-prompt" +) + +var authTree = &tctree{ + leafs: []*leaf{ + exitLeaf, + { + validator: stringValidator("project"), + ret: []goPrompt.Suggest{ + { + Text: "project", + Description: "show project options", + }, + }, + jump: func(p Prompt) string { + return "/auth/project" + }, + handler: func(p Prompt, args []string) error { + p.SetPath("/auth/project") + return nil + }, + }, + { + validator: stringValidator("hook"), + ret: []goPrompt.Suggest{ + { + Text: "hook", + Description: "show hook options", + }, + }, + jump: func(p Prompt) string { + return "/auth/hook" + }, + handler: func(p Prompt, args []string) error { + p.SetPath("/auth/hook") + return nil + }, + }, + { + validator: stringValidator("repo"), + ret: []goPrompt.Suggest{ + { + Text: "repo", + Description: "show repo options", + }, + }, + jump: func(p Prompt) string { + return "/auth/repo" + }, + handler: func(p Prompt, args []string) error { + p.SetPath("/auth/repo") + return nil + }, + }, + { + validator: stringValidator("acme"), + ret: []goPrompt.Suggest{ + { + Text: "acme", + Description: "show acme options", + }, + }, + jump: func(p Prompt) string { + return "/auth/acme" + }, + handler: func(p Prompt, args []string) error { + p.SetPath("/auth/acme") + return nil + }, + }, + }, +} diff --git a/tools/taucorder/prompt/defs.go b/tools/taucorder/prompt/defs.go new file mode 100644 index 00000000..8d826c25 --- /dev/null +++ b/tools/taucorder/prompt/defs.go @@ -0,0 +1,180 @@ +package prompt + +import ( + "fmt" + "os" + "strings" + + goPrompt "github.com/c-bata/go-prompt" +) + +var stringValidator = func(strs ...string) func(Prompt, string, bool) bool { + return func(p Prompt, w string, exact bool) bool { + strs = append(strs, "") + for _, s := range strs { + if exact { + if s != "" && s == w { + return true + } + } else { + if strings.HasPrefix(s, w) { + return true + } + } + } + return false + } +} + +var forest = tcforest{ + "/": mainTree, + "/p2p": p2pTree, + "/p2p/swarm": swarmTree, + "/p2p/discover": discoverTree, + "/auth": authTree, + "/auth/project": projectTree, + "/auth/repo": repoTree, + "/auth/acme": acmeTree, + "/auth/hook": hooksTree, + "/hoarder": hoarderTree, + "/patrick": patrickTree, + "/seer": seerTree, + "/monkey": monkeyTree, + "/tns": tnsTree, +} + +var mainTree = &tctree{ + leafs: []*leaf{ + { + validator: stringValidator("p2p"), + ret: []goPrompt.Suggest{ + { + Text: "p2p", + Description: "p2p utils", + }, + }, + jump: func(p Prompt) string { + return "/p2p" + }, + handler: func(p Prompt, args []string) error { + p.SetPath("/p2p") + return nil + }, + }, + { + validator: stringValidator("auth"), + ret: []goPrompt.Suggest{ + { + Text: "auth", + Description: "auth client", + }, + }, + jump: func(p Prompt) string { + return "/auth" + }, + handler: func(p Prompt, args []string) error { + p.SetPath("/auth") + return nil + }, + }, + { + validator: stringValidator("hoarder"), + ret: []goPrompt.Suggest{ + { + Text: "hoarder", + Description: "hoarder client", + }, + }, + jump: func(p Prompt) string { + return "/hoarder" + }, + handler: func(p Prompt, args []string) error { + p.SetPath("/hoarder") + return nil + }, + }, + { + validator: stringValidator("patrick"), + ret: []goPrompt.Suggest{ + { + Text: "patrick", + Description: "patrick client", + }, + }, + jump: func(p Prompt) string { + return "/patrick" + }, + handler: func(p Prompt, args []string) error { + p.SetPath("/patrick") + return nil + }, + }, + { + validator: stringValidator("monkey"), + ret: []goPrompt.Suggest{ + { + Text: "monkey", + Description: "monkey client", + }, + }, + jump: func(p Prompt) string { + return "/monkey" + }, + handler: func(p Prompt, args []string) error { + p.SetPath("/monkey") + return nil + }, + }, + { + validator: stringValidator("seer"), + ret: []goPrompt.Suggest{ + { + Text: "seer", + Description: "seer client", + }, + }, + jump: func(p Prompt) string { + return "/seer" + }, + handler: func(p Prompt, args []string) error { + p.SetPath("/seer") + return nil + }, + }, + { + validator: stringValidator("tns"), + ret: []goPrompt.Suggest{ + { + Text: "tns", + Description: "tns client", + }, + }, + jump: func(p Prompt) string { + return "/tns" + }, + handler: func(p Prompt, args []string) error { + p.SetPath("/tns") + return nil + }, + }, + { + validator: stringValidator("exit", "bye"), + ret: []goPrompt.Suggest{ + { + Text: "exit", + Description: "exit", + }, + { + Text: "bye", + Description: "exit", + }, + }, + handler: func(p Prompt, args []string) error { + fmt.Println("BYE") + p.Done() + os.Exit(0) + return nil + }, + }, + }, +} diff --git a/tools/taucorder/prompt/discover.go b/tools/taucorder/prompt/discover.go new file mode 100644 index 00000000..7661faad --- /dev/null +++ b/tools/taucorder/prompt/discover.go @@ -0,0 +1,87 @@ +package prompt + +import ( + "context" + "errors" + "fmt" + "time" + + goPrompt "github.com/c-bata/go-prompt" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/protocol" +) + +var discoverTree = &tctree{ + leafs: []*leaf{ + exitLeaf, + { + validator: stringValidator("healthy"), + ret: []goPrompt.Suggest{ + { + Text: "healthy", + Description: "show providers with status check", + }, + }, + handler: discoverWithCheckCMD, + }, + }, +} + +func discoverCMD(p Prompt, args []string) error { + if len(args) < 2 { + p.SetPath("/p2p/discover") + return nil + } + + service := args[1] + + ctx, ctxC := context.WithTimeout(context.Background(), 3*time.Second) + defer ctxC() + + peers, err := prompt.Node().Discovery().FindPeers(ctx, service) + if err != nil { + fmt.Printf("Failed to discover `%s` with %s\n", service, err.Error()) + return err + } + + for p := range peers { + fmt.Printf("- %s %v\n", p.ID.String(), p.Addrs) + } + + return nil +} + +func discoverWithCheckCMD(p Prompt, args []string) error { + if len(args) < 2 { + return errors.New("must provide Service") + } + + service := args[1] + + ctx, ctxC := context.WithTimeout(context.Background(), 5*time.Second) + defer ctxC() + + peers, err := prompt.Node().Discovery().FindPeers(ctx, service) + if err != nil { + fmt.Printf("Failed to discover `%s` with %s\n", service, err.Error()) + return err + } + + for p := range peers { + go func(p0 peer.AddrInfo) { + _ctx, _ctxC := context.WithTimeout(ctx, 300*time.Millisecond) + defer _ctxC() + s, err := prompt.Node().Peer().NewStream(_ctx, p0.ID, protocol.ID(service)) + status := "[...]" + if err != nil { + status = fmt.Sprintf("[ERROR] %s", err) + } else { + status = fmt.Sprintf("[OK|%s]", s.Stat().Direction.String()) + s.Close() + } + fmt.Printf("- %s %v %s\n", p0.ID.String(), p0.Addrs, status) + }(p) + } + + return nil +} diff --git a/tools/taucorder/prompt/forest.go b/tools/taucorder/prompt/forest.go new file mode 100644 index 00000000..9fa8d855 --- /dev/null +++ b/tools/taucorder/prompt/forest.go @@ -0,0 +1,60 @@ +package prompt + +import ( + "fmt" + "path" + + goPrompt "github.com/c-bata/go-prompt" +) + +type tcforest map[string]*tctree + +var _ Forest = forest + +func (f tcforest) Complete(p Prompt, d goPrompt.Document) []goPrompt.Suggest { + path := p.CurrentPath() + tree, ok := f[path] + if !ok { + return nil + } + return tree.Complete(p, d) +} + +func (f tcforest) Execute(p Prompt, in []string) error { + path := p.CurrentPath() + tree, ok := f[path] + if !ok { + return fmt.Errorf("unknow path `%s`", path) + } + return tree.Execute(p, in) +} + +func (f tcforest) FindBranch(path string) ([]*leaf, error) { + tree, ok := f[path] + if !ok { + return nil, fmt.Errorf("unknow path `%s`", path) + } + return tree.leafs, nil +} + +var exitLeaf = &leaf{ + validator: stringValidator("exit", "bye"), + ret: []goPrompt.Suggest{ + { + Text: "exit", + Description: "exit", + }, + { + Text: "bye", + Description: "exit", + }, + }, + handler: func(p Prompt, args []string) error { + parent, _ := path.Split(p.CurrentPath()) + if parent != "/" && parent[len(parent)-1] == '/' { + parent = parent[:len(parent)-1] + } + p.SetPath(parent) + return nil + }, +} diff --git a/tools/taucorder/prompt/hoarder.go b/tools/taucorder/prompt/hoarder.go new file mode 100644 index 00000000..98e2d8ee --- /dev/null +++ b/tools/taucorder/prompt/hoarder.go @@ -0,0 +1,40 @@ +package prompt + +import ( + "fmt" + + goPrompt "github.com/c-bata/go-prompt" + list "github.com/taubyte/tau/tools/taucorder/helpers" +) + +var hoarderTree = &tctree{ + leafs: []*leaf{ + exitLeaf, + { + validator: stringValidator("list"), + ret: []goPrompt.Suggest{ + { + Text: "list", + Description: "show all stashed cids", + }, + }, + handler: listCids, + }, + }, +} + +func listCids(p Prompt, args []string) error { + ids, err := p.TaubyteHoarderClient().List() + if err != nil { + return fmt.Errorf("failed listing hoarder cids with error: %w", err) + } + + if len(ids) == 0 { + fmt.Println("No ids are currently stored") + return nil + } + + list.CreateTableIds(ids, "Hoarder List") + + return nil +} diff --git a/tools/taucorder/prompt/hooks.go b/tools/taucorder/prompt/hooks.go new file mode 100644 index 00000000..13cb72a8 --- /dev/null +++ b/tools/taucorder/prompt/hooks.go @@ -0,0 +1,73 @@ +package prompt + +import ( + "errors" + "fmt" + + goPrompt "github.com/c-bata/go-prompt" + "github.com/ipfs/go-cid" + list "github.com/taubyte/tau/tools/taucorder/helpers" +) + +var hooksTree = &tctree{ + leafs: []*leaf{ + exitLeaf, + { + validator: stringValidator("list"), + ret: []goPrompt.Suggest{ + { + Text: "list", + Description: "show registered hooks", + }, + }, + handler: hookList, + }, + { + validator: stringValidator("get"), + ret: []goPrompt.Suggest{ + { + Text: "get", + Description: "get repository by hook id", + }, + }, + handler: getHook, + }, + }, +} + +func hookList(p Prompt, args []string) error { + hooks, err := p.TaubyteAuthClient().Hooks().List() + if err != nil { + return fmt.Errorf("failed fetching hooks with error: %v", err) + } + + if len(hooks) == 0 { + fmt.Println("No projects are currently stored") + return nil + } + + list.CreateTableIds(hooks, "Hooks List") + + return nil +} + +func getHook(p Prompt, args []string) error { + if len(args) < 2 { + fmt.Println("Must provide hook id") + return errors.New("must provide PID") + } + pid := args[1] + _, err := cid.Decode(pid) + if err != nil { + return fmt.Errorf("project id `%s` is invalid", pid) + } + + hook, err := p.TaubyteAuthClient().Hooks().Get(pid) + if err != nil { + return fmt.Errorf("failed fetching hook `%s` with error: %v", pid, err) + } + + fmt.Println(hook) + + return nil +} diff --git a/tools/taucorder/prompt/ifaces.go b/tools/taucorder/prompt/ifaces.go new file mode 100644 index 00000000..79672ec3 --- /dev/null +++ b/tools/taucorder/prompt/ifaces.go @@ -0,0 +1,36 @@ +package prompt + +import ( + goPrompt "github.com/c-bata/go-prompt" + "github.com/taubyte/p2p/peer" + auth "github.com/taubyte/tau/clients/p2p/auth" + hoarder "github.com/taubyte/tau/clients/p2p/hoarder" + monkey "github.com/taubyte/tau/clients/p2p/monkey" + patrick "github.com/taubyte/tau/clients/p2p/patrick" + seer "github.com/taubyte/tau/clients/p2p/seer" + tnsIface "github.com/taubyte/tau/core/services/tns" +) + +type Prompt interface { + Run(peer.Node) error + Done() + Node() peer.Node + TaubyteAuthClient() *auth.Client + TaubyteSeerClient() *seer.Client + TaubytePatrickClient() *patrick.Client + TaubyteHoarderClient() *hoarder.Client + TaubyteMonkeyClient() *monkey.Client + TaubyteTnsClient() tnsIface.Client + CurrentPath() string + SetPath(string) +} + +type Tree interface { + Complete(Prompt, goPrompt.Document) []goPrompt.Suggest + Execute(p Prompt, args []string) error +} + +type Forest interface { + Tree + FindBranch(path string) ([]*leaf, error) +} diff --git a/tools/taucorder/prompt/monkey.go b/tools/taucorder/prompt/monkey.go new file mode 100644 index 00000000..0f7b4257 --- /dev/null +++ b/tools/taucorder/prompt/monkey.go @@ -0,0 +1,76 @@ +package prompt + +import ( + "errors" + "fmt" + "os" + + "github.com/jedib0t/go-pretty/v6/table" + + goPrompt "github.com/c-bata/go-prompt" + list "github.com/taubyte/tau/tools/taucorder/helpers" +) + +var monkeyTree = &tctree{ + leafs: []*leaf{ + exitLeaf, + { + validator: stringValidator("list"), + ret: []goPrompt.Suggest{ + { + Text: "list", + Description: "show all jobs that monkey has", + }, + }, + handler: listMonkeyJobs, + }, + { + validator: stringValidator("get"), + ret: []goPrompt.Suggest{ + { + Text: "get", + Description: "show data about a job", + }, + }, + handler: getMonkeyJobs, + }, + }, +} + +func listMonkeyJobs(p Prompt, args []string) error { + ids, err := p.TaubyteMonkeyClient().List() + if err != nil { + return fmt.Errorf("failed listing jobs cids with error: %w", err) + } + + if len(ids) == 0 { + fmt.Println("No jobs are currently running in monkey") + return nil + } + + list.CreateTableIds(ids, "Monkey job Id's") + + return nil +} + +func getMonkeyJobs(p Prompt, args []string) error { + t := table.NewWriter() + t.SetStyle(table.StyleLight) + t.SetOutputMirror(os.Stdout) + if len(args) < 2 { + fmt.Println("Must provide job ID") + return errors.New("must provide job ID") + } + jid := args[1] + resp, err := p.TaubyteMonkeyClient().Status(jid) + if err != nil { + return fmt.Errorf("failed listing jobs cids with error: %w", err) + } + + t.AppendRows([]table.Row{{"Jobs", resp.Jid}}, table.RowConfig{}) + t.AppendRows([]table.Row{{"LogCID", resp.Logs}}, table.RowConfig{}) + t.AppendRows([]table.Row{{"Status", resp.Status.String()}}, table.RowConfig{}) + + t.Render() + return nil +} diff --git a/tools/taucorder/prompt/new.go b/tools/taucorder/prompt/new.go new file mode 100644 index 00000000..a94a9703 --- /dev/null +++ b/tools/taucorder/prompt/new.go @@ -0,0 +1,174 @@ +package prompt + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/taubyte/p2p/peer" + auth "github.com/taubyte/tau/clients/p2p/auth" + + //billingApi "bitbucket.org/taubyte/billing/api/p2p" + + goPrompt "github.com/c-bata/go-prompt" + "github.com/google/shlex" + dreamland "github.com/taubyte/tau/clients/http/dream" + hoarder "github.com/taubyte/tau/clients/p2p/hoarder" + monkey "github.com/taubyte/tau/clients/p2p/monkey" + patrick "github.com/taubyte/tau/clients/p2p/patrick" + seer "github.com/taubyte/tau/clients/p2p/seer" + tns "github.com/taubyte/tau/clients/p2p/tns" + tnsIface "github.com/taubyte/tau/core/services/tns" + "github.com/taubyte/tau/tools/taucorder/common" +) + +type tcprompt struct { + ctx context.Context + ctxC context.CancelFunc + engine *goPrompt.Prompt + path string + node peer.Node + authClient *auth.Client + seerClient *seer.Client + hoarderClient *hoarder.Client + monkeyClient *monkey.Client + tnsClient tnsIface.Client + //billingClient billingiFace.Client + patrickClient *patrick.Client + dreamlandClient *dreamland.Client +} + +var prompt Prompt + +func New(ctx context.Context) (Prompt, error) { + if prompt != nil { + return prompt, nil + } + + p := &tcprompt{ + path: "/", + } + + prompt = p + + p.ctx, p.ctxC = common.GlobalContext, common.GlobalContextCancel + + p.engine = goPrompt.New( + func(s string) { + args, err := shlex.Split(s) + if err != nil { + args = strings.Split(strings.TrimSpace(s), " ") + } + forest.Execute(p, args) + }, + func(in goPrompt.Document) []goPrompt.Suggest { + ret := forest.Complete(prompt, in) + return ret + }, + //goPrompt.OptionPrefix("tau> "), + goPrompt.OptionLivePrefix(func() (prefix string, useLivePrefix bool) { + return prompt.CurrentPath() + "> ", true + }), + goPrompt.OptionTitle("taucorder"), + goPrompt.OptionCompletionOnDown(), + ) + + return prompt, nil +} + +func (p *tcprompt) Run(node peer.Node) error { + if node == nil { + return errors.New("you need to select a cloud") + } + + p.node = node + + err := p.node.WaitForSwarm(10 * time.Second) + if err != nil { + return err + } + + p.authClient, err = auth.New(p.ctx, p.node) + if err != nil { + return err + } + + p.seerClient, err = seer.New(p.ctx, p.node) + if err != nil { + return err + } + + pc, err := patrick.New(p.ctx, p.node) + if err != nil { + return err + } + p.patrickClient = pc.(*patrick.Client) + + p.hoarderClient, err = hoarder.New(p.ctx, p.node) + if err != nil { + return err + } + + p.monkeyClient, err = monkey.New(p.ctx, p.node) + if err != nil { + return err + } + + p.tnsClient, err = tns.New(p.ctx, p.node) + if err != nil { + return err + } + + p.dreamlandClient, err = dreamland.New(p.ctx) + if err != nil { + return err + } + + p.engine.Run() + + fmt.Println() + + return nil +} + +func (p *tcprompt) Done() { + p.ctxC() +} + +func (p *tcprompt) Node() peer.Node { + return p.node +} + +func (p *tcprompt) CurrentPath() string { + return p.path +} + +func (p *tcprompt) SetPath(path string) { + p.path = path +} + +func (p *tcprompt) TaubyteAuthClient() *auth.Client { + return p.authClient +} + +func (p *tcprompt) TaubyteSeerClient() *seer.Client { + return p.seerClient +} + +func (p *tcprompt) TaubytePatrickClient() *patrick.Client { + return p.patrickClient +} + +func (p *tcprompt) TaubyteHoarderClient() *hoarder.Client { + return p.hoarderClient +} + +func (p *tcprompt) TaubyteMonkeyClient() *monkey.Client { + return p.monkeyClient +} + +func (p *tcprompt) TaubyteTnsClient() tnsIface.Client { + return p.tnsClient +} diff --git a/tools/taucorder/prompt/p2p.go b/tools/taucorder/prompt/p2p.go new file mode 100644 index 00000000..55a7295d --- /dev/null +++ b/tools/taucorder/prompt/p2p.go @@ -0,0 +1,52 @@ +package prompt + +import goPrompt "github.com/c-bata/go-prompt" + +var p2pTree = &tctree{ + leafs: []*leaf{ + exitLeaf, + { + validator: stringValidator("ping"), + ret: []goPrompt.Suggest{ + { + Text: "ping", + Description: "ping a node", + }, + }, + handler: pingCMD, + }, + { + validator: stringValidator("swarm"), + ret: []goPrompt.Suggest{ + { + Text: "swarm", + Description: "show swarm", + }, + }, + jump: func(p Prompt) string { + return "/p2p/swarm" + }, + handler: func(p Prompt, args []string) error { + p.SetPath("/p2p/swarm") + return nil + }, + }, + { + validator: stringValidator("discover", "find"), + ret: []goPrompt.Suggest{ + { + Text: "discover", + Description: "discover a service", + }, + { + Text: "find", + Description: "find a service", + }, + }, + jump: func(p Prompt) string { + return "/p2p/discover" + }, + handler: discoverCMD, + }, + }, +} diff --git a/tools/taucorder/prompt/patrick.go b/tools/taucorder/prompt/patrick.go new file mode 100644 index 00000000..2659bc85 --- /dev/null +++ b/tools/taucorder/prompt/patrick.go @@ -0,0 +1,100 @@ +package prompt + +import ( + "errors" + "fmt" + "os" + + "github.com/jedib0t/go-pretty/v6/table" + + goPrompt "github.com/c-bata/go-prompt" + list "github.com/taubyte/tau/tools/taucorder/helpers" +) + +var patrickTree = &tctree{ + leafs: []*leaf{ + exitLeaf, + { + validator: stringValidator("list"), + ret: []goPrompt.Suggest{ + { + Text: "list", + Description: "show all jobs cids", + }, + }, + handler: listJobs, + }, + { + validator: stringValidator("get"), + ret: []goPrompt.Suggest{ + { + Text: "get", + Description: "show a job", + }, + }, + handler: getJobs, + }, + }, +} + +func listJobs(p Prompt, args []string) error { + ids, err := p.TaubytePatrickClient().List() + if err != nil { + return fmt.Errorf("failed listing jobs cids with error: %w", err) + } + + if len(ids) == 0 { + fmt.Println("No jobs are currently stored") + return nil + } + list.CreateTableIds(ids, "Job Id's") + + return nil +} + +func getJobs(p Prompt, args []string) error { + t := table.NewWriter() + t.SetStyle(table.StyleLight) + t.SetOutputMirror(os.Stdout) + if len(args) < 2 { + fmt.Println("Must provide job ID") + return errors.New("must provide job ID") + } + jid := args[1] + job, err := p.TaubytePatrickClient().Get(jid) + if err != nil { + t.AppendRows([]table.Row{{"--"}}, + table.RowConfig{}) + + return fmt.Errorf("failed listing jobs cids with error: %w", err) + } + t.AppendRows([]table.Row{{"Id ", job.Id}}, + table.RowConfig{}) + t.AppendSeparator() + t.AppendRows([]table.Row{{"TimeStamp ", job.Timestamp}}, + table.RowConfig{}) + t.AppendSeparator() + t.AppendRows([]table.Row{{"Logs ", job.Logs}}, + table.RowConfig{}) + t.AppendSeparator() + t.AppendRows([]table.Row{{"Status", job.Status}}, + table.RowConfig{}) + t.AppendSeparator() + t.AppendRows([]table.Row{{"Meta:\n"}}, + table.RowConfig{}) + t.AppendRows([]table.Row{{"\tBefore", job.Meta.Before + "\n"}, + {"\tAfter ", job.Meta.After + "\n"}, + {"\tHeadCommitId", job.Meta.HeadCommit.ID + "\n"}, + {"\tRef", job.Meta.Ref + "\n"}, + {"\tRepository:", "\n"}, + {"\t\tId", job.Meta.Repository.ID}, + {"\t\tProvider", job.Meta.Repository.Provider}, + {"\t\tShURL", job.Meta.Repository.SSHURL}}, + table.RowConfig{}) + t.AppendSeparator() + t.AppendRows([]table.Row{{"Attempt", job.Attempt}}, + table.RowConfig{}) + t.AppendSeparator() + t.Render() + return nil +} diff --git a/tools/taucorder/prompt/ping.go b/tools/taucorder/prompt/ping.go new file mode 100644 index 00000000..ae66e9ca --- /dev/null +++ b/tools/taucorder/prompt/ping.go @@ -0,0 +1,62 @@ +package prompt + +import ( + "errors" + "fmt" + "os" + + "github.com/libp2p/go-libp2p/core/peer" + + "github.com/jedib0t/go-pretty/v6/table" +) + +func pingCMD(p Prompt, args []string) error { + if len(args) < 2 { + return errors.New("must provide PID") + } + pid := args[1] + _pid, err := peer.Decode(pid) + if err != nil { + return fmt.Errorf("peer id `%s` is invalid", pid) + } + + count, time, err := prompt.Node().Ping(pid, 3) + + t := table.NewWriter() + t.SetStyle(table.StyleLight) + t.SetOutputMirror(os.Stdout) + t.SetColumnConfigs([]table.ColumnConfig{ + { + Number: 1, + AutoMerge: true, + }, + }) + + t.AppendRows([]table.Row{ + {"Host", pid, pid}, + }, table.RowConfig{AutoMerge: true}) + t.AppendSeparator() + + _addrs := prompt.Node().Peer().Peerstore().PeerInfo(_pid).Addrs + _row := make([]table.Row, len(_addrs)) + for i, _addr := range _addrs { + _row[i] = table.Row{"Addresses", _addr.String(), _addr.String()} + } + t.AppendRows(_row, table.RowConfig{AutoMerge: true}) + + t.AppendSeparator() + if err != nil { + t.AppendRows([]table.Row{ + {"Error", err}, + }) + } else { + t.AppendRows([]table.Row{ + {"Stats", "Count", count}, + {"Stats", "Time", time}, + }, table.RowConfig{AutoMerge: true}) + } + + t.Render() + + return nil +} diff --git a/tools/taucorder/prompt/project.go b/tools/taucorder/prompt/project.go new file mode 100644 index 00000000..c52b14bb --- /dev/null +++ b/tools/taucorder/prompt/project.go @@ -0,0 +1,72 @@ +package prompt + +import ( + "errors" + "fmt" + + goPrompt "github.com/c-bata/go-prompt" + "github.com/ipfs/go-cid" + list "github.com/taubyte/tau/tools/taucorder/helpers" +) + +var projectTree = &tctree{ + leafs: []*leaf{ + exitLeaf, + { + validator: stringValidator("list"), + ret: []goPrompt.Suggest{ + { + Text: "list", + Description: "show registered project ids", + }, + }, + handler: listProjects, + }, + { + validator: stringValidator("get"), + ret: []goPrompt.Suggest{ + { + Text: "get", + Description: "show a project's data", + }, + }, + handler: getProject, + }, + }, +} + +func listProjects(p Prompt, args []string) error { + prj, err := p.TaubyteAuthClient().Projects().List() + if err != nil { + return fmt.Errorf("failed listing repos with error: %v", err) + } + + if len(prj) == 0 { + fmt.Println("No projects are currently stored") + return nil + } + + list.CreateTableIds(prj, "Project Id's") + + return nil +} +func getProject(p Prompt, args []string) error { + if len(args) < 2 { + fmt.Println("Must provide project id") + return errors.New("must provide project id") + } + pid := args[1] + _, err := cid.Decode(pid) + if err != nil { + return fmt.Errorf("project id `%s` is invalid", pid) + } + + prj := p.TaubyteAuthClient().Projects().Get(pid) + if prj == nil { + return fmt.Errorf("failed fetching project `%s`", pid) + } + + fmt.Println(prj) + + return nil +} diff --git a/tools/taucorder/prompt/repo.go b/tools/taucorder/prompt/repo.go new file mode 100644 index 00000000..96527df9 --- /dev/null +++ b/tools/taucorder/prompt/repo.go @@ -0,0 +1,67 @@ +package prompt + +import ( + "errors" + "fmt" + "strconv" + + goPrompt "github.com/c-bata/go-prompt" + list "github.com/taubyte/tau/tools/taucorder/helpers" +) + +var repoTree = &tctree{ + leafs: []*leaf{ + exitLeaf, + { + validator: stringValidator("list"), + ret: []goPrompt.Suggest{ + { + Text: "list", + Description: "show registered repo ids", + }, + }, + handler: listRepo, + }, + { + validator: stringValidator("get"), + ret: []goPrompt.Suggest{ + { + Text: "get", + Description: "show a repo's data", + }, + }, + handler: getRepo, + }, + }, +} + +func listRepo(p Prompt, args []string) error { + repo, err := p.TaubyteAuthClient().Repositories().Github().List() + if err != nil { + return fmt.Errorf("failed listing repos with error: %v", err) + } + + list.CreateTableIds(repo, "Repo List") + + return nil +} + +func getRepo(p Prompt, args []string) error { + if len(args) < 2 { + fmt.Println("Must provide repo id") + return errors.New("must provide repo id") + } + rid, err := strconv.Atoi(args[1]) + if err != nil { + return fmt.Errorf("failed converting %d to int with error: %v", rid, err) + } + + repo, err := p.TaubyteAuthClient().Repositories().Github().Get(rid) + if err != nil { + return fmt.Errorf("failed getting repo %d with error: %v", rid, err) + } + + fmt.Println(repo) + + return nil +} diff --git a/tools/taucorder/prompt/seer.go b/tools/taucorder/prompt/seer.go new file mode 100644 index 00000000..c88f3a52 --- /dev/null +++ b/tools/taucorder/prompt/seer.go @@ -0,0 +1,144 @@ +package prompt + +import ( + "errors" + "fmt" + "os" + + "github.com/jedib0t/go-pretty/v6/table" + + goPrompt "github.com/c-bata/go-prompt" + list "github.com/taubyte/tau/tools/taucorder/helpers" + "github.com/taubyte/utils/maps" +) + +var seerTree = &tctree{ + leafs: []*leaf{ + exitLeaf, + { + validator: stringValidator("list"), + ret: []goPrompt.Suggest{ + { + Text: "list", + Description: "show all registered node ids", + }, + }, + handler: listUsage, + }, + { + validator: stringValidator("get"), + ret: []goPrompt.Suggest{ + { + Text: "get", + Description: "get usage data for a node", + }, + }, + handler: getUsage, + }, + { + validator: stringValidator("listServiceId"), + ret: []goPrompt.Suggest{ + { + Text: "listServiceId", + Description: "show all registered ids in seers sql for a specific service", + }, + }, + handler: listServiceId, + }, + }, +} + +func listUsage(p Prompt, args []string) error { + ids, err := p.TaubyteSeerClient().Usage().List() + if err != nil { + return fmt.Errorf("failed listing usage ids with error: %w", err) + } + + if len(ids) == 0 { + fmt.Println("No usages are currently stored") + return nil + } + + list.CreateTableIds(ids, "Node Id's") + + return nil +} + +func listServiceId(p Prompt, args []string) error { + if len(args) < 2 { + fmt.Println("Must provide service name") + return errors.New("must provide service name") + } + + ids, err := p.TaubyteSeerClient().Usage().ListServiceId(args[1]) + if err != nil { + return fmt.Errorf("failed listing usage ids with error: %w", err) + } + + serviceIds, err := maps.StringArray(ids, "ids") + if err != nil { + return fmt.Errorf("failed map string aray ids with error: %w", err) + } + + if len(serviceIds) == 0 { + return fmt.Errorf("currently no entries for %s", args[1]) + } + + title := fmt.Sprintf("%s Id's", args[1]) + list.CreateTableIds(serviceIds, title) + + return nil +} + +func getUsage(p Prompt, args []string) error { + t := table.NewWriter() + t.SetStyle(table.StyleLight) + t.SetOutputMirror(os.Stdout) + if len(args) < 2 { + fmt.Println("Must provide node ID") + return errors.New("must provide node ID") + } + id := args[1] + usg, err := p.TaubyteSeerClient().Usage().Get(id) + freemem := usg.FreeMem / 1073741824 + totalmem := usg.TotalMem / 1073741824 + usedmem := usg.UsedMem / 1048576 + if err != nil { + t.AppendRows([]table.Row{{"--"}}, + table.RowConfig{}) + return fmt.Errorf("failed listing jobs cids with error: %w", err) + } + + t.AppendRows([]table.Row{{"Id", usg.Id}}, + table.RowConfig{}) + t.AppendSeparator() + t.AppendRows([]table.Row{{"Name", usg.Name}}, + table.RowConfig{}) + t.AppendSeparator() + t.AppendRows([]table.Row{{"Type", usg.Type}}, + table.RowConfig{}) + t.AppendSeparator() + t.AppendRows([]table.Row{{"Timestamp", usg.Timestamp}}, + table.RowConfig{}) + t.AppendSeparator() + t.AppendRows([]table.Row{{"Address", usg.Address}}, + table.RowConfig{}) + t.AppendSeparator() + t.AppendRows([]table.Row{{"Memory:\n"}}, + table.RowConfig{}) + t.AppendRows([]table.Row{{"\tMemory Total", fmt.Sprintf("%d GB", totalmem)}, + {"\tMemory Free", fmt.Sprintf("%d GB", freemem)}, + {"\tMemory Used", fmt.Sprintf("%d MB", usedmem)}}, + table.RowConfig{}) + t.AppendSeparator() + t.AppendRows([]table.Row{{"CPU:\n"}}, + table.RowConfig{}) + t.AppendRows([]table.Row{{"\tCPU Threads", usg.CpuCount}, + {"\tCPU Usage", usg.TotalCpu}, + {"\tCPU User", usg.CpuUser}, + {"\tCPU Idle", usg.CpuIdle}}, + table.RowConfig{}) + t.AppendSeparator() + t.Render() + return nil +} diff --git a/tools/taucorder/prompt/swarm.go b/tools/taucorder/prompt/swarm.go new file mode 100644 index 00000000..bb9be5ab --- /dev/null +++ b/tools/taucorder/prompt/swarm.go @@ -0,0 +1,85 @@ +package prompt + +import ( + "fmt" + "sync" + + goPrompt "github.com/c-bata/go-prompt" + "github.com/jedib0t/go-pretty/v6/table" + "github.com/libp2p/go-libp2p/core/peer" +) + +var swarmTree = &tctree{ + leafs: []*leaf{ + exitLeaf, + { + validator: stringValidator("all"), + ret: []goPrompt.Suggest{ + { + Text: "all", + Description: "show swarm", + }, + }, + handler: swarmList, + }, + { + validator: stringValidator("healthy"), + ret: []goPrompt.Suggest{ + { + Text: "healthy", + Description: "show swarm with ping status", + }, + }, + handler: swarmHealth, + }, + }, +} + +func swarmList(p Prompt, args []string) error { + t := table.NewWriter() + t.AppendHeader(table.Row{"PID", "Address"}) + t.SetStyle(table.StyleLight) + for _, pid := range prompt.Node().Peer().Peerstore().Peers() { + peerInfo := prompt.Node().Peer().Peerstore().PeerInfo(pid) + t.AppendRows([]table.Row{{pid.String(), peerInfo.Addrs[0].String()}}, + table.RowConfig{}) + t.AppendSeparator() + } + + fmt.Println(t.Render()) + return nil +} + +func swarmHealth(p Prompt, args []string) error { + var ( + wg sync.WaitGroup + lock sync.Mutex + ) + t := table.NewWriter() + t.AppendHeader(table.Row{"PID", "Address", "Count", "time"}) + t.SetStyle(table.StyleLight) + for _, pid := range prompt.Node().Peer().Peerstore().Peers() { + wg.Add(1) + go func(_pid peer.ID) { + peerInfo := prompt.Node().Peer().Peerstore().PeerInfo(_pid) + count, time, err := prompt.Node().Ping(_pid.String(), 3) + pid := _pid.String() + addr := peerInfo.Addrs[0].String() + lock.Lock() + if err != nil { + t.AppendRows([]table.Row{{pid, addr, "--", "--"}}, + table.RowConfig{}) + } else { + t.AppendRows([]table.Row{{pid, addr, count, time}}, + table.RowConfig{}) + } + t.AppendSeparator() + lock.Unlock() + wg.Done() + }(pid) + } + wg.Wait() + + fmt.Println(t.Render()) + return nil +} diff --git a/tools/taucorder/prompt/tns.go b/tools/taucorder/prompt/tns.go new file mode 100644 index 00000000..39d6d01d --- /dev/null +++ b/tools/taucorder/prompt/tns.go @@ -0,0 +1,101 @@ +package prompt + +import ( + "errors" + "fmt" + "path" + "strings" + + "github.com/taubyte/tau/core/services/tns" + + goPrompt "github.com/c-bata/go-prompt" + spec "github.com/taubyte/tau/pkg/specs/common" + list "github.com/taubyte/tau/tools/taucorder/helpers" +) + +var tnsTree = &tctree{ + leafs: []*leaf{ + exitLeaf, + { + validator: stringValidator("list"), + ret: []goPrompt.Suggest{ + { + Text: "list", + Description: "show all keys registered", + }, + }, + handler: listKeys, + }, + { + validator: stringValidator("fetch"), + ret: []goPrompt.Suggest{ + { + Text: "fetch", + Description: "fetch a key", + }, + }, + handler: fetchValue, + }, + { + validator: stringValidator("lookup"), + ret: []goPrompt.Suggest{ + { + Text: "lookup", + Description: "lookup a key", + }, + }, + handler: lookupValue, + }, + }, +} + +func listKeys(p Prompt, args []string) error { + keys, err := p.TaubyteTnsClient().List(1) + if err != nil { + return fmt.Errorf("failed listing tns keys with error: %w", err) + } + + if len(keys) == 0 { + fmt.Println("No keys are currently stored") + return nil + } + + list.CreateTableIds(keys, "Keys List") + + return nil +} + +func fetchValue(p Prompt, args []string) error { + if len(args) == 1 { + return errors.New("no arguments provided to fetch") + } + + iface, err := p.TaubyteTnsClient().Fetch(spec.NewTnsPath([]string{args[1]})) + if err != nil { + return fmt.Errorf("failed listing tns keys with error: %w", err) + } + + return list.CreateTableInterface("Fetch", iface.Interface()) +} + +func lookupValue(p Prompt, args []string) error { + if len(args) == 1 { + return errors.New("no arguments provided to lookup") + } + + iface, err := p.TaubyteTnsClient().Lookup(tns.Query{Prefix: args[1:]}) + if err != nil { + return fmt.Errorf("failed listing tns keys with error: %w", err) + } + + _iface := make([]string, 0) + for _, item := range iface.([]string) { + start := path.Join(args[1:]...) + if !strings.HasPrefix(start, "/") { + start = "/" + start + } + _iface = append(_iface, strings.TrimPrefix(item, start)) + } + + return list.CreateTableInterface("Lookup", _iface) +} diff --git a/tools/taucorder/prompt/tree.go b/tools/taucorder/prompt/tree.go new file mode 100644 index 00000000..c4b20f43 --- /dev/null +++ b/tools/taucorder/prompt/tree.go @@ -0,0 +1,93 @@ +package prompt + +import ( + "fmt" + "strings" + + goPrompt "github.com/c-bata/go-prompt" + + "github.com/google/shlex" +) + +type leaf struct { + validator func(Prompt, string, bool) bool + handler func(p Prompt, args []string) error + jump func(p Prompt) string + ret []goPrompt.Suggest + leafs []*leaf +} + +type tctree struct { + leafs []*leaf +} + +var _ Tree = mainTree + +func (c tctree) Complete(p Prompt, d goPrompt.Document) []goPrompt.Suggest { + blocks, err := shlex.Split(d.Text) + if err != nil { + blocks = strings.Split(strings.TrimSpace(d.Text), " ") + } + if !strings.HasSuffix(d.Text, " ") && len(blocks) > 1 { + blocks = blocks[:len(blocks)-1] + } + tree := c.leafs + ret := make([]goPrompt.Suggest, 0) + if len(blocks) > 0 { + for _, b := range blocks { + if len(b) > 0 && (b[0] == '"' || b[0] == '\'') { + b = b[1:] + } + if tree == nil { + // reached a dead-end + // maybe not + return nil + } + for _, s := range tree { + if s.validator == nil || s.validator(p, b, true) { + if s.leafs != nil { + tree = s.leafs + } else if s.jump != nil { + newpath := s.jump(p) + tree, _ = forest.FindBranch(newpath) + } + } + } + } + } + b := d.GetWordBeforeCursor() + if len(b) > 0 && (b[0] == '"' || b[0] == '\'') { + b = b[1:] + } + for _, s := range tree { + if s.validator == nil || s.validator(p, b, false) { + ret = append(ret, s.ret...) + } + } + + return ret +} + +func (c tctree) Execute(p Prompt, in []string) error { + tree := c.leafs + for i, b := range in { + for _, s := range tree { + if s.validator == nil || s.validator(p, b, true) { + if s.leafs == nil && s.jump != nil && i+1 < len(in) { + newpath := s.jump(p) + tree, _ = forest.FindBranch(newpath) + continue + } + if s.leafs == nil || i+1 == len(in) { + if s.handler == nil { + return fmt.Errorf("no handler for `%s`", b) + } + return s.handler(prompt, in[i:]) + } + tree = s.leafs + } + } + } + + return fmt.Errorf("can't find match for `%v`", in) +}