updated mly with batch mode

.gitignore

@@ -1,2 +1,3 @@
 examples/
-.idea
+.idea
+local_test.go

example/README.md

@@ -0,0 +1,21 @@
+## Running example 
+
+### Prerequisites
+
+
+### Running e2e test
+
+
+
+### Troubleshooting datastore caching
+
+* http://localhost:8086/v1/api/model/sls/meta/config
+* http://localhost:8086/v1/api/model/sls/meta/dictionary
+
+
+
+```bash
+docker exec -it aero aql
+SELECT * FROM test.vec
+SELECT * FROM test.sls
+```

example/client/cli.go

@@ -0,0 +1,17 @@
+package client
+
+import (
+	"github.com/jessevdk/go-flags"
+	"log"
+)
+
+func Run(args []string) {
+	options := &Options{}
+	_, err := flags.ParseArgs(options, args)
+	if err != nil {
+		log.Fatal(err)
+	}
+	if err = RunWithOptions(options); err != nil {
+		log.Fatal(err)
+	}
+}

example/client/mlyc/mlyc.go

@@ -0,0 +1,13 @@
+package main
+
+import (
+	"github.com/viant/mly/example/client"
+	"os"
+)
+
+func main() {
+	os.Args = []string{
+		"", "-m=vec", "-t=a", "-t=b", "-s=d", "-s=d",
+	}
+	client.Run(os.Args[1:])
+}

example/client/option.go

@@ -0,0 +1,43 @@
+package client
+
+import (
+	"fmt"
+	"github.com/viant/mly/shared/client"
+)
+
+type Options struct {
+	Sa []string `short:"a" long:"sa" description:"sls model input" `
+	Sl []string `short:"s" long:"sl" description:"sls/vec model input"`
+	X  []string `short:"x" long:"x_input" description:"auxiliary input input"`
+
+	Tv    []string `short:"t" long:"tv" description:"vec model input" `
+	Model string   `short:"m" long:"model" description:"model" `
+	Host  string   `short:"h" long:"host" description:"endpoint host" `
+	Port  int      `short:"p" long:"port" description:"endpoint port" `
+}
+
+func (o *Options) Init() {
+	if o.Host == "" {
+		o.Host = "localhost"
+	}
+	if o.Port == 0 {
+		o.Port = 8086
+	}
+	if len(o.Tv) > 0 && o.Model == "" {
+		o.Model = "vec"
+	}
+	if len(o.Sa) > 0 && o.Model == "" {
+		o.Model = "sls"
+	}
+}
+
+func (o *Options) Validate() error {
+	if o.Model == "" {
+		return fmt.Errorf("model was empty")
+	}
+	return nil
+}
+
+func (o *Options) Hosts() []*client.Host {
+	return []*client.Host{{Name: o.Host, Port: o.Port}}
+}

example/client/runner.go

@@ -0,0 +1,49 @@
+package client
+
+import (
+	"context"
+	"fmt"
+	"github.com/viant/mly/example/sls"
+	"github.com/viant/mly/example/vec"
+	"github.com/viant/mly/shared/client"
+	"github.com/viant/toolbox"
+)
+
+func RunWithOptions(options *Options) error {
+	options.Init()
+	if err := options.Validate(); err != nil {
+		return err
+	}
+	srv, err := client.New(options.Model, options.Hosts())
+	if err != nil {
+		return err
+	}
+
+	message := srv.NewMessage()
+	defer message.Release()
+	response := &client.Response{}
+
+	if len(options.Sa) > 0 {
+		message.StringKey("sa", options.Sa[0])
+		message.StringKey("sl", options.Sl[0])
+		message.StringKey("x", options.X[0])
+		response.Data = &sls.Record{}
+	} else {
+		if len(options.Tv) == 0 {
+			return fmt.Errorf("no input data")
+		}
+		//multi input mode (batch mode)
+		message.StringsKey("sl", options.Sl)
+		message.StringsKey("tv", options.Tv)
+		records := vec.Records{}
+		response.Data = &records
+
+	}
+
+	err = srv.Run(context.Background(), message, response)
+	if err != nil {
+		return err
+	}
+	toolbox.Dump(response)
+	return err
+}

example/e2e/build.yaml

@@ -0,0 +1,21 @@
+
+pipeline:
+
+  deploy:
+    set_sdk:
+      action: sdk.set
+      target: $target
+      sdk: go:1.17
+
+    package:
+      action: exec:run
+      target: $target
+      checkError: true
+      commands:
+        - mkdir -p /tmp/e2e
+        - cd ${appPath}/example/server/mly
+        - go build
+        - mv mly /tmp/e2e/
+        - cd ${appPath}/example/client/mlyc
+        - go build
+        - mv mlyc /tmp/e2e/

example/e2e/regression/app.yaml

@@ -0,0 +1,25 @@
+pipeline:
+
+  copyConfig:
+    action: storage:copy
+    expand: true
+    source:
+      URL: ${appPath}/example/server/etc/config.yaml
+    dest:
+      URL: /tmp/e2e/config.yaml
+
+  stop:
+    action: process:stop
+    target: $target
+    input: datly
+
+  start:
+    action: process:start
+    sleepTimeMs: 3000
+    target: $target
+    directory: /tmp/e2e
+    checkError: true
+    immuneToHangups: true
+    env:
+      TEST: 1
+    command: ./mly -c=/tmp/e2e/config.yaml

example/e2e/regression/cases/001_sls_client/expect.json

@@ -0,0 +1,7 @@
+{
+  "data": {
+    "Value": 30,
+    "Sl": "b",
+    "X": "x"
+  }
+}

example/e2e/regression/cases/001_sls_client/test.yaml

@@ -0,0 +1,17 @@
+init:
+  parentPath: $parent.path
+  expect: $LoadJSON('${parentPath}/expect.json')
+pipeline:
+
+  test:
+    action: exec:run
+    target: $target
+    checkError: true
+    commands:
+      - /tmp/e2e/mlyc -m=sls -a=a -s=b -x=x
+  assert:
+    action: validator:assert
+    init:
+      actual: $AsJSON($test.Cmd[0].Stdout)
+    actual: $actual
+    expect: $expect

example/e2e/regression/cases/002_vec_client/expect.json

@@ -0,0 +1,13 @@
+{
+  "status": "ok",
+  "data": [
+    {
+      "Value": 12,
+      "Sa": "d"
+    },
+    {
+      "Value": 28,
+      "Sa": "d"
+    }
+  ]
+}

example/e2e/regression/cases/002_vec_client/test.yaml

@@ -0,0 +1,20 @@
+init:
+  parentPath: $parent.path
+  expect: $LoadJSON('${parentPath}/expect.json')
+pipeline:
+
+  test:
+    action: exec:run
+    target: $target
+    checkError: true
+    commands:
+      - /tmp/e2e/mlyc -m=vec -t=a -t=b -s=d -s=d
+  info:
+    action: print
+    message:
+  assert:
+    action: validator:assert
+    init:
+      actual: $AsJSON($test.Cmd[0].Stdout)
+    actual: $actual
+    expect: $expect

example/e2e/regression/datastore.yaml

@@ -0,0 +1,23 @@
+
+pipeline:
+
+  reset_aero:
+    init:
+      action: dsunit:init
+      recreate: true
+      datastore: aero
+      config:
+        driverName: aerospike
+        descriptor: tcp([host]:3000)/test
+        parameters:
+          dbname: aero
+          namespace: test
+          host: 127.0.0.1
+          port: 3000
+          keyColumnName: id
+          excludedColumns: id
+
+    populate:
+      action: dsunit:prepare
+      datastore: aero
+      URL: ${appPath}/example/e2e/regression/reset

example/e2e/regression/regression.yaml

@@ -0,0 +1,30 @@
+init:
+
+pipeline:
+
+  datastore:
+    description: start datly app with rule generted from SQLs
+    action: run
+    request: '@datastore'
+
+  app:
+    description: start datly app with rule generted from SQLs
+    action: run
+    request: '@app'
+
+  test:
+    tag: $pathMatch
+    description: '@info'
+
+    subPath: 'cases/${index}_*'
+
+    range: 1..002
+    template:
+      checkSkip:
+        action: nop
+        comments: use case init
+        skip: $HasResource(${path}/skip.txt)
+      test:
+        action: run
+        request: '@test'
+

example/e2e/regression/reset/sls.json

@@ -0,0 +1 @@
+[{}]

example/e2e/regression/reset/vec.json

@@ -0,0 +1 @@
+[{}]

example/e2e/run.yaml

@@ -0,0 +1,25 @@
+init:
+  target:
+    URL: ssh://localhost/
+    credentials: localhost
+  appPath: $WorkingDirectory(../..)
+
+
+pipeline:
+  init:
+    description: initialise test (docker,database,build app)
+    system:
+      action: run
+      request: '@system'
+      tasks: '*'
+
+    build:
+      action: run
+      request: '@build'
+      tasks: '*'
+
+
+  test:
+    action: run
+    description: run regression test
+    request: '@regression/regression'

example/e2e/system.yaml

@@ -0,0 +1,16 @@
+pipeline:
+  stop:
+    services:
+      action: docker:stop
+      images:
+        - aerospike-server
+
+  start:
+    aerospike:
+      action: docker:run
+      image: 'aerospike/aerospike-server'
+      name: aero
+      ports:
+        3000: 3000
+        3001: 3001
+        3002: 3002