Updates.

.gitignore

@@ -1,3 +1,4 @@
 dist/
 deps/
-*.db
+*.db
+benchmarks

Makefile

@@ -1,2 +1,9 @@
+COMMIT=$(shell git rev-parse HEAD)
+VERSION=$(shell git describe --tags --exact-match --always)
+DATE=$(shell date +'%FT%TZ%z')
+
 dist/http.so:  $(shell find . -type f -name '*.go')
-	go build -buildmode=c-shared -o $@ -tags="shared" shared.go
+	go build \
+	-buildmode=c-shared -o $@ -tags="shared" \
+	-ldflags '-X main.Version=$(VERSION) -X main.Commit=$(COMMIT) -X main.Date=$(DATE)' \
+	shared.go

README.md

@@ -104,24 +104,75 @@ Examples:
 
 ## TODO
 
+### Immediate
+
+- [ ] Code cleanup
+  - http_do -> get.go, post.go, do.go
+  - New internal package?
+  - cookies package?
+  - url package?
+- [ ] TVF for headers, ex `select name, value from http_headers_each('...')`
+  - how handle case-insensitive lookup?
+- [ ] Timings
+  - move to end
+  - add marker for body start, body end? might have to add to cursor
+  - document caveats, e.g. sqlite .Next()/.Column, need to access fields, etc.
+- [ ] `select http_settings('fail_on', )`
+  - if request fails, what do
+  - ex ssl error, site not exist, timeout
+  - for now, lets do udf for changing failure behavior and other settings
+  - `select http_timeout_set(5000)`
+
 ### POST/DO body utilities
-  - multipart forms
-  - other types of POSTs idk
-  - `http_post_multiform(name1, file1, ...)`
+
+- multipart forms
+- other types of POSTs idk
+- `http_post_multiform(name1, file1, ...)`
+- `application/x-www-form-urlencoded`
 
 ### More cookie utility functions
-  - `http_cookie_name(cookie)`, `http_cookie_expires(cookie)`
+
+- `http_cookie_name(cookie)`, `http_cookie_expires(cookie)`
 
 ### URL utility functions
-  - query parameters, `url_query_parameters`
-  - host, path
-  - scheme
-  - username/password
+
+- query parameters, `url_query_parameters`
+- host, path
+- scheme
+- username/password
+
+```
+http_url("https", "acs.ca.gov", "/path/to/file",
+  http_url_query_parameters(
+    "q", "books",
+    _format", "json",
+    "timeout", 1500
+  )
+)
+```
+
+```
+http_url_extract_domain('https://www.dir.ca.gov/cac/cac.html') -- 'www.dir.ca.gov'
+http_url_extract_tld('https://www.dir.ca.gov/cac/cac.html') -- 'ca.gov'
+http_url_extract_path('https://www.dir.ca.gov/cac/cac.html') -- '/cac/cac.html'
+http_url_extract_fragment('') -- ''
+http_url_query_params('') -- ''
+
+
+```
 
 ### HAR (HTTP Archive) compatibility
-  - Export table to HAR?
+
+- Export do table to HAR
   - `http_to_har('select * from archives')`
   - `http_to_har('select * from http_get(...)')`
-  - `http_get_har(...)`
-  - `http_from_har(har)` (table-valued functions)
-- [ ] Rate limiters?
+- Import from har
+  - `select * from http_har(readfile('file.har'), 'creator_name')`
+- Create HAR
+- `http_get_har(...), http_post_har(...), http_do_har(...)`
+
+### Rate Limiter?
+
+```
+select http_rate_limiter_set(100);
+```

http_do/http_get.go

@@ -26,9 +26,7 @@ func formatSqliteDatetime(t *time.Time) *string {
 }
 
 var SharedDoTableColumns = []vtab.Column{
-	{Name: "timings", Type: sqlite.SQLITE_TEXT.String()},
-	{Name: "remote_address", Type: sqlite.SQLITE_TEXT.String()},
-
+
 	{Name: "request_url", Type: sqlite.SQLITE_TEXT.String()},
 	{Name: "request_method", Type: sqlite.SQLITE_TEXT.String()},
 	{Name: "request_headers", Type: sqlite.SQLITE_TEXT.String()},
@@ -40,6 +38,9 @@ var SharedDoTableColumns = []vtab.Column{
 	{Name: "response_headers", Type: sqlite.SQLITE_TEXT.String()},
 	{Name: "response_cookies", Type: sqlite.SQLITE_TEXT.String()},
 	{Name: "response_body", Type: sqlite.SQLITE_BLOB.String()},
+	{Name: "remote_address", Type: sqlite.SQLITE_TEXT.String()},
+	{Name: "timings", Type: sqlite.SQLITE_TEXT.String()},
+	{Name: "meta", Type: sqlite.SQLITE_TEXT.String()},
 }
 
 var GetTableColumns = append([]vtab.Column{
@@ -89,9 +90,17 @@ type Timings struct {
 	TLSHandshakeStart *time.Time 
 	TLSHandshakeDone *time.Time 
 	WroteHeaders *time.Time 
+	BodyStart * time.Time
+	BodyEnd * time.Time
 }
 func (cur *HttpDoCursor) Column(ctx *sqlite.Context, c int) error {
 	col := cur.columns[c]
+
+  // TODO this should be a compile-time (?) option. response is nil bc commented out error handling in client.Do in GET
+  if(strings.HasPrefix(col.Name, "response_") && cur.response == nil) {
+    ctx.ResultNull()
+    return nil
+  }
 	switch col.Name {
 	case "url":
 		ctx.ResultText("")
@@ -100,11 +109,6 @@ func (cur *HttpDoCursor) Column(ctx *sqlite.Context, c int) error {
 	case "cookies":
 		ctx.ResultText("")
 
-	case "timings":
-		buf, _ := json.Marshal(cur.timing)
-		ctx.ResultText(string(buf))
-	case "remote_address":
-		ctx.ResultText(cur.meta.RemoteAddr)
 	case "request_url":
 		ctx.ResultText(cur.request.URL.String())
 	case "request_method":
@@ -147,8 +151,20 @@ func (cur *HttpDoCursor) Column(ctx *sqlite.Context, c int) error {
 			ctx.ResultText(string(buf))
 		}
 	case "response_body":
+		start := time.Now()
+		cur.timing.BodyStart =  &start
+
 		body, _ := io.ReadAll(cur.response.Body)
+
+		end := time.Now()
+		cur.timing.BodyEnd =  &end
+
 		ctx.ResultBlob(body)
+	case "timings":
+		buf, _ := json.Marshal(cur.timing)
+		ctx.ResultText(string(buf))
+	case "remote_address":
+		ctx.ResultText(cur.meta.RemoteAddr)
 	default:
 		fmt.Println("what the fuck")
 	}
@@ -234,7 +250,9 @@ func GetTableIterator(constraints []*vtab.Constraint, order []*sqlite.OrderBy) (
 	cursor := HttpDoCursor{
 		columns: GetTableColumns,
 	}
-	client := &http.Client{}
+	client := &http.Client{
+    Timeout: 5 * time.Second,
+  }
 
 	request, err := createRequest("GET", url, headers, nil, cookies)
 	if err != nil {
@@ -250,8 +268,9 @@ func GetTableIterator(constraints []*vtab.Constraint, order []*sqlite.OrderBy) (
 
 	response, err := client.Do(request)
 	if err != nil {
+		fmt.Println("client.Do error")
 		fmt.Println(err)
-		return nil, sqlite.SQLITE_ABORT
+		//return nil, sqlite.SQLITE_ABORT
 	}
 
 	cursor.current = -1
@@ -425,17 +444,20 @@ func readHeader(rawHeader string) (textproto.MIMEHeader, error) {
 	return header, nil
 }
 type TimingJSON struct {
-	Started * string `json:"started"`
+	Started * string `json:"start"`
 	FirstResponseByte * string `json:"first_byte"`
+	GotConn * string `json:"connection"`
+	WroteHeaders * string `json:"wrote_headers"`
 	DNSStart * string `json:"dns_start"`
-	DNSDone * string `json:"dns_done"`
-	GotConn * string `json:"got_conn"`
+	DNSDone * string `json:"dns_end"`
 	ConnectStart * string `json:"connect_start"`
-	ConnectDone * string `json:"connect_done"`
+	ConnectDone * string `json:"connect_end"`
 	TLSHandshakeStart * string `json:"tls_handshake_start"`
-	TLSHandshakeDone * string `json:"tls_handshake_done"`
-	WroteHeaders * string `json:"wrote_headers"`
+	TLSHandshakeDone * string `json:"tls_handshake_end"`
+	BodyStart * string `json:"body_start"`
+	BodyEnd * string `json:"body_end"`
 }
+
 func (t Timings) MarshalJSON() ([]byte, error) {
 	tj := TimingJSON{} 
 	if t.Started != nil {
@@ -469,6 +491,12 @@ func (t Timings) MarshalJSON() ([]byte, error) {
 	if t.WroteHeaders != nil {
 		tj.WroteHeaders = formatSqliteDatetime(t.WroteHeaders)
 	}
+	if t.BodyStart != nil {
+		tj.BodyStart = formatSqliteDatetime(t.BodyStart)
+	}
+	if t.BodyEnd != nil {
+		tj.BodyEnd = formatSqliteDatetime(t.BodyEnd)
+	}
 
 	return json.Marshal(tj)
 }

shared.go

@@ -1,14 +1,58 @@
 package main
 
 import (
+	"fmt"
+	"runtime"
+
 	headers "github.com/asg017/sqlite-http/headers"
 	http_do "github.com/asg017/sqlite-http/http_do"
 	"go.riyazali.net/sqlite"
 )
 
+// Set in Makefile
+var (
+	Commit  string
+	Date    string
+	Version string
+)
+
+
+// http_version
+type VersionFunc struct{}
+func (*VersionFunc) Deterministic() bool { return true }
+func (*VersionFunc) Args() int           { return 0 }
+func (*VersionFunc) Apply(c *sqlite.Context, values ...sqlite.Value) {
+	c.ResultText(Version)
+}
+
+// http_debug
+type DebugFunc struct{}
+func (*DebugFunc) Deterministic() bool { return true }
+func (*DebugFunc) Args() int           { return 0 }
+func (*DebugFunc) Apply(c *sqlite.Context, values ...sqlite.Value) {
+	c.ResultText(fmt.Sprintf("Version: %s\nCommit: %s\nRuntime: %s %s/%s\nDate: %s\n",
+	Version,
+	Commit,
+	runtime.Version(),
+	runtime.GOOS,
+	runtime.GOARCH,
+	Date,
+))
+}
+
+var functions = map[string]sqlite.Function{
+	"http_version": &VersionFunc{},
+	"http_debug": &DebugFunc{},
+}
+
 func init() {
 
 	sqlite.Register(func(api *sqlite.ExtensionApi) (sqlite.ErrorCode, error) {
+		for name, function := range functions {
+			if err := api.CreateFunction(name, function); err != nil {
+				return sqlite.SQLITE_ERROR, err
+			}
+		}
 		if err := http_do.Register(api); err != nil {
 			return sqlite.SQLITE_ERROR, err
 		}