From 433a32859cdc893855f774766b6759a8cebd0366 Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Wed, 20 Oct 2021 12:43:28 -0700 Subject: [PATCH] Updates. --- .gitignore | 3 +- Makefile | 9 +++++- README.md | 75 +++++++++++++++++++++++++++++++++++++-------- http_do/http_get.go | 60 ++++++++++++++++++++++++++---------- shared.go | 44 ++++++++++++++++++++++++++ 5 files changed, 161 insertions(+), 30 deletions(-) diff --git a/.gitignore b/.gitignore index 7c7499b..0c7b56f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ dist/ deps/ -*.db \ No newline at end of file +*.db +benchmarks diff --git a/Makefile b/Makefile index 4f3f6cd..fc994a8 100644 --- a/Makefile +++ b/Makefile @@ -1,2 +1,9 @@ +COMMIT=$(shell git rev-parse HEAD) +VERSION=$(shell git describe --tags --exact-match --always) +DATE=$(shell date +'%FT%TZ%z') + dist/http.so: $(shell find . -type f -name '*.go') - go build -buildmode=c-shared -o $@ -tags="shared" shared.go \ No newline at end of file + go build \ + -buildmode=c-shared -o $@ -tags="shared" \ + -ldflags '-X main.Version=$(VERSION) -X main.Commit=$(COMMIT) -X main.Date=$(DATE)' \ + shared.go \ No newline at end of file diff --git a/README.md b/README.md index 5a6bfa0..3f13961 100644 --- a/README.md +++ b/README.md @@ -104,24 +104,75 @@ Examples: ## TODO +### Immediate + +- [ ] Code cleanup + - http_do -> get.go, post.go, do.go + - New internal package? + - cookies package? + - url package? +- [ ] TVF for headers, ex `select name, value from http_headers_each('...')` + - how handle case-insensitive lookup? +- [ ] Timings + - move to end + - add marker for body start, body end? might have to add to cursor + - document caveats, e.g. sqlite .Next()/.Column, need to access fields, etc. +- [ ] `select http_settings('fail_on', )` + - if request fails, what do + - ex ssl error, site not exist, timeout + - for now, lets do udf for changing failure behavior and other settings + - `select http_timeout_set(5000)` + ### POST/DO body utilities - - multipart forms - - other types of POSTs idk - - `http_post_multiform(name1, file1, ...)` + +- multipart forms +- other types of POSTs idk +- `http_post_multiform(name1, file1, ...)` +- `application/x-www-form-urlencoded` ### More cookie utility functions - - `http_cookie_name(cookie)`, `http_cookie_expires(cookie)` + +- `http_cookie_name(cookie)`, `http_cookie_expires(cookie)` ### URL utility functions - - query parameters, `url_query_parameters` - - host, path - - scheme - - username/password + +- query parameters, `url_query_parameters` +- host, path +- scheme +- username/password + +``` +http_url("https", "acs.ca.gov", "/path/to/file", + http_url_query_parameters( + "q", "books", + _format", "json", + "timeout", 1500 + ) +) +``` + +``` +http_url_extract_domain('https://www.dir.ca.gov/cac/cac.html') -- 'www.dir.ca.gov' +http_url_extract_tld('https://www.dir.ca.gov/cac/cac.html') -- 'ca.gov' +http_url_extract_path('https://www.dir.ca.gov/cac/cac.html') -- '/cac/cac.html' +http_url_extract_fragment('') -- '' +http_url_query_params('') -- '' + + +``` ### HAR (HTTP Archive) compatibility - - Export table to HAR? + +- Export do table to HAR - `http_to_har('select * from archives')` - `http_to_har('select * from http_get(...)')` - - `http_get_har(...)` - - `http_from_har(har)` (table-valued functions) -- [ ] Rate limiters? \ No newline at end of file +- Import from har + - `select * from http_har(readfile('file.har'), 'creator_name')` +- Create HAR +- `http_get_har(...), http_post_har(...), http_do_har(...)` + +### Rate Limiter? + +``` +select http_rate_limiter_set(100); +``` diff --git a/http_do/http_get.go b/http_do/http_get.go index 5360afb..abf328c 100644 --- a/http_do/http_get.go +++ b/http_do/http_get.go @@ -26,9 +26,7 @@ func formatSqliteDatetime(t *time.Time) *string { } var SharedDoTableColumns = []vtab.Column{ - {Name: "timings", Type: sqlite.SQLITE_TEXT.String()}, - {Name: "remote_address", Type: sqlite.SQLITE_TEXT.String()}, - + {Name: "request_url", Type: sqlite.SQLITE_TEXT.String()}, {Name: "request_method", Type: sqlite.SQLITE_TEXT.String()}, {Name: "request_headers", Type: sqlite.SQLITE_TEXT.String()}, @@ -40,6 +38,9 @@ var SharedDoTableColumns = []vtab.Column{ {Name: "response_headers", Type: sqlite.SQLITE_TEXT.String()}, {Name: "response_cookies", Type: sqlite.SQLITE_TEXT.String()}, {Name: "response_body", Type: sqlite.SQLITE_BLOB.String()}, + {Name: "remote_address", Type: sqlite.SQLITE_TEXT.String()}, + {Name: "timings", Type: sqlite.SQLITE_TEXT.String()}, + {Name: "meta", Type: sqlite.SQLITE_TEXT.String()}, } var GetTableColumns = append([]vtab.Column{ @@ -89,9 +90,17 @@ type Timings struct { TLSHandshakeStart *time.Time TLSHandshakeDone *time.Time WroteHeaders *time.Time + BodyStart * time.Time + BodyEnd * time.Time } func (cur *HttpDoCursor) Column(ctx *sqlite.Context, c int) error { col := cur.columns[c] + + // TODO this should be a compile-time (?) option. response is nil bc commented out error handling in client.Do in GET + if(strings.HasPrefix(col.Name, "response_") && cur.response == nil) { + ctx.ResultNull() + return nil + } switch col.Name { case "url": ctx.ResultText("") @@ -100,11 +109,6 @@ func (cur *HttpDoCursor) Column(ctx *sqlite.Context, c int) error { case "cookies": ctx.ResultText("") - case "timings": - buf, _ := json.Marshal(cur.timing) - ctx.ResultText(string(buf)) - case "remote_address": - ctx.ResultText(cur.meta.RemoteAddr) case "request_url": ctx.ResultText(cur.request.URL.String()) case "request_method": @@ -147,8 +151,20 @@ func (cur *HttpDoCursor) Column(ctx *sqlite.Context, c int) error { ctx.ResultText(string(buf)) } case "response_body": + start := time.Now() + cur.timing.BodyStart = &start + body, _ := io.ReadAll(cur.response.Body) + + end := time.Now() + cur.timing.BodyEnd = &end + ctx.ResultBlob(body) + case "timings": + buf, _ := json.Marshal(cur.timing) + ctx.ResultText(string(buf)) + case "remote_address": + ctx.ResultText(cur.meta.RemoteAddr) default: fmt.Println("what the fuck") } @@ -234,7 +250,9 @@ func GetTableIterator(constraints []*vtab.Constraint, order []*sqlite.OrderBy) ( cursor := HttpDoCursor{ columns: GetTableColumns, } - client := &http.Client{} + client := &http.Client{ + Timeout: 5 * time.Second, + } request, err := createRequest("GET", url, headers, nil, cookies) if err != nil { @@ -250,8 +268,9 @@ func GetTableIterator(constraints []*vtab.Constraint, order []*sqlite.OrderBy) ( response, err := client.Do(request) if err != nil { + fmt.Println("client.Do error") fmt.Println(err) - return nil, sqlite.SQLITE_ABORT + //return nil, sqlite.SQLITE_ABORT } cursor.current = -1 @@ -425,17 +444,20 @@ func readHeader(rawHeader string) (textproto.MIMEHeader, error) { return header, nil } type TimingJSON struct { - Started * string `json:"started"` + Started * string `json:"start"` FirstResponseByte * string `json:"first_byte"` + GotConn * string `json:"connection"` + WroteHeaders * string `json:"wrote_headers"` DNSStart * string `json:"dns_start"` - DNSDone * string `json:"dns_done"` - GotConn * string `json:"got_conn"` + DNSDone * string `json:"dns_end"` ConnectStart * string `json:"connect_start"` - ConnectDone * string `json:"connect_done"` + ConnectDone * string `json:"connect_end"` TLSHandshakeStart * string `json:"tls_handshake_start"` - TLSHandshakeDone * string `json:"tls_handshake_done"` - WroteHeaders * string `json:"wrote_headers"` + TLSHandshakeDone * string `json:"tls_handshake_end"` + BodyStart * string `json:"body_start"` + BodyEnd * string `json:"body_end"` } + func (t Timings) MarshalJSON() ([]byte, error) { tj := TimingJSON{} if t.Started != nil { @@ -469,6 +491,12 @@ func (t Timings) MarshalJSON() ([]byte, error) { if t.WroteHeaders != nil { tj.WroteHeaders = formatSqliteDatetime(t.WroteHeaders) } + if t.BodyStart != nil { + tj.BodyStart = formatSqliteDatetime(t.BodyStart) + } + if t.BodyEnd != nil { + tj.BodyEnd = formatSqliteDatetime(t.BodyEnd) + } return json.Marshal(tj) } diff --git a/shared.go b/shared.go index 1217783..68578bf 100644 --- a/shared.go +++ b/shared.go @@ -1,14 +1,58 @@ package main import ( + "fmt" + "runtime" + headers "github.com/asg017/sqlite-http/headers" http_do "github.com/asg017/sqlite-http/http_do" "go.riyazali.net/sqlite" ) +// Set in Makefile +var ( + Commit string + Date string + Version string +) + + +// http_version +type VersionFunc struct{} +func (*VersionFunc) Deterministic() bool { return true } +func (*VersionFunc) Args() int { return 0 } +func (*VersionFunc) Apply(c *sqlite.Context, values ...sqlite.Value) { + c.ResultText(Version) +} + +// http_debug +type DebugFunc struct{} +func (*DebugFunc) Deterministic() bool { return true } +func (*DebugFunc) Args() int { return 0 } +func (*DebugFunc) Apply(c *sqlite.Context, values ...sqlite.Value) { + c.ResultText(fmt.Sprintf("Version: %s\nCommit: %s\nRuntime: %s %s/%s\nDate: %s\n", + Version, + Commit, + runtime.Version(), + runtime.GOOS, + runtime.GOARCH, + Date, +)) +} + +var functions = map[string]sqlite.Function{ + "http_version": &VersionFunc{}, + "http_debug": &DebugFunc{}, +} + func init() { sqlite.Register(func(api *sqlite.ExtensionApi) (sqlite.ErrorCode, error) { + for name, function := range functions { + if err := api.CreateFunction(name, function); err != nil { + return sqlite.SQLITE_ERROR, err + } + } if err := http_do.Register(api); err != nil { return sqlite.SQLITE_ERROR, err }