Skip to content

Commit a9f3c52

Browse files
b5b5
committed
feat(cron): store logs and files of stdout logs
I think it's worth it to get this API right, it has a number of implications for the overall design of cron.
1 parent 6f639e1 commit a9f3c52

File tree

13 files changed

+520
-252
lines changed

13 files changed

+520
-252
lines changed

‎base/cron.go‎

Lines changed: 67 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package base
22

33
import (
4-
"context"
54
"fmt"
65
"os/exec"
76
"path/filepath"
@@ -13,77 +12,80 @@ import (
1312
"github.com/qri-io/qri/cron"
1413
)
1514

16-
// DatasetSaveRunner returns a cron.RunFunc that invokes the "qri save" command
17-
func DatasetSaveRunner(basepath string) cron.RunJobFunc {
18-
return func(ctx context.Context, streams ioes.IOStreams, job *cron.Job) error {
19-
args := []string{"save", job.Name}
15+
// JobToCmd returns an operating system command that will execute the given job
16+
// wiring operating system in/out/errout to the provided iostreams.
17+
func JobToCmd(streams ioes.IOStreams, job *cron.Job) *exec.Cmd {
18+
switch job.Type {
19+
case cron.JTDataset:
20+
return datasetSaveCmd(streams, job)
21+
case cron.JTShellScript:
22+
return shellScriptCmd(streams, job)
23+
default:
24+
return nil
25+
}
26+
}
2027

21-
if o, ok := job.Options.(*cron.DatasetOptions); ok {
22-
if o.Title != "" {
23-
args = append(args, fmt.Sprintf(`--title="%s"`, o.Title))
24-
}
25-
if o.Message != "" {
26-
args = append(args, fmt.Sprintf(`--message="%s"`, o.Message))
27-
}
28-
if o.Recall != "" {
29-
args = append(args, fmt.Sprintf(`--recall="%s"`, o.Recall))
30-
}
31-
if o.BodyPath != "" {
32-
args = append(args, fmt.Sprintf(`--body="%s"`, o.BodyPath))
33-
}
34-
if len(o.FilePaths) > 0 {
35-
for _, path := range o.FilePaths {
36-
args = append(args, fmt.Sprintf(`--file="%s"`, path))
37-
}
28+
// datasetSaveCmd configures a "qri save" command based on job details
29+
// wiring operating system in/out/errout to the provided iostreams.
30+
func datasetSaveCmd(streams ioes.IOStreams, job *cron.Job) *exec.Cmd {
31+
args := []string{"save", job.Name}
32+
33+
if o, ok := job.Options.(*cron.DatasetOptions); ok {
34+
if o.Title != "" {
35+
args = append(args, fmt.Sprintf(`--title="%s"`, o.Title))
36+
}
37+
if o.Message != "" {
38+
args = append(args, fmt.Sprintf(`--message="%s"`, o.Message))
39+
}
40+
if o.Recall != "" {
41+
args = append(args, fmt.Sprintf(`--recall="%s"`, o.Recall))
42+
}
43+
if o.BodyPath != "" {
44+
args = append(args, fmt.Sprintf(`--body="%s"`, o.BodyPath))
45+
}
46+
if len(o.FilePaths) > 0 {
47+
for _, path := range o.FilePaths {
48+
args = append(args, fmt.Sprintf(`--file="%s"`, path))
3849
}
50+
}
3951

40-
// TODO (b5) - config and secrets
52+
// TODO (b5) - config and secrets
4153

42-
boolFlags := map[string]bool{
43-
"--publish": o.Publish,
44-
"--strict": o.Strict,
45-
"--force": o.Force,
46-
"--keep-format": o.ConvertFormatToPrev,
47-
"--no-render": !o.ShouldRender,
48-
}
49-
for flag, use := range boolFlags {
50-
if use {
51-
args = append(args, flag)
52-
}
54+
boolFlags := map[string]bool{
55+
"--publish": o.Publish,
56+
"--strict": o.Strict,
57+
"--force": o.Force,
58+
"--keep-format": o.ConvertFormatToPrev,
59+
"--no-render": !o.ShouldRender,
60+
}
61+
for flag, use := range boolFlags {
62+
if use {
63+
args = append(args, flag)
5364
}
54-
5565
}
5666

57-
cmd := exec.Command("qri", args...)
58-
// cmd.Dir = basepath
59-
cmd.Stderr = streams.ErrOut
60-
cmd.Stdout = streams.Out
61-
cmd.Stdin = streams.In
62-
return cmd.Run()
6367
}
68+
69+
cmd := exec.Command("qri", args...)
70+
// cmd.Dir = basepath
71+
cmd.Stderr = streams.ErrOut
72+
cmd.Stdout = streams.Out
73+
cmd.Stdin = streams.In
74+
return cmd
6475
}
6576

66-
// LocalShellScriptRunner creates a script runner anchored at a local path
67-
// The runner it wires operating sytsem command in/out/errour to the iostreams
68-
// provided by RunJobFunc. All paths are in relation to the provided base path
77+
// shellScriptCmd creates an exec.Cmd, wires operating system in/out/errout
78+
// to the provided iostreams.
6979
// Commands are executed with access to the same enviornment variables as the
7080
// process the runner is executing in
71-
// The executing command blocks until completion
72-
func LocalShellScriptRunner(basepath string) cron.RunJobFunc {
73-
return func(ctx context.Context, streams ioes.IOStreams, job *cron.Job) error {
74-
path := job.Name
75-
if qfs.PathKind(job.Name) == "local" {
76-
// TODO (b5) - need to first check that path can't be found
77-
// path = filepath.Join(basepath, path)
78-
}
79-
80-
cmd := exec.Command(path)
81-
// cmd.Dir = basepath
82-
cmd.Stderr = streams.ErrOut
83-
cmd.Stdout = streams.Out
84-
cmd.Stdin = streams.In
85-
return cmd.Run()
86-
}
81+
func shellScriptCmd(streams ioes.IOStreams, job *cron.Job) *exec.Cmd {
82+
// TODO (b5) - config and secrets as env vars
83+
84+
cmd := exec.Command(job.Name)
85+
cmd.Stderr = streams.ErrOut
86+
cmd.Stdout = streams.Out
87+
cmd.Stdin = streams.In
88+
return cmd
8789
}
8890

8991
// PossibleShellScript checks a path to see if it might be a shell script
@@ -109,10 +111,11 @@ func DatasetToJob(ds *dataset.Dataset, periodicity string, opts *cron.DatasetOpt
109111

110112
job = &cron.Job{
111113
// TODO (b5) - dataset.Dataset needs an Alias() method:
112-
Name: fmt.Sprintf("%s/%s", ds.Peername, ds.Name),
113-
Periodicity: p,
114-
Type: cron.JTDataset,
115-
LastRun: ds.Commit.Timestamp,
114+
Name: fmt.Sprintf("%s/%s", ds.Peername, ds.Name),
115+
Periodicity: p,
116+
Type: cron.JTDataset,
117+
LastRunStart: ds.Commit.Timestamp,
118+
LastRunStop: ds.Commit.Timestamp,
116119
}
117120
if opts != nil {
118121
job.Options = opts

‎cron/client.go‎

Lines changed: 89 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"bytes"
55
"context"
66
"fmt"
7+
"io"
78
"io/ioutil"
89
"net/http"
910
"strings"
@@ -46,41 +47,31 @@ func (c HTTPClient) Ping() error {
4647
if res.StatusCode == http.StatusOK {
4748
return nil
4849
}
49-
return resError(res)
50+
return maybeErrorResponse(res)
5051
}
5152

5253
// Jobs lists jobs by querying an HTTP server
5354
func (c HTTPClient) Jobs(ctx context.Context, offset, limit int) ([]*Job, error) {
54-
res, err := http.Get(fmt.Sprintf("http://%s/jobs?offset=%d&limit=&%d", c.Addr, offset, limit))
55+
res, err := http.Get(fmt.Sprintf("http://%s/jobs?offset=%d&limit=%d", c.Addr, offset, limit))
5556
if err != nil {
5657
return nil, err
5758
}
5859

59-
defer res.Body.Close()
60-
data, err := ioutil.ReadAll(res.Body)
60+
return decodeJobsResponse(res)
61+
}
62+
63+
// Job gets a job by querying an HTTP server
64+
func (c HTTPClient) Job(ctx context.Context, name string) (*Job, error) {
65+
res, err := http.Get(fmt.Sprintf("http://%s/job?name=%s", c.Addr, name))
6166
if err != nil {
6267
return nil, err
6368
}
6469

65-
js := cronfb.GetRootAsJobs(data, 0)
66-
dec := &cronfb.Job{}
67-
jobs := make([]*Job, js.ListLength())
68-
69-
for i := 0; i < js.ListLength(); i++ {
70-
js.List(dec, i)
71-
decJob := &Job{}
72-
if err := decJob.UnmarshalFlatbuffer(dec); err != nil {
73-
return nil, err
74-
}
75-
jobs[i] = decJob
70+
if res.StatusCode == 200 {
71+
return decodeJobResponse(res)
7672
}
7773

78-
return jobs, nil
79-
}
80-
81-
// Job gets a job by querying an HTTP server
82-
func (c HTTPClient) Job(ctx context.Context, name string) (*Job, error) {
83-
return nil, fmt.Errorf("not finished")
74+
return nil, maybeErrorResponse(res)
8475
}
8576

8677
// Schedule adds a job to the cron scheduler via an HTTP request
@@ -100,7 +91,45 @@ func (c HTTPClient) Unschedule(ctx context.Context, name string) error {
10091
return err
10192
}
10293

103-
return resError(res)
94+
return maybeErrorResponse(res)
95+
}
96+
97+
// Logs gives a log of executed jobs
98+
func (c HTTPClient) Logs(ctx context.Context, offset, limit int) ([]*Job, error) {
99+
res, err := http.Get(fmt.Sprintf("http://%s/logs?offset=%d&limit=%d", c.Addr, offset, limit))
100+
if err != nil {
101+
return nil, err
102+
}
103+
104+
return decodeJobsResponse(res)
105+
}
106+
107+
// LoggedJob returns a single executed job by job.LogName
108+
func (c HTTPClient) LoggedJob(ctx context.Context, logName string) (*Job, error) {
109+
res, err := http.Get(fmt.Sprintf("http://%s/log?log_name=%s", c.Addr, logName))
110+
if err != nil {
111+
return nil, err
112+
}
113+
114+
if res.StatusCode == 200 {
115+
return decodeJobResponse(res)
116+
}
117+
118+
return nil, maybeErrorResponse(res)
119+
}
120+
121+
// LoggedJobFile returns a reader for a file at the given name
122+
func (c HTTPClient) LoggedJobFile(ctx context.Context, logName string) (io.ReadCloser, error) {
123+
res, err := http.Get(fmt.Sprintf("http://%s/log/output?log_name=%s", c.Addr, logName))
124+
if err != nil {
125+
return nil, err
126+
}
127+
128+
if res.StatusCode == 200 {
129+
return res.Body, nil
130+
}
131+
132+
return nil, maybeErrorResponse(res)
104133
}
105134

106135
func (c HTTPClient) postJob(job *Job) error {
@@ -114,10 +143,10 @@ func (c HTTPClient) postJob(job *Job) error {
114143
return err
115144
}
116145

117-
return resError(res)
146+
return maybeErrorResponse(res)
118147
}
119148

120-
func resError(res *http.Response) error {
149+
func maybeErrorResponse(res *http.Response) error {
121150
if res.StatusCode == 200 {
122151
return nil
123152
}
@@ -129,3 +158,39 @@ func resError(res *http.Response) error {
129158

130159
return fmt.Errorf(string(errData))
131160
}
161+
162+
func decodeJobsResponse(res *http.Response) ([]*Job, error) {
163+
defer res.Body.Close()
164+
data, err := ioutil.ReadAll(res.Body)
165+
if err != nil {
166+
return nil, err
167+
}
168+
169+
js := cronfb.GetRootAsJobs(data, 0)
170+
dec := &cronfb.Job{}
171+
jobs := make([]*Job, js.ListLength())
172+
173+
for i := 0; i < js.ListLength(); i++ {
174+
js.List(dec, i)
175+
decJob := &Job{}
176+
if err := decJob.UnmarshalFlatbuffer(dec); err != nil {
177+
return nil, err
178+
}
179+
jobs[i] = decJob
180+
}
181+
182+
return jobs, nil
183+
}
184+
185+
func decodeJobResponse(res *http.Response) (*Job, error) {
186+
defer res.Body.Close()
187+
data, err := ioutil.ReadAll(res.Body)
188+
if err != nil {
189+
return nil, err
190+
}
191+
192+
js := cronfb.GetRootAsJob(data, 0)
193+
dec := &Job{}
194+
err = dec.UnmarshalFlatbuffer(js)
195+
return dec, err
196+
}

‎cron/cron.fbs‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ table StringMapVal {
1010
val:string;
1111
}
1212

13-
// TODO (b5): I think it would be smarter to remove all knoweldge from cron
13+
// TODO (b5): I think it would be smarter to remove all details from cron
1414
// about what exactly is being scheduled, but we would need a go implementation
1515
// of flexbuffers to do that properly, so let's leave this in for now
1616
union Options { DatasetOptions, ShellScriptOptions }
@@ -38,11 +38,14 @@ table ShellScriptOptions {
3838

3939
table Job {
4040
name:string;
41+
path:string;
4142
type:JobType;
4243
periodicity:string;
4344

44-
lastRun:string;
45+
lastRunStart:string;
46+
lastRunStop:string;
4547
lastError:string;
48+
logFilePath:string;
4649

4750
options:Options;
4851
}

0 commit comments

Comments
 (0)