Skip to content

Commit c757765

Browse files
authored
Implement actions artifacts (#22738)
Implement action artifacts server api. This change is used for supporting https://github.com/actions/upload-artifact and https://github.com/actions/download-artifact in gitea actions. It can run sample workflow from doc https://docs.github.com/en/actions/using-workflows/storing-workflow-data-as-artifacts. The api design is inspired by https://github.com/nektos/act/blob/master/pkg/artifacts/server.go and includes some changes from gitea internal structs and methods. Actions artifacts contains two parts: - Gitea server api and storage (this pr implement basic design without some complex cases supports) - Runner communicate with gitea server api (in comming) Old pr #22345 is outdated after actions merged. I create new pr from main branch. ![897f7694-3e0f-4f7c-bb4b-9936624ead45](https://user-images.githubusercontent.com/2142787/219382371-eb3cf810-e4e0-456b-a8ff-aecc2b1a1032.jpeg) Add artifacts list in actions workflow page.
1 parent 7985cde commit c757765

File tree

24 files changed

+1127
-6
lines changed

24 files changed

+1127
-6
lines changed

models/actions/artifact.go

+122
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
// Copyright 2023 The Gitea Authors. All rights reserved.
2+
// SPDX-License-Identifier: MIT
3+
4+
// This artifact server is inspired by https://github.com/nektos/act/blob/master/pkg/artifacts/server.go.
5+
// It updates url setting and uses ObjectStore to handle artifacts persistence.
6+
7+
package actions
8+
9+
import (
10+
"context"
11+
"errors"
12+
13+
"code.gitea.io/gitea/models/db"
14+
"code.gitea.io/gitea/modules/timeutil"
15+
"code.gitea.io/gitea/modules/util"
16+
)
17+
18+
const (
19+
// ArtifactStatusUploadPending is the status of an artifact upload that is pending
20+
ArtifactStatusUploadPending = 1
21+
// ArtifactStatusUploadConfirmed is the status of an artifact upload that is confirmed
22+
ArtifactStatusUploadConfirmed = 2
23+
// ArtifactStatusUploadError is the status of an artifact upload that is errored
24+
ArtifactStatusUploadError = 3
25+
)
26+
27+
func init() {
28+
db.RegisterModel(new(ActionArtifact))
29+
}
30+
31+
// ActionArtifact is a file that is stored in the artifact storage.
32+
type ActionArtifact struct {
33+
ID int64 `xorm:"pk autoincr"`
34+
RunID int64 `xorm:"index UNIQUE(runid_name)"` // The run id of the artifact
35+
RunnerID int64
36+
RepoID int64 `xorm:"index"`
37+
OwnerID int64
38+
CommitSHA string
39+
StoragePath string // The path to the artifact in the storage
40+
FileSize int64 // The size of the artifact in bytes
41+
FileCompressedSize int64 // The size of the artifact in bytes after gzip compression
42+
ContentEncoding string // The content encoding of the artifact
43+
ArtifactPath string // The path to the artifact when runner uploads it
44+
ArtifactName string `xorm:"UNIQUE(runid_name)"` // The name of the artifact when runner uploads it
45+
Status int64 `xorm:"index"` // The status of the artifact, uploading, expired or need-delete
46+
CreatedUnix timeutil.TimeStamp `xorm:"created"`
47+
UpdatedUnix timeutil.TimeStamp `xorm:"updated index"`
48+
}
49+
50+
// CreateArtifact create a new artifact with task info or get same named artifact in the same run
51+
func CreateArtifact(ctx context.Context, t *ActionTask, artifactName string) (*ActionArtifact, error) {
52+
if err := t.LoadJob(ctx); err != nil {
53+
return nil, err
54+
}
55+
artifact, err := getArtifactByArtifactName(ctx, t.Job.RunID, artifactName)
56+
if errors.Is(err, util.ErrNotExist) {
57+
artifact := &ActionArtifact{
58+
RunID: t.Job.RunID,
59+
RunnerID: t.RunnerID,
60+
RepoID: t.RepoID,
61+
OwnerID: t.OwnerID,
62+
CommitSHA: t.CommitSHA,
63+
Status: ArtifactStatusUploadPending,
64+
}
65+
if _, err := db.GetEngine(ctx).Insert(artifact); err != nil {
66+
return nil, err
67+
}
68+
return artifact, nil
69+
} else if err != nil {
70+
return nil, err
71+
}
72+
return artifact, nil
73+
}
74+
75+
func getArtifactByArtifactName(ctx context.Context, runID int64, name string) (*ActionArtifact, error) {
76+
var art ActionArtifact
77+
has, err := db.GetEngine(ctx).Where("run_id = ? AND artifact_name = ?", runID, name).Get(&art)
78+
if err != nil {
79+
return nil, err
80+
} else if !has {
81+
return nil, util.ErrNotExist
82+
}
83+
return &art, nil
84+
}
85+
86+
// GetArtifactByID returns an artifact by id
87+
func GetArtifactByID(ctx context.Context, id int64) (*ActionArtifact, error) {
88+
var art ActionArtifact
89+
has, err := db.GetEngine(ctx).ID(id).Get(&art)
90+
if err != nil {
91+
return nil, err
92+
} else if !has {
93+
return nil, util.ErrNotExist
94+
}
95+
96+
return &art, nil
97+
}
98+
99+
// UpdateArtifactByID updates an artifact by id
100+
func UpdateArtifactByID(ctx context.Context, id int64, art *ActionArtifact) error {
101+
art.ID = id
102+
_, err := db.GetEngine(ctx).ID(id).AllCols().Update(art)
103+
return err
104+
}
105+
106+
// ListArtifactsByRunID returns all artifacts of a run
107+
func ListArtifactsByRunID(ctx context.Context, runID int64) ([]*ActionArtifact, error) {
108+
arts := make([]*ActionArtifact, 0, 10)
109+
return arts, db.GetEngine(ctx).Where("run_id=?", runID).Find(&arts)
110+
}
111+
112+
// ListUploadedArtifactsByRunID returns all uploaded artifacts of a run
113+
func ListUploadedArtifactsByRunID(ctx context.Context, runID int64) ([]*ActionArtifact, error) {
114+
arts := make([]*ActionArtifact, 0, 10)
115+
return arts, db.GetEngine(ctx).Where("run_id=? AND status=?", runID, ArtifactStatusUploadConfirmed).Find(&arts)
116+
}
117+
118+
// ListArtifactsByRepoID returns all artifacts of a repo
119+
func ListArtifactsByRepoID(ctx context.Context, repoID int64) ([]*ActionArtifact, error) {
120+
arts := make([]*ActionArtifact, 0, 10)
121+
return arts, db.GetEngine(ctx).Where("repo_id=?", repoID).Find(&arts)
122+
}

models/fixtures/access_token.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -30,4 +30,4 @@
3030
token_last_eight: 69d28c91
3131
created_unix: 946687980
3232
updated_unix: 946687980
33-
#commented out tokens so you can see what they are in plaintext
33+
#commented out tokens so you can see what they are in plaintext

models/fixtures/action_run.yml

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
-
2+
id: 791
3+
title: "update actions"
4+
repo_id: 4
5+
owner_id: 1
6+
workflow_id: "artifact.yaml"
7+
index: 187
8+
trigger_user_id: 1
9+
ref: "refs/heads/master"
10+
commit_sha: "c2d72f548424103f01ee1dc02889c1e2bff816b0"
11+
event: "push"
12+
is_fork_pull_request: 0
13+
status: 1
14+
started: 1683636528
15+
stopped: 1683636626
16+
created: 1683636108
17+
updated: 1683636626
18+
need_approval: 0
19+
approved_by: 0

models/fixtures/action_run_job.yml

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
-
2+
id: 192
3+
run_id: 791
4+
repo_id: 4
5+
owner_id: 1
6+
commit_sha: c2d72f548424103f01ee1dc02889c1e2bff816b0
7+
is_fork_pull_request: 0
8+
name: job_2
9+
attempt: 1
10+
job_id: job_2
11+
task_id: 47
12+
status: 1
13+
started: 1683636528
14+
stopped: 1683636626

models/fixtures/action_task.yml

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
-
2+
id: 47
3+
job_id: 192
4+
attempt: 3
5+
runner_id: 1
6+
status: 6 # 6 is the status code for "running", running task can upload artifacts
7+
started: 1683636528
8+
stopped: 1683636626
9+
repo_id: 4
10+
owner_id: 1
11+
commit_sha: c2d72f548424103f01ee1dc02889c1e2bff816b0
12+
is_fork_pull_request: 0
13+
token_hash: 6d8ef48297195edcc8e22c70b3020eaa06c52976db67d39b4260c64a69a2cc1508825121b7b8394e48e00b1bf8718b2a867e
14+
token_salt: jVuKnSPGgy
15+
token_last_eight: eeb1a71a
16+
log_filename: artifact-test2/2f/47.log
17+
log_in_storage: 1
18+
log_length: 707
19+
log_size: 90179
20+
log_expired: 0

models/migrations/migrations.go

+2
Original file line numberDiff line numberDiff line change
@@ -491,6 +491,8 @@ var migrations = []Migration{
491491
NewMigration("Add ArchivedUnix Column", v1_20.AddArchivedUnixToRepository),
492492
// v256 -> v257
493493
NewMigration("Add is_internal column to package", v1_20.AddIsInternalColumnToPackage),
494+
// v257 -> v258
495+
NewMigration("Add Actions Artifact table", v1_20.CreateActionArtifactTable),
494496
}
495497

496498
// GetCurrentDBVersion returns the current db version

models/migrations/v1_20/v257.go

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
// Copyright 2023 The Gitea Authors. All rights reserved.
2+
// SPDX-License-Identifier: MIT
3+
4+
package v1_20 //nolint
5+
6+
import (
7+
"code.gitea.io/gitea/modules/timeutil"
8+
9+
"xorm.io/xorm"
10+
)
11+
12+
func CreateActionArtifactTable(x *xorm.Engine) error {
13+
// ActionArtifact is a file that is stored in the artifact storage.
14+
type ActionArtifact struct {
15+
ID int64 `xorm:"pk autoincr"`
16+
RunID int64 `xorm:"index UNIQUE(runid_name)"` // The run id of the artifact
17+
RunnerID int64
18+
RepoID int64 `xorm:"index"`
19+
OwnerID int64
20+
CommitSHA string
21+
StoragePath string // The path to the artifact in the storage
22+
FileSize int64 // The size of the artifact in bytes
23+
FileCompressedSize int64 // The size of the artifact in bytes after gzip compression
24+
ContentEncoding string // The content encoding of the artifact
25+
ArtifactPath string // The path to the artifact when runner uploads it
26+
ArtifactName string `xorm:"UNIQUE(runid_name)"` // The name of the artifact when runner uploads it
27+
Status int64 `xorm:"index"` // The status of the artifact
28+
CreatedUnix timeutil.TimeStamp `xorm:"created"`
29+
UpdatedUnix timeutil.TimeStamp `xorm:"updated index"`
30+
}
31+
32+
return x.Sync(new(ActionArtifact))
33+
}

models/repo.go

+15
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,12 @@ func DeleteRepository(doer *user_model.User, uid, repoID int64) error {
5959
return fmt.Errorf("find actions tasks of repo %v: %w", repoID, err)
6060
}
6161

62+
// Query the artifacts of this repo, they will be needed after they have been deleted to remove artifacts files in ObjectStorage
63+
artifacts, err := actions_model.ListArtifactsByRepoID(ctx, repoID)
64+
if err != nil {
65+
return fmt.Errorf("list actions artifacts of repo %v: %w", repoID, err)
66+
}
67+
6268
// In case is a organization.
6369
org, err := user_model.GetUserByID(ctx, uid)
6470
if err != nil {
@@ -164,6 +170,7 @@ func DeleteRepository(doer *user_model.User, uid, repoID int64) error {
164170
&actions_model.ActionRunJob{RepoID: repoID},
165171
&actions_model.ActionRun{RepoID: repoID},
166172
&actions_model.ActionRunner{RepoID: repoID},
173+
&actions_model.ActionArtifact{RepoID: repoID},
167174
); err != nil {
168175
return fmt.Errorf("deleteBeans: %w", err)
169176
}
@@ -336,6 +343,14 @@ func DeleteRepository(doer *user_model.User, uid, repoID int64) error {
336343
}
337344
}
338345

346+
// delete actions artifacts in ObjectStorage after the repo have already been deleted
347+
for _, art := range artifacts {
348+
if err := storage.ActionsArtifacts.Delete(art.StoragePath); err != nil {
349+
log.Error("remove artifact file %q: %v", art.StoragePath, err)
350+
// go on
351+
}
352+
}
353+
339354
return nil
340355
}
341356

models/unittest/testdb.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ func MainTest(m *testing.M, testOpts *TestOptions) {
126126

127127
setting.Packages.Storage.Path = filepath.Join(setting.AppDataPath, "packages")
128128

129-
setting.Actions.Storage.Path = filepath.Join(setting.AppDataPath, "actions_log")
129+
setting.Actions.LogStorage.Path = filepath.Join(setting.AppDataPath, "actions_log")
130130

131131
setting.Git.HomePath = filepath.Join(setting.AppDataPath, "home")
132132

modules/setting/actions.go

+7-2
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,8 @@ import (
1010
// Actions settings
1111
var (
1212
Actions = struct {
13-
Storage // how the created logs should be stored
13+
LogStorage Storage // how the created logs should be stored
14+
ArtifactStorage Storage // how the created artifacts should be stored
1415
Enabled bool
1516
DefaultActionsURL string `ini:"DEFAULT_ACTIONS_URL"`
1617
}{
@@ -25,5 +26,9 @@ func loadActionsFrom(rootCfg ConfigProvider) {
2526
log.Fatal("Failed to map Actions settings: %v", err)
2627
}
2728

28-
Actions.Storage = getStorage(rootCfg, "actions_log", "", nil)
29+
actionsSec := rootCfg.Section("actions.artifacts")
30+
storageType := actionsSec.Key("STORAGE_TYPE").MustString("")
31+
32+
Actions.LogStorage = getStorage(rootCfg, "actions_log", "", nil)
33+
Actions.ArtifactStorage = getStorage(rootCfg, "actions_artifacts", storageType, actionsSec)
2934
}

modules/storage/storage.go

+9-2
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,8 @@ var (
128128

129129
// Actions represents actions storage
130130
Actions ObjectStorage = uninitializedStorage
131+
// Actions Artifacts represents actions artifacts storage
132+
ActionsArtifacts ObjectStorage = uninitializedStorage
131133
)
132134

133135
// Init init the stoarge
@@ -212,9 +214,14 @@ func initPackages() (err error) {
212214
func initActions() (err error) {
213215
if !setting.Actions.Enabled {
214216
Actions = discardStorage("Actions isn't enabled")
217+
ActionsArtifacts = discardStorage("ActionsArtifacts isn't enabled")
215218
return nil
216219
}
217-
log.Info("Initialising Actions storage with type: %s", setting.Actions.Storage.Type)
218-
Actions, err = NewStorage(setting.Actions.Storage.Type, &setting.Actions.Storage)
220+
log.Info("Initialising Actions storage with type: %s", setting.Actions.LogStorage.Type)
221+
if Actions, err = NewStorage(setting.Actions.LogStorage.Type, &setting.Actions.LogStorage); err != nil {
222+
return err
223+
}
224+
log.Info("Initialising ActionsArtifacts storage with type: %s", setting.Actions.ArtifactStorage.Type)
225+
ActionsArtifacts, err = NewStorage(setting.Actions.ArtifactStorage.Type, &setting.Actions.ArtifactStorage)
219226
return err
220227
}

options/locale/locale_en-US.ini

+2
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,8 @@ unknown = Unknown
114114

115115
rss_feed = RSS Feed
116116

117+
artifacts = Artifacts
118+
117119
concept_system_global = Global
118120
concept_user_individual = Individual
119121
concept_code_repository = Repository

0 commit comments

Comments
 (0)