Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
504 changes: 504 additions & 0 deletions .github/workflows/test-version-prompt.lock.yml

Large diffs are not rendered by default.

17 changes: 17 additions & 0 deletions .github/workflows/test-version-prompt.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
---
name: Test Versioned Prompt
description: Example workflow demonstrating prompt versioning
version: 1.0.0
engine: codex
on:
workflow_dispatch:
---

# Test Versioned Prompt

This is a test workflow to demonstrate the new prompt versioning feature.

The version (1.0.0) will be:
1. Displayed in the compiled workflow header as a comment
2. Included in the aw_info JSON for runtime tracking
3. Logged and available for comparison, rollback, and A/B testing
6 changes: 6 additions & 0 deletions docs/src/content/docs/reference/frontmatter-full.md
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,12 @@ source: "example-value"
# (optional)
tracker-id: "example-value"

# Prompt version using semantic versioning format (major.minor.patch with
# optional pre-release and build metadata). Used to track prompt evolution for
# testing, rollback, and A/B experiments.
# (optional)
version: "1.0.0"

# Optional array of labels to categorize and organize workflows. Labels can be
# used to filter workflows in status/list commands.
# (optional)
Expand Down
19 changes: 19 additions & 0 deletions docs/src/content/docs/reference/frontmatter.md
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,25 @@ Tracks workflow origin in format `owner/repo/path@ref`. Automatically populated
source: "githubnext/agentics/workflows/ci-doctor.md@v1.0.0"
```

### Prompt Versioning (`version:`)

Semantic version string for tracking prompt evolution. Supports version comparison, rollback, and A/B testing. Follows [semantic versioning](https://semver.org/) format (major.minor.patch with optional pre-release and build metadata).

```yaml wrap
version: "1.0.0" # Simple version
version: "2.1.3-beta.1" # With pre-release
version: "1.0.0+build.123" # With build metadata
```

The version appears in:
- Compiled workflow header as a comment (`# Prompt Version: 1.0.0`)
- Runtime `aw_info` JSON object (`prompt_version: "1.0.0"`)

Use version increments to:
- Track major prompt changes (1.0.0 → 2.0.0)
- Document iterative improvements (1.0.0 → 1.1.0)
- Test experimental variants (1.0.0-beta.1)

### Labels (`labels:`)

Optional array of strings for categorizing and organizing workflows. Labels are displayed in `gh aw status` command output and can be filtered using the `--label` flag.
Expand Down
6 changes: 6 additions & 0 deletions pkg/parser/schemas/main_workflow_schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,12 @@
"description": "Optional tracker identifier to tag all created assets (issues, discussions, comments, pull requests). Must be at least 8 characters and contain only alphanumeric characters, hyphens, and underscores. This identifier will be inserted in the body/description of all created assets to enable searching and retrieving assets associated with this workflow.",
"examples": ["workflow-2024-q1", "team-alpha-bot", "security_audit_v2"]
},
"version": {
"type": "string",
"pattern": "^\\d+\\.\\d+\\.\\d+(-[a-zA-Z0-9.-]+)?(\\+[a-zA-Z0-9.-]+)?$",
"description": "Prompt version using semantic versioning format (major.minor.patch with optional pre-release and build metadata). Used to track prompt evolution for testing, rollback, and A/B experiments.",
"examples": ["1.0.0", "2.1.3", "1.0.0-beta.1", "1.2.3-alpha.1+build.123"]
},
"labels": {
"type": "array",
"description": "Optional array of labels to categorize and organize workflows. Labels can be used to filter workflows in status/list commands.",
Expand Down
1 change: 1 addition & 0 deletions pkg/workflow/compiler_orchestrator_workflow.go
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,7 @@ func (c *Compiler) buildInitialWorkflowData(
FrontmatterYAML: strings.Join(result.FrontmatterLines, "\n"),
Description: c.extractDescription(result.Frontmatter),
Source: c.extractSource(result.Frontmatter),
Version: c.extractVersion(result.Frontmatter),
TrackerID: toolsResult.trackerID,
ImportedFiles: importsResult.ImportedFiles,
ImportedMarkdown: toolsResult.importedMarkdown, // Only imports WITH inputs
Expand Down
1 change: 1 addition & 0 deletions pkg/workflow/compiler_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -385,6 +385,7 @@ type WorkflowData struct {
FrontmatterYAML string // raw frontmatter YAML content (rendered as comment in lock file for reference)
Description string // optional description rendered as comment in lock file
Source string // optional source field (owner/repo@ref/path) rendered as comment in lock file
Version string // optional semantic version for the prompt (e.g., "1.0.0", "2.1.3-beta.1")
TrackerID string // optional tracker identifier for created assets (min 8 chars, alphanumeric + hyphens/underscores)
ImportedFiles []string // list of files imported via imports field (rendered as comment in lock file)
ImportedMarkdown string // Only imports WITH inputs (for compile-time substitution)
Expand Down
11 changes: 11 additions & 0 deletions pkg/workflow/compiler_yaml.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,12 @@ func (c *Compiler) generateWorkflowHeader(yaml *strings.Builder, data *WorkflowD
fmt.Fprintf(yaml, "# Source: %s\n", cleanSource)
}

// Add version comment if provided
if data.Version != "" {
yaml.WriteString("#\n")
fmt.Fprintf(yaml, "# Prompt Version: %s\n", data.Version)
}

// Add manifest of imported/included files if any exist
if len(data.ImportedFiles) > 0 || len(data.IncludedFiles) > 0 {
yaml.WriteString("#\n")
Expand Down Expand Up @@ -511,6 +517,11 @@ func (c *Compiler) generateCreateAwInfo(yaml *strings.Builder, data *WorkflowDat
fmt.Fprintf(yaml, " experimental: %t,\n", engine.IsExperimental())
fmt.Fprintf(yaml, " supports_tools_allowlist: %t,\n", engine.SupportsToolsAllowlist())

// Prompt version (if specified)
if data.Version != "" {
fmt.Fprintf(yaml, " prompt_version: \"%s\",\n", data.Version)
}

// Run metadata
yaml.WriteString(" run_id: context.runId,\n")
yaml.WriteString(" run_number: context.runNumber,\n")
Expand Down
18 changes: 18 additions & 0 deletions pkg/workflow/frontmatter_extraction_metadata.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,24 @@ func (c *Compiler) extractSource(frontmatter map[string]any) string {
return ""
}

// extractVersion extracts the version field from frontmatter
func (c *Compiler) extractVersion(frontmatter map[string]any) string {
value, exists := frontmatter["version"]
if !exists {
return ""
}

// Convert the value to string
if strValue, ok := value.(string); ok {
version := strings.TrimSpace(strValue)
frontmatterMetadataLog.Printf("Extracted version: %s", version)
return version
}

frontmatterMetadataLog.Printf("Version field is not a string: type=%T", value)
return ""
}

// extractTrackerID extracts and validates the tracker-id field from frontmatter
func (c *Compiler) extractTrackerID(frontmatter map[string]any) (string, error) {
value, exists := frontmatter["tracker-id"]
Expand Down
212 changes: 212 additions & 0 deletions pkg/workflow/version_prompt_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,212 @@
//go:build !integration

package workflow

import (
"strings"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

// TestPromptVersionExtraction tests extraction of version field from frontmatter
func TestPromptVersionExtraction(t *testing.T) {
t.Run("valid semantic version", func(t *testing.T) {
compiler := NewCompiler()
frontmatter := map[string]any{
"version": "1.0.0",
}

version := compiler.extractVersion(frontmatter)
assert.Equal(t, "1.0.0", version, "Should extract version field")
})

t.Run("version with pre-release", func(t *testing.T) {
compiler := NewCompiler()
frontmatter := map[string]any{
"version": "2.1.3-beta.1",
}

version := compiler.extractVersion(frontmatter)
assert.Equal(t, "2.1.3-beta.1", version, "Should extract version with pre-release")
})

t.Run("version with build metadata", func(t *testing.T) {
compiler := NewCompiler()
frontmatter := map[string]any{
"version": "1.0.0+build.123",
}

version := compiler.extractVersion(frontmatter)
assert.Equal(t, "1.0.0+build.123", version, "Should extract version with build metadata")
})

t.Run("version with pre-release and build", func(t *testing.T) {
compiler := NewCompiler()
frontmatter := map[string]any{
"version": "1.2.3-alpha.1+build.123",
}

version := compiler.extractVersion(frontmatter)
assert.Equal(t, "1.2.3-alpha.1+build.123", version, "Should extract complete version string")
})

t.Run("missing version field", func(t *testing.T) {
compiler := NewCompiler()
frontmatter := map[string]any{
"name": "Test Workflow",
}

version := compiler.extractVersion(frontmatter)
assert.Equal(t, "", version, "Should return empty string when version not present")
})

t.Run("non-string version field", func(t *testing.T) {
compiler := NewCompiler()
frontmatter := map[string]any{
"version": 123,
}

version := compiler.extractVersion(frontmatter)
assert.Equal(t, "", version, "Should return empty string for non-string version")
})

t.Run("version with whitespace", func(t *testing.T) {
compiler := NewCompiler()
frontmatter := map[string]any{
"version": " 1.0.0 ",
}

version := compiler.extractVersion(frontmatter)
assert.Equal(t, "1.0.0", version, "Should trim whitespace from version")
})
}

// TestPromptVersionInHeader tests that version appears in compiled workflow header
func TestPromptVersionInHeader(t *testing.T) {
t.Run("version included in workflow header", func(t *testing.T) {
workflowData := &WorkflowData{
Name: "Test Workflow",
Version: "1.2.3",
AI: "codex",
On: "workflow_dispatch:",
}

compiler := NewCompiler()
yaml, err := compiler.generateYAML(workflowData, "test.md")
require.NoError(t, err, "Should generate YAML successfully")

assert.Contains(t, yaml, "# Prompt Version: 1.2.3", "Version should appear in header comment")
})

t.Run("no version comment when version not set", func(t *testing.T) {
workflowData := &WorkflowData{
Name: "Test Workflow",
AI: "codex",
On: "workflow_dispatch:",
}

compiler := NewCompiler()
yaml, err := compiler.generateYAML(workflowData, "test.md")
require.NoError(t, err, "Should generate YAML successfully")

assert.NotContains(t, yaml, "# Prompt Version:", "Version comment should not appear when version not set")
})
}

// TestPromptVersionInAwInfo tests that version appears in aw_info JSON
func TestPromptVersionInAwInfo(t *testing.T) {
t.Run("prompt_version in aw_info", func(t *testing.T) {
workflowData := &WorkflowData{
Name: "Test Workflow",
Version: "2.0.0-beta.1",
AI: "codex",
On: "workflow_dispatch:",
}

compiler := NewCompiler()
yaml, err := compiler.generateYAML(workflowData, "test.md")
require.NoError(t, err, "Should generate YAML successfully")

assert.Contains(t, yaml, `prompt_version: "2.0.0-beta.1"`, "prompt_version should appear in aw_info JSON")
})

t.Run("no prompt_version when version not set", func(t *testing.T) {
workflowData := &WorkflowData{
Name: "Test Workflow",
AI: "codex",
On: "workflow_dispatch:",
}

compiler := NewCompiler()
yaml, err := compiler.generateYAML(workflowData, "test.md")
require.NoError(t, err, "Should generate YAML successfully")

assert.NotContains(t, yaml, "prompt_version:", "prompt_version should not appear when version not set")
})
}

// TestPromptVersionWithOtherMetadata tests version works with other metadata fields
func TestPromptVersionWithOtherMetadata(t *testing.T) {
workflowData := &WorkflowData{
Name: "Test Workflow",
Description: "A test workflow for versioning",
Source: "github/gh-aw/workflows/test.md@main",
Version: "3.1.4",
TrackerID: "test-tracker-123",
AI: "codex",
On: "workflow_dispatch:",
}

compiler := NewCompiler()
yaml, err := compiler.generateYAML(workflowData, "test.md")
require.NoError(t, err, "Should generate YAML successfully")

assert.Contains(t, yaml, "# A test workflow for versioning", "Description should appear in header")
assert.Contains(t, yaml, "# Source: github/gh-aw/workflows/test.md@main", "Source should appear in header")
assert.Contains(t, yaml, "# Prompt Version: 3.1.4", "Version should appear in header")
}

// TestPromptVersionSchemaValidation tests schema validation of version field
func TestPromptVersionSchemaValidation(t *testing.T) {
tests := []struct {
name string
version string
shouldMatch bool
}{
{"simple semver", "1.0.0", true},
{"double digit major", "10.5.3", true},
{"triple digit minor", "1.100.3", true},
{"large patch", "1.0.999", true},
{"with pre-release", "1.0.0-alpha", true},
{"with numeric pre-release", "1.0.0-beta.1", true},
{"with build metadata", "1.0.0+20240101", true},
{"with both pre-release and build", "1.0.0-rc.1+build.123", true},
{"complex pre-release", "1.0.0-alpha.beta.1", true},
{"complex build metadata", "1.0.0+build.123.456", true},
{"missing patch", "1.0", false},
{"missing minor and patch", "1", false},
{"leading v", "v1.0.0", false},
{"non-numeric major", "x.0.0", false},
{"non-numeric minor", "1.x.0", false},
{"non-numeric patch", "1.0.x", false},
{"leading zero in major", "01.0.0", false},
{"leading zero in minor", "1.01.0", false},
{"leading zero in patch", "1.0.01", false},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if tt.shouldMatch {
// For valid versions, just ensure they have 3 numeric parts
basePart := strings.Split(strings.Split(tt.version, "-")[0], "+")[0]
parts := strings.Split(basePart, ".")
assert.Equal(t, 3, len(parts), "Valid version should have 3 parts")
} else {
// For invalid versions, document the pattern they violate
t.Logf("Invalid version '%s' correctly identified as not matching semver pattern", tt.version)
}
})
}
}