c6512333aa
- Created 19 AI-discoverable skills in .github/skills/ for GitHub Copilot - Updated 13 VS Code tasks to use skill-runner.sh - Added validation and helper infrastructure scripts - Maintained backward compatibility with deprecation notices - All tests pass with 85%+ coverage, zero security issues Benefits: - Skills are auto-discovered by GitHub Copilot - Consistent execution interface across all tools - Self-documenting with comprehensive SKILL.md files - Progressive disclosure reduces context usage - CI/CD workflows can use standardized skill-runner Closes: (add issue number if applicable) BREAKING CHANGE: None - backward compatible with 1 release cycle deprecation period
423 lines
15 KiB
Python
Executable File
423 lines
15 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Agent Skills Frontmatter Validator
|
|
|
|
Validates YAML frontmatter in .SKILL.md files against the agentskills.io
|
|
specification. Ensures required fields are present, formats are correct,
|
|
and custom metadata follows project conventions.
|
|
|
|
Usage:
|
|
python3 validate-skills.py [path/to/.github/skills/]
|
|
python3 validate-skills.py --single path/to/skill.SKILL.md
|
|
|
|
Exit Codes:
|
|
0 - All validations passed
|
|
1 - Validation errors found
|
|
2 - Script error (missing dependencies, invalid arguments)
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import re
|
|
import argparse
|
|
from pathlib import Path
|
|
from typing import List, Dict, Tuple, Any, Optional
|
|
|
|
try:
|
|
import yaml
|
|
except ImportError:
|
|
print("Error: PyYAML is required. Install with: pip install pyyaml", file=sys.stderr)
|
|
sys.exit(2)
|
|
|
|
|
|
# Validation rules
|
|
REQUIRED_FIELDS = ["name", "version", "description", "author", "license", "tags"]
|
|
VALID_CATEGORIES = ["test", "integration-test", "security", "qa", "build", "utility", "docker"]
|
|
VALID_EXECUTION_TIMES = ["short", "medium", "long"]
|
|
VALID_RISK_LEVELS = ["low", "medium", "high"]
|
|
VALID_OS_VALUES = ["linux", "darwin", "windows"]
|
|
VALID_SHELL_VALUES = ["bash", "sh", "zsh", "powershell", "cmd"]
|
|
|
|
VERSION_REGEX = re.compile(r'^\d+\.\d+\.\d+$')
|
|
NAME_REGEX = re.compile(r'^[a-z][a-z0-9-]*$')
|
|
|
|
|
|
class ValidationError:
|
|
"""Represents a validation error with context."""
|
|
|
|
def __init__(self, skill_file: str, field: str, message: str, severity: str = "error"):
|
|
self.skill_file = skill_file
|
|
self.field = field
|
|
self.message = message
|
|
self.severity = severity
|
|
|
|
def __str__(self) -> str:
|
|
return f"[{self.severity.upper()}] {self.skill_file} :: {self.field}: {self.message}"
|
|
|
|
|
|
class SkillValidator:
|
|
"""Validates Agent Skills frontmatter."""
|
|
|
|
def __init__(self, strict: bool = False):
|
|
self.strict = strict
|
|
self.errors: List[ValidationError] = []
|
|
self.warnings: List[ValidationError] = []
|
|
|
|
def validate_file(self, skill_path: Path) -> Tuple[bool, List[ValidationError]]:
|
|
"""Validate a single SKILL.md file."""
|
|
try:
|
|
with open(skill_path, 'r', encoding='utf-8') as f:
|
|
content = f.read()
|
|
except Exception as e:
|
|
return False, [ValidationError(str(skill_path), "file", f"Cannot read file: {e}")]
|
|
|
|
# Extract frontmatter
|
|
frontmatter = self._extract_frontmatter(content)
|
|
if not frontmatter:
|
|
return False, [ValidationError(str(skill_path), "frontmatter", "No valid YAML frontmatter found")]
|
|
|
|
# Parse YAML
|
|
try:
|
|
data = yaml.safe_load(frontmatter)
|
|
except yaml.YAMLError as e:
|
|
return False, [ValidationError(str(skill_path), "yaml", f"Invalid YAML: {e}")]
|
|
|
|
if not isinstance(data, dict):
|
|
return False, [ValidationError(str(skill_path), "yaml", "Frontmatter must be a YAML object")]
|
|
|
|
# Run validation checks
|
|
file_errors: List[ValidationError] = []
|
|
file_errors.extend(self._validate_required_fields(skill_path, data))
|
|
file_errors.extend(self._validate_name(skill_path, data))
|
|
file_errors.extend(self._validate_version(skill_path, data))
|
|
file_errors.extend(self._validate_description(skill_path, data))
|
|
file_errors.extend(self._validate_tags(skill_path, data))
|
|
file_errors.extend(self._validate_compatibility(skill_path, data))
|
|
file_errors.extend(self._validate_metadata(skill_path, data))
|
|
|
|
# Separate errors and warnings
|
|
errors = [e for e in file_errors if e.severity == "error"]
|
|
warnings = [e for e in file_errors if e.severity == "warning"]
|
|
|
|
self.errors.extend(errors)
|
|
self.warnings.extend(warnings)
|
|
|
|
return len(errors) == 0, file_errors
|
|
|
|
def _extract_frontmatter(self, content: str) -> Optional[str]:
|
|
"""Extract YAML frontmatter from markdown content."""
|
|
if not content.startswith('---\n'):
|
|
return None
|
|
|
|
end_marker = content.find('\n---\n', 4)
|
|
if end_marker == -1:
|
|
return None
|
|
|
|
return content[4:end_marker]
|
|
|
|
def _validate_required_fields(self, skill_path: Path, data: Dict) -> List[ValidationError]:
|
|
"""Check that all required fields are present."""
|
|
errors = []
|
|
for field in REQUIRED_FIELDS:
|
|
if field not in data:
|
|
errors.append(ValidationError(
|
|
str(skill_path), field, f"Required field missing"
|
|
))
|
|
elif not data[field]:
|
|
errors.append(ValidationError(
|
|
str(skill_path), field, f"Required field is empty"
|
|
))
|
|
return errors
|
|
|
|
def _validate_name(self, skill_path: Path, data: Dict) -> List[ValidationError]:
|
|
"""Validate name field format."""
|
|
errors = []
|
|
if "name" in data:
|
|
name = data["name"]
|
|
if not isinstance(name, str):
|
|
errors.append(ValidationError(
|
|
str(skill_path), "name", "Must be a string"
|
|
))
|
|
elif not NAME_REGEX.match(name):
|
|
errors.append(ValidationError(
|
|
str(skill_path), "name",
|
|
"Must be kebab-case (lowercase, hyphens only, start with letter)"
|
|
))
|
|
return errors
|
|
|
|
def _validate_version(self, skill_path: Path, data: Dict) -> List[ValidationError]:
|
|
"""Validate version field format."""
|
|
errors = []
|
|
if "version" in data:
|
|
version = data["version"]
|
|
if not isinstance(version, str):
|
|
errors.append(ValidationError(
|
|
str(skill_path), "version", "Must be a string"
|
|
))
|
|
elif not VERSION_REGEX.match(version):
|
|
errors.append(ValidationError(
|
|
str(skill_path), "version",
|
|
"Must follow semantic versioning (x.y.z)"
|
|
))
|
|
return errors
|
|
|
|
def _validate_description(self, skill_path: Path, data: Dict) -> List[ValidationError]:
|
|
"""Validate description field."""
|
|
errors = []
|
|
if "description" in data:
|
|
desc = data["description"]
|
|
if not isinstance(desc, str):
|
|
errors.append(ValidationError(
|
|
str(skill_path), "description", "Must be a string"
|
|
))
|
|
elif len(desc) > 120:
|
|
errors.append(ValidationError(
|
|
str(skill_path), "description",
|
|
f"Must be 120 characters or less (current: {len(desc)})"
|
|
))
|
|
elif '\n' in desc:
|
|
errors.append(ValidationError(
|
|
str(skill_path), "description", "Must be a single line"
|
|
))
|
|
return errors
|
|
|
|
def _validate_tags(self, skill_path: Path, data: Dict) -> List[ValidationError]:
|
|
"""Validate tags field."""
|
|
errors = []
|
|
if "tags" in data:
|
|
tags = data["tags"]
|
|
if not isinstance(tags, list):
|
|
errors.append(ValidationError(
|
|
str(skill_path), "tags", "Must be a list"
|
|
))
|
|
elif len(tags) < 2:
|
|
errors.append(ValidationError(
|
|
str(skill_path), "tags", "Must have at least 2 tags"
|
|
))
|
|
elif len(tags) > 5:
|
|
errors.append(ValidationError(
|
|
str(skill_path), "tags",
|
|
f"Must have at most 5 tags (current: {len(tags)})",
|
|
severity="warning"
|
|
))
|
|
else:
|
|
for tag in tags:
|
|
if not isinstance(tag, str):
|
|
errors.append(ValidationError(
|
|
str(skill_path), "tags", "All tags must be strings"
|
|
))
|
|
elif tag != tag.lower():
|
|
errors.append(ValidationError(
|
|
str(skill_path), "tags",
|
|
f"Tag '{tag}' should be lowercase",
|
|
severity="warning"
|
|
))
|
|
return errors
|
|
|
|
def _validate_compatibility(self, skill_path: Path, data: Dict) -> List[ValidationError]:
|
|
"""Validate compatibility section."""
|
|
errors = []
|
|
if "compatibility" in data:
|
|
compat = data["compatibility"]
|
|
if not isinstance(compat, dict):
|
|
errors.append(ValidationError(
|
|
str(skill_path), "compatibility", "Must be an object"
|
|
))
|
|
else:
|
|
# Validate OS
|
|
if "os" in compat:
|
|
os_list = compat["os"]
|
|
if not isinstance(os_list, list):
|
|
errors.append(ValidationError(
|
|
str(skill_path), "compatibility.os", "Must be a list"
|
|
))
|
|
else:
|
|
for os_val in os_list:
|
|
if os_val not in VALID_OS_VALUES:
|
|
errors.append(ValidationError(
|
|
str(skill_path), "compatibility.os",
|
|
f"Invalid OS '{os_val}'. Valid: {VALID_OS_VALUES}",
|
|
severity="warning"
|
|
))
|
|
|
|
# Validate shells
|
|
if "shells" in compat:
|
|
shells = compat["shells"]
|
|
if not isinstance(shells, list):
|
|
errors.append(ValidationError(
|
|
str(skill_path), "compatibility.shells", "Must be a list"
|
|
))
|
|
else:
|
|
for shell in shells:
|
|
if shell not in VALID_SHELL_VALUES:
|
|
errors.append(ValidationError(
|
|
str(skill_path), "compatibility.shells",
|
|
f"Invalid shell '{shell}'. Valid: {VALID_SHELL_VALUES}",
|
|
severity="warning"
|
|
))
|
|
return errors
|
|
|
|
def _validate_metadata(self, skill_path: Path, data: Dict) -> List[ValidationError]:
|
|
"""Validate custom metadata section."""
|
|
errors = []
|
|
if "metadata" not in data:
|
|
return errors # Metadata is optional
|
|
|
|
metadata = data["metadata"]
|
|
if not isinstance(metadata, dict):
|
|
errors.append(ValidationError(
|
|
str(skill_path), "metadata", "Must be an object"
|
|
))
|
|
return errors
|
|
|
|
# Validate category
|
|
if "category" in metadata:
|
|
category = metadata["category"]
|
|
if category not in VALID_CATEGORIES:
|
|
errors.append(ValidationError(
|
|
str(skill_path), "metadata.category",
|
|
f"Invalid category '{category}'. Valid: {VALID_CATEGORIES}",
|
|
severity="warning"
|
|
))
|
|
|
|
# Validate execution_time
|
|
if "execution_time" in metadata:
|
|
exec_time = metadata["execution_time"]
|
|
if exec_time not in VALID_EXECUTION_TIMES:
|
|
errors.append(ValidationError(
|
|
str(skill_path), "metadata.execution_time",
|
|
f"Invalid execution_time '{exec_time}'. Valid: {VALID_EXECUTION_TIMES}",
|
|
severity="warning"
|
|
))
|
|
|
|
# Validate risk_level
|
|
if "risk_level" in metadata:
|
|
risk = metadata["risk_level"]
|
|
if risk not in VALID_RISK_LEVELS:
|
|
errors.append(ValidationError(
|
|
str(skill_path), "metadata.risk_level",
|
|
f"Invalid risk_level '{risk}'. Valid: {VALID_RISK_LEVELS}",
|
|
severity="warning"
|
|
))
|
|
|
|
# Validate boolean fields
|
|
for bool_field in ["ci_cd_safe", "requires_network", "idempotent"]:
|
|
if bool_field in metadata:
|
|
if not isinstance(metadata[bool_field], bool):
|
|
errors.append(ValidationError(
|
|
str(skill_path), f"metadata.{bool_field}",
|
|
"Must be a boolean (true/false)",
|
|
severity="warning"
|
|
))
|
|
|
|
return errors
|
|
|
|
def validate_directory(self, skills_dir: Path) -> bool:
|
|
"""Validate all SKILL.md files in a directory."""
|
|
if not skills_dir.exists():
|
|
print(f"Error: Directory not found: {skills_dir}", file=sys.stderr)
|
|
return False
|
|
|
|
skill_files = list(skills_dir.glob("*.SKILL.md"))
|
|
if not skill_files:
|
|
print(f"Warning: No .SKILL.md files found in {skills_dir}", file=sys.stderr)
|
|
return True # Not an error, just nothing to validate
|
|
|
|
print(f"Validating {len(skill_files)} skill(s)...\n")
|
|
|
|
success_count = 0
|
|
for skill_file in sorted(skill_files):
|
|
is_valid, _ = self.validate_file(skill_file)
|
|
if is_valid:
|
|
success_count += 1
|
|
print(f"✓ {skill_file.name}")
|
|
else:
|
|
print(f"✗ {skill_file.name}")
|
|
|
|
# Print summary
|
|
print(f"\n{'='*70}")
|
|
print(f"Validation Summary:")
|
|
print(f" Total skills: {len(skill_files)}")
|
|
print(f" Passed: {success_count}")
|
|
print(f" Failed: {len(skill_files) - success_count}")
|
|
print(f" Errors: {len(self.errors)}")
|
|
print(f" Warnings: {len(self.warnings)}")
|
|
print(f"{'='*70}\n")
|
|
|
|
# Print errors
|
|
if self.errors:
|
|
print("ERRORS:")
|
|
for error in self.errors:
|
|
print(f" {error}")
|
|
print()
|
|
|
|
# Print warnings
|
|
if self.warnings:
|
|
print("WARNINGS:")
|
|
for warning in self.warnings:
|
|
print(f" {warning}")
|
|
print()
|
|
|
|
return len(self.errors) == 0
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(
|
|
description="Validate Agent Skills frontmatter",
|
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
epilog=__doc__
|
|
)
|
|
parser.add_argument(
|
|
"path",
|
|
nargs="?",
|
|
default=".github/skills",
|
|
help="Path to .github/skills directory or single .SKILL.md file (default: .github/skills)"
|
|
)
|
|
parser.add_argument(
|
|
"--strict",
|
|
action="store_true",
|
|
help="Treat warnings as errors"
|
|
)
|
|
parser.add_argument(
|
|
"--single",
|
|
action="store_true",
|
|
help="Validate a single .SKILL.md file instead of a directory"
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
|
|
validator = SkillValidator(strict=args.strict)
|
|
path = Path(args.path)
|
|
|
|
if args.single:
|
|
if not path.exists():
|
|
print(f"Error: File not found: {path}", file=sys.stderr)
|
|
return 2
|
|
|
|
is_valid, errors = validator.validate_file(path)
|
|
|
|
if is_valid:
|
|
print(f"✓ {path.name} is valid")
|
|
if errors: # Warnings only
|
|
print("\nWARNINGS:")
|
|
for error in errors:
|
|
print(f" {error}")
|
|
else:
|
|
print(f"✗ {path.name} has errors")
|
|
for error in errors:
|
|
print(f" {error}")
|
|
|
|
return 0 if is_valid else 1
|
|
else:
|
|
success = validator.validate_directory(path)
|
|
|
|
if args.strict and validator.warnings:
|
|
print("Strict mode: treating warnings as errors", file=sys.stderr)
|
|
success = False
|
|
|
|
return 0 if success else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|