mirror of
https://github.com/github/spec-kit.git
synced 2026-03-17 02:43:08 +00:00
feat(extensions): implement multi-catalog stack support
- Add CatalogEntry dataclass to represent catalog entries - Add get_active_catalogs() reading SPECKIT_CATALOG_URL, project config, user config, or built-in default stack (org-approved + community) - Add _load_catalog_config() to parse .specify/extension-catalogs.yml - Add _validate_catalog_url() HTTPS validation helper - Add _fetch_single_catalog() with per-URL caching, backward-compat for DEFAULT_CATALOG_URL - Add _get_merged_extensions() that merges all catalogs (priority wins on conflict) - Update search() and get_extension_info() to use merged results annotated with _catalog_name and _install_allowed - Update clear_cache() to also remove per-URL hash cache files - Add extension_catalogs CLI command to list active catalogs - Add catalog add/remove sub-commands for .specify/extension-catalogs.yml - Update extension_add to enforce install_allowed=false policy - Update extension_search to show source catalog per result - Update extension_info to show source catalog with install_allowed status - Add 13 new tests covering catalog stack, merge conflict resolution, install_allowed enforcement, and catalog metadata Co-authored-by: mnriem <15701806+mnriem@users.noreply.github.com>
This commit is contained in:
committed by
Manfred Riem
parent
623cd79b38
commit
50c605ed5f
@@ -1772,6 +1772,13 @@ extension_app = typer.Typer(
|
||||
)
|
||||
app.add_typer(extension_app, name="extension")
|
||||
|
||||
catalog_app = typer.Typer(
|
||||
name="catalog",
|
||||
help="Manage extension catalogs",
|
||||
add_completion=False,
|
||||
)
|
||||
extension_app.add_typer(catalog_app, name="catalog")
|
||||
|
||||
|
||||
def get_speckit_version() -> str:
|
||||
"""Get current spec-kit version."""
|
||||
@@ -1837,6 +1844,155 @@ def extension_list(
|
||||
console.print(" [cyan]specify extension add <name>[/cyan]")
|
||||
|
||||
|
||||
@extension_app.command("catalogs")
|
||||
def extension_catalogs():
|
||||
"""List all active extension catalogs."""
|
||||
from .extensions import ExtensionCatalog, ValidationError
|
||||
|
||||
project_root = Path.cwd()
|
||||
|
||||
specify_dir = project_root / ".specify"
|
||||
if not specify_dir.exists():
|
||||
console.print("[red]Error:[/red] Not a spec-kit project (no .specify/ directory)")
|
||||
console.print("Run this command from a spec-kit project root")
|
||||
raise typer.Exit(1)
|
||||
|
||||
catalog = ExtensionCatalog(project_root)
|
||||
|
||||
try:
|
||||
active_catalogs = catalog.get_active_catalogs()
|
||||
except ValidationError as e:
|
||||
console.print(f"[red]Error:[/red] {e}")
|
||||
raise typer.Exit(1)
|
||||
|
||||
console.print("\n[bold cyan]Active Extension Catalogs:[/bold cyan]\n")
|
||||
for entry in active_catalogs:
|
||||
install_str = (
|
||||
"[green]install allowed[/green]"
|
||||
if entry.install_allowed
|
||||
else "[yellow]discovery only[/yellow]"
|
||||
)
|
||||
console.print(f" [bold]{entry.name}[/bold] (priority {entry.priority})")
|
||||
console.print(f" URL: {entry.url}")
|
||||
console.print(f" Install: {install_str}")
|
||||
console.print()
|
||||
|
||||
config_path = project_root / ".specify" / "extension-catalogs.yml"
|
||||
if config_path.exists():
|
||||
console.print(f"[dim]Config: {config_path.relative_to(project_root)}[/dim]")
|
||||
else:
|
||||
console.print("[dim]Using built-in default catalog stack.[/dim]")
|
||||
console.print(
|
||||
"[dim]Add .specify/extension-catalogs.yml to customize.[/dim]"
|
||||
)
|
||||
|
||||
|
||||
@catalog_app.command("add")
|
||||
def catalog_add(
|
||||
url: str = typer.Argument(help="Catalog URL (must use HTTPS)"),
|
||||
name: str = typer.Option(..., "--name", help="Catalog name"),
|
||||
priority: int = typer.Option(10, "--priority", help="Priority (lower = higher priority)"),
|
||||
install_allowed: bool = typer.Option(
|
||||
False, "--install-allowed/--no-install-allowed",
|
||||
help="Allow extensions from this catalog to be installed",
|
||||
),
|
||||
):
|
||||
"""Add a catalog to .specify/extension-catalogs.yml."""
|
||||
from .extensions import ExtensionCatalog, ValidationError
|
||||
|
||||
project_root = Path.cwd()
|
||||
|
||||
specify_dir = project_root / ".specify"
|
||||
if not specify_dir.exists():
|
||||
console.print("[red]Error:[/red] Not a spec-kit project (no .specify/ directory)")
|
||||
console.print("Run this command from a spec-kit project root")
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Validate URL
|
||||
tmp_catalog = ExtensionCatalog(project_root)
|
||||
try:
|
||||
tmp_catalog._validate_catalog_url(url)
|
||||
except ValidationError as e:
|
||||
console.print(f"[red]Error:[/red] {e}")
|
||||
raise typer.Exit(1)
|
||||
|
||||
config_path = specify_dir / "extension-catalogs.yml"
|
||||
|
||||
# Load existing config
|
||||
if config_path.exists():
|
||||
try:
|
||||
config = yaml.safe_load(config_path.read_text()) or {}
|
||||
except Exception:
|
||||
config = {}
|
||||
else:
|
||||
config = {}
|
||||
|
||||
catalogs = config.get("catalogs", [])
|
||||
|
||||
# Check for duplicate name
|
||||
for existing in catalogs:
|
||||
if existing.get("name") == name:
|
||||
console.print(f"[yellow]Warning:[/yellow] A catalog named '{name}' already exists.")
|
||||
console.print("Use 'specify extension catalog remove' first, or choose a different name.")
|
||||
raise typer.Exit(1)
|
||||
|
||||
catalogs.append({
|
||||
"name": name,
|
||||
"url": url,
|
||||
"priority": priority,
|
||||
"install_allowed": install_allowed,
|
||||
})
|
||||
|
||||
config["catalogs"] = catalogs
|
||||
config_path.write_text(yaml.dump(config, default_flow_style=False, sort_keys=False))
|
||||
|
||||
install_label = "install allowed" if install_allowed else "discovery only"
|
||||
console.print(f"\n[green]✓[/green] Added catalog '[bold]{name}[/bold]' ({install_label})")
|
||||
console.print(f" URL: {url}")
|
||||
console.print(f" Priority: {priority}")
|
||||
console.print(f"\nConfig saved to {config_path.relative_to(project_root)}")
|
||||
|
||||
|
||||
@catalog_app.command("remove")
|
||||
def catalog_remove(
|
||||
name: str = typer.Argument(help="Catalog name to remove"),
|
||||
):
|
||||
"""Remove a catalog from .specify/extension-catalogs.yml."""
|
||||
project_root = Path.cwd()
|
||||
|
||||
specify_dir = project_root / ".specify"
|
||||
if not specify_dir.exists():
|
||||
console.print("[red]Error:[/red] Not a spec-kit project (no .specify/ directory)")
|
||||
console.print("Run this command from a spec-kit project root")
|
||||
raise typer.Exit(1)
|
||||
|
||||
config_path = specify_dir / "extension-catalogs.yml"
|
||||
if not config_path.exists():
|
||||
console.print(f"[red]Error:[/red] No catalog config found. Nothing to remove.")
|
||||
raise typer.Exit(1)
|
||||
|
||||
try:
|
||||
config = yaml.safe_load(config_path.read_text()) or {}
|
||||
except Exception:
|
||||
console.print("[red]Error:[/red] Failed to read catalog config.")
|
||||
raise typer.Exit(1)
|
||||
|
||||
catalogs = config.get("catalogs", [])
|
||||
original_count = len(catalogs)
|
||||
catalogs = [c for c in catalogs if c.get("name") != name]
|
||||
|
||||
if len(catalogs) == original_count:
|
||||
console.print(f"[red]Error:[/red] Catalog '{name}' not found.")
|
||||
raise typer.Exit(1)
|
||||
|
||||
config["catalogs"] = catalogs
|
||||
config_path.write_text(yaml.dump(config, default_flow_style=False, sort_keys=False))
|
||||
|
||||
console.print(f"[green]✓[/green] Removed catalog '{name}'")
|
||||
if not catalogs:
|
||||
console.print("\n[dim]No catalogs remain in config. Built-in defaults will be used.[/dim]")
|
||||
|
||||
|
||||
@extension_app.command("add")
|
||||
def extension_add(
|
||||
extension: str = typer.Argument(help="Extension name or path"),
|
||||
@@ -1925,6 +2081,19 @@ def extension_add(
|
||||
console.print(" specify extension search")
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Enforce install_allowed policy
|
||||
if not ext_info.get("_install_allowed", True):
|
||||
catalog_name = ext_info.get("_catalog_name", "community")
|
||||
console.print(
|
||||
f"[red]Error:[/red] '{extension}' is available in the "
|
||||
f"'{catalog_name}' catalog but installation is not allowed from that catalog."
|
||||
)
|
||||
console.print(
|
||||
f"\nTo enable installation, add '{extension}' to an approved catalog "
|
||||
f"(install_allowed: true) in .specify/extension-catalogs.yml."
|
||||
)
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Download extension ZIP
|
||||
console.print(f"Downloading {ext_info['name']} v{ext_info.get('version', 'unknown')}...")
|
||||
zip_path = catalog.download_extension(extension)
|
||||
@@ -2069,6 +2238,15 @@ def extension_search(
|
||||
tags_str = ", ".join(ext['tags'])
|
||||
console.print(f" [dim]Tags:[/dim] {tags_str}")
|
||||
|
||||
# Source catalog
|
||||
catalog_name = ext.get("_catalog_name", "")
|
||||
install_allowed = ext.get("_install_allowed", True)
|
||||
if catalog_name:
|
||||
if install_allowed:
|
||||
console.print(f" [dim]Catalog:[/dim] {catalog_name}")
|
||||
else:
|
||||
console.print(f" [dim]Catalog:[/dim] {catalog_name} [yellow](discovery only — not installable)[/yellow]")
|
||||
|
||||
# Stats
|
||||
stats = []
|
||||
if ext.get('downloads') is not None:
|
||||
@@ -2082,8 +2260,15 @@ def extension_search(
|
||||
if ext.get('repository'):
|
||||
console.print(f" [dim]Repository:[/dim] {ext['repository']}")
|
||||
|
||||
# Install command
|
||||
# Install command (show warning if not installable)
|
||||
if install_allowed:
|
||||
console.print(f"\n [cyan]Install:[/cyan] specify extension add {ext['id']}")
|
||||
else:
|
||||
console.print(f"\n [yellow]⚠[/yellow] Not directly installable from '{catalog_name}'.")
|
||||
console.print(
|
||||
f" Add to an approved catalog with install_allowed: true, "
|
||||
f"or use: specify extension add --from <url>"
|
||||
)
|
||||
console.print()
|
||||
|
||||
except ExtensionError as e:
|
||||
@@ -2132,6 +2317,12 @@ def extension_info(
|
||||
# Author and License
|
||||
console.print(f"[dim]Author:[/dim] {ext_info.get('author', 'Unknown')}")
|
||||
console.print(f"[dim]License:[/dim] {ext_info.get('license', 'Unknown')}")
|
||||
|
||||
# Source catalog
|
||||
if ext_info.get("_catalog_name"):
|
||||
install_allowed = ext_info.get("_install_allowed", True)
|
||||
install_note = "" if install_allowed else " [yellow](discovery only)[/yellow]"
|
||||
console.print(f"[dim]Source catalog:[/dim] {ext_info['_catalog_name']}{install_note}")
|
||||
console.print()
|
||||
|
||||
# Requirements
|
||||
@@ -2188,12 +2379,21 @@ def extension_info(
|
||||
|
||||
# Installation status and command
|
||||
is_installed = manager.registry.is_installed(ext_info['id'])
|
||||
install_allowed = ext_info.get("_install_allowed", True)
|
||||
if is_installed:
|
||||
console.print("[green]✓ Installed[/green]")
|
||||
console.print(f"\nTo remove: specify extension remove {ext_info['id']}")
|
||||
else:
|
||||
elif install_allowed:
|
||||
console.print("[yellow]Not installed[/yellow]")
|
||||
console.print(f"\n[cyan]Install:[/cyan] specify extension add {ext_info['id']}")
|
||||
else:
|
||||
catalog_name = ext_info.get("_catalog_name", "community")
|
||||
console.print("[yellow]Not installed[/yellow]")
|
||||
console.print(
|
||||
f"\n[yellow]⚠[/yellow] '{ext_info['id']}' is available in the '{catalog_name}' catalog "
|
||||
f"but not in your approved catalog. Add it to .specify/extension-catalogs.yml "
|
||||
f"with install_allowed: true to enable installation."
|
||||
)
|
||||
|
||||
except ExtensionError as e:
|
||||
console.print(f"\n[red]Error:[/red] {e}")
|
||||
|
||||
@@ -8,9 +8,11 @@ without bloating the core framework.
|
||||
|
||||
import json
|
||||
import hashlib
|
||||
import os
|
||||
import tempfile
|
||||
import zipfile
|
||||
import shutil
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Optional, Dict, List, Any
|
||||
from datetime import datetime, timezone
|
||||
@@ -36,6 +38,15 @@ class CompatibilityError(ExtensionError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass
|
||||
class CatalogEntry:
|
||||
"""Represents a single catalog entry in the catalog stack."""
|
||||
url: str
|
||||
name: str
|
||||
priority: int
|
||||
install_allowed: bool
|
||||
|
||||
|
||||
class ExtensionManifest:
|
||||
"""Represents and validates an extension manifest (extension.yml)."""
|
||||
|
||||
@@ -976,6 +987,7 @@ class ExtensionCatalog:
|
||||
"""Manages extension catalog fetching, caching, and searching."""
|
||||
|
||||
DEFAULT_CATALOG_URL = "https://raw.githubusercontent.com/github/spec-kit/main/extensions/catalog.json"
|
||||
COMMUNITY_CATALOG_URL = "https://raw.githubusercontent.com/github/spec-kit/main/extensions/catalog.community.json"
|
||||
CACHE_DURATION = 3600 # 1 hour in seconds
|
||||
|
||||
def __init__(self, project_root: Path):
|
||||
@@ -990,43 +1002,82 @@ class ExtensionCatalog:
|
||||
self.cache_file = self.cache_dir / "catalog.json"
|
||||
self.cache_metadata_file = self.cache_dir / "catalog-metadata.json"
|
||||
|
||||
def get_catalog_url(self) -> str:
|
||||
"""Get catalog URL from config or use default.
|
||||
def _validate_catalog_url(self, url: str) -> None:
|
||||
"""Validate that a catalog URL uses HTTPS (localhost HTTP allowed).
|
||||
|
||||
Checks in order:
|
||||
1. SPECKIT_CATALOG_URL environment variable
|
||||
2. Default catalog URL
|
||||
|
||||
Returns:
|
||||
URL to fetch catalog from
|
||||
Args:
|
||||
url: URL to validate
|
||||
|
||||
Raises:
|
||||
ValidationError: If custom URL is invalid (non-HTTPS)
|
||||
ValidationError: If URL is invalid or uses non-HTTPS scheme
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
from urllib.parse import urlparse
|
||||
|
||||
# Environment variable override (useful for testing)
|
||||
if env_value := os.environ.get("SPECKIT_CATALOG_URL"):
|
||||
catalog_url = env_value.strip()
|
||||
parsed = urlparse(catalog_url)
|
||||
|
||||
# Require HTTPS for security (prevent man-in-the-middle attacks)
|
||||
# Allow http://localhost for local development/testing
|
||||
parsed = urlparse(url)
|
||||
is_localhost = parsed.hostname in ("localhost", "127.0.0.1", "::1")
|
||||
if parsed.scheme != "https" and not (parsed.scheme == "http" and is_localhost):
|
||||
raise ValidationError(
|
||||
f"Invalid SPECKIT_CATALOG_URL: must use HTTPS (got {parsed.scheme}://). "
|
||||
f"Catalog URL must use HTTPS (got {parsed.scheme}://). "
|
||||
"HTTP is only allowed for localhost."
|
||||
)
|
||||
|
||||
if not parsed.netloc:
|
||||
raise ValidationError(
|
||||
"Invalid SPECKIT_CATALOG_URL: must be a valid URL with a host."
|
||||
)
|
||||
raise ValidationError("Catalog URL must be a valid URL with a host.")
|
||||
|
||||
# Warn users when using a non-default catalog (once per instance)
|
||||
def _load_catalog_config(self, config_path: Path) -> Optional[List[CatalogEntry]]:
|
||||
"""Load catalog stack configuration from a YAML file.
|
||||
|
||||
Args:
|
||||
config_path: Path to extension-catalogs.yml
|
||||
|
||||
Returns:
|
||||
Ordered list of CatalogEntry objects, or None if file doesn't exist
|
||||
or contains no valid catalog entries.
|
||||
"""
|
||||
if not config_path.exists():
|
||||
return None
|
||||
try:
|
||||
data = yaml.safe_load(config_path.read_text()) or {}
|
||||
catalogs_data = data.get("catalogs", [])
|
||||
if not catalogs_data:
|
||||
return None
|
||||
entries: List[CatalogEntry] = []
|
||||
for idx, item in enumerate(catalogs_data):
|
||||
url = str(item.get("url", "")).strip()
|
||||
if not url:
|
||||
continue
|
||||
self._validate_catalog_url(url)
|
||||
entries.append(CatalogEntry(
|
||||
url=url,
|
||||
name=str(item.get("name", f"catalog-{idx + 1}")),
|
||||
priority=int(item.get("priority", idx + 1)),
|
||||
install_allowed=bool(item.get("install_allowed", True)),
|
||||
))
|
||||
entries.sort(key=lambda e: e.priority)
|
||||
return entries if entries else None
|
||||
except (yaml.YAMLError, OSError):
|
||||
return None
|
||||
|
||||
def get_active_catalogs(self) -> List[CatalogEntry]:
|
||||
"""Get the ordered list of active catalogs.
|
||||
|
||||
Resolution order:
|
||||
1. SPECKIT_CATALOG_URL env var — single catalog replacing all defaults
|
||||
2. Project-level .specify/extension-catalogs.yml
|
||||
3. User-level ~/.specify/extension-catalogs.yml
|
||||
4. Built-in default stack (org-approved + community)
|
||||
|
||||
Returns:
|
||||
List of CatalogEntry objects sorted by priority (ascending)
|
||||
|
||||
Raises:
|
||||
ValidationError: If a catalog URL is invalid
|
||||
"""
|
||||
import sys
|
||||
|
||||
# 1. SPECKIT_CATALOG_URL env var replaces all defaults for backward compat
|
||||
if env_value := os.environ.get("SPECKIT_CATALOG_URL"):
|
||||
catalog_url = env_value.strip()
|
||||
self._validate_catalog_url(catalog_url)
|
||||
if catalog_url != self.DEFAULT_CATALOG_URL:
|
||||
if not getattr(self, "_non_default_catalog_warning_shown", False):
|
||||
print(
|
||||
@@ -1035,11 +1086,160 @@ class ExtensionCatalog:
|
||||
file=sys.stderr,
|
||||
)
|
||||
self._non_default_catalog_warning_shown = True
|
||||
return [CatalogEntry(url=catalog_url, name="custom", priority=1, install_allowed=True)]
|
||||
|
||||
return catalog_url
|
||||
# 2. Project-level config overrides all defaults
|
||||
project_config_path = self.project_root / ".specify" / "extension-catalogs.yml"
|
||||
catalogs = self._load_catalog_config(project_config_path)
|
||||
if catalogs is not None:
|
||||
return catalogs
|
||||
|
||||
# TODO: Support custom catalogs from .specify/extension-catalogs.yml
|
||||
return self.DEFAULT_CATALOG_URL
|
||||
# 3. User-level config
|
||||
user_config_path = Path.home() / ".specify" / "extension-catalogs.yml"
|
||||
catalogs = self._load_catalog_config(user_config_path)
|
||||
if catalogs is not None:
|
||||
return catalogs
|
||||
|
||||
# 4. Built-in default stack
|
||||
return [
|
||||
CatalogEntry(url=self.DEFAULT_CATALOG_URL, name="org-approved", priority=1, install_allowed=True),
|
||||
CatalogEntry(url=self.COMMUNITY_CATALOG_URL, name="community", priority=2, install_allowed=False),
|
||||
]
|
||||
|
||||
def get_catalog_url(self) -> str:
|
||||
"""Get the primary catalog URL.
|
||||
|
||||
Returns the URL of the highest-priority catalog. Kept for backward
|
||||
compatibility. Use get_active_catalogs() for full multi-catalog support.
|
||||
|
||||
Returns:
|
||||
URL of the primary catalog
|
||||
|
||||
Raises:
|
||||
ValidationError: If a catalog URL is invalid
|
||||
"""
|
||||
active = self.get_active_catalogs()
|
||||
return active[0].url if active else self.DEFAULT_CATALOG_URL
|
||||
|
||||
def _fetch_single_catalog(self, entry: CatalogEntry, force_refresh: bool = False) -> Dict[str, Any]:
|
||||
"""Fetch a single catalog with per-URL caching.
|
||||
|
||||
For the DEFAULT_CATALOG_URL, uses legacy cache files (self.cache_file /
|
||||
self.cache_metadata_file) for backward compatibility. For all other URLs,
|
||||
uses URL-hash-based cache files in self.cache_dir.
|
||||
|
||||
Args:
|
||||
entry: CatalogEntry describing the catalog to fetch
|
||||
force_refresh: If True, bypass cache
|
||||
|
||||
Returns:
|
||||
Catalog data dictionary
|
||||
|
||||
Raises:
|
||||
ExtensionError: If catalog cannot be fetched or has invalid format
|
||||
"""
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
|
||||
# Determine cache file paths (backward compat for default catalog)
|
||||
if entry.url == self.DEFAULT_CATALOG_URL:
|
||||
cache_file = self.cache_file
|
||||
cache_meta_file = self.cache_metadata_file
|
||||
is_valid = not force_refresh and self.is_cache_valid()
|
||||
else:
|
||||
url_hash = hashlib.sha256(entry.url.encode()).hexdigest()[:16]
|
||||
cache_file = self.cache_dir / f"catalog-{url_hash}.json"
|
||||
cache_meta_file = self.cache_dir / f"catalog-{url_hash}-metadata.json"
|
||||
is_valid = False
|
||||
if not force_refresh and cache_file.exists() and cache_meta_file.exists():
|
||||
try:
|
||||
metadata = json.loads(cache_meta_file.read_text())
|
||||
cached_at = datetime.fromisoformat(metadata.get("cached_at", ""))
|
||||
age = (datetime.now(timezone.utc) - cached_at).total_seconds()
|
||||
is_valid = age < self.CACHE_DURATION
|
||||
except (json.JSONDecodeError, ValueError, KeyError):
|
||||
pass
|
||||
|
||||
# Use cache if valid
|
||||
if is_valid:
|
||||
try:
|
||||
return json.loads(cache_file.read_text())
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
# Fetch from network
|
||||
try:
|
||||
with urllib.request.urlopen(entry.url, timeout=10) as response:
|
||||
catalog_data = json.loads(response.read())
|
||||
|
||||
if "schema_version" not in catalog_data or "extensions" not in catalog_data:
|
||||
raise ExtensionError(f"Invalid catalog format from {entry.url}")
|
||||
|
||||
# Save to cache
|
||||
self.cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
cache_file.write_text(json.dumps(catalog_data, indent=2))
|
||||
cache_meta_file.write_text(json.dumps({
|
||||
"cached_at": datetime.now(timezone.utc).isoformat(),
|
||||
"catalog_url": entry.url,
|
||||
}, indent=2))
|
||||
|
||||
return catalog_data
|
||||
|
||||
except urllib.error.URLError as e:
|
||||
raise ExtensionError(f"Failed to fetch catalog from {entry.url}: {e}")
|
||||
except json.JSONDecodeError as e:
|
||||
raise ExtensionError(f"Invalid JSON in catalog from {entry.url}: {e}")
|
||||
|
||||
def _get_merged_extensions(self, force_refresh: bool = False) -> List[Dict[str, Any]]:
|
||||
"""Fetch and merge extensions from all active catalogs.
|
||||
|
||||
Higher-priority (lower priority number) catalogs win on conflicts
|
||||
(same extension id in two catalogs). Each extension dict is annotated with:
|
||||
- _catalog_name: name of the source catalog
|
||||
- _install_allowed: whether installation is allowed from this catalog
|
||||
|
||||
Catalogs that fail to fetch are skipped. Raises ExtensionError only if
|
||||
ALL catalogs fail.
|
||||
|
||||
Args:
|
||||
force_refresh: If True, bypass all caches
|
||||
|
||||
Returns:
|
||||
List of merged extension dicts
|
||||
|
||||
Raises:
|
||||
ExtensionError: If all catalogs fail to fetch
|
||||
"""
|
||||
import sys
|
||||
|
||||
active_catalogs = self.get_active_catalogs()
|
||||
merged: Dict[str, Dict[str, Any]] = {}
|
||||
any_success = False
|
||||
|
||||
for catalog_entry in active_catalogs:
|
||||
try:
|
||||
catalog_data = self._fetch_single_catalog(catalog_entry, force_refresh)
|
||||
any_success = True
|
||||
except ExtensionError as e:
|
||||
print(
|
||||
f"Warning: Could not fetch catalog '{catalog_entry.name}': {e}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
continue
|
||||
|
||||
for ext_id, ext_data in catalog_data.get("extensions", {}).items():
|
||||
if ext_id not in merged: # Higher-priority catalog wins
|
||||
merged[ext_id] = {
|
||||
"id": ext_id,
|
||||
**ext_data,
|
||||
"_catalog_name": catalog_entry.name,
|
||||
"_install_allowed": catalog_entry.install_allowed,
|
||||
}
|
||||
|
||||
if not any_success and active_catalogs:
|
||||
raise ExtensionError("Failed to fetch any extension catalog")
|
||||
|
||||
return list(merged.values())
|
||||
|
||||
def is_cache_valid(self) -> bool:
|
||||
"""Check if cached catalog is still valid.
|
||||
@@ -1116,7 +1316,7 @@ class ExtensionCatalog:
|
||||
author: Optional[str] = None,
|
||||
verified_only: bool = False,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Search catalog for extensions.
|
||||
"""Search catalog for extensions across all active catalogs.
|
||||
|
||||
Args:
|
||||
query: Search query (searches name, description, tags)
|
||||
@@ -1125,14 +1325,16 @@ class ExtensionCatalog:
|
||||
verified_only: If True, show only verified extensions
|
||||
|
||||
Returns:
|
||||
List of matching extension metadata
|
||||
List of matching extension metadata, each annotated with
|
||||
``_catalog_name`` and ``_install_allowed`` from its source catalog.
|
||||
"""
|
||||
catalog = self.fetch_catalog()
|
||||
extensions = catalog.get("extensions", {})
|
||||
all_extensions = self._get_merged_extensions()
|
||||
|
||||
results = []
|
||||
|
||||
for ext_id, ext_data in extensions.items():
|
||||
for ext_data in all_extensions:
|
||||
ext_id = ext_data["id"]
|
||||
|
||||
# Apply filters
|
||||
if verified_only and not ext_data.get("verified", False):
|
||||
continue
|
||||
@@ -1158,25 +1360,26 @@ class ExtensionCatalog:
|
||||
if query_lower not in searchable_text:
|
||||
continue
|
||||
|
||||
results.append({"id": ext_id, **ext_data})
|
||||
results.append(ext_data)
|
||||
|
||||
return results
|
||||
|
||||
def get_extension_info(self, extension_id: str) -> Optional[Dict[str, Any]]:
|
||||
"""Get detailed information about a specific extension.
|
||||
|
||||
Searches all active catalogs in priority order.
|
||||
|
||||
Args:
|
||||
extension_id: ID of the extension
|
||||
|
||||
Returns:
|
||||
Extension metadata or None if not found
|
||||
Extension metadata (annotated with ``_catalog_name`` and
|
||||
``_install_allowed``) or None if not found.
|
||||
"""
|
||||
catalog = self.fetch_catalog()
|
||||
extensions = catalog.get("extensions", {})
|
||||
|
||||
if extension_id in extensions:
|
||||
return {"id": extension_id, **extensions[extension_id]}
|
||||
|
||||
all_extensions = self._get_merged_extensions()
|
||||
for ext_data in all_extensions:
|
||||
if ext_data["id"] == extension_id:
|
||||
return ext_data
|
||||
return None
|
||||
|
||||
def download_extension(self, extension_id: str, target_dir: Optional[Path] = None) -> Path:
|
||||
@@ -1236,11 +1439,18 @@ class ExtensionCatalog:
|
||||
raise ExtensionError(f"Failed to save extension ZIP: {e}")
|
||||
|
||||
def clear_cache(self):
|
||||
"""Clear the catalog cache."""
|
||||
"""Clear the catalog cache (both legacy and URL-hash-based files)."""
|
||||
if self.cache_file.exists():
|
||||
self.cache_file.unlink()
|
||||
if self.cache_metadata_file.exists():
|
||||
self.cache_metadata_file.unlink()
|
||||
# Also clear any per-URL hash-based cache files
|
||||
if self.cache_dir.exists():
|
||||
for extra_cache in self.cache_dir.glob("catalog-*.json"):
|
||||
if extra_cache != self.cache_file:
|
||||
extra_cache.unlink(missing_ok=True)
|
||||
for extra_meta in self.cache_dir.glob("catalog-*-metadata.json"):
|
||||
extra_meta.unlink(missing_ok=True)
|
||||
|
||||
|
||||
class ConfigManager:
|
||||
|
||||
@@ -6,6 +6,7 @@ Tests cover:
|
||||
- Extension registry operations
|
||||
- Extension manager installation/removal
|
||||
- Command registration
|
||||
- Catalog stack (multi-catalog support)
|
||||
"""
|
||||
|
||||
import pytest
|
||||
@@ -16,6 +17,7 @@ from pathlib import Path
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from specify_cli.extensions import (
|
||||
CatalogEntry,
|
||||
ExtensionManifest,
|
||||
ExtensionRegistry,
|
||||
ExtensionManager,
|
||||
@@ -880,10 +882,29 @@ class TestExtensionCatalog:
|
||||
|
||||
def test_search_all_extensions(self, temp_dir):
|
||||
"""Test searching all extensions without filters."""
|
||||
import yaml as yaml_module
|
||||
|
||||
project_dir = temp_dir / "project"
|
||||
project_dir.mkdir()
|
||||
(project_dir / ".specify").mkdir()
|
||||
|
||||
# Use a single-catalog config so community extensions don't interfere
|
||||
config_path = project_dir / ".specify" / "extension-catalogs.yml"
|
||||
with open(config_path, "w") as f:
|
||||
yaml_module.dump(
|
||||
{
|
||||
"catalogs": [
|
||||
{
|
||||
"name": "test-catalog",
|
||||
"url": ExtensionCatalog.DEFAULT_CATALOG_URL,
|
||||
"priority": 1,
|
||||
"install_allowed": True,
|
||||
}
|
||||
]
|
||||
},
|
||||
f,
|
||||
)
|
||||
|
||||
catalog = ExtensionCatalog(project_dir)
|
||||
|
||||
# Create mock catalog
|
||||
@@ -1133,3 +1154,373 @@ class TestExtensionCatalog:
|
||||
|
||||
assert not catalog.cache_file.exists()
|
||||
assert not catalog.cache_metadata_file.exists()
|
||||
|
||||
|
||||
# ===== CatalogEntry Tests =====
|
||||
|
||||
class TestCatalogEntry:
|
||||
"""Test CatalogEntry dataclass."""
|
||||
|
||||
def test_catalog_entry_creation(self):
|
||||
"""Test creating a CatalogEntry."""
|
||||
entry = CatalogEntry(
|
||||
url="https://example.com/catalog.json",
|
||||
name="test",
|
||||
priority=1,
|
||||
install_allowed=True,
|
||||
)
|
||||
assert entry.url == "https://example.com/catalog.json"
|
||||
assert entry.name == "test"
|
||||
assert entry.priority == 1
|
||||
assert entry.install_allowed is True
|
||||
|
||||
|
||||
# ===== Catalog Stack Tests =====
|
||||
|
||||
class TestCatalogStack:
|
||||
"""Test multi-catalog stack support."""
|
||||
|
||||
def _make_project(self, temp_dir: Path) -> Path:
|
||||
"""Create a minimal spec-kit project directory."""
|
||||
project_dir = temp_dir / "project"
|
||||
project_dir.mkdir()
|
||||
(project_dir / ".specify").mkdir()
|
||||
return project_dir
|
||||
|
||||
def _write_catalog_config(self, project_dir: Path, catalogs: list) -> None:
|
||||
"""Write extension-catalogs.yml to project .specify dir."""
|
||||
import yaml as yaml_module
|
||||
|
||||
config_path = project_dir / ".specify" / "extension-catalogs.yml"
|
||||
with open(config_path, "w") as f:
|
||||
yaml_module.dump({"catalogs": catalogs}, f)
|
||||
|
||||
def _write_valid_cache(
|
||||
self, catalog: ExtensionCatalog, extensions: dict, url: str = "http://test.com"
|
||||
) -> None:
|
||||
"""Populate the primary cache file with mock extension data."""
|
||||
catalog_data = {"schema_version": "1.0", "extensions": extensions}
|
||||
catalog.cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
catalog.cache_file.write_text(json.dumps(catalog_data))
|
||||
catalog.cache_metadata_file.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"cached_at": datetime.now(timezone.utc).isoformat(),
|
||||
"catalog_url": url,
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
# --- get_active_catalogs ---
|
||||
|
||||
def test_default_stack(self, temp_dir):
|
||||
"""Default stack includes org-approved and community catalogs."""
|
||||
project_dir = self._make_project(temp_dir)
|
||||
catalog = ExtensionCatalog(project_dir)
|
||||
|
||||
entries = catalog.get_active_catalogs()
|
||||
|
||||
assert len(entries) == 2
|
||||
assert entries[0].url == ExtensionCatalog.DEFAULT_CATALOG_URL
|
||||
assert entries[0].name == "org-approved"
|
||||
assert entries[0].priority == 1
|
||||
assert entries[0].install_allowed is True
|
||||
assert entries[1].url == ExtensionCatalog.COMMUNITY_CATALOG_URL
|
||||
assert entries[1].name == "community"
|
||||
assert entries[1].priority == 2
|
||||
assert entries[1].install_allowed is False
|
||||
|
||||
def test_env_var_overrides_default_stack(self, temp_dir, monkeypatch):
|
||||
"""SPECKIT_CATALOG_URL replaces the entire default stack."""
|
||||
project_dir = self._make_project(temp_dir)
|
||||
custom_url = "https://example.com/catalog.json"
|
||||
monkeypatch.setenv("SPECKIT_CATALOG_URL", custom_url)
|
||||
|
||||
catalog = ExtensionCatalog(project_dir)
|
||||
entries = catalog.get_active_catalogs()
|
||||
|
||||
assert len(entries) == 1
|
||||
assert entries[0].url == custom_url
|
||||
assert entries[0].install_allowed is True
|
||||
|
||||
def test_env_var_invalid_url_raises(self, temp_dir, monkeypatch):
|
||||
"""SPECKIT_CATALOG_URL with http:// (non-localhost) raises ValidationError."""
|
||||
project_dir = self._make_project(temp_dir)
|
||||
monkeypatch.setenv("SPECKIT_CATALOG_URL", "http://example.com/catalog.json")
|
||||
|
||||
catalog = ExtensionCatalog(project_dir)
|
||||
with pytest.raises(ValidationError, match="HTTPS"):
|
||||
catalog.get_active_catalogs()
|
||||
|
||||
def test_project_config_overrides_defaults(self, temp_dir):
|
||||
"""Project-level extension-catalogs.yml overrides default stack."""
|
||||
project_dir = self._make_project(temp_dir)
|
||||
self._write_catalog_config(
|
||||
project_dir,
|
||||
[
|
||||
{
|
||||
"name": "custom",
|
||||
"url": "https://example.com/catalog.json",
|
||||
"priority": 1,
|
||||
"install_allowed": True,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
catalog = ExtensionCatalog(project_dir)
|
||||
entries = catalog.get_active_catalogs()
|
||||
|
||||
assert len(entries) == 1
|
||||
assert entries[0].url == "https://example.com/catalog.json"
|
||||
assert entries[0].name == "custom"
|
||||
|
||||
def test_project_config_sorted_by_priority(self, temp_dir):
|
||||
"""Catalog entries are sorted by priority (ascending)."""
|
||||
project_dir = self._make_project(temp_dir)
|
||||
self._write_catalog_config(
|
||||
project_dir,
|
||||
[
|
||||
{
|
||||
"name": "secondary",
|
||||
"url": "https://example.com/secondary.json",
|
||||
"priority": 5,
|
||||
"install_allowed": False,
|
||||
},
|
||||
{
|
||||
"name": "primary",
|
||||
"url": "https://example.com/primary.json",
|
||||
"priority": 1,
|
||||
"install_allowed": True,
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
catalog = ExtensionCatalog(project_dir)
|
||||
entries = catalog.get_active_catalogs()
|
||||
|
||||
assert len(entries) == 2
|
||||
assert entries[0].name == "primary"
|
||||
assert entries[1].name == "secondary"
|
||||
|
||||
def test_project_config_invalid_url_raises(self, temp_dir):
|
||||
"""Project config with HTTP (non-localhost) URL raises ValidationError."""
|
||||
project_dir = self._make_project(temp_dir)
|
||||
self._write_catalog_config(
|
||||
project_dir,
|
||||
[
|
||||
{
|
||||
"name": "bad",
|
||||
"url": "http://example.com/catalog.json",
|
||||
"priority": 1,
|
||||
"install_allowed": True,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
catalog = ExtensionCatalog(project_dir)
|
||||
with pytest.raises(ValidationError, match="HTTPS"):
|
||||
catalog.get_active_catalogs()
|
||||
|
||||
def test_empty_project_config_falls_back_to_defaults(self, temp_dir):
|
||||
"""Empty catalogs list in config falls back to default stack."""
|
||||
import yaml as yaml_module
|
||||
|
||||
project_dir = self._make_project(temp_dir)
|
||||
config_path = project_dir / ".specify" / "extension-catalogs.yml"
|
||||
with open(config_path, "w") as f:
|
||||
yaml_module.dump({"catalogs": []}, f)
|
||||
|
||||
catalog = ExtensionCatalog(project_dir)
|
||||
entries = catalog.get_active_catalogs()
|
||||
|
||||
# Falls back to default stack
|
||||
assert len(entries) == 2
|
||||
assert entries[0].url == ExtensionCatalog.DEFAULT_CATALOG_URL
|
||||
|
||||
# --- _load_catalog_config ---
|
||||
|
||||
def test_load_catalog_config_missing_file(self, temp_dir):
|
||||
"""Returns None when config file doesn't exist."""
|
||||
project_dir = self._make_project(temp_dir)
|
||||
catalog = ExtensionCatalog(project_dir)
|
||||
|
||||
result = catalog._load_catalog_config(project_dir / ".specify" / "nonexistent.yml")
|
||||
assert result is None
|
||||
|
||||
def test_load_catalog_config_localhost_allowed(self, temp_dir):
|
||||
"""Localhost HTTP URLs are allowed in config."""
|
||||
project_dir = self._make_project(temp_dir)
|
||||
self._write_catalog_config(
|
||||
project_dir,
|
||||
[
|
||||
{
|
||||
"name": "local",
|
||||
"url": "http://localhost:8000/catalog.json",
|
||||
"priority": 1,
|
||||
"install_allowed": True,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
catalog = ExtensionCatalog(project_dir)
|
||||
entries = catalog.get_active_catalogs()
|
||||
|
||||
assert len(entries) == 1
|
||||
assert entries[0].url == "http://localhost:8000/catalog.json"
|
||||
|
||||
# --- Merge conflict resolution ---
|
||||
|
||||
def test_merge_conflict_higher_priority_wins(self, temp_dir):
|
||||
"""When same extension id is in two catalogs, higher priority wins."""
|
||||
import yaml as yaml_module
|
||||
|
||||
project_dir = self._make_project(temp_dir)
|
||||
|
||||
# Write project config with two catalogs
|
||||
self._write_catalog_config(
|
||||
project_dir,
|
||||
[
|
||||
{
|
||||
"name": "primary",
|
||||
"url": ExtensionCatalog.DEFAULT_CATALOG_URL,
|
||||
"priority": 1,
|
||||
"install_allowed": True,
|
||||
},
|
||||
{
|
||||
"name": "secondary",
|
||||
"url": ExtensionCatalog.COMMUNITY_CATALOG_URL,
|
||||
"priority": 2,
|
||||
"install_allowed": False,
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
catalog = ExtensionCatalog(project_dir)
|
||||
|
||||
# Write primary cache with jira v2.0.0
|
||||
primary_data = {
|
||||
"schema_version": "1.0",
|
||||
"extensions": {
|
||||
"jira": {
|
||||
"name": "Jira Integration",
|
||||
"id": "jira",
|
||||
"version": "2.0.0",
|
||||
"description": "Primary Jira",
|
||||
}
|
||||
},
|
||||
}
|
||||
catalog.cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
catalog.cache_file.write_text(json.dumps(primary_data))
|
||||
catalog.cache_metadata_file.write_text(
|
||||
json.dumps({"cached_at": datetime.now(timezone.utc).isoformat(), "catalog_url": "http://test.com"})
|
||||
)
|
||||
|
||||
# Write secondary cache (URL-hash-based) with jira v1.0.0 (should lose)
|
||||
import hashlib
|
||||
|
||||
url_hash = hashlib.sha256(ExtensionCatalog.COMMUNITY_CATALOG_URL.encode()).hexdigest()[:16]
|
||||
secondary_cache = catalog.cache_dir / f"catalog-{url_hash}.json"
|
||||
secondary_meta = catalog.cache_dir / f"catalog-{url_hash}-metadata.json"
|
||||
secondary_data = {
|
||||
"schema_version": "1.0",
|
||||
"extensions": {
|
||||
"jira": {
|
||||
"name": "Jira Integration Community",
|
||||
"id": "jira",
|
||||
"version": "1.0.0",
|
||||
"description": "Community Jira",
|
||||
},
|
||||
"linear": {
|
||||
"name": "Linear",
|
||||
"id": "linear",
|
||||
"version": "0.9.0",
|
||||
"description": "Linear from secondary",
|
||||
},
|
||||
},
|
||||
}
|
||||
secondary_cache.write_text(json.dumps(secondary_data))
|
||||
secondary_meta.write_text(
|
||||
json.dumps({"cached_at": datetime.now(timezone.utc).isoformat(), "catalog_url": ExtensionCatalog.COMMUNITY_CATALOG_URL})
|
||||
)
|
||||
|
||||
results = catalog.search()
|
||||
jira_results = [r for r in results if r["id"] == "jira"]
|
||||
assert len(jira_results) == 1
|
||||
# Primary catalog wins
|
||||
assert jira_results[0]["version"] == "2.0.0"
|
||||
assert jira_results[0]["_catalog_name"] == "primary"
|
||||
assert jira_results[0]["_install_allowed"] is True
|
||||
|
||||
# linear comes from secondary
|
||||
linear_results = [r for r in results if r["id"] == "linear"]
|
||||
assert len(linear_results) == 1
|
||||
assert linear_results[0]["_catalog_name"] == "secondary"
|
||||
assert linear_results[0]["_install_allowed"] is False
|
||||
|
||||
def test_install_allowed_false_from_get_extension_info(self, temp_dir):
|
||||
"""get_extension_info includes _install_allowed from source catalog."""
|
||||
project_dir = self._make_project(temp_dir)
|
||||
|
||||
# Single catalog that is install_allowed=False
|
||||
self._write_catalog_config(
|
||||
project_dir,
|
||||
[
|
||||
{
|
||||
"name": "discovery",
|
||||
"url": ExtensionCatalog.DEFAULT_CATALOG_URL,
|
||||
"priority": 1,
|
||||
"install_allowed": False,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
catalog = ExtensionCatalog(project_dir)
|
||||
self._write_valid_cache(
|
||||
catalog,
|
||||
{
|
||||
"jira": {
|
||||
"name": "Jira Integration",
|
||||
"id": "jira",
|
||||
"version": "1.0.0",
|
||||
"description": "Jira integration",
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
info = catalog.get_extension_info("jira")
|
||||
assert info is not None
|
||||
assert info["_install_allowed"] is False
|
||||
assert info["_catalog_name"] == "discovery"
|
||||
|
||||
def test_search_results_include_catalog_metadata(self, temp_dir):
|
||||
"""Search results include _catalog_name and _install_allowed."""
|
||||
project_dir = self._make_project(temp_dir)
|
||||
self._write_catalog_config(
|
||||
project_dir,
|
||||
[
|
||||
{
|
||||
"name": "org",
|
||||
"url": ExtensionCatalog.DEFAULT_CATALOG_URL,
|
||||
"priority": 1,
|
||||
"install_allowed": True,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
catalog = ExtensionCatalog(project_dir)
|
||||
self._write_valid_cache(
|
||||
catalog,
|
||||
{
|
||||
"jira": {
|
||||
"name": "Jira Integration",
|
||||
"id": "jira",
|
||||
"version": "1.0.0",
|
||||
"description": "Jira integration",
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
results = catalog.search()
|
||||
assert len(results) == 1
|
||||
assert results[0]["_catalog_name"] == "org"
|
||||
assert results[0]["_install_allowed"] is True
|
||||
|
||||
Reference in New Issue
Block a user