solidworks_mcp.agents.smoke_test

solidworks_mcp.agents.smoke_test

CLI for running validated prompt smoke tests against custom agent files.

Attributes

app module-attribute

app = Typer(name='smoke-test', help='Run validated custom-agent prompt tests against the SolidWorks MCP agents.', no_args_is_help=True)

Classes

DocsPlan

Bases: BaseModel

Validation shape for docs-engineering responses.

Attributes:

Name	Type	Description
audience	str	The audience value.
decisions	list[ToolRoutingDecision]	The decisions value.
demo_steps	list[str]	The demo steps value.
objective	str	The objective value.
sections	list[str]	The sections value.

ManufacturabilityReview

Bases: BaseModel

Validation shape for printability-focused agent responses.

Attributes:

Name	Type	Description
assumptions	list[Assumption]	The assumptions value.
build_volume_check	str	The build volume check value.
orientation_guidance	str	The orientation guidance value.
recommendations	list[Recommendation]	The recommendations value.
summary	str	The summary value.
tolerance_clearance_notes	list[str]	The tolerance clearance notes value.

ReconstructionPlan

Bases: BaseModel

Structured plan for recreating a SolidWorks part from scratch using MCP tools.

Attributes:

Name	Type	Description
analysis_summary	str	The analysis summary value.
assembly_mates	list[str]	The assembly mates value.
complexity_tier	Literal[1, 2, 3, 4]	The complexity tier value.
feature_sequence	list[FeatureStep]	The feature sequence value.
part_name	str	The part name value.
validation_strategy	str	The validation strategy value.
vba_required	bool	The vba required value.

RecoverableFailure

Bases: BaseModel

Typed failure output used when agent needs user-guided retry.

Attributes:

Name	Type	Description
explanation	str	The explanation value.
remediation_steps	list[str]	The remediation steps value.
retry_focus	str | None	The retry focus value.
should_retry	bool	The should retry value.

SchemaChoice

Bases: StrEnum

Handle schema choice.

Attributes:

Name	Type	Description
docs	Any	The docs value.
manufacturability	Any	The manufacturability value.
reconstruction	Any	The reconstruction value.

Functions

_ensure_provider_credentials

_ensure_provider_credentials(model: str) -> None

Provide actionable credential guidance before model invocation.

Parameters:

Name	Type	Description	Default
model	str	The model value.	required

Returns:

Name	Type	Description
None	None	None.

Raises:

Type	Description
BadParameter	OPENAI_API_KEY is not set. Copilot subscription cannot be reused as an OpenAI API key for local pydantic-ai scripts. Use --github-models instead.

Source code in src/solidworks_mcp/agents/smoke_test.py

def _ensure_provider_credentials(model: str) -> None:
    """Provide actionable credential guidance before model invocation.

    Args:
        model (str): The model value.

    Returns:
        None: None.

    Raises:
        BadParameter: OPENAI_API_KEY is not set. Copilot subscription cannot be reused as an
                      OpenAI API key for local pydantic-ai scripts. Use --github-models
                      instead.
    """
    if model.startswith("github:"):
        github_token = os.getenv("GITHUB_API_KEY") or os.getenv("GH_TOKEN")
        if not github_token:
            # Fall back to `gh auth token` if the gh CLI is available and authenticated.
            try:
                import subprocess

                result = subprocess.run(
                    ["gh", "auth", "token"], capture_output=True, text=True, timeout=5
                )
                if result.returncode == 0:
                    github_token = result.stdout.strip()
            except Exception:
                pass
        if not github_token:
            raise typer.BadParameter(
                "GitHub Models requires GITHUB_API_KEY (or GH_TOKEN) with models:read scope.\n"
                "Options:\n"
                "  1. Run: gh auth login  (GitHub CLI — free with Copilot subscription)\n"
                "  2. Or set: $env:GH_TOKEN = 'github_pat_...'",
                param_hint="'--github-models'",
            )
        # pydantic-ai GitHubProvider looks for GITHUB_API_KEY.
        os.environ.setdefault("GITHUB_API_KEY", github_token)
        return

    if model.startswith("anthropic:") and not os.getenv("ANTHROPIC_API_KEY"):
        raise typer.BadParameter(
            "ANTHROPIC_API_KEY is not set.\n"
            "Export your key before running smoke_test:\n"
            "  $env:ANTHROPIC_API_KEY = 'sk-ant-...'\n"
            "Or use --github-models to run via GitHub Models with a GitHub PAT.",
            param_hint="'--anthropic'",
        )

    if model.startswith("openai:") and not os.getenv("OPENAI_API_KEY"):
        raise typer.BadParameter(
            "OPENAI_API_KEY is not set. Copilot subscription cannot be reused as an "
            "OpenAI API key for local pydantic-ai scripts. "
            "Use --github-models instead.",
            param_hint="'--model'",
        )

_resolve_model

_resolve_model(anthropic: bool, claude_model: str, github_models: bool, github_model: str, model: str | None) -> str

Resolve effective model id from CLI options.

Parameters:

Name	Type	Description	Default
anthropic	bool	The anthropic value.	required
claude_model	str	The claude model value.	required
github_models	bool	The github models value.	required
github_model	str	The github model value.	required
model	str | None	The model value.	required

Returns:

Name	Type	Description
str	str	The resulting text value.

Raises:

Type	Description
BadParameter	--model is required unless --github-models or --anthropic is enabled. Recommended: --github-models (requires GH_TOKEN with models:read scope).

Source code in src/solidworks_mcp/agents/smoke_test.py

def _resolve_model(
    anthropic: bool,
    claude_model: str,
    github_models: bool,
    github_model: str,
    model: str | None,
) -> str:
    """Resolve effective model id from CLI options.

    Args:
        anthropic (bool): The anthropic value.
        claude_model (str): The claude model value.
        github_models (bool): The github models value.
        github_model (str): The github model value.
        model (str | None): The model value.

    Returns:
        str: The resulting text value.

    Raises:
        BadParameter: --model is required unless --github-models or --anthropic is enabled.
                      Recommended: --github-models (requires GH_TOKEN with models:read
                      scope).
    """
    if github_models:
        return f"github:{github_model}"

    if anthropic:
        return f"anthropic:{claude_model}"

    if not model:
        raise typer.BadParameter(
            "--model is required unless --github-models or --anthropic is enabled. "
            "Recommended: --github-models (requires GH_TOKEN with models:read scope).",
            param_hint="'--model'",
        )

    return model

_run async

_run(agent_file: str, model_name: str, prompt: str, schema: SchemaChoice, max_retries_on_recoverable: int) -> int

Build internal run.

Parameters:

Name	Type	Description	Default
agent_file	str	The agent file value.	required
model_name	str	Embedding model name to use.	required
prompt	str	The prompt value.	required
schema	SchemaChoice	The schema value.	required
max_retries_on_recoverable	int	The max retries on recoverable value.	required

Returns:

Name	Type	Description
int	int	The computed numeric result.

Source code in src/solidworks_mcp/agents/smoke_test.py

async def _run(
    agent_file: str,
    model_name: str,
    prompt: str,
    schema: SchemaChoice,
    max_retries_on_recoverable: int,
) -> int:
    """Build internal run.

    Args:
        agent_file (str): The agent file value.
        model_name (str): Embedding model name to use.
        prompt (str): The prompt value.
        schema (SchemaChoice): The schema value.
        max_retries_on_recoverable (int): The max retries on recoverable value.

    Returns:
        int: The computed numeric result.
    """

    schema_type: (
        type[ManufacturabilityReview] | type[ReconstructionPlan] | type[DocsPlan]
    )

    if schema == SchemaChoice.manufacturability:
        schema_type = ManufacturabilityReview
    elif schema == SchemaChoice.reconstruction:
        schema_type = ReconstructionPlan
    else:
        schema_type = DocsPlan
    result = await run_validated_prompt(
        agent_file_name=agent_file,
        model_name=model_name,
        user_prompt=prompt,
        result_type=schema_type,
        max_retries_on_recoverable=max_retries_on_recoverable,
    )

    if isinstance(result, RecoverableFailure):
        typer.echo("RecoverableFailure returned after retry attempts:")
    typer.echo(pretty_json(result))
    return 0

main

main() -> None

Handle main.

Returns:

Name	Type	Description
None	None	None.

Source code in src/solidworks_mcp/agents/smoke_test.py

def main() -> None:
    """Handle main.

    Returns:
        None: None.
    """

    app()

pretty_json

pretty_json(model: BaseModel) -> str

Return pretty JSON for test output snapshots.

Parameters:

Name	Type	Description	Default
model	BaseModel	The model value.	required

Returns:

Name	Type	Description
str	str	The resulting text value.

Source code in src/solidworks_mcp/agents/harness.py

def pretty_json(model: BaseModel) -> str:
    """Return pretty JSON for test output snapshots.

    Args:
        model (BaseModel): The model value.

    Returns:
        str: The resulting text value.
    """
    return json.dumps(model.model_dump(mode="json"), indent=2)

run

run(agent_file: Annotated[str, Option('--agent-file', help='Agent filename in .github/agents/', show_default=False)], prompt: Annotated[str, Option('--prompt', help='User prompt to run', show_default=False)], github_models: Annotated[bool, Option('--github-models', help='Use GitHub Models provider (recommended). Requires GH_TOKEN or GITHUB_API_KEY with models:read scope.')] = False, github_model: Annotated[str, Option('--github-model', help='GitHub Models catalog id, e.g. openai/gpt-4.1 or mistral-ai/mistral-large.')] = 'openai/gpt-4.1', anthropic: Annotated[bool, Option('--anthropic', help='Use Anthropic Claude (requires ANTHROPIC_API_KEY with active billing).')] = False, claude_model: Annotated[str, Option('--claude-model', help='Anthropic model id, e.g. claude-sonnet-4-6 or claude-opus-4-6.')] = 'claude-sonnet-4-6', model: Annotated[str | None, Option('--model', help='Explicit PydanticAI model spec, e.g. openai:gpt-4.1. Overridden by --anthropic/--github-models.')] = None, schema: Annotated[SchemaChoice, Option('--schema', help='Validation schema expected from the model output.')] = SchemaChoice.manufacturability, max_retries_on_recoverable: Annotated[int, Option('--max-retries-on-recoverable', help='Automatic retry attempts when model returns RecoverableFailure.')] = 1) -> None

Run a validated custom-agent prompt test and print structured JSON output.

Parameters:

Name	Type	Description	Default
agent_file	Annotated[str, Option('--agent-file', help='Agent filename in .github/agents/', show_default=False)]	Agent filename under .github/agents/.	required
prompt	Annotated[str, Option('--prompt', help='User prompt to run', show_default=False)]	User prompt to execute against the agent.	required
github_models	Annotated[bool, Option('--github-models', help='Use GitHub Models provider (recommended). Requires GH_TOKEN or GITHUB_API_KEY with models:read scope.')]	Use GitHub Models provider.	False
github_model	Annotated[str, Option('--github-model', help='GitHub Models catalog id, e.g. openai/gpt-4.1 or mistral-ai/mistral-large.')]	GitHub model id when github_models is enabled.	'openai/gpt-4.1'
anthropic	Annotated[bool, Option('--anthropic', help='Use Anthropic Claude (requires ANTHROPIC_API_KEY with active billing).')]	Use Anthropic provider.	False
claude_model	Annotated[str, Option('--claude-model', help='Anthropic model id, e.g. claude-sonnet-4-6 or claude-opus-4-6.')]	Anthropic model id when anthropic is enabled.	'claude-sonnet-4-6'
model	Annotated[str | None, Option('--model', help='Explicit PydanticAI model spec, e.g. openai:gpt-4.1. Overridden by --anthropic/--github-models.')]	Explicit provider-qualified model (for example openai:gpt-4.1).	None
schema	Annotated[SchemaChoice, Option('--schema', help='Validation schema expected from the model output.')]	Expected structured output schema.	manufacturability
max_retries_on_recoverable	Annotated[int, Option('--max-retries-on-recoverable', help='Automatic retry attempts when model returns RecoverableFailure.')]	Retries for RecoverableFailure outputs.	1

Returns:

Name	Type	Description
None	None	None.

Raises:

Type	Description
SystemExit	Always raised with the command exit code.

Source code in src/solidworks_mcp/agents/smoke_test.py

@app.command()
def run(
    agent_file: Annotated[
        str,
        typer.Option(
            "--agent-file", help="Agent filename in .github/agents/", show_default=False
        ),
    ],
    prompt: Annotated[
        str,
        typer.Option("--prompt", help="User prompt to run", show_default=False),
    ],
    github_models: Annotated[
        bool,
        typer.Option(
            "--github-models",
            help="Use GitHub Models provider (recommended). Requires GH_TOKEN or GITHUB_API_KEY with models:read scope.",
        ),
    ] = False,
    github_model: Annotated[
        str,
        typer.Option(
            "--github-model",
            help="GitHub Models catalog id, e.g. openai/gpt-4.1 or mistral-ai/mistral-large.",
        ),
    ] = "openai/gpt-4.1",
    anthropic: Annotated[
        bool,
        typer.Option(
            "--anthropic",
            help="Use Anthropic Claude (requires ANTHROPIC_API_KEY with active billing).",
        ),
    ] = False,
    claude_model: Annotated[
        str,
        typer.Option(
            "--claude-model",
            help="Anthropic model id, e.g. claude-sonnet-4-6 or claude-opus-4-6.",
        ),
    ] = "claude-sonnet-4-6",
    model: Annotated[
        str | None,
        typer.Option(
            "--model",
            help="Explicit PydanticAI model spec, e.g. openai:gpt-4.1. Overridden by --anthropic/--github-models.",
        ),
    ] = None,
    schema: Annotated[
        SchemaChoice,
        typer.Option(
            "--schema", help="Validation schema expected from the model output."
        ),
    ] = SchemaChoice.manufacturability,
    max_retries_on_recoverable: Annotated[
        int,
        typer.Option(
            "--max-retries-on-recoverable",
            help="Automatic retry attempts when model returns RecoverableFailure.",
        ),
    ] = 1,
) -> None:
    """Run a validated custom-agent prompt test and print structured JSON output.

    Args:
        agent_file: Agent filename under `.github/agents/`.
        prompt: User prompt to execute against the agent.
        github_models: Use GitHub Models provider.
        github_model: GitHub model id when `github_models` is enabled.
        anthropic: Use Anthropic provider.
        claude_model: Anthropic model id when `anthropic` is enabled.
        model: Explicit provider-qualified model (for example `openai:gpt-4.1`).
        schema: Expected structured output schema.
        max_retries_on_recoverable: Retries for `RecoverableFailure` outputs.

    Returns:
        None: None.

    Raises:
        SystemExit: Always raised with the command exit code.
    """
    model_name = _resolve_model(
        anthropic, claude_model, github_models, github_model, model
    )
    _ensure_provider_credentials(model_name)
    raise SystemExit(
        asyncio.run(
            _run(agent_file, model_name, prompt, schema, max_retries_on_recoverable)
        )
    )

run_validated_prompt async

run_validated_prompt(*, agent_file_name: str, model_name: str, user_prompt: str, result_type: type[TModel], max_retries_on_recoverable: int = 1, db_path: Path | None = None) -> TModel | RecoverableFailure

Run one prompt through PydanticAI and validate the output schema.

Parameters:

Name	Type	Description	Default
agent_file_name	str	The agent file name value.	required
model_name	str	Embedding model name to use.	required
user_prompt	str	The user prompt value.	required
result_type	type[TModel]	The result type value.	required
max_retries_on_recoverable	int	The max retries on recoverable value. Defaults to 1.	1
db_path	Path | None	The db path value. Defaults to None.	None

Returns:

Type	Description
TModel | RecoverableFailure	TModel | RecoverableFailure: The result produced by the operation.

Raises:

Type	Description
RuntimeError	Pydantic_ai is not importable. Install dependencies and retry.

Source code in src/solidworks_mcp/agents/harness.py

async def run_validated_prompt(
    *,
    agent_file_name: str,
    model_name: str,
    user_prompt: str,
    result_type: type[TModel],
    max_retries_on_recoverable: int = 1,
    db_path: Path | None = None,
) -> TModel | RecoverableFailure:
    """Run one prompt through PydanticAI and validate the output schema.

    Args:
        agent_file_name (str): The agent file name value.
        model_name (str): Embedding model name to use.
        user_prompt (str): The user prompt value.
        result_type (type[TModel]): The result type value.
        max_retries_on_recoverable (int): The max retries on recoverable value. Defaults to
                                          1.
        db_path (Path | None): The db path value. Defaults to None.

    Returns:
        TModel | RecoverableFailure: The result produced by the operation.

    Raises:
        RuntimeError: Pydantic_ai is not importable. Install dependencies and retry.
    """
    if Agent is None:  # pragma: no cover
        raise RuntimeError(
            "pydantic_ai is not importable. Install dependencies and retry."
        ) from IMPORT_ERROR

    base_run_id = str(uuid.uuid4())
    instructions = _load_agent_prompt(agent_file_name)
    system_prompt = (
        f"{instructions}\n\n"
        "If you cannot safely produce the requested structured output, return "
        "a RecoverableFailure with concrete remediation steps and a focused retry hint."
    )

    attempt = 0
    current_prompt = user_prompt

    while True:
        attempt += 1
        run_id = f"{base_run_id}-a{attempt}"

        prompt_snapshot = current_prompt

        agent = Agent(
            model_name,
            system_prompt=system_prompt,
            output_type=[result_type, RecoverableFailure],
        )

        try:
            result = await agent.run(current_prompt)
            payload = _extract_data(result)

            if isinstance(payload, RecoverableFailure):
                insert_run(
                    run_id=run_id,
                    agent_name=agent_file_name,
                    prompt=prompt_snapshot,
                    status="recoverable_failure",
                    output_json=payload.model_dump_json(indent=2),
                    model_name=model_name,
                    db_path=db_path,
                )
                insert_error(
                    ErrorRecord(
                        source="pydantic_ai",
                        tool_name="run_validated_prompt",
                        error_type="RecoverableFailure",
                        error_message=payload.explanation,
                        root_cause=payload.explanation,
                        remediation="; ".join(payload.remediation_steps)
                        if payload.remediation_steps
                        else "Follow retry_focus guidance and retry with narrower scope.",
                    ),
                    run_id=run_id,
                    db_path=db_path,
                )
                if payload.should_retry and attempt <= max_retries_on_recoverable:
                    retry_hint = (
                        payload.retry_focus or "Narrow scope and clarify assumptions."
                    )
                    remediation = "\n".join(
                        f"- {step}" for step in payload.remediation_steps
                    )
                    current_prompt = (
                        f"{user_prompt}\n\n"
                        "Retry context from previous recoverable failure:\n"
                        f"Explanation: {payload.explanation}\n"
                        f"Retry focus: {retry_hint}\n"
                        f"Remediation steps:\n{remediation}"
                    )
                    continue
                return payload

            validated = (
                payload
                if isinstance(payload, result_type)
                else result_type.model_validate(payload)
            )

            insert_run(
                run_id=run_id,
                agent_name=agent_file_name,
                prompt=prompt_snapshot,
                status="success",
                output_json=validated.model_dump_json(indent=2),
                model_name=model_name,
                db_path=db_path,
            )
            return validated
        except Exception as exc:
            insert_run(
                run_id=run_id,
                agent_name=agent_file_name,
                prompt=current_prompt,
                status="error",
                output_json=None,
                model_name=model_name,
                db_path=db_path,
            )
            insert_error(
                ErrorRecord(
                    source="pydantic_ai",
                    tool_name="run_validated_prompt",
                    error_type=exc.__class__.__name__,
                    error_message=str(exc),
                    root_cause="Agent output failed schema validation or model invocation",
                    remediation=(
                        "Review response schema, verify model provider credentials, and re-run with a narrower prompt."
                    ),
                ),
                run_id=run_id,
                db_path=db_path,
            )
            raise