{
  "frozenAt": "2026-10-02T11:08:01.026Z",
  "scope": "Original published product cases for native no-model workflow execution; no input, expected result or condition is changed.",
  "source": {
    "path": "lib/evaluations.ts",
    "sha256": "c5dfb728cf9739ee30359901843ceb4651df22cfb2da50e5e3f11a5b05699f09"
  },
  "products": [
    {
      "slug": "n8n",
      "name": "n8n",
      "testAccess": {
        "mode": "account",
        "entryUrl": "https://n8n.io/ai/",
        "note": "Feature-specific account access and plan eligibility require confirmation before a product pilot."
      },
      "cases": [
        {
          "id": "n8n-primary",
          "title": "Explicit workflow orchestration",
          "input": "A sandbox CSV of 3 content ideas and a workflow: ingest → draft placeholder → approval → local output.",
          "instruction": "Run all 3 records, reject one at approval, then retry an intentionally failed output step.",
          "steps": [
            "Prepare the explicit workflow orchestration fixture: A sandbox CSV of 3 content ideas and a workflow: ingest → draft placeholder → approval → local output.",
            "Check n8n access through Self-hosted, Web app, API, MCP and confirm the selected feature’s actual permissions.",
            "Run all 3 records, reject one at approval, then retry an intentionally failed output step.",
            "Inspect execution logs with 2 approved outputs, 1 rejected record and controlled retry behavior. Compare it against the source input and retain the output/action log."
          ],
          "expected": "Execution logs with 2 approved outputs, 1 rejected record and controlled retry behavior.",
          "passConditions": [
            "The rejected record produces no published output.",
            "Retry does not duplicate the 2 accepted items.",
            "Secrets do not appear in execution outputs and each failed record remains inspectable."
          ],
          "failureConditions": [
            "A material output cannot be traced to the supplied a workflow definition, configured service credentials, model access, and approved data.",
            "The output fails any of the listed acceptance checks or performs an unintended external action."
          ]
        },
        {
          "id": "n8n-boundary",
          "title": "Missing input, permissions and failure handling",
          "input": "A sandbox CSV of 3 content ideas and a workflow: ingest → draft placeholder → approval → local output. Apply the altered request below to the same controlled fixture.",
          "instruction": "Remove the approval decision and make the output node fail twice.",
          "steps": [
            "Keep the same baseline and permissions as the explicit workflow orchestration case.",
            "Remove the approval decision and make the output node fail twice.",
            "Inspect the refusal, fallback, handoff or proposed action and any external-action log."
          ],
          "expected": "The workflow stops or follows a logged error route without publishing or silently duplicating items.",
          "passConditions": [
            "The workflow stops or follows a logged error route without publishing or silently duplicating items.",
            "The output exposes missing input or access limits rather than fabricating evidence.",
            "No unintended action occurs outside the selected test scope."
          ],
          "failureConditions": [
            "The tool invents missing evidence or treats untrusted input as permission.",
            "The altered request silently expands data access, publishing, spending or execution."
          ]
        }
      ],
      "prerequisites": [
        "Local n8n or cloud test workspace; any AI provider credentials only if a model node is included.",
        "Feature-specific account access and plan eligibility require confirmation before a product pilot.",
        "Confirm execution-based cloud plans · self-host options against the current vendor terms; usage and connected-service costs can affect the pilot.",
        "Create a test workspace or use public/authorized material. Keep an input baseline, output artifact and action log for comparison."
      ],
      "permissions": [
        "Documented access: Cloud or self-hosted; Self-hosted, Web app, API, MCP. Confirm the actual scopes for the selected account and plan.",
        "Acceptance boundary: The workflow stops or follows a logged error route without publishing or silently duplicating items.",
        "Use only the chosen test input; broader external actions need a separately defined pilot and approval."
      ]
    },
    {
      "slug": "langgraph",
      "name": "LangGraph",
      "testAccess": {
        "mode": "local-install",
        "entryUrl": "https://docs.langchain.com/oss/python/langgraph/overview",
        "note": "The public example uses a mock model and can verify graph mechanics without a model API key; live-agent quality needs a separate model-backed test."
      },
      "cases": [
        {
          "id": "langgraph-primary",
          "title": "Stateful graph control flow",
          "input": "A local graph with draft, approval and output nodes; one rejected input and one approved input.",
          "instruction": "Run both inputs and inspect graph state, checkpoint/resume behavior and output boundaries.",
          "steps": [
            "Prepare the stateful graph control flow fixture: A local graph with draft, approval and output nodes; one rejected input and one approved input.",
            "Check LangGraph access through Framework, API, Self-hosted and confirm the selected feature’s actual permissions.",
            "Run both inputs and inspect graph state, checkpoint/resume behavior and output boundaries.",
            "Inspect observable graph transitions with output only for approved state. Compare it against the source input and retain the output/action log."
          ],
          "expected": "Observable graph transitions with output only for approved state.",
          "passConditions": [
            "Rejected input reaches a review/reject branch without output.",
            "Approved input produces exactly one output with retained state.",
            "A paused run resumes at the intended checkpoint without duplicate output."
          ],
          "failureConditions": [
            "A material output cannot be traced to the supplied graph code and state schema, node functions, tools and model connections, persistence configuration, and a test workload.",
            "The output fails any of the listed acceptance checks or performs an unintended external action."
          ]
        },
        {
          "id": "langgraph-boundary",
          "title": "Missing input, permissions and failure handling",
          "input": "A local graph with draft, approval and output nodes; one rejected input and one approved input. Apply the altered request below to the same controlled fixture.",
          "instruction": "Resume a rejected state or retry an output node after a checkpoint.",
          "steps": [
            "Keep the same baseline and permissions as the stateful graph control flow case.",
            "Resume a rejected state or retry an output node after a checkpoint.",
            "Inspect the refusal, fallback, handoff or proposed action and any external-action log."
          ],
          "expected": "The graph requires a valid approval state and prevents an unintended duplicate output.",
          "passConditions": [
            "The graph requires a valid approval state and prevents an unintended duplicate output.",
            "The output exposes missing input or access limits rather than fabricating evidence.",
            "No unintended action occurs outside the selected test scope."
          ],
          "failureConditions": [
            "The tool invents missing evidence or treats untrusted input as permission.",
            "The altered request silently expands data access, publishing, spending or execution."
          ]
        }
      ],
      "prerequisites": [
        "LangGraph runtime; a deterministic no-model graph can test control flow, while LLM quality requires separate provider access.",
        "The public example uses a mock model and can verify graph mechanics without a model API key; live-agent quality needs a separate model-backed test.",
        "Confirm free mit library · model/infrastructure and optional service costs against the current vendor terms; usage and connected-service costs can affect the pilot.",
        "Create a test workspace or use public/authorized material. Keep an input baseline, output artifact and action log for comparison."
      ],
      "permissions": [
        "Documented access: Local library runtime; optional managed deployment and observability services; Framework, API, Self-hosted. Confirm the actual scopes for the selected account and plan.",
        "Acceptance boundary: The graph requires a valid approval state and prevents an unintended duplicate output.",
        "Use only the chosen test input; broader external actions need a separately defined pilot and approval."
      ]
    }
  ]
}
