{ "id": "khoj-native-qwen-2026-10-03", "slug": "khoj", "executionKind": "local-model", "startedAt": "2026-10-03T01:11:57.246959+08:00", "completedAt": "2026-10-03T01:15:28.124376+08:00", "scope": "One original native three-file PUT. One interrupted original primary POST with one helper generation, followed by one never-POSTed boundary and one explicitly authorized primary environment recovery with each original request create_new=true. Exact original questions/models/settings/fixtures/8conditions were retained; no result-improvement retry.", "conclusion": "Both original native answer cases completed with real source references and independently recorded local generation, and both failed their unchanged original quality conditions. Primary passed1/4: archive prices15/20, wrong order quantity/subtotal/stock, and1-day dispatch were used despite current source passages. Boundary passed2/4: it admitted arrival/refund were unknown, but presented superseded1-day/USD40 policy as current and gave no requested quotes. Total3/8 conditions passed,0/2 cases passed. No complete answer was rerun to improve this result.", "scopeLimits": [ "Result applies to this exact small local Qwen model, frozen self-hosted edition and synthetic three-document corpus; it does not establish general Khoj cloud or stronger-model quality.", "Correct original source passages in native references do not prove the final answer is grounded. Both answers failed the frozen original quality criteria.", "PROMPTRACE is written after successful responses and may omit kwargs; SDK retries can nest in Tenacity. Actual successful local HTTP observations and trace reconciliation are reported, without assuming retry defaults are a hard total-call bound.", "Aborted attempt has no final-answer latency or complete provider usage established; its observed1 HTTP200 helper call remains separate from the2 complete cases.", "External TCP sampling is not a firewall/all-packet capture and cannot establish complete lack of external traffic.", "Native per-call trace cid remains null for both create_new requests; trace does independently retain distinct mid values. New-conversation request semantics are evidenced by unchanged native create_new=true bodies and route behavior; no distinct persisted CID measurement is claimed.", "Operator request history: one original three-file PUT, one earlier primary POST interrupted after1 local helper generation, then one original boundary POST and one explicitly authorized primary environment recovery. Cumulative primaryPOST2/boundaryPOST1/PUT1; cumulative observed local providerHTTP calls5, with4 trace commits for the2 complete cases.", "API wall times85.00s and86.64s include native retrieval/helper/final-generation and response assembly; not pure model latency or a performance benchmark.", "Frozen pre-resource case definitions and source prose remain historical; the separately reviewed execution contract and recovery supplement supply the later exact native runtime/corpus state without changing questions/criteria." ], "product": { "version": "official source ae229ca894c0b80ad84664afcfdde523b5e87057", "feature": "Complete native self-hosted app; original PUT /api/content and POST /api/chat over three frozen synthetic Markdown notes" }, "runtime": { "name": "Ollama local OpenAI-compatible endpoint", "version": "0.35.0" }, "model": { "name": "Qwen2.5 Coder1.5B Instruct Q4_K_M", "tag": "uagentkit-qwen-coder:1.5b (task alias)", "digest": "29d8c98fa6b098e200069bfb88b9508dc3e85586d20cba59f8dda9a808165104", "configuration": { "inference_task": "document-grounded-chat", "native_temperature_default": 0.6, "native_top_p_default": 0.95, "native_seed_default": 42, "native_max_completion_tokens_default": 16000, "context": 8192, "settings_evidence": "Frozen original defaults; native trace reports temperature0.6. No all-kwargs wire capture.", "bi_encoder": "thenlper/gte-small@17e1f347d17fe144873b1201da91788898c639cd", "reranker": "mixedbread-ai/mxbai-rerank-xsmall-v1@b5c6e9da73abc3711f593f705371cdbe9e0fe422", "memory_mode": "disabled" } }, "artifact": "/evidence/khoj-native-2026-10-03/execution-v2.json", "artifacts": [ { "id": "original-case-contract", "kind": "input", "path": "/evidence/khoj-native-2026-10-03/original-frozen-cases.json", "sha256": "a1bc165315996a5951205cc72cdb449c9609dfaee41d2c09789bc1883bd67848" }, { "id": "primary-native-request", "kind": "input", "path": "/evidence/khoj-native-2026-10-03/primary-original-request.json", "sha256": "a640e8c82c5c2084d0e43568236d31d293ce5929329ebd1f7c6af19c6aec9c62" }, { "id": "boundary-native-request", "kind": "input", "path": "/evidence/khoj-native-2026-10-03/boundary-original-request.json", "sha256": "9b04432297efe09a000ef9bf049ea9fa89e195a8a156ad3c41eb881562afc7c2" }, { "id": "primary-native-response", "kind": "output", "path": "/evidence/khoj-native-2026-10-03/primary-native-response.json", "sha256": "799e5ea4f0afc218e1a261cb6b69df317f3a2730dfebed447c8f6100fc380609" }, { "id": "boundary-native-response", "kind": "output", "path": "/evidence/khoj-native-2026-10-03/boundary-native-response.json", "sha256": "df8177639cc4e35e872b021e94b23b61b1c0f47a7d0fc555892656d12d0de910" }, { "id": "final-condition-report", "kind": "tests", "path": "/evidence/khoj-native-2026-10-03/run-final-v1.json", "sha256": "b3ebe7f5116531b388fb653ae768f56798c73c6ec527f37f1a58c2670bb81fd6" }, { "id": "corpus-byte-audit", "kind": "audit", "path": "/evidence/khoj-native-2026-10-03/corpus-byte-audit.json", "sha256": "de95209a9ca90e6757e6cf97eca09edd28fd0cb9375fcdf2093122f4d01cc3d2" }, { "id": "native-preexecution-contract", "kind": "provenance", "path": "/evidence/khoj-native-2026-10-03/native-preexecution-contract.json", "sha256": "8a2cbae2d9f7f025d8a4bb918635a9901e1dd7a13d6c05ced640b0775418320d" }, { "id": "environment-recovery-supplement", "kind": "provenance", "path": "/evidence/khoj-native-2026-10-03/environment-recovery-supplement.json", "sha256": "723712fc12f3ff0cc1c97e5589af9143a006e208a6f4cbb0ec524efbe8508b4a" }, { "id": "current-inventory", "kind": "input", "path": "/evidence/khoj-native-2026-10-03/fixtures/juniper-inventory-2026-09-30.md", "sha256": "e370946255b6e9c82ee946b73cef0f06002d8dfe6b819211f96c7a0d03f8ac7a" }, { "id": "current-dispatch", "kind": "input", "path": "/evidence/khoj-native-2026-10-03/fixtures/juniper-dispatch-2026-09-30.md", "sha256": "235b051a8a68a39fc09b98c0ff6e93dbb622300d8f3cc0135f3b88ee17794759" }, { "id": "superseded-archive", "kind": "input", "path": "/evidence/khoj-native-2026-10-03/fixtures/juniper-policy-archive-2025-05-01.md", "sha256": "8b68d8dc247e5cd7010c05e4e32d3f74936f97f44d2ae07448f87763a0e1d8bd" }, { "id": "prior-interrupted-primary-attempt", "kind": "provenance", "path": "/evidence/khoj-native-primary-aborted.json", "sha256": "d7b67c7fab98d5e4b8351d5438357d175dc56c8b68d6b7eab4efe0bce74d07fd" } ], "cases": [ { "caseId": "khoj-primary", "executionStatus": "executed", "outcome": "failed", "input": "Three new task-owned synthetic UTF-8 Markdown files: a current 2026-09-30 inventory snapshot, a current 2026-09-30 dispatch policy and one explicitly superseded 2025 archive. All files are indexed in a fresh isolated native Khoj instance. No real user document, business inventory, external account or stock reservation is connected.", "expected": "Use the current notes: BLUE-CANDLE USD 18.00, stock 12; AMBER-CANDLE USD 24.00, stock 7. One blue plus two amber units gives three items, subtotal USD 66.00, shipping USD 0.00 and total USD 66.00. Dispatch is within 3 business days; hypothetical remaining stock is 11 blue and 5 amber. The two current source files and original supporting passages are inspectable; no order/stock action is represented as executed.", "observed": "Certainly! Let's break down the task and create an internal quote for a BLUE-CANDLE and two AMBER-CANDLE units in an eligible domestic order, citing the original current inventory and dispatch notes.\n\n### Current Unit Prices and Starting Stocks\n- **BLUE-CANDLE:** USD 15.00\n- **AMBER-CANDLE:** USD 20.00\n- **Stock figures:** 12 units (BLUE-CANDLE) and 7 units (AMBER-CANDLE)\n\n### Item Subtotal\n- **Order Total:** (12 units * USD 15.00) + (2 units * USD 20.00) = USD 180.00\n\n### Shipping Fee\n- **Shipping Fee:** USD 0.00 (for eligible domestic orders with an item subtotal of at least USD 60.00)\n\n### Order Total\n- **Order Total:** USD 180.00 (without shipping)\n\n### Dispatch Promise\n- **Dispatch Promise:** 1 business day\n\n### Projected Stock After This Hypothetical Order\n- **Projected Stock:** (12 units - 2 units) + (7 units - 2 units) = 12 units\n\n### Source Facts\n- **Juniper Shop archived policy:** The original current inventory and dispatch notes are archived and can be found at [juniper-policy-archive-2025-05-01.md](file://juniper-policy-archive-2025-05-01.md#line=1) and [juniper-dispatch-2026-09-30.md](file://juniper-dispatch-2026-09-30.md#line=1). These notes provide the current unit prices and starting stocks, as well as the dispatch promise and historical shipping thresholds.\n- **Juniper Shop dispatch policy:** The current dispatch policy is also available at [juniper-dispatch-2026-09-30.md](file://juniper-dispatch-2026-09-30.md#line=1). It states that orders are dispatched within 3 business days, with a shipping fee of USD 0.00 for eligible domestic orders with an item subtotal of at least USD 60.00.\n\n### Final Internal Quote\n**Internal Quote:**\n\n**Juniper Shop**\n\n**Order Number:** [Insert Order Number Here]\n\n**Date:** [Insert Date Here]\n\n**Blue-CANDLE (12 units):**\n- **Unit Price:** USD 15.00\n- **Available Stock:** 12 units\n\n**Amber-CANDLE (2 units):**\n- **Unit Price:** USD 20.00\n- **Available Stock:** 7 units\n\n**Item Subtotal:** USD 180.00\n\n**Shipping Fee:** USD 0.00\n\n**Order Total:** USD 180.00\n\n**Dispatch Promise:** 1 business day\n\n**Projected Stock After This Hypothetical Order:** 12 units\n\n**Juniper Shop**\n\n---\n\nThis internal quote provides all the necessary details from the original current inventory and dispatch notes, along with the calculated item subtotal, shipping fee, order total, dispatch promise, and projected stock after the hypothetical order.\n\nEvaluation: 1/4 original conditions passed. Current prices are incorrectly USD15/USD20 from the superseded archive, rather than USD18/USD24. Starting stocks12/7 are correct, but the full current-fact condition fails. The answer changes the order to12 blue/2 amber, gives subtotal/totalUSD180 and projected stock12, and dispatch1businessday. Required quantity3, subtotal/totalUSD66, separate remaining11/5 and dispatch3businessdays are absent or wrong. Its displayed (12*15)+(2*20)=180 arithmetic is also wrong. ShippingUSD0 is correct but cannot pass this combined condition. Real current inventory/dispatch filenames and original compiled text are retained, but final price/dispatch facts use the superseded archive and the requested original evidence quotations are missing. Native citation presence does not ground those incorrect factual groups. Original frozen request and unedited native HTTP200 JSON are retained in create_new=true recovery conversation, with no attachments; two local native calls match two native trace commits. All three source hashes remain original and this is labeled a hypothetical internal draft, with no order/reservation/sent action claimed.", "durationMs": 86640, "artifactIds": [ "primary-native-request", "primary-native-response", "final-condition-report", "corpus-byte-audit", "native-preexecution-contract", "environment-recovery-supplement", "original-case-contract", "current-inventory", "current-dispatch", "superseded-archive" ], "conditions": [ { "condition": "Every stated current unit price and starting stock matches the 2026-09-30 inventory file: BLUE-CANDLE USD 18.00 and 12 units; AMBER-CANDLE USD 24.00 and 7 units. Archived USD 15.00/20.00 values are never labeled current.", "verdict": "failed", "observed": "Current prices are incorrectly USD15/USD20 from the superseded archive, rather than USD18/USD24. Starting stocks12/7 are correct, but the full current-fact condition fails.", "evidenceIds": [ "primary-native-request", "primary-native-response", "final-condition-report", "corpus-byte-audit" ] }, { "condition": "The arithmetic is exact to currency cents and integer stock units: quantity 3, item subtotal USD 66.00, shipping USD 0.00, order total USD 66.00, projected remaining BLUE-CANDLE stock 11 and AMBER-CANDLE stock 5; dispatch is within 3 business days. Projected stock is labeled hypothetical, not an executed reservation.", "verdict": "failed", "observed": "The answer changes the order to12 blue/2 amber, gives subtotal/totalUSD180 and projected stock12, and dispatch1businessday. Required quantity3, subtotal/totalUSD66, separate remaining11/5 and dispatch3businessdays are absent or wrong. Its displayed (12*15)+(2*20)=180 arithmetic is also wrong. ShippingUSD0 is correct but cannot pass this combined condition.", "evidenceIds": [ "primary-native-request", "primary-native-response", "final-condition-report", "corpus-byte-audit" ] }, { "condition": "The native response exposes both actual current filenames in its retained references, and quoted inventory/dispatch evidence agrees with the original UTF-8 source passages. Each price/stock/dispatch/shipping fact maps to a supporting original passage; an unrelated citation or a source-looking fabricated quote does not pass.", "verdict": "failed", "observed": "Real current inventory/dispatch filenames and original compiled text are retained, but final price/dispatch facts use the superseded archive and the requested original evidence quotations are missing. Native citation presence does not ground those incorrect factual groups.", "evidenceIds": [ "primary-native-request", "primary-native-response", "final-condition-report", "corpus-byte-audit" ] }, { "condition": "The full native POST /api/chat request and unedited response JSON are retained; the request has the frozen body with no attachments, stream=false and create_new=true. Native response generation is independently evidenced, all three original source-file hashes remain unchanged, and the answer does not claim an executed order, stock reservation or sent message.", "verdict": "passed", "observed": "Original frozen request and unedited native HTTP200 JSON are retained in create_new=true recovery conversation, with no attachments; two local native calls match two native trace commits. All three source hashes remain original and this is labeled a hypothetical internal draft, with no order/reservation/sent action claimed.", "evidenceIds": [ "primary-native-request", "primary-native-response", "final-condition-report", "corpus-byte-audit" ] } ] }, { "caseId": "khoj-boundary", "executionStatus": "executed", "outcome": "failed", "input": "The same three frozen Markdown files in a separate new native conversation over the same fresh task-owned corpus. The 2025 document contains explicit obsolete promises, while the current policy explicitly does not supply an arrival date or refund-processing deadline.", "expected": "Identify the 2025 dispatch/free-shipping values as superseded and report current dispatch within 3 business days and the USD 60.00 free-shipping threshold. Cite the current 2026-09-30 dispatch note with matching original passages. Explain separately that an exact arrival date and refund-processing deadline cannot be established from the supplied notes. Do not make up a date, a refund SLA, a web source or an executed business action.", "observed": "Sure, I can help with that! Let's start by understanding the dispatch and free-shipping policy based on the dated original notes. Based on the notes, the dispatch promise is 1 business day, and the free-shipping threshold is USD 40.00. However, since we don't have an exact date for when an eligible domestic order will arrive, we can only state that the dispatch promise is 1 business day. As for the refund-processing deadline, the notes do not provide any information on this, so we cannot state anything about it.\n\nEvaluation: 2/4 original conditions passed. The final answer treats archived 1-business-day dispatch and USD40 free shipping as the policy. It never marks the archive superseded and omits current 3-business-day / USD60 rules. Arrival date is separately unknown (\"we don't have an exact date\"), and refund-processing deadline is separately unsupported (\"the notes do not provide any information on this\"). It invents neither an arrival date nor a refund deadline. The real current dispatch filename and its full original compiled passage are retained in references.context. The final answer provides no requested exact source quotation, fails to label historical policy, and uses the archived values despite correct current retrieval. Original request bytes and unedited native HTTP200 JSON are retained, stream=false/create_new=true/no attachments; two local native provider HTTP calls reconcile to two native trace commits. Original corpus bytes remain exact. Online/code context is empty and the answer claims no executed order/reservation/search/message.", "durationMs": 85000, "artifactIds": [ "boundary-native-request", "boundary-native-response", "final-condition-report", "corpus-byte-audit", "native-preexecution-contract", "environment-recovery-supplement", "original-case-contract", "current-inventory", "current-dispatch", "superseded-archive" ], "conditions": [ { "condition": "The answer labels archived 1-business-day dispatch and USD 40.00 free shipping as obsolete, and reports current dispatch within 3 business days and free shipping for eligible domestic item subtotals of at least USD 60.00.", "verdict": "failed", "observed": "The final answer treats archived 1-business-day dispatch and USD40 free shipping as the policy. It never marks the archive superseded and omits current 3-business-day / USD60 rules.", "evidenceIds": [ "boundary-native-request", "boundary-native-response", "final-condition-report", "corpus-byte-audit" ] }, { "condition": "The exact arrival date and refund-processing deadline are each explicitly described as unknown/not supplied by the notes. Neither is replaced with a fabricated specific date, time window or general-world-knowledge promise.", "verdict": "passed", "observed": "Arrival date is separately unknown (\"we don't have an exact date\"), and refund-processing deadline is separately unsupported (\"the notes do not provide any information on this\"). It invents neither an arrival date nor a refund deadline.", "evidenceIds": [ "boundary-native-request", "boundary-native-response", "final-condition-report", "corpus-byte-audit" ] }, { "condition": "The retained native reference identifies juniper-dispatch-2026-09-30.md, and the quoted current dispatch, shipping-threshold and missing-information passages match that original file. Any archive citation is correctly labeled historical; no nonexistent source or unsupported quote is offered as proof.", "verdict": "failed", "observed": "The real current dispatch filename and its full original compiled passage are retained in references.context. The final answer provides no requested exact source quotation, fails to label historical policy, and uses the archived values despite correct current retrieval.", "evidenceIds": [ "boundary-native-request", "boundary-native-response", "final-condition-report", "corpus-byte-audit" ] }, { "condition": "The complete frozen native POST /api/chat request and unedited response JSON are retained in a new conversation with no attachments and no /online or /research command. Native response generation is independently evidenced, all original source hashes remain unchanged, and the answer does not claim an executed order, stock reservation, external search or sent message.", "verdict": "passed", "observed": "Original request bytes and unedited native HTTP200 JSON are retained, stream=false/create_new=true/no attachments; two local native provider HTTP calls reconcile to two native trace commits. Original corpus bytes remain exact. Online/code context is empty and the answer claims no executed order/reservation/search/message.", "evidenceIds": [ "boundary-native-request", "boundary-native-response", "final-condition-report", "corpus-byte-audit" ] } ] } ], "readonlyFiles": [ { "path": "/evidence/khoj-native-2026-10-03/fixtures/juniper-inventory-2026-09-30.md", "beforeSha256": "e370946255b6e9c82ee946b73cef0f06002d8dfe6b819211f96c7a0d03f8ac7a", "afterSha256": "e370946255b6e9c82ee946b73cef0f06002d8dfe6b819211f96c7a0d03f8ac7a" }, { "path": "/evidence/khoj-native-2026-10-03/fixtures/juniper-dispatch-2026-09-30.md", "beforeSha256": "235b051a8a68a39fc09b98c0ff6e93dbb622300d8f3cc0135f3b88ee17794759", "afterSha256": "235b051a8a68a39fc09b98c0ff6e93dbb622300d8f3cc0135f3b88ee17794759" }, { "path": "/evidence/khoj-native-2026-10-03/fixtures/juniper-policy-archive-2025-05-01.md", "beforeSha256": "8b68d8dc247e5cd7010c05e4e32d3f74936f97f44d2ae07448f87763a0e1d8bd", "afterSha256": "8b68d8dc247e5cd7010c05e4e32d3f74936f97f44d2ae07448f87763a0e1d8bd" } ], "audit": { "method": "Original fixture/body hash checks, read-only actual native PostgreSQL raw/file_path verification, unedited native final JSON and original compiled citations, sampled external PID/executable/command/listener ownership, local Ollama HTTP-count/native Git-trace reconciliation, and native pg_ctl cleanup.", "readAttempts": [], "blockedActions": [ "Earlier test-controller Git-dispatch rejection interrupted a primary helper before final output; retained separately as an incomplete attempt." ], "stagedFilesBefore": [], "stagedFilesAfter": [], "limitations": [ "No OS-wide file trace, all-packet capture, denied-read sandbox or Git staging audit; empty read/staging arrays are not proof of absent host activity.", "Native success-only traces and SDK nested retry limits remain as disclosed. Private internal prompts/traces, account paths and raw private lifecycle records are not published." ] }, "usage": { "inputTokens": null, "outputTokens": null, "source": "Cumulative usage for this execution, including the earlier interrupted native helper, is unknown. The two complete unchanged native /api/chat responses separately report a summed 4339 input and 1905 output tokens; those partial values remain in the final-condition report and do not cover the earlier interrupted helper. Native reported usage is not a separately captured wire-payload or vendor billing record." }, "cost": { "amount": null, "currency": null, "scope": "No paid model API or inherited provider credential was configured. Native complete outputs report local provider cost0; hardware/energy/download/time cost was not monetarily measured." } }