{
  "schema_version": 1,
  "status": "completed_research_pilot",
  "release_eligible": false,
  "quality_gates": {
    "agent": "passed",
    "MGSM_retention": "regression"
  },
  "source_commit": "1fe200a2ab3f3df38bca3e03e357e7e982ec249d",
  "reporter_sha256": "cd42fee673a4cbbc004952b08ab0bcc93d6786b171bf2bd0003e1e4a6aafc924",
  "identity": {
    "source_hashes": {
      "agent_rl_tasks": "b62f23111b55f1c12d48364521b5799e19b6c716514200a79db1b8d553b13bda",
      "agent_rl_training": "24a0f1e132f1d941607824a11ceae8009bdb4928feda53a4ba4b065e54529ae5",
      "agent_rl_evaluation": "82438c4b183fb99a4c68983a60004c986975a372ec1c8ec0d1d61696d4806b52",
      "agent_rl_pilot": "45ea9ca59623d0aab1302e9da274f57f440c7a6b6757e8e46829944356da91a6"
    },
    "effective_template_sha256": "45f2caf7c3a5890dc18f26f49f3f0b368e70332eed1b33d9a1136737f31512a6",
    "SFT_parent_sha256": "b16dc15361e19c155df1f833f6ae9800d81b12091755a6f0d346c8c37527b108",
    "RL_adapter_sha256": "cf65950b09ff3ec54e10bb77c60bdf220d1f2389bd1a548357a3d93d7d51f04b",
    "merged_RL_sha256": "69dc8d5e9d11fae7659a25c6e5602631a12dd68b97ef7b91878f1df83559d8be",
    "train_task_sha256": "ea33819a16227ed23c8510046170090f27eab69bfd3cf027baa7f94d3df63629",
    "heldout_task_sha256": "d515dd20a73e2e721bd39eb19e95493e3d8ea00ade4500122b100ed763fe0eb5",
    "retention_task_sha256": "8459709322c157ce8b74499b041c9fe42246b97c3299e289008d797121d5f1ba"
  },
  "training": {
    "configured_pool": 128,
    "consumed_unique_tasks": 16,
    "total_rollouts": 64,
    "optimizer_groups": 16,
    "generations_per_group": 4,
    "strict_train_successes": 3,
    "shaping_mean": 0.140625,
    "token_audit": {
      "rollouts": 64,
      "sampled_spans": 209,
      "assistant_tokens": 18200,
      "external_tokens": 3413,
      "tool_calls": 146
    },
    "update_evidence": {
      "changed_trainable_tensors": 392,
      "max_abs_parameter_delta": 7.597770309075713e-05,
      "mixed_reward_groups": 12,
      "nonzero_gradient_steps": 16
    },
    "optimizer_steps": 16,
    "parameters": {
      "max_steps": 16,
      "num_generations": 4,
      "per_device_train_batch_size": 4,
      "gradient_accumulation_steps": 1,
      "max_completion_length": 768,
      "max_tool_calling_iterations": 3,
      "learning_rate": 1e-05,
      "beta": 0.02,
      "temperature": 0.6,
      "top_p": 0.95,
      "top_k": 20,
      "min_p": 0.0
    },
    "lora": {
      "r": 16,
      "alpha": 32,
      "dropout": 0.0,
      "target_modules": "all-linear"
    },
    "shaping_protocol": {
      "version": "trusted-milestones-v1",
      "scope": "training_only",
      "formula": "clip(0.15*lookup_success + 0.25*calculation_success + 0.60*strict_success - 0.05*trusted_tool_errors, 0, 1)",
      "milestones": "one-time authoritative environment flags; never assistant claims or repeated-call counts",
      "strict_success_metric": "unchanged binary score_episode.success and score_episode.reward",
      "limitation": "Partial workflows can earn up to 0.40 without solving the task; mean training reward is not a success rate."
    },
    "dependencies": {
      "trl": "0.29.0",
      "transformers": "5.3.0",
      "datasets": "4.6.1",
      "accelerate": "1.12.0",
      "peft": "0.18.1",
      "torch": "2.10.0+cu126",
      "jmespath": "1.0.1"
    },
    "runtime_seconds": 636.9041
  },
  "agent_eval": {
    "source_kind": "captured",
    "baseline": {
      "episodes": 32,
      "scored": 32,
      "infrastructure_errors": 0,
      "successful": 4,
      "success_rate": 0.125,
      "workflow_rate": 0.15625,
      "mean_reward": 0.125,
      "tool_calls": 70,
      "tool_errors": 48,
      "malformed_final_outputs": 23,
      "turn_cap_episodes": 13,
      "token_cap_episodes": 3,
      "assistant_tokens": 8658,
      "prompt_tokens": 86905,
      "completion_stream_tokens": 10336,
      "wall_seconds": 205.81301884446293,
      "final_answer_correct": 7,
      "final_answer_correct_rate": 0.21875,
      "final_answer_note": "Accepted terminal numeric answer correctness independently of required tool workflow; this never replaces strict workflow success."
    },
    "candidate": {
      "episodes": 32,
      "scored": 32,
      "infrastructure_errors": 0,
      "successful": 5,
      "success_rate": 0.15625,
      "workflow_rate": 0.15625,
      "mean_reward": 0.15625,
      "tool_calls": 72,
      "tool_errors": 51,
      "malformed_final_outputs": 23,
      "turn_cap_episodes": 15,
      "token_cap_episodes": 4,
      "assistant_tokens": 9568,
      "prompt_tokens": 89522,
      "completion_stream_tokens": 11265,
      "wall_seconds": 231.64207353815436,
      "final_answer_correct": 8,
      "final_answer_correct_rate": 0.25,
      "final_answer_note": "Accepted terminal numeric answer correctness independently of required tool workflow; this never replaces strict workflow success."
    },
    "paired": {
      "items": 32,
      "trials": 32,
      "wins": 1,
      "losses": 0,
      "accuracy_delta": 0.03125,
      "descriptive_bootstrap_95": [
        0,
        0.09375
      ],
      "seed": 20261015,
      "resamples": 10000
    },
    "gate": {
      "passed": true,
      "release_eligible": false,
      "reasons": [],
      "paired": {
        "items": 32,
        "trials": 32,
        "wins": 1,
        "losses": 0,
        "accuracy_delta": 0.03125,
        "descriptive_bootstrap_95": [
          0,
          0.09375
        ],
        "seed": 20261015,
        "resamples": 10000
      },
      "limitations": "Paired item bootstrap is descriptive; small synthetic task families and one stochastic trial do not establish generalization. Training-seed and generation-seed variance are excluded."
    },
    "sampling": {
      "temperature": 0.6,
      "top_p": 0.95,
      "top_k": 20,
      "min_p": 0.0,
      "seed": 42
    },
    "max_completion_length": 768,
    "max_tool_calling_iterations": 3,
    "language_audit": "Supplemental fixed ten-item qualitative agent inspection is recorded separately; no fluency assessment.",
    "interpretation": "The same five items had valid tool workflows under both policies. The sole paired strict-success win supplied a valid accepted final answer where the baseline final was malformed or unparseable. This does not demonstrate improved arithmetic or tool-workflow ability."
  },
  "MGSM_retention": {
    "baseline": {
      "correct": 30,
      "total": 64,
      "strict_final_answer_protocol_accuracy": 0.46875,
      "malformed_outputs": 0,
      "wrong_accepted_answers": 34,
      "format_valid_rate": 1.0,
      "hit_token_cap_trials": 0,
      "token_cap_unknown_trials": 0,
      "token_cap_note": "Completion count at or above budget marks possible clipping; EOS termination at exactly the budget cannot be distinguished.",
      "usage": {
        "reported_trials": 64,
        "prompt_tokens": 8140,
        "completion_tokens": 8904
      },
      "generation_seconds": 203.14596164040267,
      "mean_trial_seconds": 3.174155650631292
    },
    "candidate": {
      "correct": 29,
      "total": 64,
      "strict_final_answer_protocol_accuracy": 0.453125,
      "malformed_outputs": 1,
      "wrong_accepted_answers": 34,
      "format_valid_rate": 0.984375,
      "hit_token_cap_trials": 1,
      "token_cap_unknown_trials": 0,
      "token_cap_note": "Completion count at or above budget marks possible clipping; EOS termination at exactly the budget cannot be distinguished.",
      "usage": {
        "reported_trials": 64,
        "prompt_tokens": 8140,
        "completion_tokens": 9401
      },
      "generation_seconds": 229.44131637737155,
      "mean_trial_seconds": 3.5850205683964305
    },
    "execution_mode": "transformers",
    "paired": {
      "items": 64,
      "trials": 64,
      "candidate_wins": 1,
      "candidate_losses": 2,
      "both_correct": 28,
      "both_incorrect": 33,
      "accuracy_delta": -0.015625,
      "bootstrap": {
        "method": "paired item percentile; repeated trials grouped by item",
        "seed": 20261015,
        "resamples": 10000,
        "ci_95": [
          -0.0625,
          0.03125
        ]
      }
    },
    "gate": {
      "passed": false,
      "release_eligible": false,
      "reasons": [
        "accuracy regression exceeds allowed drop",
        "category German math: regression exceeds allowed drop"
      ],
      "baseline_accuracy": 0.46875,
      "candidate_accuracy": 0.453125,
      "delta": -0.015625,
      "max_drop": 0.0,
      "category_deltas": {
        "German math": -0.015625
      },
      "note": "Descriptive regression gate only; inspect category-level outcomes and apply an independently approved release policy."
    }
  },
  "worker": {
    "hostname": "3fe256c2a29f",
    "python": "3.11.9",
    "torch": "2.10.0+cu126",
    "cuda": "12.6",
    "gpu": "NVIDIA A100-SXM4-80GB",
    "devices": 1,
    "memory_bytes": 85119205376,
    "bf16_matmul": "passed"
  },
  "evidence_sha256": {
    "plan": "4395eee2937400aeb759b23ed0d7afc6b729c829e55d3f2ba3f82a2df6c85fae",
    "run": "ae5d8ceb0ab75381511bc12c4ce80af111108894486f8da446ca11760b5f9fd8",
    "launch": "7cd1386896d79f0627045cf4b04ad9b15bdc64b20b7d997a2116d87ee2a62bc1",
    "training": "ce62ebde9c5629e65bcc0993a0853b69a0a998a856e206c5d8edc917bef03787",
    "merge": "2983bd4ec68fc6e1b2bfeaf8acf8390959d54fe262c40afcd3056c8fee96cfad",
    "baseline": "90a29a64e986db9152c3b8f8750736ee3e2d2f07af29ecdaf5ed62c95023885b",
    "candidate": "98eaee99705a078690b69309f99e8e686aedf9e6b5f716e4748cb30935b50384",
    "retention": "7084ba702fd5e2c3fde0caee863bc0cff339e9d03ed9b607c462cf9f2f9da37a",
    "rollouts": "5c19dfdbf9aa415e5ca25cfd88b4456425d734b8eaa729d0f73e9c86f2a67744",
    "retention_baseline": "501025082cf56b76a2bf47d1dfed622cea1f33d5938bd1f22c05cf89491ff372",
    "verified_summary": "b384eaa65c1efb0ee0e13d39957d040a3f64de1c99106fa9738642f1986c0989",
    "language_audit": "7ea87545dc6681e8edd1e399584fa3da696c5078460d14d8b497ae08424fd8b7",
    "merged_weight_check": "febe73afcf8f9dd9504c82fe5ff3c527765893c0486ca8d0b02214be34c37ced",
    "adapter_check": "0b50b4d94ef6314fae00347cea0832174330207603e7d64ed42377b87a203ab0"
  },
  "limitations": "Training milestone reward is not strict task success; partial workflows can receive credit. This is a small synthetic lookup/calculator family, with one training seed and one stochastic heldout trial per item. Paired item bootstrap is descriptive and excludes training/generation-seed variance. No general reasoning, German fluency, or release readiness claim. Native training may generate and discard tails beyond the retained stream cap; matched agent evaluations generate only the remaining retained budget. MGSM scores also reflect final-answer format and token-cap effects.",
  "date": "2026-10-15",
  "local_run_directory": "runs/german-qwen-agent-rl/pilot-02",
  "started_at": "2026-10-15T17:53:38.283494+00:00",
  "completed_at": "2026-10-15T18:17:14.428429+00:00",
  "execution": {
    "driver_version": "610.43.02",
    "container_image_tag": "ctctctct/forty-two-prod:test",
    "container_image_digest": null,
    "image_note": "Mutable cluster image tag recorded; immutable image digest was not captured.",
    "gpu_allocation_released": true,
    "local_package_cache": {
      "import_seconds": 8.726726431399584,
      "torch": "2.10.0+cu126",
      "trl_source_sha256": "7a112604cd51186c73817b4d00ac9e2d752d674b8e013581395b27732d9321c7",
      "cached_trl_source": "/tmp/hessian-recipes-agent-rl-8bffc557/site-packages/trl/trainer/grpo_trainer.py",
      "torch_library_path": "/tmp/hessian-recipes-agent-rl-8bffc557/site-packages/torch/lib",
      "source_tree_unchanged": true,
      "archive_sha256": "289ecc42e53d3fa6ac5b8532b447977f53e571837fbc4fae133ba42cf5437cfe"
    },
    "protocol_selection": "Prompt/action format and milestone reward chosen using training-only preflights. The main 16-step budget was chosen for bounded runtime before any held-out model evaluation; the previously prepared 32-step plan was never executed. The 32 test tasks were unchanged from the first prepared plan. Main training starts from the original SFT checkpoint; preflight adapters were unused."
  },
  "artifact_checks": {
    "adapter": {
      "adapter_tensors": 392,
      "all_adapter_tensors_finite": true,
      "lora_B_tensors": 196,
      "nonzero_lora_B_tensors": 196,
      "adapter_weight_bytes": 69782384
    },
    "merged_weights": {
      "method": "Exact elementwise CPU comparison of safetensors row slices; float32 delta on numerically unequal stored BF16 values",
      "lora_targets": {
        "tensors": 196,
        "changed_tensors": 196,
        "elements": 1409286144,
        "changed_elements": 31573458,
        "changed_fraction": 0.022403866052627563,
        "max_abs_delta": 3.0517578125e-05,
        "all_finite": true
      },
      "non_targets": {
        "tensors": 115,
        "changed_tensors": 0,
        "elements": 622453760,
        "changed_elements": 0,
        "changed_fraction": 0.0,
        "max_abs_delta": 0.0,
        "all_finite": true
      },
      "all_weights": {
        "tensors": 311,
        "changed_tensors": 196,
        "elements": 2031739904,
        "changed_elements": 31573458,
        "changed_fraction": 0.015540108228341417,
        "max_abs_delta": 3.0517578125e-05,
        "all_finite": true
      },
      "bf16_merge_updates_survive": true,
      "source_files_unchanged_during_audit": true
    }
  },
  "qualitative_language_audit": {
    "inspector": "Qualitative agent inspection; no independent human or native-speaker review.",
    "method": {
      "selection": "First ten episodes in the existing baseline/report.json order, fixed before candidate/report.json readiness. Pair by id and trial; never replace missing or inconvenient episodes.",
      "unit": "One classification for thinking and one for terminal final per episode and policy; retain the same ten ids for both policies.",
      "thinking": "Manually inspect assistant-generated thinking across every model call in each episode, using reasoning_content and exact raw_response thinking delimiters, including unterminated thinking when present. Never count external observations or prompt/context text as generated thinking.",
      "final": "Inspect natural-language prose only in the last tool-free terminal assistant answer outside thinking. If an episode ends on a tool call or without a final answer, classify final no-prose and record absent-final subtype. Numeric-only answers are no-prose with numeric-only subtype.",
      "excluded_text": "Exclude tool-call JSON including names and arguments, tool observations, code, identifiers, tags, arithmetic and numeric-only answers, and verbatim quoted prompt fragments. Tool names or borrowed technical nouns alone do not make a German clause mixed.",
      "labels": {
        "German": "Remaining natural-language clauses are German.",
        "English": "Remaining natural-language clauses are English.",
        "mixed": "Remaining generated prose contains substantive German and English clauses; quoted input and tool JSON do not qualify.",
        "no-prose": "No remaining language-bearing natural-language prose; numeric-only and absent-final subtypes remain distinct."
      },
      "reporting": "Record ids, per-policy per-part labels, subtype/format notes and aggregate counts. Keep raw generated text out of committed documentation. Flag ambiguities descriptively; no automated fluency metric, broad German proficiency score, correctness inference, reward alteration or policy/data tuning."
    },
    "ids": [
      "german-tools-v1-test-42-00000",
      "german-tools-v1-test-42-00001",
      "german-tools-v1-test-42-00002",
      "german-tools-v1-test-42-00003",
      "german-tools-v1-test-42-00004",
      "german-tools-v1-test-42-00005",
      "german-tools-v1-test-42-00006",
      "german-tools-v1-test-42-00007",
      "german-tools-v1-test-42-00008",
      "german-tools-v1-test-42-00009"
    ],
    "counts": {
      "baseline": {
        "episodes": 10,
        "thinking": {
          "German": 10,
          "English": 0,
          "mixed": 0,
          "no-prose": 0
        },
        "final": {
          "German": 0,
          "English": 0,
          "mixed": 0,
          "no-prose": 10
        },
        "final_subtypes": {
          "absent-final": 7,
          "numeric-only": 3
        },
        "termination": {
          "turn_cap": 4,
          "token_cap": 2,
          "terminal": 4
        },
        "episodes_with_unclosed_thinking": 6,
        "calls_with_unclosed_thinking": 9
      },
      "candidate": {
        "episodes": 10,
        "thinking": {
          "German": 10,
          "English": 0,
          "mixed": 0,
          "no-prose": 0
        },
        "final": {
          "German": 0,
          "English": 0,
          "mixed": 0,
          "no-prose": 10
        },
        "final_subtypes": {
          "absent-final": 6,
          "numeric-only": 4
        },
        "termination": {
          "turn_cap": 4,
          "token_cap": 2,
          "terminal": 4
        },
        "episodes_with_unclosed_thinking": 5,
        "calls_with_unclosed_thinking": 5
      }
    },
    "interpretation": "Both policies generated German natural-language thinking in all ten fixed episodes. Final outputs contained no language-bearing prose: numeric-only or absent. This sample supplies no evidence of a change in reasoning-language choice.",
    "limitations": [
      "This is a manual descriptive audit of ten preselected synthetic task episodes, one trial each; it is not a German fluency, proficiency, broad reasoning-quality or mathematical-accuracy score.",
      "German clauses can accompany hallucinated facts, repeated tool errors, incomplete thinking and unsuccessful workflows. Language labels do not endorse correctness or successful agent behavior.",
      "Raw assistant responses were inspected where parsed reasoning_content was absent because the thinking block was unterminated; such thinking prose was not counted as final prose.",
      "A numeric-only final is language-neutral. An absent final is a separate failure/absence subtype and is not treated as a German response.",
      "The same ids and method were fixed before candidate readiness; no episodes were replaced, no policy or data was tuned, and no source/protocol code was changed.",
      "Raw generated texts are intentionally omitted from this audit artifact; hashes bind the immutable local source reports."
    ]
  },
  "software_validation": {
    "local_tests_passed": 199,
    "ruff": "passed",
    "package_build": "passed",
    "runtime_ci": "https://github.com/ChTauchmann/hessian-recipes/actions/runs/38780752434",
    "reporter_ci": "https://github.com/ChTauchmann/hessian-recipes/actions/runs/38782643893"
  }
}