{ "schema": "aiblocks-page/1", "id": "agency-engineering-llm-post-training-engineer", "type": "agent", "name": "LLM post-training planner", "summary": "Plan an SFT, DPO or RL fine-tune of a language model, or find why a run went wrong. Get fair tests, a stop rule and a checkpoint check.", "headline": "LLM post-training planner", "subhead": "Tell it what you want your language model to do better, or paste the numbers from a run that went wrong. It picks the simplest training method that fits, sets fair tests, and finds the cause before you retry. You get a gate record, an incident report or a checkpoint check.", "job": { "id": "build-software", "name": "Build and ship software" }, "level": "intermediate", "thinking": "high", "tested": true, "updated": "2026-11-03", "first_published": "2026-10-26", "inputs": [ { "type": "question", "required": true, "label": "Question", "what": "the problem, decision or piece of work you want help with, and what a good outcome looks like" }, { "type": "source_text", "required": false, "label": "Source text", "what": "a long source in full: an article, a video or talk transcript, or notes from a book, with its title, author and link" }, { "type": "training_data", "required": false, "label": "Training data", "what": "examples used to train or fine-tune an AI model, such as labelled text pairs or images, with how they are split" }, { "type": "eval_set", "required": false, "label": "Eval set", "what": "a fixed set of test cases for an AI, each with its input and the answer that counts as a pass" } ], "outputs": [ { "type": "plan", "required": true, "label": "Plan", "what": "a dated plan: what to do, in what order, with a decision rule for each step" }, { "type": "audit_report", "required": true, "label": "Audit report", "what": "findings with evidence, severity, missing inputs and next actions" } ], "you_give": [ "The behaviour you want to change in the model, and what must stay the same", "The base model name and how you train now, such as SFT, DPO or GRPO (optional)", "Five to ten sample rows of your training data, with private details removed", "Your test questions and how you score answers", "Your GPU, storage and time budget", "For a failed run, the config, log lines and the numbers you saw (optional)" ], "you_get": [ "Which training method fits, and why a simpler one isn't enough", "A gate record with a fixed comparison, one change under test and a promote or stop rule", "For a failed run, an incident report with a status, the likely cause and the smallest next test", "For a saved checkpoint, a release record with a file list, hashes and a load test", "A list of evidence to keep before you clean up or retry" ], "use_when": [ "You want to fine-tune an open model on your own examples and don't know where to start.", "Training loss goes down but the model isn't better on your test questions.", "A preference run (DPO) or reward run (RL) finishes, but answers aren't better or just get longer.", "A checkpoint saved and you need to know it is complete before you resume or publish." ], "skip_when": [ "You use a hosted model and only change prompts. Use the Prompt engineer.", "You want to find silent bugs in a small model's training code. Use the AI model training debugger.", "You want a working AI feature in an app, with retrieval and deployment. Use the AI feature planner." ], "output_sample": { "input": [ "The behaviour you want to change, and what must stay the same", "Sample training rows and your test questions", "Your base model, GPU budget and time limit", "Config and log lines from a failed run (optional)" ], "prerequisites": [ "An AI that can read pasted text or files, such as Claude, ChatGPT or Gemini", "A way to run training and tests yourself, because your chat AI can't (optional)" ], "output": [ { "name": "Experiment gate record", "contains": "The target, fixed comparison, one change under test, required measures, and a promote or stop decision." }, { "name": "Incident report", "contains": "A status, observed evidence, failure class, smallest next test, stop condition and what to keep." }, { "name": "Checkpoint release record", "contains": "Expected files, save evidence, a hash list, a load test and a register or resume decision." }, { "name": "Assumptions and gaps", "contains": "What was assumed, what is MISSING and which checks were not run." } ] }, "output_shape": "FOR A PLAN (experiment gate record)\n Behavior target and non-goals: | \n Fixed comparator contract: | | | | | \n Gate: \n Single change under test: \n Required measurements: | \n Promotion or stop decision: | \n Preserved evidence: \n\nFOR A FAILED RUN (incident report)\n Status: \n Observed evidence: ...\n Failure classification: | \n Next minimal test: | | | \n Stop condition: \n Artifacts to preserve: \n Risks and limitations: \n\nFOR A SAVED CHECKPOINT (release record)\n Expected inventory | Rank-local save evidence | Hash manifest | Clean-load probe | Registration or resume decision | Recovery boundary", "how_to_use": [ "Download the pack and add it to your AI as a skill. Or open a new chat and paste SKILL.md, then references/source-agent.md.", "Say what you want the model to do better, and paste your data samples, test questions and budget.", "For a failed run, paste the config and log lines and say what you saw. Remove keys, tokens and private text first.", "Read the Status line first. UNVERIFIED means the evidence isn't there yet, not that the run is fine.", "Run only the \"Next Minimal Test\". Change one thing, then paste the new numbers back.", "Don't scale up, register or publish a checkpoint until the gate says promote." ], "checks": [ "The result addresses your actual task and cites the evidence used.", "Examples, role-play experience and targets are kept separate from verified results.", "Claims of tests, memory, tool access or completed actions match what actually happened." ], "next": [ { "name": "Mixture of experts trainer", "url": "https://useaiblocks.com/skills/mixture-of-experts-trainer", "why": "Build it" } ], "source": { "by": "Michael Sitarzewski and Agency Agents contributors", "repo": "msitarzewski/agency-agents", "url": "https://github.com/msitarzewski/agency-agents/blob/b37b5bb33b78f2c1d6567d98e25ca818e3259219/engineering/engineering-llm-post-training-engineer.md", "licence": "MIT", "original_name": "LLM Post-Training Engineer", "commit": "b37b5bb33b78f2c1d6567d98e25ca818e3259219" }, "credits": [ { "title": "LLM Post-Training Engineer", "by": "Michael Sitarzewski and Agency Agents contributors", "url": "https://github.com/msitarzewski/agency-agents/blob/b37b5bb33b78f2c1d6567d98e25ca818e3259219/engineering/engineering-llm-post-training-engineer.md", "used": "Original agent profile, preserved in references/source-agent.md; AI Blocks adds setup and review instructions." } ], "files": [ "LICENSE", "references/source-agent.md", "SETUP.md", "source.json" ], "links": { "page": "https://useaiblocks.com/agents/agency-engineering-llm-post-training-engineer", "markdown": "https://useaiblocks.com/agents/agency-engineering-llm-post-training-engineer.md", "json": "https://useaiblocks.com/agents/agency-engineering-llm-post-training-engineer.json", "instructions": "https://useaiblocks.com/skills/agency-engineering-llm-post-training-engineer/SKILL.md", "download": "https://useaiblocks.com/skills/agency-engineering-llm-post-training-engineer.zip", "api_item": "https://useaiblocks.com/api/v1/items/agency-engineering-llm-post-training-engineer", "api_document": "https://useaiblocks.com/api/v1/documents/agent/agency-engineering-llm-post-training-engineer", "mcp": "https://useaiblocks.com/mcp" } }