{ "schema": "aiblocks-page/1", "id": "agency-engineering-data-engineer", "type": "agent", "name": "Data pipeline designer", "summary": "Get a data pipeline design with raw, cleaned and report-ready layers, quality checks, freshness targets, alerts and a plan for when loads fail.", "headline": "Data pipeline designer", "subhead": "List your data sources, who uses the data and how fresh it must be. It designs the pipeline in layers, adds checks and alerts, and writes what to do when a load fails or a source changes.", "job": { "id": "build-software", "name": "Build and ship software" }, "level": "intermediate", "thinking": "high", "tested": true, "updated": "2026-11-03", "first_published": "2026-10-26", "inputs": [ { "type": "question", "required": true, "label": "Question", "what": "the problem, decision or piece of work you want help with, and what a good outcome looks like" }, { "type": "codebase", "required": false, "label": "Codebase", "what": "a project's code, in a folder or repo the AI can open" } ], "outputs": [ { "type": "plan", "required": true, "label": "Plan", "what": "a dated plan: what to do, in what order, with a decision rule for each step" } ], "you_give": [ "Your data sources, such as apps, databases, files or event feeds, and roughly how much data each holds.", "How often each source changes and whether it can tell you what changed.", "Who uses the data and the questions they need answered.", "How fresh the data must be, such as daily or within 15 minutes.", "Where the data will live and the tools you use now (optional).", "Anything private in the data, such as customer emails or card details." ], "you_get": [ "A source list with the expected shape of each source and who owns it.", "A choice between batch, changes-only and streaming loads, with the reason.", "A design for the raw, cleaned and business-ready layers.", "A set of quality checks, with what each does when it fails.", "A freshness target for each output and the alerts that guard it.", "A recovery plan for reruns, backfills and source changes, and a short runbook for each pipeline." ], "use_when": [ "You are moving data from several systems, such as a shop, a CRM and ad accounts, into one warehouse.", "A report is wrong or late and nobody knows which step broke.", "You must choose between a nightly batch and a near-live feed.", "You want a standard for quality checks and alerts before more teams depend on the data." ], "skip_when": [ "You want working pipeline code for a smaller job. Use the Data pipeline builder.", "You need dashboards or charts. Use the Data visualization engineer agent.", "You are reviewing a business question or a SQL query on a warehouse. Use the Data warehouse query reviewer.", "One query is slow. Use the Database performance analyst.", "You have a single messy file to clean once. Ask your AI to clean it directly. A full pipeline design is more than you need." ], "output_sample": { "input": [ "Your data sources and how much data each holds", "Who uses the data, the questions they ask and how fresh it must be", "Where the data will live and the tools you use (optional)", "Anything private in the data" ], "prerequisites": [ "An AI you can paste a long brief into, such as Claude, ChatGPT or Gemini", "An engineer to build and test the pipeline, because the agent designs it and runs nothing" ], "output": [ { "name": "Sources and contracts", "contains": "Each source with its owner, volume, change rate, expected fields and any private data." }, { "name": "Load choice", "contains": "Full, changes-only or streaming for each source, with the reason and a cost note." }, { "name": "Layer design", "contains": "What the raw, cleaned and business-ready layers hold, their rules and who may read them." }, { "name": "Quality checks", "contains": "Each check, where it runs and what happens when it fails." }, { "name": "Freshness and alerts", "contains": "A freshness target for each output, the alert that guards it and its owner." }, { "name": "Recovery plan", "contains": "Safe reruns, backfills and the runbook note for each kind of failure." }, { "name": "Tool options", "contains": "Options for storage, running jobs and checks, with a reminder to confirm current prices." } ] }, "output_shape": "SOURCES AND CONTRACTS\n | | | | \n\nLOAD CHOICE\n | | | \n\nLAYER DESIGN\n | | | \n\nQUALITY CHECKS\n | | \n\nFRESHNESS AND ALERTS\n | | | \n\nRECOVERY PLAN\n | | \n\nTOOL OPTIONS\n | \n\nOPEN QUESTIONS AND CHECKS STILL NEEDED\n | ", "how_to_use": [ "Download the pack from this page and add it to your AI as a skill. Or paste SKILL.md into a new chat, then paste references/source-agent.md after it.", "List your sources and describe each in a line. Add the questions your team asks of the data and how fresh answers must be.", "Say \"Design the pipeline\" for a new setup, or \"Review this pipeline\" if you paste an existing design or code.", "Your AI asks once for anything missing. Unknown volumes stay marked as assumptions.", "Read the load choice and the quality checks first. Make sure each check matches a real way your data goes wrong.", "Hand the design to the engineer who will build it. Code in it is an example to adapt, and it needs testing on your data." ], "checks": [ "The result addresses your actual task and cites the evidence used.", "Examples, role-play experience and targets are kept separate from verified results.", "Claims of tests, memory, tool access or completed actions match what actually happened." ], "next": [ { "name": "Data pipeline builder", "url": "https://useaiblocks.com/skills/data-pipeline-builder", "why": "Build it" }, { "name": "Database schema designer", "url": "https://useaiblocks.com/skills/database-schema-designer", "why": "Build it" }, { "name": "RAG pipeline implementation planner", "url": "https://useaiblocks.com/skills/agency-engineering-rag-pipeline-engineer", "why": "Build it" } ], "source": { "by": "Michael Sitarzewski and Agency Agents contributors", "repo": "msitarzewski/agency-agents", "url": "https://github.com/msitarzewski/agency-agents/blob/b37b5bb33b78f2c1d6567d98e25ca818e3259219/engineering/engineering-data-engineer.md", "licence": "MIT", "original_name": "Data Engineer", "commit": "b37b5bb33b78f2c1d6567d98e25ca818e3259219" }, "credits": [ { "title": "Data Engineer", "by": "Michael Sitarzewski and Agency Agents contributors", "url": "https://github.com/msitarzewski/agency-agents/blob/b37b5bb33b78f2c1d6567d98e25ca818e3259219/engineering/engineering-data-engineer.md", "used": "Original agent profile, preserved in references/source-agent.md; AI Blocks adds setup and review instructions." } ], "files": [ "LICENSE", "references/source-agent.md", "SETUP.md", "source.json" ], "links": { "page": "https://useaiblocks.com/agents/agency-engineering-data-engineer", "markdown": "https://useaiblocks.com/agents/agency-engineering-data-engineer.md", "json": "https://useaiblocks.com/agents/agency-engineering-data-engineer.json", "instructions": "https://useaiblocks.com/skills/agency-engineering-data-engineer/SKILL.md", "download": "https://useaiblocks.com/skills/agency-engineering-data-engineer.zip", "api_item": "https://useaiblocks.com/api/v1/items/agency-engineering-data-engineer", "api_document": "https://useaiblocks.com/api/v1/documents/agent/agency-engineering-data-engineer", "mcp": "https://useaiblocks.com/mcp" } }