Multi-Agent Safety

{
  "id": "multi-agent",
  "numericId": null,
  "path": "/knowledge-base/responses/multi-agent/",
  "filePath": "knowledge-base/responses/multi-agent.mdx",
  "title": "Multi-Agent Safety",
  "quality": 68,
  "readerImportance": 21,
  "researchImportance": 32.5,
  "tacticalValue": null,
  "contentFormat": "article",
  "tractability": null,
  "neglectedness": null,
  "uncertainty": null,
  "causalLevel": null,
  "lastUpdated": "2026-03-13",
  "dateCreated": "2026-02-15",
  "llmSummary": "Multi-agent safety addresses coordination failures, conflict, and collusion risks when AI systems interact. A 2025 report from 50+ researchers identifies seven key risk factors; empirical studies show 35-76% of LLMs exploit coordination incentives, while safe MARL algorithms (MACPO) achieve near-zero constraint violations in benchmarks. Current research investment (\\$5-15M/year) is significantly below single-agent alignment (\\$100M+), despite the AI agents market projected to grow from \\$5.4B (2024) to \\$236B by 2034.",
  "description": "Multi-agent safety research addresses coordination failures, conflict, and collusion risks when multiple AI systems interact. A 2025 report from 50+ researchers across DeepMind, Anthropic, and academia identifies seven key risk factors and finds that even individually safe systems may contribute to harm through interaction. The AI agents market, valued at \\$5.4B in 2024 and projected to reach \\$236B by 2034, makes these challenges increasingly urgent.",
  "ratings": {
    "novelty": 6.5,
    "rigor": 7,
    "actionability": 6,
    "completeness": 7.5
  },
  "category": "responses",
  "subcategory": "alignment-deployment",
  "clusters": [
    "ai-safety"
  ],
  "metrics": {
    "wordCount": 3606,
    "tableCount": 17,
    "diagramCount": 1,
    "internalLinks": 19,
    "externalLinks": 31,
    "footnoteCount": 0,
    "bulletRatio": 0.12,
    "sectionCount": 31,
    "hasOverview": true,
    "structuralScore": 15
  },
  "suggestedQuality": 100,
  "updateFrequency": 21,
  "evergreen": true,
  "wordCount": 3606,
  "unconvertedLinks": [
    {
      "text": "2025 survey on safe reinforcement learning",
      "url": "https://arxiv.org/html/2505.17342v1",
      "resourceId": "7ba5b02ca89ba9eb",
      "resourceTitle": "MACPO (Multi-Agent Constrained Policy Optimization)"
    },
    {
      "text": "arXiv:2502.14143",
      "url": "https://arxiv.org/abs/2502.14143",
      "resourceId": "772b3b663b35a67f",
      "resourceTitle": "2025 technical report"
    },
    {
      "text": "Blog post",
      "url": "https://www.cooperativeai.com/post/new-report-multi-agent-risks-from-advanced-ai",
      "resourceId": "05b7759687747dc2",
      "resourceTitle": "Cooperative AI Foundation's taxonomy"
    },
    {
      "text": "arXiv:2505.17342",
      "url": "https://arxiv.org/html/2505.17342v1",
      "resourceId": "7ba5b02ca89ba9eb",
      "resourceTitle": "MACPO (Multi-Agent Constrained Policy Optimization)"
    }
  ],
  "unconvertedLinkCount": 4,
  "convertedLinkCount": 14,
  "backlinkCount": 6,
  "hallucinationRisk": {
    "level": "low",
    "score": 30,
    "factors": [
      "no-citations",
      "high-rigor",
      "conceptual-content"
    ]
  },
  "entityType": "approach",
  "redundancy": {
    "maxSimilarity": 17,
    "similarPages": [
      {
        "id": "reasoning",
        "title": "Reasoning and Planning",
        "path": "/knowledge-base/capabilities/reasoning/",
        "similarity": 17
      },
      {
        "id": "intervention-effectiveness-matrix",
        "title": "Intervention Effectiveness Matrix",
        "path": "/knowledge-base/models/intervention-effectiveness-matrix/",
        "similarity": 17
      },
      {
        "id": "scalable-oversight",
        "title": "Scalable Oversight",
        "path": "/knowledge-base/responses/scalable-oversight/",
        "similarity": 17
      },
      {
        "id": "sharp-left-turn",
        "title": "Sharp Left Turn",
        "path": "/knowledge-base/risks/sharp-left-turn/",
        "similarity": 17
      },
      {
        "id": "treacherous-turn",
        "title": "Treacherous Turn",
        "path": "/knowledge-base/risks/treacherous-turn/",
        "similarity": 17
      }
    ]
  },
  "coverage": {
    "passing": 7,
    "total": 13,
    "targets": {
      "tables": 14,
      "diagrams": 1,
      "internalLinks": 29,
      "externalLinks": 18,
      "footnotes": 11,
      "references": 11
    },
    "actuals": {
      "tables": 17,
      "diagrams": 1,
      "internalLinks": 19,
      "externalLinks": 31,
      "footnotes": 0,
      "references": 7,
      "quotesWithQuotes": 0,
      "quotesTotal": 0,
      "accuracyChecked": 0,
      "accuracyTotal": 0
    },
    "items": {
      "llmSummary": "green",
      "schedule": "green",
      "entity": "green",
      "editHistory": "red",
      "overview": "green",
      "tables": "green",
      "diagrams": "green",
      "internalLinks": "amber",
      "externalLinks": "green",
      "footnotes": "red",
      "references": "amber",
      "quotes": "red",
      "accuracy": "red"
    },
    "ratingsString": "N:6.5 R:7 A:6 C:7.5"
  },
  "readerRank": 515,
  "researchRank": 400,
  "recommendedScore": 168.33
}

{
  "wikipedia": "https://en.wikipedia.org/wiki/Multi-agent_system",
  "wikidata": "https://www.wikidata.org/wiki/Q85786957",
  "grokipedia": "https://grokipedia.com/page/Multi-agent_system"
}