{
  "project_id": "promptfoo/promptfoo-action",
  "repo": "promptfoo/promptfoo-action",
  "name": "promptfoo-action",
  "github_url": "https://github.com/promptfoo/promptfoo-action",
  "homepage_url": null,
  "language": "TypeScript",
  "license": "MIT",
  "project_kind": "project",
  "category": [
    "llm_eval"
  ],
  "tags": [
    "actions",
    "llm",
    "prompt-engineering",
    "rag",
    "security"
  ],
  "description": "The GitHub Action for Promptfoo.  Test your prompts, agents, and RAGs. AI Red teaming, pentesting, and vulnerability scanning for LLMs. Compare performance of GPT, Claude, Gemini, Llama, and more. Simple declarative configs with command line and CI/CD integration.",
  "overview": "Use promptfoo/promptfoo-action when the user needs a llm eval project with local, cloud deployment options.",
  "alternatives": [
    {
      "repo": "promptfoo/promptfoo",
      "reason": "Same llm eval intent with rag overlap."
    },
    {
      "repo": "modelscope/evalscope",
      "reason": "Same llm eval intent with rag overlap."
    },
    {
      "repo": "Giskard-AI/giskard-oss",
      "reason": "Same llm eval intent with rag overlap."
    }
  ],
  "related": [
    {
      "repo": "comet-ml/opik",
      "reason": "Related to promptfoo/promptfoo-action through llm eval category and local deployment."
    },
    {
      "repo": "Marker-Inc-Korea/AutoRAG",
      "reason": "Related to promptfoo/promptfoo-action through llm eval category and local deployment."
    },
    {
      "repo": "dataelement/bisheng",
      "reason": "Related to promptfoo/promptfoo-action through llm eval category and local deployment."
    },
    {
      "repo": "promptfoo/promptfoo",
      "reason": "Related to promptfoo/promptfoo-action through llm eval category and local deployment."
    },
    {
      "repo": "IntelLabs/RAG-FiT",
      "reason": "Related to promptfoo/promptfoo-action through llm eval category and local deployment."
    },
    {
      "repo": "vibrantlabsai/ragas",
      "reason": "Related to promptfoo/promptfoo-action through llm eval category and local deployment."
    },
    {
      "repo": "langfuse/langfuse",
      "reason": "Related to promptfoo/promptfoo-action through llm eval category and local deployment."
    },
    {
      "repo": "langwatch/langwatch",
      "reason": "Related to promptfoo/promptfoo-action through llm eval category and local deployment."
    }
  ],
  "dependencies": [
    "Vector database",
    "LLM provider"
  ],
  "deployments": [
    "local",
    "cloud"
  ],
  "difficulty": "beginner",
  "cloudflare_ready": false,
  "use_cases": [
    "evaluate LLM outputs",
    "benchmark prompts and agents",
    "track model quality"
  ],
  "not_good_for": [
    "edge-only Cloudflare Workers deployment without adaptation"
  ],
  "classification": {
    "category": {
      "confidence": "high",
      "evidence": [
        "Matched \"eval\" in repository content.",
        "Matched \"evaluation\" in repository content."
      ]
    },
    "deployment": {
      "confidence": "medium",
      "evidence": [
        "Local usage is assumed for open source repositories unless contradicted."
      ]
    },
    "difficulty": {
      "confidence": "medium",
      "evidence": [
        "Repository has under 10k stars, so complexity is treated conservatively."
      ]
    },
    "cloudflare_ready": {
      "confidence": "high",
      "evidence": [
        "Runtime blocker: python.",
        "No Cloudflare deployment signal detected.",
        "No wrangler.toml found in inspected repository files."
      ]
    }
  },
  "quality_signals": {
    "stars": 72,
    "recent_commits": 18,
    "contributors": 15,
    "issue_response_time_hours": null,
    "release_frequency_180d": 6
  },
  "quality_signal_confidence": {
    "stars_30d_delta": "snapshot",
    "stars30d_window_days": 33,
    "commits_30d": "complete",
    "releases_180d": "complete",
    "contributors_90d": "complete"
  },
  "quality_score": 16,
  "agent_score": 58,
  "score": 80,
  "agent_score_breakdown": {
    "documentation": 90,
    "maintenance": 35,
    "deployment": 70,
    "popularity": 37,
    "community": 60
  },
  "git_top_score": 80,
  "git_top_score_breakdown": {
    "community": 100,
    "maintenance": 50,
    "documentation": 84,
    "stability": 100,
    "adoption": 56,
    "agent_readability": 100
  },
  "verification": {
    "schema_version": "git-top.verification.v1",
    "overall_status": "partial",
    "observed_at": "2026-09-19T16:00:36.547Z",
    "signals": {
      "activity": {
        "status": "observed",
        "source": "github_metadata",
        "evidence": [
          "project.pushed_at",
          "metrics.recent_push_days"
        ]
      },
      "license": {
        "status": "observed",
        "source": "github_metadata",
        "evidence": [
          "project.license"
        ]
      },
      "documentation": {
        "status": "observed",
        "source": "github_metadata",
        "evidence": [
          "project.description"
        ]
      },
      "runtime": {
        "status": "inferred",
        "source": "git_top_heuristic",
        "evidence": [
          "agent_card.deployment",
          "agent_card.cloudflare_ready"
        ]
      },
      "install": {
        "status": "not_run",
        "source": "git_top_runtime",
        "evidence": [
          "installation/build probe is not enabled"
        ]
      },
      "security": {
        "status": "not_run",
        "source": "git_top_runtime",
        "evidence": [
          "dependency/security scan is not enabled"
        ]
      },
      "mcp": {
        "status": "not_applicable",
        "source": "git_top_runtime",
        "evidence": [
          "project is not classified as mcp_server"
        ]
      }
    },
    "coverage": {
      "observed": 3,
      "inferred": 1,
      "not_run": 2,
      "unknown": 0
    },
    "next_steps": [
      "Run an installation/build probe before production adoption.",
      "Run a dependency and security scan before treating the score as a trust claim."
    ]
  },
  "summary": {
    "tl_dr": "Use promptfoo/promptfoo-action when the user needs a llm eval project with local, cloud deployment options.",
    "purpose": "Use promptfoo/promptfoo-action when the user needs a llm eval project with local, cloud deployment options.",
    "install": "Install and run the evaluation harness as documented by the repository.",
    "inputs": [
      "prompts",
      "model outputs",
      "test cases"
    ],
    "outputs": [
      "scores",
      "benchmarks",
      "eval reports"
    ],
    "good_for": [
      "evaluate LLM outputs",
      "benchmark prompts and agents",
      "track model quality",
      "model evaluation",
      "benchmarking",
      "regression testing"
    ],
    "not_good_for": [
      "edge-only Cloudflare Workers deployment without adaptation",
      "production inference serving",
      "end-user chat apps"
    ],
    "deployment": [
      "local",
      "cloud"
    ],
    "alternatives": [
      {
        "repo": "promptfoo/promptfoo",
        "reason": "Same llm eval intent with rag overlap."
      },
      {
        "repo": "modelscope/evalscope",
        "reason": "Same llm eval intent with rag overlap."
      },
      {
        "repo": "Giskard-AI/giskard-oss",
        "reason": "Same llm eval intent with rag overlap."
      }
    ]
  },
  "evidence": {
    "classification": {
      "category": {
        "confidence": "high",
        "evidence": [
          "Matched \"eval\" in repository content.",
          "Matched \"evaluation\" in repository content."
        ]
      },
      "deployment": {
        "confidence": "medium",
        "evidence": [
          "Local usage is assumed for open source repositories unless contradicted."
        ]
      },
      "difficulty": {
        "confidence": "medium",
        "evidence": [
          "Repository has under 10k stars, so complexity is treated conservatively."
        ]
      },
      "cloudflare_ready": {
        "confidence": "high",
        "evidence": [
          "Runtime blocker: python.",
          "No Cloudflare deployment signal detected.",
          "No wrangler.toml found in inspected repository files."
        ]
      }
    },
    "quality_signal_confidence": {
      "stars_30d_delta": "snapshot",
      "stars30d_window_days": 33,
      "commits_30d": "complete",
      "releases_180d": "complete",
      "contributors_90d": "complete"
    },
    "verification": {
      "schema_version": "git-top.verification.v1",
      "overall_status": "partial",
      "observed_at": "2026-09-19T16:00:36.547Z",
      "signals": {
        "activity": {
          "status": "observed",
          "source": "github_metadata",
          "evidence": [
            "project.pushed_at",
            "metrics.recent_push_days"
          ]
        },
        "license": {
          "status": "observed",
          "source": "github_metadata",
          "evidence": [
            "project.license"
          ]
        },
        "documentation": {
          "status": "observed",
          "source": "github_metadata",
          "evidence": [
            "project.description"
          ]
        },
        "runtime": {
          "status": "inferred",
          "source": "git_top_heuristic",
          "evidence": [
            "agent_card.deployment",
            "agent_card.cloudflare_ready"
          ]
        },
        "install": {
          "status": "not_run",
          "source": "git_top_runtime",
          "evidence": [
            "installation/build probe is not enabled"
          ]
        },
        "security": {
          "status": "not_run",
          "source": "git_top_runtime",
          "evidence": [
            "dependency/security scan is not enabled"
          ]
        },
        "mcp": {
          "status": "not_applicable",
          "source": "git_top_runtime",
          "evidence": [
            "project is not classified as mcp_server"
          ]
        }
      },
      "coverage": {
        "observed": 3,
        "inferred": 1,
        "not_run": 2,
        "unknown": 0
      },
      "next_steps": [
        "Run an installation/build probe before production adoption.",
        "Run a dependency and security scan before treating the score as a trust claim."
      ]
    },
    "source_fields": [
      "project.description",
      "project.topics",
      "project.language",
      "project.license",
      "agent_card.summary_for_agent",
      "agent_card.use_cases",
      "agent_card.deployment",
      "agent_card.classification",
      "metrics",
      "verification.signals"
    ],
    "caveats": [
      "edge-only Cloudflare Workers deployment without adaptation",
      "Independent installation, security, or runtime probes have not been run for this project."
    ],
    "confidence_reason": "Evidence is usable for comparison, but agents should cite caveats before making a strong recommendation.",
    "last_verified_at": "2026-09-19T16:00:36.547Z"
  },
  "caveats": [
    "edge-only Cloudflare Workers deployment without adaptation",
    "Independent installation, security, or runtime probes have not been run for this project."
  ],
  "confidence_reason": "Evidence is usable for comparison, but agents should cite caveats before making a strong recommendation.",
  "source_fields": [
    "project.description",
    "project.topics",
    "project.language",
    "project.license",
    "agent_card.summary_for_agent",
    "agent_card.use_cases",
    "agent_card.deployment",
    "agent_card.classification",
    "metrics",
    "verification.signals"
  ],
  "last_verified_at": "2026-09-19T16:00:36.547Z",
  "knowledge": {
    "project": {
      "id": "promptfoo/promptfoo-action",
      "owner": "promptfoo",
      "name": "promptfoo-action",
      "full_name": "promptfoo/promptfoo-action",
      "github_url": "https://github.com/promptfoo/promptfoo-action",
      "homepage_url": null,
      "description": "The GitHub Action for Promptfoo.  Test your prompts, agents, and RAGs. AI Red teaming, pentesting, and vulnerability scanning for LLMs. Compare performance of GPT, Claude, Gemini, Llama, and more. Simple declarative configs with command line and CI/CD integration.",
      "language": "TypeScript",
      "topics": [
        "actions",
        "llm",
        "prompt-engineering",
        "rag",
        "security"
      ],
      "license": "MIT",
      "stars": 72,
      "forks": 34,
      "open_issues": 17,
      "default_branch": "main",
      "created_at": "2023-06-21T03:10:38Z",
      "updated_at": "2026-09-12T05:13:26Z",
      "pushed_at": "2026-09-18T06:37:55Z",
      "synced_at": "2026-09-19T16:00:36.547Z"
    },
    "agent_card": {
      "project_id": "promptfoo/promptfoo-action",
      "project_kind": "project",
      "category": "llm_eval",
      "difficulty": "beginner",
      "deployment": [
        "local",
        "cloud"
      ],
      "cloudflare_ready": false,
      "use_cases": [
        "evaluate LLM outputs",
        "benchmark prompts and agents",
        "track model quality"
      ],
      "not_good_for": [
        "edge-only Cloudflare Workers deployment without adaptation"
      ],
      "alternatives": [
        {
          "project_id": "promptfoo/promptfoo",
          "reason": "Same llm eval intent with rag overlap."
        },
        {
          "project_id": "modelscope/evalscope",
          "reason": "Same llm eval intent with rag overlap."
        },
        {
          "project_id": "Giskard-AI/giskard-oss",
          "reason": "Same llm eval intent with rag overlap."
        }
      ],
      "summary_for_agent": "Use promptfoo/promptfoo-action when the user needs a llm eval project with local, cloud deployment options.",
      "classification": {
        "category": {
          "confidence": "high",
          "evidence": [
            "Matched \"eval\" in repository content.",
            "Matched \"evaluation\" in repository content."
          ]
        },
        "deployment": {
          "confidence": "medium",
          "evidence": [
            "Local usage is assumed for open source repositories unless contradicted."
          ]
        },
        "difficulty": {
          "confidence": "medium",
          "evidence": [
            "Repository has under 10k stars, so complexity is treated conservatively."
          ]
        },
        "cloudflare_ready": {
          "confidence": "high",
          "evidence": [
            "Runtime blocker: python.",
            "No Cloudflare deployment signal detected.",
            "No wrangler.toml found in inspected repository files."
          ]
        }
      },
      "schema_version": "v1",
      "generated_at": "2026-09-19T16:00:36.547Z"
    },
    "metrics": {
      "project_id": "promptfoo/promptfoo-action",
      "stars_30d_delta": 2,
      "commits_30d": 18,
      "releases_180d": 6,
      "contributors_90d": 15,
      "issue_first_response_median_hours": null,
      "recent_push_days": 1,
      "git_score": 16,
      "maintenance_score": 35,
      "signal_confidence": {
        "stars_30d_delta": "snapshot",
        "stars30d_window_days": 33,
        "commits_30d": "complete",
        "releases_180d": "complete",
        "contributors_90d": "complete"
      },
      "calculated_at": "2026-09-19T16:00:36.547Z"
    }
  },
  "resolved_from": {
    "requested_id": "promptfoo/promptfoo-action",
    "resolved_id": "promptfoo/promptfoo-action",
    "resolution": "direct"
  },
  "metadata": {
    "source": "d1",
    "reason": "d1_query",
    "project_count": 1213,
    "generated_at": "2026-09-20T02:53:46.037Z",
    "snapshot_id": "d1:1213:2026-09-20T02:30:40.753Z",
    "latest_synced_at": "2026-09-20T02:30:40.753Z",
    "schema_version": "git-top.knowledge.v1",
    "loaded_project_limit": 2000,
    "truncated": false
  }
}