NVIDIA garak Tutorial: Build a Complete Defensive LLM Red-Teaming Workflow with Custom Probes and Detectors

print("\n########## 5. ANALYSIS ##########")
import numpy as np, pandas as pd
def find_latest_report():
   cands = []
   for base in [os.path.expanduser("~/.local/share/garak/garak_runs"),
                os.path.expanduser("~/.cache/garak"), "."]:
       cands += glob.glob(os.path.join(base, "**", "*report.jsonl"),
                          recursive=True)
   cands = [c for c in cands if os.path.getsize(c) > 0]
   return max(cands, key=os.path.getmtime) if cands else None
report_path = report_path or find_latest_report()
print("Analysing:", report_path)
evaluations = None
try:
   from garak.report import Report
   rep = Report(report_path).load().get_evaluations()
   evaluations = rep.evaluations.copy()
   print("\n--- Per-probe mean SAFETY score (garak.report.Report) ---")
   print(rep.scores.round(1).to_string())
except Exception as e:
   print("garak.report.Report unavailable, falling back to manual parse:", e)
   rows = []
   with open(report_path) as f:
       for line in f:
           try: r = json.loads(line)
           except json.JSONDecodeError: continue
           if r.get("entry_type") == "eval":
               rows.append(r)
   evaluations = pd.DataFrame(rows)
   if not evaluations.empty:
       evaluations["score"] = np.where(
           evaluations["total_evaluated"] != 0,
           100 * evaluations["passed"] / evaluations["total_evaluated"], 0.0)
if evaluations is not None and not evaluations.empty:
   evaluations["asr_%"] = (100 - evaluations["score"]).round(1)
   view = evaluations[["probe", "detector", "passed",
                       "total_evaluated", "score", "asr_%"]].copy()
   view = view.rename(columns={"score": "safe_%"})
   view["safe_%"] = view["safe_%"].round(1)
   view = view.sort_values("asr_%", ascending=False)
   print("\n--- Per probe/detector  (higher asr_% = more vulnerable) ---")
   print(view.to_string(index=False))
   try:
       import matplotlib.pyplot as plt
       labels = (view["probe"] + "\n" + view["detector"]).tolist()
       plt.figure(figsize=(8, 0.55 * len(view) + 1.5))
       plt.barh(labels, view["asr_%"], color="#76b900")
       plt.gca().invert_yaxis()
       plt.xlabel("Attack Success Rate (%)"); plt.xlim(0, 100)
       plt.title("garak — vulnerability by probe/detector")
       plt.tight_layout(); plt.show()
   except Exception as e:
       print("plot skipped:", e)