Getting Started with vfairness

# Core (numpy, pandas, scipy, scikit-learn, requests)
pip install vfairness

# With visualization support (matplotlib, seaborn)
pip install "vfairness[viz]"

# With interactive dashboards (Plotly + Dash)
pip install "vfairness[interactive]"

# Plotly charts only
pip install "vfairness[dashboard]"

# Experiment tracking (MLflow, Weights & Biases)
pip install "vfairness[mlops]"

# Full installation (all optional dependencies)
pip install "vfairness[all]"

import vfairness
print(f"vfairness {vfairness.__version__}")

import numpy as np
from vfairness import FairnessAnalyzer, demographic_parity_difference

# Sample data
y_true = np.array([1, 0, 1, 0, 1, 0, 1, 0, 1, 1] * 100)
y_pred = np.array([1, 0, 1, 1, 1, 0, 0, 0, 1, 1] * 100)
gender = np.array(['M','M','M','M','M','F','F','F','F','F'] * 100)

# Single metric
dp = demographic_parity_difference(y_true, y_pred, gender)
print(f"Demographic Parity Difference: {dp:.3f}")

# Full analyzer with a confidence interval.
# random_state pins the bootstrap, so the interval is reproducible.
analyzer = FairnessAnalyzer(y_true, y_pred, gender, min_group_size=30)
result = analyzer.demographic_parity_difference(
    include_ci=True, n_bootstrap=5000, confidence_level=0.95, random_state=0
)
lo, hi = result.confidence_interval
print(f"DP Diff: {result.value:.3f}  95% CI: [{lo:.3f}, {hi:.3f}]")

# All metrics at once
for name, value in analyzer.compute_all_metrics().items():
    print(f"  {name}: {value:.3f}")

# The verdict, and the rows it was computed over
print(analyzer.get_report()["assessment"]["summary"])

Demographic Parity Difference: 0.400
DP Diff: 0.400  95% CI: [0.346, 0.456]
  demographic_parity_difference: 0.400
  equalized_odds_difference: 0.500
  equal_opportunity_difference: 0.333
  demographic_parity_ratio: 0.500
  predictive_parity_difference: 0.250
0/5 metrics within thresholds (data provenance: 1000 of 1000 rows assessed, 0 excluded for missing values, 0 in 0 group(s) withheld by the group-size floor, missing_strategy='exclude')

import numpy as np
from vfairness import FairnessAnalyzer

# The same 1,000 rows as above, plus 12 nonbinary applicants.
y_true = np.array([1, 0, 1, 0, 1, 0, 1, 0, 1, 1] * 100)
y_pred = np.array([1, 0, 1, 1, 1, 0, 0, 0, 1, 1] * 100)
gender = np.array(['M','M','M','M','M','F','F','F','F','F'] * 100)

y_true = np.append(y_true, [1, 0] * 6)
y_pred = np.append(y_pred, [1, 1] * 6)
gender = np.append(gender, ['NB'] * 12)

assessment = FairnessAnalyzer(y_true, y_pred, gender).get_report()["assessment"]

print(assessment["summary"])

group = assessment["insufficient_evidence_groups"][0]
print(f"{group['group']}: n={group['n']}, verdict={group['verdict']}")
print(group["reason"])

UserWarning: Excluding 1 group(s) below min_group_size=30: {'NB': 12}. That leaves
12 of 1012 rows (1.2 percent) out of this metric. The metric is computed over the
remaining groups only, so it measures nothing about the excluded group(s): they are
could not check, not a measured pass. To include them, lower min_group_size.

0/5 metrics within thresholds (1 group(s) with insufficient evidence, excluded from the verdict: ['NB']) (data provenance: 1012 of 1012 rows assessed, 0 excluded for missing values, 12 in 1 group(s) withheld by the group-size floor, missing_strategy='exclude')
NB: n=12, verdict=insufficient_evidence
n=12 is below min_group_size=30; this group is excluded from the disparity metrics, so there is insufficient evidence to assess it. Underpowered (n<30): the rate is noisy; treat as indicative only.

import numpy as np
from vfairness import FairnessAnalyzer

# 400 men and 3 women: only one group is large enough to measure, so there is
# nothing to compare it against.
gender = np.array(['M'] * 400 + ['F'] * 3)
y_true = np.append(np.random.default_rng(1).integers(0, 2, 400), [1, 0, 1])
y_pred = np.append(np.random.default_rng(2).integers(0, 2, 400), [1, 1, 0])

assessment = FairnessAnalyzer(y_true, y_pred, gender).get_report()["assessment"]

print("assessable:    ", assessment["assessable"])
print("fairness_score:", assessment["fairness_score"])
print("not assessable:", [m["metric"] for m in assessment["not_assessable_metrics"]])
print(assessment["summary"])

assessable:     False
fairness_score: None
not assessable: ['demographic_parity_difference', 'demographic_parity_ratio', 'equalized_odds_difference', 'equal_opportunity_difference', 'predictive_parity_difference']
NOT ASSESSABLE: 1 valid group(s) after filtering (need at least 2): disparity metrics are vacuous and do not certify fairness. (1 group(s) with insufficient evidence, excluded from the verdict: ['F']) (5 metric(s) not assessable, excluded from the verdict) (data provenance: 403 of 403 rows assessed, 0 excluded for missing values, 3 in 1 group(s) withheld by the group-size floor, missing_strategy='exclude')

from vfairness.llm import OutputAnalyzer

analyzer = OutputAnalyzer(alpha=0.05)
result = analyzer.analyze_sentiment(texts_a, texts_b)

# Every result has audit metadata
print(result.metadata.timestamp)        # ISO 8601 UTC
print(result.metadata.library_version)  # current vfairness.__version__
print(result.metadata.parameters)       # {"alpha": 0.05, "metric": "sentiment", ...}

# Serialize for database storage
data = result.to_dict()    # Plain dict
json_str = result.to_json()  # JSON string

# Structured logging (configure once)
import logging
logging.basicConfig(level=logging.INFO)
analyzer.analyze_all(texts_a, texts_b)

INFO:vfairness.llm.output_analysis:analyze_all: group_a=group_a (18 samples), group_b=group_b (18 samples), correction=benjamini_hochberg
WARNING:vfairness.llm.output_analysis:Sample size (18, 18) below recommended minimum of 25 for metric 'semantic_quality'
WARNING:vfairness.llm.output_analysis:Sample size (18, 18) below recommended minimum of 25 for metric 'sentiment'
...
INFO:vfairness.llm.output_analysis:LLM judge scorer not configured, skipping analyze_llm_judge
INFO:vfairness.llm.output_analysis:analyze_all complete: 11 metrics evaluated

from tqdm import tqdm
from vfairness.llm import LLMApiProxy

proxy = LLMApiProxy(endpoint_url=..., api_format="openai", model_name=...)

# With tqdm progress bar
pbar = tqdm(total=100)
proxy.send_batch(
    prompts=my_prompts,
    n_runs=25,
    progress_callback=lambda current, total: pbar.update(1)
)
pbar.close()

import pandas as pd
from vfairness import BiasDetector, detect_historical_patterns, identify_proxy_variables

# Full audit
detector = BiasDetector(
    df,
    protected_attributes=['gender', 'age'],
    outcome_column='approved',
    benchmarks={'gender': {'M': 0.49, 'F': 0.51}}
)
report = detector.full_audit()
print(report.summary())

# Historical discrimination patterns (scans all column names)
for r in detect_historical_patterns(df):
    print(f"  {r.feature}: {r.risk_level.value} - {r.pattern_type}")

# Proxy variable detection
for p in identify_proxy_variables(df, protected_attributes=['gender'], correlation_threshold=0.3):
    print(f"  {p.feature} -> {p.protected_attribute}  (r={p.correlation:.3f})")

Overall Risk Score: 50.0%

SUMMARY OF FINDINGS
Historical Pattern Issues: 1
Representation Bias Issues: 2
Statistical Disparities: 0
Proxy Variables Identified: 0

TOP RECOMMENDATIONS
1. CRITICAL: Review and consider removing high-risk features: zip_code
...

  zip_code: high - Geographic Redlining

from vfairness.post_processing.calibration import CalibrationAnalyzer

analyzer = CalibrationAnalyzer(
    y_true=labels, y_prob=model_probabilities,
    protected_attr=gender, attribute_name='gender'
)

report = analyzer.full_analysis(context='lending')
print(f"Well Calibrated: {report.is_well_calibrated}")
print(f"Significant Disparity: {report.has_significant_disparity}")
print(f"Overall ECE: {report.overall_metrics['ece']:.4f}")

# Apply group-specific recalibration
analyzer.fit_calibrator(method='isotonic')
calibrated_probs = analyzer.transform(y_prob_test, gender_test)

from vfairness import FairnessAnalyzer

# FairExplAIner mode for per-metric explanations
analyzer = FairnessAnalyzer(y_true, y_pred, gender, fair_explainer=True)
report = analyzer.get_report(include_ci=True)

for name, expl in report['explanations']['metrics'].items():
    print(f"{name}: {expl['severity']} - {expl['evaluation']}")

# FairnessExplainer for cross-module explanations
from vfairness.explainer import FairnessExplainer

explanation = FairnessExplainer.explain(audit_report)  # Takes a vfairness report object (BiasAuditReport, CalibrationReport, GateDecision, ...)
print(explanation.severity)           # 'info' | 'low' | 'medium' | 'high' | 'critical'
print(explanation.recommendations)    # Actionable next steps

from vfairness import classification_fairness_report, plot_fairness_metrics, create_fairness_dashboard
from vfairness.rendering import radar_chart_to_svg, fairness_detailed_report_to_svg

report = classification_fairness_report(y_true, y_pred, gender, include_ci=True)

# Matplotlib (static): returns a Matplotlib Axes
ax = plot_fairness_metrics(report, style='academic', show_thresholds=True)
ax.figure.savefig('fairness_metrics.png', dpi=300, bbox_inches='tight')

# Plotly (interactive)
dashboard = create_fairness_dashboard(report, style='modern')
dashboard.write_html('fairness_dashboard.html')

# SVG templates (needs the [rendering] extra: Jinja2)
svg = radar_chart_to_svg(metrics_data, save_path='radar.svg')

import pandas as pd
from vfairness.operations.monitoring import FairnessMonitor, FairnessDriftDetector

# Real-time monitoring: each batch is a DataFrame with
# 'prediction', 'label', and one 'group_<attr>' column per protected attribute
batch_df = pd.DataFrame({
    'prediction': predictions,
    'label': labels,
    'group_gender': gender,
})

monitor = FairnessMonitor(window_size=1000)
window = monitor.update_and_check(batch_df)
if window.any_alert:
    print(f"Alert! Flagged metrics: {[k for k, v in window.alerts.items() if v]}")

# Multi-scale drift detection: set a known-good baseline once,
# then compare each new stretch of metric history against it
detector = FairnessDriftDetector()
detector.set_baseline(pd.Series(reference_metrics))
drift = detector.check_drift(pd.Series(current_metrics), metric="demographic_parity")
print(f"Drift detected: {drift.drift_detected}  (score: {drift.overall_drift_score:.3f})")

from vfairness.operations.reporting import MetricsStore, FairnessDashboard, ReportGenerator

store = MetricsStore()
store.ingest_from_monitor(monitor)

# Interactive Plotly dashboard
dashboard = FairnessDashboard(store)
fig = dashboard.create_executive_view()
fig.write_html("dashboard.html")

# Automated multi-format report (HTML, PDF, JSON)
gen = ReportGenerator(store, dashboard)
report = gen.generate_executive_report()
report.save("executive_report.html")

from vfairness.operations.experimentation import (
    FairnessExperiment, FairnessPowerAnalyzer, ExperimentAnalysis
)

exp = FairnessExperiment(
    control_data=df_ctrl, treatment_data=df_treat,
    protected_attributes=['gender', 'race'], outcome_column='approved',
)
result = exp.run_full_analysis()

# Per-intersection power analysis
power = FairnessPowerAnalyzer(exp)
print(power.get_power_summary())

# Multi-objective analysis and deployment decision
analysis = ExperimentAnalysis(result, experiment=exp)
rec = analysis.decision_recommendation()
print(f"Decision: {rec.decision}")

Decision: RecommendationDecision.EXTEND_EXPERIMENT

from vfairness.operations.cicd import (
    ModelFairnessGate, HierarchicalGateConfig, FairnessReportCard
)

# Hierarchical gate: overall → single-attribute → intersectional.
# Name only metrics this gate computes: demographic_parity_difference,
# equalized_odds_difference, false_positive_rate_difference,
# predictive_parity_difference, disparate_impact_ratio. Any other name is
# treated as a check that did not happen, and the gate BLOCKS on it.
gate = ModelFairnessGate(
    metrics=['demographic_parity_difference', 'equalized_odds_difference'],
    thresholds={'demographic_parity_difference': 0.1, 'equalized_odds_difference': 0.1}
)
config = HierarchicalGateConfig(
    check_intersections=True, intersection_depth=2
)
decision = gate.evaluate_hierarchical(
    y_true, y_pred,
    protected_attrs={'gender': gender, 'race': race},  # arrays, keyed by attribute
    hierarchical_config=config
)

# Generate PR-ready report card
card = FairnessReportCard(decision, model_name='loan-approval-v2.1')
print(card.to_markdown())  # Paste into PR comment

## Fairness Report Card: loan-approval-v2.1

**Status**: `BLOCKED`
**Result**: Deployment blocked

### Hierarchy Summary

| Level | Status | Failures |
|-------|--------|----------|
| overall | BLOCKED | 2 |
| attr:gender | BLOCKED | 2 |
| attr:race | APPROVED | 0 |
| intersection:gender_x_race | BLOCKED | 2 |

#### overall: Failed Metrics

- **demographic_parity_difference**: 0.2183 (threshold: 0.1000)
- **equalized_odds_difference**: 0.1576 (threshold: 0.1000)

#### intersection:gender_x_race: Failed Metrics

- **demographic_parity_difference**: 0.2916 (threshold: 0.1200)
- **equalized_odds_difference**: 0.2537 (threshold: 0.1200)

from vfairness import log_fairness_to_wandb, auto_log_fairness

# W&B experiment tracking
log_fairness_to_wandb(analyzer, prefix='fairness')

# Auto-log decorator: the wrapped function returns (y_pred, y_true, sensitive_attr)
@auto_log_fairness(backend='mlflow')
def evaluate_model(X, y, sensitive):
    model = LogisticRegression().fit(X, y)
    return model.predict(X), y, sensitive  # fairness metrics logged automatically