Source: xref/out/r2/review_metrics.json
{
"resolver_sha256": "ee25f313096dbd2dd9102047711f9a50328e81d81c1cc954b80c4469b20cc686",
"precision": {
"act_anaphor": {
"reviewed": 50,
"resolved_predictions": 50,
"correct_resolutions": 50,
"precision": 1.0,
"wilson_95": [
0.9286524008666414,
1.0
],
"reviewed_abstentions": 0,
"correct_abstentions": 0
},
"case_id": {
"reviewed": 50,
"resolved_predictions": 50,
"correct_resolutions": 50,
"precision": 1.0,
"wilson_95": [
0.9286524008666414,
1.0
],
"reviewed_abstentions": 0,
"correct_abstentions": 0
},
"case_short": {
"reviewed": 50,
"resolved_predictions": 50,
"correct_resolutions": 50,
"precision": 1.0,
"wilson_95": [
0.9286524008666414,
1.0
],
"reviewed_abstentions": 0,
"correct_abstentions": 0
},
"cfr": {
"reviewed": 50,
"resolved_predictions": 50,
"correct_resolutions": 50,
"precision": 1.0,
"wilson_95": [
0.9286524008666414,
1.0
],
"reviewed_abstentions": 0,
"correct_abstentions": 0
},
"named_act": {
"reviewed": 50,
"resolved_predictions": 50,
"correct_resolutions": 50,
"precision": 1.0,
"wilson_95": [
0.9286524008666414,
1.0
],
"reviewed_abstentions": 0,
"correct_abstentions": 0
},
"session_law": {
"reviewed": 50,
"resolved_predictions": 50,
"correct_resolutions": 50,
"precision": 1.0,
"wilson_95": [
0.9286524008666414,
1.0
],
"reviewed_abstentions": 0,
"correct_abstentions": 0
},
"statute_short": {
"reviewed": 50,
"resolved_predictions": 50,
"correct_resolutions": 50,
"precision": 1.0,
"wilson_95": [
0.9286524008666414,
1.0
],
"reviewed_abstentions": 0,
"correct_abstentions": 0
}
},
"recall": {
"reviewed_pages": 40,
"reviewed_paragraphs": 79,
"gold_events": 33,
"resolvable_gold_events": 30,
"evidence": [
{
"url": "https://www.neochart.com/catalog/cases/f-supp-2d/volume_526/0968_01/index.html",
"start": 43174,
"end": 43181,
"pattern": "act_anaphor",
"expected": "Act: Legal Arizona Workers Act",
"detected": true,
"correctly_resolved": false
},
{
"url": "https://www.neochart.com/catalog/cases/f-supp-2d/volume_526/0968_01/index.html",
"start": 43277,
"end": 43295,
"pattern": "statute_short",
"expected": "A.R.S. \u00a7 23-212(C)",
"detected": true,
"correctly_resolved": false
},
{
"url": "https://www.neochart.com/catalog/cases/f-supp-2d/volume_526/0968_01/index.html",
"start": 43382,
"end": 43389,
"pattern": "act_anaphor",
"expected": "Act: Legal Arizona Workers Act",
"detected": true,
"correctly_resolved": false
},
{
"url": "https://www.neochart.com/catalog/cases/f-supp-2d/volume_526/0968_01/index.html",
"start": 43465,
"end": 43499,
"pattern": "session_law",
"expected": "2007 Ariz. Sess. Laws, ch. 279 \u00a7 3",
"detected": false,
"correctly_resolved": false
},
{
"url": "https://www.neochart.com/catalog/cases/f-supp-2d/volume_526/0968_01/index.html",
"start": 43496,
"end": 43499,
"pattern": "statute_short",
"expected": null,
"detected": true,
"correctly_resolved": false
},
{
"url": "https://www.neochart.com/catalog/cases/f-supp-2d/volume_526/0968_01/index.html",
"start": 43671,
"end": 43678,
"pattern": "act_anaphor",
"expected": "Act: Legal Arizona Workers Act",
"detected": true,
"correctly_resolved": false
},
{
"url": "https://www.neochart.com/catalog/cases/f-supp-2d/volume_526/0968_01/index.html",
"start": 43698,
"end": 43705,
"pattern": "act_anaphor",
"expected": "Act: Legal Arizona Workers Act",
"detected": true,
"correctly_resolved": false
},
{
"url": "https://www.neochart.com/catalog/cases/f-supp-2d/volume_526/0968_01/index.html",
"start": 44212,
"end": 44219,
"pattern": "act_anaphor",
"expected": "Act: Legal Arizona Workers Act",
"detected": true,
"correctly_resolved": false
},
{
"url": "https://www.neochart.com/catalog/cases/f-supp/volume_19/0340_01/index.html",
"start": 2254,
"end": 2259,
"pattern": "statute_short",
"expected": "Judicial Code \u00a7 266",
"detected": true,
"correctly_resolved": false
},
{
"url": "https://www.neochart.com/catalog/cases/f3d/volume_688/0751_01/index.html",
"start": 13561,
"end": 13573,
"pattern": "case_id",
"expected": null,
"detected": true,
"correctly_resolved": false
},
{
"url": "https://www.neochart.com/catalog/cases/f-supp/volume_811/0164_01/index.html",
"start": 11372,
"end": 11375,
"pattern": "case_id",
"expected": "440 U.S. 648",
"detected": true,
"correctly_resolved": true
},
{
"url": "https://www.neochart.com/catalog/cases/f-supp/volume_811/0164_01/index.html",
"start": 11406,
"end": 11412,
"pattern": "case_short",
"expected": "440 U.S. 648",
"detected": false,
"correctly_resolved": false
},
{
"url": "https://www.neochart.com/catalog/cases/f-supp/volume_811/0164_01/index.html",
"start": 12255,
"end": 12275,
"pattern": "case_short",
"expected": "439 U.S. 128",
"detected": true,
"correctly_resolved": true
},
{
"url": "https://www.neochart.com/catalog/cases/f-supp/volume_811/0164_01/index.html",
"start": 12277,
"end": 12297,
"pattern": "case_short",
"expected": "99 S. Ct. 421",
"detected": true,
"correctly_resolved": false
},
{
"url": "https://www.neochart.com/catalog/cases/f-supp/volume_88/0337_01/index.html",
"start": 2597,
"end": 2611,
"pattern": "statute_short",
"expected": null,
"detected": true,
"correctly_resolved": false
},
{
"url": "https://www.neochart.com/catalog/cases/so3d/volume_97/0265_01/index.html",
"start": 2843,
"end": 2853,
"pattern": "case_id",
"expected": "123 So. 561",
"detected": true,
"correctly_resolved": true
},
{
"url": "https://www.neochart.com/catalog/cases/so3d/volume_97/0265_01/index.html",
"start": 3171,
"end": 3186,
"pattern": "case_short",
"expected": "79 So. 3d 845",
"detected": true,
"correctly_resolved": true
},
{
"url": "https://www.neochart.com/catalog/cases/so3d/volume_97/0265_01/index.html",
"start": 3467,
"end": 3484,
"pattern": "case_short",
"expected": "816 So. 2d 1256",
"detected": true,
"correctly_resolved": true
},
{
"url": "https://www.neochart.com/catalog/cases/f-supp/volume_580/1111_01/index.html",
"start": 11484,
"end": 11494,
"pattern": "case_id",
"expected": "436 F.2d 971",
"detected": true,
"correctly_resolved": true
},
{
"url": "https://www.neochart.com/catalog/cases/f-supp/volume_580/1111_01/index.html",
"start": 11626,
"end": 11629,
"pattern": "case_id",
"expected": "436 F.2d 971",
"detected": true,
"correctly_resolved": true
},
{
"url": "https://www.neochart.com/catalog/cases/so3d/volume_198/0698_01/index.html",
"start": 4649,
"end": 4652,
"pattern": "case_id",
"expected": "160 So. 3d 63",
"detected": true,
"correctly_resolved": true
},
{
"url": "https://www.neochart.com/catalog/cases/so3d/volume_198/0698_01/index.html",
"start": 4948,
"end": 4955,
"pattern": "case_short",
"expected": "160 So. 3d 63",
"detected": false,
"correctly_resolved": false
},
{
"url": "https://www.neochart.com/catalog/cases/f3d/volume_520/0912_01/index.html",
"start": 11694,
"end": 11711,
"pattern": "case_short",
"expected": "515 F.3d 1234",
"detected": false,
"correctly_resolved": false
},
{
"url": "https://www.neochart.com/catalog/cases/f3d/volume_520/0912_01/index.html",
"start": 11789,
"end": 11800,
"pattern": "case_id",
"expected": "515 F.3d 1234",
"detected": true,
"correctly_resolved": false
},
{
"url": "https://www.neochart.com/catalog/cases/f3d/volume_520/0912_01/index.html",
"start": 12031,
"end": 12042,
"pattern": "case_id",
"expected": "515 F.3d 1234",
"detected": true,
"correctly_resolved": false
},
{
"url": "https://www.neochart.com/catalog/cases/f3d/volume_520/0912_01/index.html",
"start": 12263,
"end": 12276,
"pattern": "statute_short",
"expected": "18 U.S.C. \u00a7 1028A",
"detected": true,
"correctly_resolved": false
},
{
"url": "https://www.neochart.com/catalog/cases/f3d/volume_520/0912_01/index.html",
"start": 5319,
"end": 5332,
"pattern": "statute_short",
"expected": "18 U.S.C. \u00a7 1028A",
"detected": true,
"correctly_resolved": false
},
{
"url": "https://www.neochart.com/catalog/cases/f-supp-2d/volume_221/1000_01/index.html",
"start": 24837,
"end": 24868,
"pattern": "named_act",
"expected": "Act: Uniform Fraudulent Transfer Act",
"detected": true,
"correctly_resolved": false
},
{
"url": "https://www.neochart.com/catalog/cases/f-supp-2d/volume_221/1000_01/index.html",
"start": 24936,
"end": 24949,
"pattern": "statute_short",
"expected": "Iowa Code \u00a7 684.4",
"detected": true,
"correctly_resolved": false
},
{
"url": "https://www.neochart.com/catalog/cases/f2d/volume_539/0663_01/index.html",
"start": 8148,
"end": 8153,
"pattern": "statute_short",
"expected": "ABA Standards \u00a7 4.1",
"detected": true,
"correctly_resolved": false
},
{
"url": "https://www.neochart.com/catalog/cases/f-supp-3d/volume_304/0992_01/index.html",
"start": 7284,
"end": 7300,
"pattern": "case_short",
"expected": "142 F.3d 1150",
"detected": true,
"correctly_resolved": true
},
{
"url": "https://www.neochart.com/catalog/cases/ad3d/volume_107/0700_01/index.html",
"start": 3753,
"end": 3767,
"pattern": "case_short",
"expected": "45 N.Y.2d 493",
"detected": true,
"correctly_resolved": true
},
{
"url": "https://www.neochart.com/catalog/cases/cal-app-2d/volume_225/0676_01/index.html",
"start": 13569,
"end": 13580,
"pattern": "case_id",
"expected": "112 Cal. App. 431",
"detected": false,
"correctly_resolved": false
}
],
"detection_recall": {
"estimate": 0.8438320869867246,
"page_bootstrap_95": [
0.7074018923921565,
0.9297112459916663
],
"valid_bootstrap_draws": 2000,
"weighted_successes": 2252.5707384403036,
"weighted_events": 2669.453761214631
},
"resolution_recall": {
"estimate": 0.19055686933187962,
"page_bootstrap_95": [
0.040265838578918675,
0.5882467708410914
],
"valid_bootstrap_draws": 2000,
"weighted_successes": 431.57073844030367,
"weighted_events": 2264.787094547964
},
"ci_scope": "Exploratory weighted ratio; whole-page percentile bootstrap, conditional on phase-one frame. Does not quantify phase-one frame variability, annotator error, or unseen patterns. Sparse events can give degenerate intervals."
}
}