{ "site_meta": { "maintainer": "Alejandro Pena", "repo_slug": "adpena/comma-lab", "github_url": "https://github.com/adpena/comma-lab", "challenge_url": "https://github.com/commaai/comma_video_compression_challenge", "updated_at_utc": "2026-04-06T18:18:29.643526Z", "updated_at_local": "Apr 6, 2026, 1:18 PM CDT" }, "tracks": { "exact_current": { "latest_current_workflow": { "archive_bytes": 167, "artifacts": { "raw_report": "reports/raw/exact_current-current_workflow-cpu-report.txt", "summary_json": "reports/raw/exact_current-current_workflow-cpu-summary.json" }, "config": { "mode": "repo-side inflate exploit" }, "current_workflow_score": 0.0, "delta_vs_published_baseline": -4.39, "device": "cpu", "notes": [ "official scorer path via upstream evaluate.sh", "not rule-faithful; keep alive only as Track A" ], "packaging_view": "current_workflow", "posenet_distortion": 0.0, "published_baseline_score": 4.39, "rate": 4.45e-06, "rule_faithful_bundle_bytes": null, "rule_faithful_rate": null, "rule_faithful_score": null, "rule_faithful_status": "invalid_repo_side_dependency", "run_id": "exact_current-baseline-cpu-2026-04-03", "segnet_distortion": 0.0, "track": "exact_current", "ts_utc": "2026-04-04T16:19:02.890361+00:00", "upstream_commit": "ec82c291ffeae5212e9a38253791d58995518a80" } }, "robust_current": { "canonical_summary_path": "reports/raw/robust_current-current_workflow-cpu-summary.json", "canonical_current_workflow": { "copied_report_path": "reports/raw/2026-04-06-hardening/robust_current-hardening-current_workflow-cpu-report.txt", "current_workflow_archive_bytes": 864486, "current_workflow_rate": 0.02302503, "current_workflow_score": 2.12, "device": "cpu", "original_uncompressed_bytes": 37545489, "pose_distortion": 0.09384175, "report_path": "/Users/adpena/Projects/pact/workspace/upstream/comma_video_compression_challenge/submissions/robust_current/report.txt", "rule_faithful_bundle_bytes": 897745, "rule_faithful_bundle_paths": [ "archive.zip", "inflate.sh", "config.env", "analyze_roi.py" ], "rule_faithful_rate": 0.02391086183482655, "rule_faithful_score": 2.1418040615200598, "rule_faithful_status": "estimated_from_scorer_distortions_plus_installed_runtime_payload", "seg_distortion": 0.00575313, "track": "robust_current" }, "best_run_id": "robust_current-av1-524x394-colorspace-hardening-promoted-cpu-2026-04-06" } }, "promotion_runs": [ { "archive_bytes": 920457, "artifacts": { "raw_report": "reports/raw/2026-04-05-av1-floor-promotion/robust_current-av1-floor-current_workflow-cpu-report.txt", "summary_json": "reports/raw/2026-04-05-av1-floor-promotion/robust_current-av1-floor-current_workflow-cpu-summary.json" }, "config": { "bframes": null, "crf": 33, "film_grain": 22, "filters": "lanczos/bicubic", "gop": 180, "inflate_postfilter": "unsharp=9:9:0.35:9:9:0.0", "preset": "svtav1-p0", "ref": null, "scale_h": 394, "scale_w": 524, "video_codec": "libsvtav1" }, "current_workflow_score": 2.2, "delta_vs_published_baseline": -2.19, "device": "cpu", "notes": [ "public-style AV1 reproduction repaired", "rgb24 rawvideo fix", "new promoted floor" ], "packaging_view": "current_workflow", "posenet_distortion": 0.10690599, "published_baseline_score": 4.39, "rate": 0.02451578, "rule_faithful_bundle_bytes": 956424, "rule_faithful_rate": 0.025473739335236785, "rule_faithful_score": 2.2277190123173027, "rule_faithful_status": "estimated_from_scorer_distortions_plus_honest_bytes", "run_id": "robust_current-av1-524x394-rgb24-promoted-cpu-2026-04-05", "segnet_distortion": 0.00556922, "track": "robust_current", "ts_utc": "2026-04-05T18:20:08.762518+00:00", "upstream_commit": "ec82c291ffeae5212e9a38253791d58995518a80" }, { "archive_bytes": 864455, "artifacts": { "raw_report": "reports/raw/2026-04-06-av1-floor-promotion/robust_current-av1-crf34-floor-current_workflow-cpu-report.txt", "summary_json": "reports/raw/2026-04-06-av1-floor-promotion/robust_current-av1-crf34-floor-current_workflow-cpu-summary.json" }, "config": { "bframes": null, "crf": 34, "film_grain": 22, "filters": "lanczos/bicubic", "gop": 180, "inflate_postfilter": "unsharp=9:9:0.35:9:9:0.0", "preset": "svtav1-p0", "ref": null, "scale_h": 394, "scale_w": 524, "video_codec": "libsvtav1" }, "current_workflow_score": 2.19, "delta_vs_published_baseline": -2.2, "device": "cpu", "notes": [ "AV1 neighborhood follow-up", "hypothesis held narrowly", "new promoted floor" ], "packaging_view": "current_workflow", "posenet_distortion": 0.10903914, "published_baseline_score": 4.39, "rate": 0.0230242, "rule_faithful_bundle_bytes": 900954, "rule_faithful_rate": 0.02399633149004931, "rule_faithful_score": 2.2146293679221243, "rule_faithful_status": "estimated_from_scorer_distortions_plus_honest_bytes", "run_id": "robust_current-av1-524x394-crf34-promoted-cpu-2026-04-06", "segnet_distortion": 0.00570503, "track": "robust_current", "ts_utc": "2026-04-06T04:40:00.694888+00:00", "upstream_commit": "ec82c291ffeae5212e9a38253791d58995518a80" }, { "archive_bytes": 864455, "artifacts": { "raw_report": "reports/raw/2026-04-06-av1-lanczos-floor-promotion/robust_current-av1-lanczos-floor-current_workflow-cpu-report.txt", "smoke_json": "reports/raw/2026-04-06-av1-upscale-lanczos/robust_current-av1-upscale-lanczos-smoke.json", "summary_json": "reports/raw/2026-04-06-av1-lanczos-floor-promotion/robust_current-av1-lanczos-floor-current_workflow-cpu-summary.json" }, "config": { "bframes": null, "crf": 34, "film_grain": 22, "filters": "lanczos/lanczos", "gop": 180, "inflate_postfilter": "unsharp=9:9:0.35:9:9:0.0", "preset": "svtav1-p0", "ref": null, "scale_h": 394, "scale_w": 524, "video_codec": "libsvtav1" }, "current_workflow_score": 2.18, "delta_vs_published_baseline": -2.21, "device": "cpu", "notes": [ "AV1 neighborhood probe", "hypothesis: sharper upscale kernel might improve task fidelity at fixed bytes", "new promoted floor" ], "packaging_view": "current_workflow", "posenet_distortion": 0.106568, "published_baseline_score": 4.39, "rate": 0.0230242, "rule_faithful_bundle_bytes": 892472, "rule_faithful_rate": 0.02377041886443402, "rule_faithful_score": 2.196195252141633, "rule_faithful_status": "estimated_from_scorer_distortions_plus_installed_runtime_payload", "run_id": "robust_current-av1-524x394-upscale-lanczos-promoted-cpu-2026-04-06", "segnet_distortion": 0.00569617, "track": "robust_current", "ts_utc": "2026-04-06T13:59:06.248174+00:00", "upstream_commit": "ec82c291ffeae5212e9a38253791d58995518a80" }, { "archive_bytes": 864486, "artifacts": { "ffprobe_json": "reports/raw/2026-04-06-hardening/encoded-ffprobe.json", "raw_report": "reports/raw/2026-04-06-hardening/robust_current-hardening-current_workflow-cpu-report.txt", "smoke_json": "reports/raw/2026-04-06-hardening/robust_current-hardening-smoke.json", "summary_json": "reports/raw/2026-04-06-hardening/robust_current-hardening-current_workflow-cpu-summary.json" }, "config": { "bframes": null, "crf": 34, "film_grain": 22, "filters": "lanczos/lanczos + explicit bt709/tv->rgb(pc)", "gop": 180, "inflate_postfilter": "unsharp=9:9:0.35:9:9:0.0", "preset": "svtav1-p0", "ref": null, "scale_h": 394, "scale_w": 524, "video_codec": "libsvtav1" }, "current_workflow_score": 2.12, "delta_vs_published_baseline": -2.27, "device": "cpu", "notes": [ "production-hardening pass", "hypothesis: explicit colorspace/range handling would reduce evaluator mismatch", "new promoted floor" ], "packaging_view": "current_workflow", "posenet_distortion": 0.09384175, "published_baseline_score": 4.39, "rate": 0.02302503, "rule_faithful_bundle_bytes": 897745, "rule_faithful_rate": 0.02391086183482655, "rule_faithful_score": 2.1418040615200598, "rule_faithful_status": "estimated_from_scorer_distortions_plus_installed_runtime_payload", "run_id": "robust_current-av1-524x394-colorspace-hardening-promoted-cpu-2026-04-06", "segnet_distortion": 0.00575313, "track": "robust_current", "ts_utc": "2026-04-06T15:43:00+00:00", "upstream_commit": "ec82c291ffeae5212e9a38253791d58995518a80" } ], "timeline": [ { "current_workflow_score": 3.62, "event_id": "cold-start-baseline-promoted-medium23", "summary": "Promoted robust_current medium/23 at 512x384 after CRF sweep.", "track": "robust_current", "ts_utc": "2026-04-03T22:00:00Z", "type": "promotion" }, { "current_workflow_score": 3.56, "event_id": "resolution-sweep-promoted-448x336", "summary": "Promoted robust_current 448x336 / medium / 23 after resolution sweep.", "track": "robust_current", "ts_utc": "2026-04-03T23:00:00Z", "type": "promotion" }, { "event_id": "bat00-side-lane-bootstrapped", "host": "BAT00", "ip": "192.168.1.216", "summary": "Confirmed bat00 access and bootstrapped /home/adpena/pact-side with uv sync under WSL Ubuntu 24.04.", "ts_utc": "2026-04-04T01:49:11.290324+00:00", "type": "infra" }, { "current_workflow_score": 3.56, "event_id": "gop-sweep-promoted-keyint48", "summary": "Promoted robust_current 448x336 / medium / 23 / keyint 48 after GOP sweep.", "track": "robust_current", "ts_utc": "2026-04-04T09:29:11.298606+00:00", "type": "promotion" }, { "current_workflow_score": 3.63, "event_id": "bat00-runtime-benchmark-captured", "host": "BAT00", "summary": "Captured non-authoritative bat00 side benchmark for the then-promoted 448x336 / medium / 23 / keyint32 config.", "ts_utc": "2026-04-04T09:29:11.298612+00:00", "type": "infra" }, { "current_workflow_score": 3.54, "event_id": "filter-sweep-promoted-lanczos-lanczos", "summary": "Promoted robust_current 448x336 / medium / 23 / keyint48 / lanczos+lanczos after filter sweep.", "track": "robust_current", "ts_utc": "2026-04-04T14:18:58.594023+00:00", "type": "promotion" }, { "current_workflow_score": 3.54, "event_id": "bframe-ref-sweep-no-promotion", "summary": "Completed B-frame/ref sweep around the 3.54 floor; no candidate beat the promoted config.", "track": "robust_current", "ts_utc": "2026-04-04T16:19:02.891695+00:00", "type": "decision" }, { "current_workflow_score": 3.56, "event_id": "bat00-runtime-benchmark-keyint48-lanczos", "host": "BAT00", "summary": "Captured non-authoritative bat00 side benchmark for the current promoted keyint48/lanczos config.", "ts_utc": "2026-04-04T16:19:02.891698+00:00", "type": "infra" }, { "current_workflow_score": 3.54, "event_id": "runtime-note-local-vs-bat00-complete", "summary": "Captured a like-for-like local vs BAT00 runtime note for the promoted 3.54 floor.", "track": "robust_current", "ts_utc": "2026-04-04T16:57:19.828536+00:00", "type": "analysis" }, { "event_id": "bat00-jax-cuda-ready", "host": "BAT00", "summary": "BAT00 JAX/CUDA surrogate environment is ready under uv with JAX seeing CudaDevice(0).", "ts_utc": "2026-04-04T16:57:19.828726+00:00", "type": "infra" }, { "event_id": "bat00-surrogate-prototype-complete", "host": "BAT00", "summary": "Completed first BAT00 JAX/CUDA surrogate prototype; environment works but dataset is too small for decision authority.", "ts_utc": "2026-04-04T17:06:44.600707+00:00", "type": "research" }, { "current_workflow_score": 5.73, "event_id": "roi-two-pass-prototype-rejected", "summary": "Tiny fixed-ROI two-pass prototype cut bytes but regressed badly on the official scorer, so it stays rejected.", "track": "robust_current", "ts_utc": "2026-04-04T17:46:16.346354+00:00", "type": "research" }, { "current_workflow_score": 3.33, "event_id": "tiny-resolution-revisit-promoted-432x324", "summary": "Promoted robust_current 432x324 / medium / 23 / keyint48 / lanczos+lanczos after the tiny local resolution revisit.", "track": "robust_current", "ts_utc": "2026-04-04T19:11:30.675723+00:00", "type": "promotion" }, { "event_id": "dynamic-main-roi-plan-ready", "summary": "Prepared execution-ready dynamic main-ROI design/plan artifacts; main ROI remains mandatory and central, auxiliary ROI additive only.", "track": "robust_current", "ts_utc": "2026-04-05T00:00:00+00:00", "type": "planning" }, { "current_workflow_score": 4.47, "event_id": "dynamic-main-roi-rejected", "summary": "Measured dynamic main-ROI prototype rejected: main ROI was preserved explicitly, but score regressed to 4.47 and bytes rose to about 2.66 MB.", "track": "robust_current", "ts_utc": "2026-04-05T05:53:25.231667+00:00", "type": "research" }, { "current_workflow_score": 3.33, "event_id": "floor-regression-reverified-2026-04-05", "summary": "Fresh local CPU regression confirmed the restored promoted floor still scores 3.33.", "track": "robust_current", "ts_utc": "2026-04-05T06:35:49.299885+00:00", "type": "verification" }, { "event_id": "bat00-surrogate-v2-codec-only-usable", "host": "BAT00", "summary": "BAT00 surrogate v2 remained noisy on the full mixed set, but the codec-only subset improved enough to use as a research-only ranking aid.", "ts_utc": "2026-04-05T06:35:49.299885+00:00", "type": "research" }, { "event_id": "bat00-codec-shortlist-generated", "host": "BAT00", "summary": "BAT00 codec-only surrogate ranked a small codec shortlist, then the top two local CPU candidates were tested sequentially.", "ts_utc": "2026-04-05T07:42:28.847777+00:00", "type": "research" }, { "current_workflow_score": 3.43, "event_id": "codec-shortlist-top1-rejected", "summary": "Local CPU test of surrogate-ranked #1 candidate (432x324 / crf23 / g48 / b3 / r4) scored 3.43 and was rejected.", "track": "robust_current", "ts_utc": "2026-04-05T07:42:28.847777+00:00", "type": "decision" }, { "current_workflow_score": 3.38, "event_id": "codec-shortlist-top2-rejected", "summary": "Local CPU test of surrogate-ranked #2 candidate (432x324 / crf23 / g64 / b4 / r4) scored 3.38 and was rejected.", "track": "robust_current", "ts_utc": "2026-04-05T07:42:28.847777+00:00", "type": "decision" }, { "current_workflow_score": 3.25, "event_id": "codec-shortlist-top3-promoted", "summary": "Local CPU test of surrogate-ranked #3 candidate (424x318 / crf23 / g48 / b4 / r4) scored 3.25 and became the new promoted floor.", "track": "robust_current", "ts_utc": "2026-04-05T08:13:15.198319+00:00", "type": "promotion" }, { "current_workflow_score": 3.27, "event_id": "nearby-424x318-b3-rejected", "summary": "Nearby follow-up on the new 424x318 floor (bframes3) scored 3.27 and was rejected.", "track": "robust_current", "ts_utc": "2026-04-05T08:48:02.864256+00:00", "type": "decision" }, { "current_workflow_score": 3.26, "event_id": "nearby-424x318-g64-rejected", "summary": "Nearby follow-up on the 424x318 floor (keyint64) scored 3.26 and was rejected.", "track": "robust_current", "ts_utc": "2026-04-05T09:47:41.998084+00:00", "type": "decision" }, { "current_workflow_score": 3.44, "event_id": "lower-res-416x312-rejected", "summary": "Lower-resolution BAT00-ranked follow-up at 416x312 scored 3.44 and was rejected.", "track": "robust_current", "ts_utc": "2026-04-05T14:29:14.555005+00:00", "type": "decision" }, { "current_workflow_score": 3.32, "event_id": "nearby-scale-428x320-rejected", "summary": "BAT00-ranked nearby-scale follow-up at 428x320 scored 3.32 and was rejected.", "track": "robust_current", "ts_utc": "2026-04-05T15:10:07.425100+00:00", "type": "decision" }, { "current_workflow_score": 3.38, "event_id": "nearby-scale-426x320-rejected", "summary": "BAT00-ranked nearby-scale follow-up at 426x320 scored 3.38 and was rejected.", "track": "robust_current", "ts_utc": "2026-04-05T16:07:20.968299+00:00", "type": "decision" }, { "current_workflow_score": 97.45, "event_id": "av1-frontier-524x394-rejected", "summary": "Public-style AV1/SVT-AV1 reproduction attempt at 524x394 regressed severely to 97.45 and was rejected.", "track": "robust_current", "ts_utc": "2026-04-05T17:12:42.981688+00:00", "type": "research" }, { "current_workflow_score": 2.2, "event_id": "av1-rgb24-root-cause-identified", "summary": "AV1 failure root cause was a rawvideo pixel-format mismatch: inflate emitted yuv444p bytes instead of rgb24. Forcing rgb24 recovered the same 524x394 SVT-AV1 recipe from 97.45 to 2.20.", "track": "robust_current", "ts_utc": "2026-04-05T18:20:08.762518+00:00", "type": "analysis" }, { "current_workflow_score": 2.2, "event_id": "av1-frontier-524x394-promoted", "summary": "Canonical robust_current AV1 floor at 524x394 / svtav1-p0 / crf33 / film-grain22 / bicubic / unsharp scored 2.20 and became the new promoted floor.", "track": "robust_current", "ts_utc": "2026-04-05T18:20:08.762518+00:00", "type": "promotion" }, { "current_workflow_score": 2.19, "event_id": "av1-neighborhood-crf34-analysis", "summary": "CRF34 AV1 neighborhood probe cut bytes by 56,002 (6.08%) versus the 2.20 CRF33 AV1 floor, while pose and seg distortion rose slightly; net score improved by 0.01 to 2.19.", "track": "robust_current", "ts_utc": "2026-04-06T04:41:42.160831+00:00", "type": "analysis" }, { "current_workflow_score": 2.19, "event_id": "av1-neighborhood-crf34-promoted", "summary": "Canonical robust_current AV1 floor at 524x394 / svtav1-p0 / crf34 / film-grain22 / bicubic / unsharp scored 2.19 and replaced the prior 2.20 CRF33 floor.", "track": "robust_current", "ts_utc": "2026-04-06T04:41:42.160831+00:00", "type": "promotion" }, { "current_workflow_score": 2.21, "event_id": "av1-neighborhood-crf35-analysis", "summary": "CRF35 was estimated to maybe land around 2.17-2.20 if byte savings still dominated, but it regressed to 2.21. Bytes fell by 56,419 (6.53%), yet pose and seg rose too far.", "track": "robust_current", "ts_utc": "2026-04-06T05:26:53.551355+00:00", "type": "analysis" }, { "current_workflow_score": 2.21, "event_id": "av1-neighborhood-crf35-rejected", "summary": "One-axis AV1 probe at crf35 scored 2.21 and was rejected; the 2.19 crf34 floor remains canonical.", "track": "robust_current", "ts_utc": "2026-04-06T05:26:53.551355+00:00", "type": "decision" }, { "current_workflow_score": 2.2, "event_id": "av1-unsharp030-analysis", "summary": "Unsharp 0.30 was estimated to maybe land around 2.18-2.20 by reducing postfilter aggression, but it stayed at 864,455 bytes and regressed to 2.20. Pose and seg both worsened slightly.", "track": "robust_current", "ts_utc": "2026-04-06T06:13:03.889813+00:00", "type": "analysis" }, { "current_workflow_score": 2.2, "event_id": "av1-unsharp030-rejected", "summary": "One-axis AV1 probe with lighter unsharp (0.30) scored 2.20 and was rejected; the 2.19 unsharp 0.35 floor remains canonical.", "track": "robust_current", "ts_utc": "2026-04-06T06:13:03.889813+00:00", "type": "decision" }, { "current_workflow_score": 3.33, "event_id": "av1-filmgrain0-analysis", "summary": "Film-grain 0 was estimated to maybe land around 2.16-2.24 if synthetic grain was hurting task metrics, but it regressed badly to 3.33. Bytes fell by 145,359 (16.82%), yet PoseNet distortion exploded.", "track": "robust_current", "ts_utc": "2026-04-06T07:12:44.279648+00:00", "type": "analysis" }, { "current_workflow_score": 3.33, "event_id": "av1-filmgrain0-rejected", "summary": "One-axis AV1 probe with film-grain disabled scored 3.33 and was rejected; the 2.19 film-grain22 floor remains canonical.", "track": "robust_current", "ts_utc": "2026-04-06T07:12:44.279648+00:00", "type": "decision" }, { "current_workflow_score": 2.23, "event_id": "av1-522x392-analysis", "summary": "522x392 was estimated to maybe land around 2.17-2.22 by trimming geometry slightly, but it regressed to 2.23. Bytes fell by 2,217 (0.26%), while pose and seg both worsened.", "track": "robust_current", "ts_utc": "2026-04-06T12:58:11.103191+00:00", "type": "analysis" }, { "current_workflow_score": 2.23, "event_id": "av1-522x392-rejected", "summary": "One-axis AV1 geometry probe at 522x392 scored 2.23 and was rejected; the 2.19 524x394 floor remains canonical.", "track": "robust_current", "ts_utc": "2026-04-06T12:58:11.103191+00:00", "type": "decision" }, { "current_workflow_score": 2.18, "event_id": "av1-upscale-lanczos-analysis", "summary": "Lanczos upscale was estimated to maybe land around 2.16-2.19 by improving reconstructed edge fidelity at fixed bytes. It scored 2.18 at unchanged bytes, with both pose and seg improving slightly.", "track": "robust_current", "ts_utc": "2026-04-06T13:59:06.248174+00:00", "type": "analysis" }, { "current_workflow_score": 2.18, "event_id": "av1-upscale-lanczos-promoted", "summary": "Canonical robust_current AV1 floor at 524x394 / svtav1-p0 / crf34 / film-grain22 / lanczos upscale / unsharp scored 2.18 and replaced the prior 2.19 bicubic-upscale floor.", "track": "robust_current", "ts_utc": "2026-04-06T13:59:06.248174+00:00", "type": "promotion" }, { "current_workflow_score": 2.12, "event_id": "av1-colorspace-hardening-analysis", "summary": "Explicit bt709/tv tagging on encode plus explicit rgb24(pc) decode conversion reduced evaluator mismatch. Archive bytes rose by 31, SegNet distortion worsened slightly, but PoseNet distortion improved sharply enough to move the score from 2.18 to 2.12.", "track": "robust_current", "ts_utc": "2026-04-06T15:43:00+00:00", "type": "analysis" }, { "current_workflow_score": 2.12, "event_id": "av1-colorspace-hardening-promoted", "summary": "Canonical robust_current AV1 floor at 524x394 / svtav1-p0 / crf34 / film-grain22 / lanczos / unsharp with explicit bt709/tv encode tags and explicit rgb24(pc) decode scored 2.12 and replaced the prior 2.18 floor.", "track": "robust_current", "ts_utc": "2026-04-06T15:43:00+00:00", "type": "promotion" } ], "site_artifacts": { "dashboard_data": "reports/graphs/dashboard_data.json", "experiment_graph": "reports/graphs/experiment_graph.json", "promotion_accounting": "reports/graphs/promotion_accounting.md", "judges_one_pager": "reports/graphs/judges_one_pager.md", "submission_packet": "reports/graphs/submission_packet.md", "evidence_index": "reports/graphs/evidence_index.md" } }