{ "id": "AIBD-0214", "title": "Higher algebra scores in the trial’s second year", "summary": "Students using Cognitive Tutor Algebra I scored 0.21 standard deviations higher in the second year, with no statistically significant difference after correction for multiple comparisons.", "date_observed": "2009-01-01", "date_basis": "event", "date_precision": "year", "benefit_status": "mixed", "source_collection": null, "source_entry": null, "account": "Researchers randomly assigned 147 schools in seven U.S. states to Cognitive Tutor Algebra I or their usual curriculum during the 2007–08 and 2008–09 school years. The program combined classroom teaching, materials, and adaptive tutoring. The revised analysis included 11,066 high-school students and 5,519 middle-school students. It estimated a gain of 0.21 standard deviations for the second high-school cohort. First-year and middle-school results were not statistically significant. The revised What Works Clearinghouse review found that the second-year result also failed its corrected significance threshold. Each year involved a new cohort, rather than a second year of exposure for the same students.", "category": "general benefit", "unexpected": false, "purpose": "CTAI was built to teach first-year algebra with personalized, mastery-based software. The test-score gain is that purpose.", "system": "Cognitive Tutor Algebra I, with an ACT-R-based cognitive model, classroom instruction, and curriculum materials", "model_version": null, "developer": "Carnegie Learning", "deployer": "Participating public and Catholic parochial schools in 51 districts across seven states (study sites)", "location": null, "domain": "Education", "segment": "Secondary-school algebra", "benefit_type": "Improved Learning", "taxonomy_note": null, "registry_ids": {}, "ratings": [ { "group": "personal", "depth": 2, "reason": "The second high-school cohort had a measurable algebra gain, although the corrected significance test was inconclusive." } ], "ai_role": { "status": "assessed", "statement": "The tutor provided adaptive exercises and feedback within a complete algebra curriculum. The trial did not separate its contribution from classroom teaching, materials, or training." }, "evidence": [ { "kind": "peer-reviewed study", "citation": "Pane, John F., Beth Ann Griffin, Daniel F. McCaffrey, and Rita Karam. Educational Evaluation and Policy Analysis 36, no. 2 (2014): 127–44. https://doi.org/10.3102/0162373713507480.", "title": "Effectiveness of Cognitive Tutor Algebra I at Scale", "url": "https://journals.sagepub.com/doi/10.3102/0162373713507480", "date": null, "written_by": "independent", "quote": null }, { "kind": "report", "citation": "Pane, John F., et al. Addendum to “Effectiveness of Cognitive Tutor Algebra I at Scale.” RAND WR-1050, May 30, 2014.", "title": "Addendum to Effectiveness of Cognitive Tutor Algebra I at Scale", "url": "https://www.rand.org/content/dam/rand/pubs/working_papers/WR1000/WR1050/RAND_WR1050.pdf", "date": "2014-05-30", "written_by": "independent", "quote": null }, { "kind": "report", "citation": "What Works Clearinghouse. Cognitive Tutor® Algebra I. June 2016, revised December 21, 2016.", "title": "Cognitive Tutor® Algebra I", "url": "https://ies.ed.gov/ncee/wwc/Docs/InterventionReports/wwc_cognitivetutor_122116.pdf", "date": "2016-12-21", "written_by": "independent", "quote": null }, { "kind": "peer-reviewed study", "citation": "Ritter, Steven, John R. Anderson, Kenneth R. Koedinger, and Albert Corbett. Psychonomic Bulletin & Review 14, no. 2 (2007): 249–55. https://doi.org/10.3758/BF03194060.", "title": "Cognitive Tutor: Applied Research in Mathematics Education", "url": "https://link.springer.com/content/pdf/10.3758/BF03194060.pdf", "date": null, "written_by": "makers", "quote": null }, { "kind": "report", "citation": "Daugherty, Lindsay, Andrea Phillips, John F. Pane, and Rita Karam. Analysis of Costs in an Algebra I Curriculum Effectiveness Study. RAND TR-1171-1-DEIES, 2012.", "title": "Analysis of Costs in an Algebra I Curriculum Effectiveness Study", "url": "https://www.rand.org/content/dam/rand/pubs/technical_reports/2012/RAND_TR1171-1.pdf", "date": "2012-07-12", "written_by": "independent", "quote": null }, { "kind": "peer-reviewed study", "citation": "Mann, Charlotte Z., Jiaying Wang, Adam Sales, and Johann A. Gagnon-Bartsch. “Using Publicly Available Auxiliary Data to Improve Precision of Treatment Effect Estimation in a Randomized Efficacy Trial.” Proceedings of the 17th International Conference on Educational Data Mining, 2024, short paper 52.", "title": "Using Publicly Available Auxiliary Data to Improve Precision of Treatment Effect Estimation in a Randomized Efficacy Trial", "url": "https://educationaldatamining.org/EDM2024/proceedings/2024.EDM-short-papers.52/index.html", "date": null, "written_by": "independent", "quote": null } ], "replication": { "produced_by": [ "A curriculum that combined intelligent tutoring with classroom instruction", "Teacher training and regular computer sessions", "School-pair randomization over two annual cohorts" ], "must_hold": [ "Staff and equipment for regular tutor use", "Time to implement the curriculum" ], "echoes": [] }, "paired_incident": null, "headline": "Higher algebra scores in the trial’s second year", "metric": "Second-year algebra gain: 0.21 standard deviations; inconclusive after correction", "comparator": { "against": "Schools using their usual algebra curriculum", "before": "Comparison schools in the same implementation year", "after": "Second high-school cohort: +0.21 standard deviations; p = .04 before correction, not significant at the Clearinghouse’s corrected threshold of .025", "basis": "measured", "quote": "the WWC found that this difference was no longer statistically significant.", "url": "https://ies.ed.gov/ncee/wwc/Docs/InterventionReports/wwc_cognitivetutor_122116.pdf" }, "later": { "status": "challenged", "date": "2016-12-21", "note": "The revised What Works Clearinghouse review rated the wider algebra evidence as mixed. It found that this trial’s second-year estimate was not statistically significant after correction for multiple comparisons.", "quote": null, "url": "https://ies.ed.gov/ncee/wwc/Docs/InterventionReports/wwc_cognitivetutor_122116.pdf" }, "value": null, "developer_country": "United States", "deployer_country": null, "unrated_because": null, "brief_notes": null, "brief": null, "revisions": [ { "date": "2026-10-20", "change": "Record drafted from the cited sources.", "source": null, "editor": "AIBD" }, { "date": "2026-10-21", "change": "Added what the benefit is measured against, what happened since from the cited sources.", "source": "https://www.rand.org/pubs/research_briefs/RB9746.html; https://ies.ed.gov/ncee/wwc/Docs/InterventionReports/wwc_cognitivetutor_122116.pdf", "editor": "AIBD" }, { "date": "2026-10-21", "change": "Country of the developer added from the organization's own site or a registry.", "source": "https://www.carnegielearning.com/contact/", "editor": "AIBD" }, { "date": "2026-10-21", "change": "Prose pass: account, purpose, the system's part rewritten in shorter sentences with abbreviations expanded; facts, figures and quotations unchanged.", "source": null, "editor": "AIBD" }, { "date": "2026-10-25", "change": "Reviewed the original evidence and supporting material; updated system details, methods, outcomes, limitations, costs, and source lineage.", "source": "https://journals.sagepub.com/doi/10.3102/0162373713507480", "editor": "AIBD" } ], "confidence": "studied", "shade": "high", "claim_status": "Independently studied", "claim_basis": "Sources: two independent peer-reviewed studies, three independent reports and the makers' peer-reviewed study. The cited evidence includes no finding by a regulator or court.", "profile": { "case_id": "AIBD-0214", "catalog_version": "1.0.0", "source_fingerprint": "d0d3b347b993966f7ab31c89ecba4b49bddfe59ef866a45bb3f2774dab45228c", "collection_refs": [ { "collection_id": "aibd", "version": "snapshot-8bf7cfec9cb185aa" }, { "collection_id": "aibd", "version": "2026-10-25" }, { "collection_id": "querist", "version": "2026-10-25" } ], "source_ids": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002", "AIBD-0214-source-0003", "AIBD-0214-source-0004", "AIBD-0214-source-0005", "AIBD-0214-source-0006" ], "families": { "strategic assessments": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial", "record_ids": [] }, "evaluations": { "state": "value_present", "value": { "kind": "structured", "data": { "records": [ { "id": "AIBD-0214-E01", "design": "matched-pair cluster randomized trial", "schools": 147, "high_schools": 73, "middle_schools": 74, "school_years": [ "2007–08", "2008–09" ], "revised_high_school_n": 11066, "revised_middle_school_n": 5519, "outcomes": [ { "setting": "high school, first implementation year", "effect": -0.07, "se": 0.1, "p": 0.46, "unit": "standard deviation" }, { "setting": "high school, second implementation year", "effect": 0.21, "se": 0.1, "p": 0.04, "unit": "standard deviation", "wwc_corrected_alpha": 0.025, "significant_after_correction": false }, { "setting": "middle school, second implementation year", "effect": 0.1, "se": 0.14, "p": 0.47, "unit": "standard deviation" } ], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002", "AIBD-0214-source-0003", "AIBD-0214-source-0006" ], "locator": "C2 Tables 3–4; C3 p. 31 n. c; C6 §2." } ] } }, "provenance": { "kind": "source_review", "basis": "Original sources and locators are preserved with each observation.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002", "AIBD-0214-source-0003", "AIBD-0214-source-0006" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "The original evaluation and any distinct follow-up analyses." }, "record_ids": [ "AIBD-0214-E01" ] }, "monetary observations": { "state": "value_present", "value": { "kind": "structured", "data": { "records": [ { "id": "AIBD-0214-C01", "type": "estimated_adoption_cost", "currency": "USD", "unit": "student-year", "treatment": 97.18, "comparison": 27.88, "difference": 69.3, "scenario": "Houston excluded", "source_refs": [ "AIBD-0214-source-0005" ], "locator": "Table 3.13" }, { "id": "AIBD-0214-C02", "type": "estimated_adoption_cost", "currency": "USD", "unit": "student-year", "treatment": 136.15, "comparison": 16.37, "difference": 119.78, "scenario": "Houston included", "source_refs": [ "AIBD-0214-source-0005" ], "locator": "Table 3.14" } ] } }, "provenance": { "kind": "source_review", "basis": "Original sources and locators are preserved with each observation.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0005" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "The reported costs, forecasts, or savings, with their separate scopes." }, "record_ids": [ "AIBD-0214-C01", "AIBD-0214-C02" ] }, "institutional decisions": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial", "record_ids": [] } }, "fields": { "actors[]": { "state": "value_present", "value": { "kind": "text", "text": "Carnegie Learning supplied the curriculum. Participating schools taught it, and RAND researchers evaluated it." }, "provenance": { "kind": "source_review", "basis": "C1 methods and acknowledgments.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "Carnegie Learning supplied the curriculum. Participating schools taught it, and RAND researchers evaluated it." } }, "actors[].ownership_control": { "state": "value_present", "value": { "kind": "text", "text": "Carnegie Learning supplied the commercial curriculum and tutoring software." }, "provenance": { "kind": "source_review", "basis": "C1 intervention description.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "Carnegie Learning supplied the commercial curriculum and tutoring software." } }, "ai_contribution": { "state": "value_present", "value": { "kind": "text", "text": "The trial compared complete curricula. It did not separate the tutor’s effect from the textbook, activities, and teacher training." }, "provenance": { "kind": "source_review", "basis": "C1 methods.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "The trial compared complete curricula. It did not separate the tutor’s effect from the textbook, activities, and teacher training." } }, "ai_relationship": { "state": "value_present", "value": { "kind": "text", "text": "The intelligent tutor used a cognitive model of students’ algebra skills." }, "provenance": { "kind": "source_review", "basis": "C1 intervention description.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "The intelligent tutor used a cognitive model of students’ algebra skills." } }, "assessments[].alternatives": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "assessments[].assessor_date": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "assessments[].depth": { "state": "value_present", "value": { "kind": "structured", "data": { "ratings": [ { "group": "personal", "depth": 2, "reason": "The second high-school cohort had a measurable algebra gain, although the corrected significance test was inconclusive." } ] } }, "provenance": { "kind": "source_review", "basis": "Application of the existing AIBD depth rubric to the reported benefit.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0002", "AIBD-0214-source-0003" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "Personal: 2" } }, "assessments[].display_confidence": { "state": "value_present", "value": { "kind": "structured", "data": { "confidence": "high", "evidence_word": "studied", "claim_status": "Independently studied", "basis": "Sources: two independent peer-reviewed studies, three independent reports and the makers' peer-reviewed study. The cited evidence includes no finding by a regulator or court." } }, "provenance": { "kind": "source_review", "basis": "Derived from the corrected source inventory; authorship and original findings are documented separately.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002", "AIBD-0214-source-0003", "AIBD-0214-source-0004", "AIBD-0214-source-0005", "AIBD-0214-source-0006" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "Independently studied" } }, "assessments[].rationale": { "state": "value_present", "value": { "kind": "text", "text": "The second high-school cohort had a measurable algebra gain, although the corrected significance test was inconclusive." }, "provenance": { "kind": "source_review", "basis": "Benefit depth is assessed separately from strength of evidence.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0002", "AIBD-0214-source-0003" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "The second high-school cohort had a measurable algebra gain, although the corrected significance test was inconclusive." } }, "assessments[].rubric_version": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "assessments[].status_basis": { "state": "value_present", "value": { "kind": "text", "text": "The second-year gain was positive, but it was not statistically significant after the Clearinghouse adjusted for multiple comparisons." }, "provenance": { "kind": "source_review", "basis": "Methods, results, and Tables 2–5.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "The second-year gain was positive, but it was not statistically significant after the Clearinghouse adjusted for multiple comparisons." } }, "assessments[].target": { "state": "value_present", "value": { "kind": "text", "text": "High-school students’ algebra learning." }, "provenance": { "kind": "source_review", "basis": "C2 Table 3.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "High-school students’ algebra learning." } }, "availability": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "brief": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "case_kind": { "state": "value_present", "value": { "kind": "text", "text": "A randomized evaluation of a full algebra curriculum." }, "provenance": { "kind": "source_review", "basis": "C1 methods.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "A randomized evaluation of a full algebra curriculum." } }, "case_links[]": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "claims[].evidence_links": { "state": "value_present", "value": { "kind": "structured", "data": { "links": [ { "proposition": "Original school-randomized evaluation of the full curriculum.", "source_refs": [ "AIBD-0214-source-0001" ], "locator": "Methods, results, and Tables 2–5." }, { "proposition": "Reanalysis excluding students without observed baseline scores, with revised samples and effects.", "source_refs": [ "AIBD-0214-source-0002" ], "locator": "pp. 1–6, Tables 1–4." }, { "proposition": "Independent evidence review and correction for multiple comparisons.", "source_refs": [ "AIBD-0214-source-0003" ], "locator": "p. 6; pp. 23–24, Appendix A.4; p. 31, Table B.1, n. c." }, { "proposition": "Original account of the cognitive model, step-by-step model tracing, and skill tracking.", "source_refs": [ "AIBD-0214-source-0004" ], "locator": "pp. 249–52." }, { "proposition": "Annualized adoption costs, cost assumptions, and the effect of including Houston.", "source_refs": [ "AIBD-0214-source-0005" ], "locator": "pp. ix–x, 5–11, 26–28; Tables 3.13–3.14." }, { "proposition": "The paper establishes the original trial’s school years and links public reanalysis code while explaining that school identities remain protected.", "source_refs": [ "AIBD-0214-source-0006" ], "locator": "§2 establishes the school years; §7 and the linked code repository describe available replication materials." } ] } }, "provenance": { "kind": "source_review", "basis": "Original-source lineage.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002", "AIBD-0214-source-0003", "AIBD-0214-source-0004", "AIBD-0214-source-0005", "AIBD-0214-source-0006" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "The sources are linked to the findings that they support." } }, "claims[].evidence_strength": { "state": "value_present", "value": { "kind": "text", "text": "A large randomized trial supports a positive second-year estimate. An independent review found that it did not survive correction for multiple comparisons." }, "provenance": { "kind": "source_review", "basis": "C2 Table 3; C3 p. 31, n. c.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002", "AIBD-0214-source-0003" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "A large randomized trial supports a positive second-year estimate. An independent review found that it did not survive correction for multiple comparisons." } }, "claims[].missingness": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "claims[].proposition": { "state": "value_present", "value": { "kind": "text", "text": "The second high-school cohort had higher algebra scores under the combined curriculum." }, "provenance": { "kind": "source_review", "basis": "C2 Table 3.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "The second high-school cohort had higher algebra scores under the combined curriculum." } }, "cross_border_flows[]": { "state": "not_reported", "provenance": { "kind": "source_review", "basis": "Institutions and settings.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0004" ], "reviewed_on": "2026-10-25" }, "reason": "Development and school deployment were in the United States. The study does not document an international flow of student data.", "review_state": "reviewed", "coverage": "complete", "display": { "answer": "Development and school deployment were in the United States. The study does not document an international flow of student data." } }, "decisions[].action_outcome": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial", "family": "institutional decisions" }, "decisions[].grounds": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial", "family": "institutional decisions" }, "decisions[].institution_task": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial", "family": "institutional decisions" }, "decisions[].success_criterion": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial", "family": "institutional decisions" }, "deployment_stage": { "state": "value_present", "value": { "kind": "text", "text": "Schools used the curriculum for two consecutive years." }, "provenance": { "kind": "source_review", "basis": "C1 methods.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "Schools used the curriculum for two consecutive years." } }, "economics[].amount": { "state": "value_present", "value": { "kind": "text", "text": "Excluding Houston, estimated annual cost was US$97.18 per student for Cognitive Tutor and US$27.88 for comparison curricula." }, "provenance": { "kind": "source_review", "basis": "Table 3.13.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0005" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "Excluding Houston, estimated annual cost was US$97.18 per student for Cognitive Tutor and US$27.88 for comparison curricula." } }, "economics[].costs[]": { "state": "value_present", "value": { "kind": "text", "text": "Costs include textbooks, software, computers, and curriculum-specific teacher training." }, "provenance": { "kind": "source_review", "basis": "Cost methods.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0005" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "Costs include textbooks, software, computers, and curriculum-specific teacher training." } }, "economics[].counterfactual": { "state": "value_present", "value": { "kind": "text", "text": "The comparison is the cost of the curricula that control schools used." }, "provenance": { "kind": "source_review", "basis": "Cost methods.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0005" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "The comparison is the cost of the curricula that control schools used." } }, "economics[].currency_price_basis": { "state": "value_present", "value": { "kind": "text", "text": "U.S. dollars. The estimate used the vendor’s 2006 price guide to represent ordinary district adoption." }, "provenance": { "kind": "source_review", "basis": "pp. 10–11.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0005" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "U.S. dollars. The estimate used the vendor’s 2006 price guide to represent ordinary district adoption." } }, "economics[].gross_net": { "state": "value_present", "value": { "kind": "text", "text": "The estimates compare expenditure. They do not value learning gains or calculate net benefits." }, "provenance": { "kind": "source_review", "basis": "Cost methods and conclusion.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0005" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "The estimates compare expenditure. They do not value learning gains or calculate net benefits." } }, "economics[].measure_type": { "state": "value_present", "value": { "kind": "text", "text": "Annualized curriculum adoption cost." }, "provenance": { "kind": "source_review", "basis": "Tables 3.13–3.14.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0005" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "Annualized curriculum adoption cost." } }, "economics[].period_unit": { "state": "value_present", "value": { "kind": "text", "text": "Per student per year, with five-year adoption and equipment lives." }, "provenance": { "kind": "source_review", "basis": "Cost methods.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0005" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "Per student per year, with five-year adoption and equipment lives." } }, "economics[].realization": { "state": "value_present", "value": { "kind": "text", "text": "These are estimated adoption costs, not the trial’s discounted invoices." }, "provenance": { "kind": "source_review", "basis": "pp. 10–11.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0005" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "These are estimated adoption costs, not the trial’s discounted invoices." } }, "economics[].recipient_payer": { "state": "value_present", "value": { "kind": "text", "text": "School districts would pay for materials, software, equipment, and training." }, "provenance": { "kind": "source_review", "basis": "Cost methods.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0005" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "School districts would pay for materials, software, equipment, and training." } }, "economics[].return_payback": { "state": "not_reported", "provenance": { "kind": "source_review", "basis": "Conclusion.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0005" ], "reviewed_on": "2026-10-25" }, "reason": "The analysis does not estimate financial returns or a payback period.", "review_state": "reviewed", "coverage": "complete", "display": { "answer": "The analysis does not estimate financial returns or a payback period." } }, "economics[].valuation_method": { "state": "value_present", "value": { "kind": "text", "text": "RAND combined vendor prices and district reports, annualized the costs, and weighted them by the number of schools." }, "provenance": { "kind": "source_review", "basis": "pp. 5–11, 26–28.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0005" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "RAND combined vendor prices and district reports, annualized the costs, and weighted them by the number of schools." } }, "evaluations[].ai_uplift": { "state": "value_present", "value": { "kind": "text", "text": "The software’s separate contribution was not estimated." }, "provenance": { "kind": "source_review", "basis": "C1 intervention and design.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "The software’s separate contribution was not estimated." } }, "evaluations[].comparator": { "state": "value_present", "value": { "kind": "text", "text": "Schools continued the algebra curriculum that they normally used." }, "provenance": { "kind": "source_review", "basis": "C1 methods.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "Schools continued the algebra curriculum that they normally used." } }, "evaluations[].design": { "state": "value_present", "value": { "kind": "text", "text": "Researchers assigned matched pairs of schools to Cognitive Tutor or their usual curriculum." }, "provenance": { "kind": "source_review", "basis": "C1 methods.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "Researchers assigned matched pairs of schools to Cognitive Tutor or their usual curriculum." } }, "evaluations[].effect": { "state": "value_present", "value": { "kind": "text", "text": "The revised high-school estimate was −0.07 standard deviations in year one and +0.21 in year two." }, "provenance": { "kind": "source_review", "basis": "C2 Table 3, preferred model.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "The revised high-school estimate was −0.07 standard deviations in year one and +0.21 in year two." } }, "evaluations[].exposure": { "state": "value_present", "value": { "kind": "text", "text": "Students generally used the tutor for about 40% of class time during the school year." }, "provenance": { "kind": "source_review", "basis": "C1 intervention description.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "Students generally used the tutor for about 40% of class time during the school year." } }, "evaluations[].id": { "state": "value_present", "value": { "kind": "identifiers", "identifiers": [ "AIBD-0214-E01" ], "role": "evaluations" }, "provenance": { "kind": "source_review", "basis": "Identifiers assigned to the evaluations described here.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002", "AIBD-0214-source-0003", "AIBD-0214-source-0006" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "AIBD-0214-E01" } }, "evaluations[].measurement_method": { "state": "value_present", "value": { "kind": "text", "text": "Students took standardized algebra readiness and proficiency tests. Researchers adjusted the end-of-year comparison for baseline scores and other characteristics." }, "provenance": { "kind": "source_review", "basis": "C1 measures; C2 analysis.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "Students took standardized algebra readiness and proficiency tests. Researchers adjusted the end-of-year comparison for baseline scores and other characteristics." } }, "evaluations[].metric": { "state": "value_present", "value": { "kind": "text", "text": "Algebra proficiency." }, "provenance": { "kind": "source_review", "basis": "C1 measures.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "Algebra proficiency." } }, "evaluations[].reproduction_materials": { "state": "value_present", "value": { "kind": "text", "text": "The 2024 reanalysis provides public code. Its full replication data are withheld to protect the participating schools’ identities." }, "provenance": { "kind": "source_review", "basis": "Mann et al. (2024), §7; the linked aeis-aux-rct repository, README.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0006" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "The 2024 reanalysis provides public code. Its full replication data are withheld to protect the participating schools’ identities." } }, "evaluations[].setting_sample": { "state": "value_present", "value": { "kind": "text", "text": "The trial included 73 high schools and 74 middle schools. The revised analysis excluded students without observed baseline tests." }, "provenance": { "kind": "source_review", "basis": "C1 sample; C2 pp. 1–3.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "The trial included 73 high schools and 74 middle schools. The revised analysis excluded students without observed baseline tests." } }, "evaluations[].uncertainty_limitations": { "state": "value_present", "value": { "kind": "text", "text": "The second-year estimate depended on adjustment for student characteristics. Its p-value of .04 did not meet the Clearinghouse’s corrected threshold of .025." }, "provenance": { "kind": "source_review", "basis": "C2 Table 3; C3 p. 31, n. c.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0002", "AIBD-0214-source-0003" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "The second-year estimate depended on adjustment for student characteristics. Its p-value of .04 did not meet the Clearinghouse’s corrected threshold of .025." } }, "external_references[]": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "follow_up[]": { "state": "value_present", "value": { "kind": "text", "text": "The revised Clearinghouse review rated the wider algebra evidence as mixed and removed this trial’s statistical significance after correcting for multiple comparisons." }, "provenance": { "kind": "source_review", "basis": "p. 6; p. 31, n. c.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0003" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "The revised Clearinghouse review rated the wider algebra evidence as mixed and removed this trial’s statistical significance after correcting for multiple comparisons." } }, "human_oversight": { "state": "value_present", "value": { "kind": "text", "text": "Teachers received three days of training before implementation and another day during the school year." }, "provenance": { "kind": "source_review", "basis": "C1 intervention description.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "Teachers received three days of training before implementation and another day during the school year." } }, "id": { "state": "value_present", "value": { "kind": "record_fields", "fields": { "id": "AIBD-0214" } }, "provenance": { "kind": "legacy_record", "basis": "Preserved from the existing AIBD record; external sources were not rechecked.", "record_fields": [ "id" ], "source_refs": [] }, "review_state": "record_mapped", "coverage": "partial" }, "infrastructure_dependencies[]": { "state": "value_present", "value": { "kind": "text", "text": "Schools needed computers for regular sessions, the curriculum materials, and trained algebra teachers." }, "provenance": { "kind": "source_review", "basis": "C1 intervention description.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "Schools needed computers for regular sessions, the curriculum materials, and trained algebra teachers." } }, "outcomes[].beneficiaries": { "state": "value_present", "value": { "kind": "text", "text": "Students who took Algebra I at participating high schools." }, "provenance": { "kind": "source_review", "basis": "C2 Table 3.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "Students who took Algebra I at participating high schools." } }, "outcomes[].beneficiary_count": { "state": "value_present", "value": { "kind": "text", "text": "The revised analysis included 11,066 high-school students and 5,519 middle-school students across two annual cohorts." }, "provenance": { "kind": "source_review", "basis": "C2 Tables 1–2.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "The revised analysis included 11,066 high-school students and 5,519 middle-school students across two annual cohorts." } }, "outcomes[].benefit_type": { "state": "value_present", "value": { "kind": "text", "text": "Improved learning." }, "provenance": { "kind": "source_review", "basis": "C2 Tables 3–4.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "Improved learning." } }, "outcomes[].claim": { "state": "value_present", "value": { "kind": "text", "text": "High-school algebra scores improved in the second year of implementation." }, "provenance": { "kind": "source_review", "basis": "C2 Table 3.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "High-school algebra scores improved in the second year of implementation." } }, "outcomes[].distribution": { "state": "value_present", "value": { "kind": "text", "text": "The reported gain was in the second high-school cohort. First-year and middle-school estimates were not statistically significant." }, "provenance": { "kind": "source_review", "basis": "C2 Tables 3–4.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "The reported gain was in the second high-school cohort. First-year and middle-school estimates were not statistically significant." } }, "outcomes[].duration": { "state": "value_present", "value": { "kind": "text", "text": "Each cohort studied Algebra I for one school year." }, "provenance": { "kind": "source_review", "basis": "C1 methods.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "Each cohort studied Algebra I for one school year." } }, "outcomes[].id": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "outcomes[].stakeholder_group": { "state": "value_present", "value": { "kind": "text", "text": "Students." }, "provenance": { "kind": "source_review", "basis": "C1 sample description.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "Students." } }, "outcomes[].status": { "state": "value_present", "value": { "kind": "text", "text": "The second-year gain was positive, but it was not statistically significant after the Clearinghouse adjusted for multiple comparisons." }, "provenance": { "kind": "source_review", "basis": "p. 31, Table B.1, n. c.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0003" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "The second-year gain was positive, but it was not statistically significant after the Clearinghouse adjusted for multiple comparisons." } }, "outcomes[].tradeoffs": { "state": "value_present", "value": { "kind": "text", "text": "Schools incurred higher adoption costs and needed time for implementation. The trial did not show significant gains in every setting." }, "provenance": { "kind": "source_review", "basis": "C1 discussion; C2 Tables 3–4; C5 Tables 3.13–3.14.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002", "AIBD-0214-source-0005" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "Schools incurred higher adoption costs and needed time for implementation. The trial did not show significant gains in every setting." } }, "outcomes[].unexpected": { "state": "value_present", "value": { "kind": "text", "text": "Improved algebra learning was the intended benefit." }, "provenance": { "kind": "source_review", "basis": "C1 introduction.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "Improved algebra learning was the intended benefit." } }, "places[]": { "state": "value_present", "value": { "kind": "text", "text": "The study covered 147 schools in 51 districts across Alabama, Connecticut, Kentucky, Louisiana, Michigan, New Jersey, and Texas." }, "provenance": { "kind": "source_review", "basis": "C1 sample description.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "The study covered 147 schools in 51 districts across Alabama, Connecticut, Kentucky, Louisiana, Michigan, New Jersey, and Texas." } }, "purpose": { "state": "value_present", "value": { "kind": "text", "text": "Cognitive Tutor combined classroom algebra lessons with practice that adapted to each student." }, "provenance": { "kind": "source_review", "basis": "C1 intervention description.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "Cognitive Tutor combined classroom algebra lessons with practice that adapted to each student." } }, "record_dates": { "state": "value_present", "value": { "kind": "structured", "data": { "start": "2007", "end": "2009", "precision": "school years", "display": "2007–08 and 2008–09", "refs": [ "AIBD-0214-source-0006" ], "current_date_observed": "2013-11-18", "instruction": "Use the observed result period for the case date. Keep the full study period separately; the normalized sort date does not imply a more precise observation.", "observed_result_period": "2009", "observed_result_precision": "year", "observed_result_basis": "End of the second school year, 2008–09." } }, "provenance": { "kind": "source_review", "basis": "Original study timing.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0006" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "The trial covered the 2007–08 and 2008–09 school years." } }, "replication.attempts[]": { "state": "value_present", "value": { "kind": "text", "text": "The Clearinghouse reviewed five qualifying Algebra I studies and rated their combined results as mixed." }, "provenance": { "kind": "source_review", "basis": "Overview and pp. 5–6.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0003" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "The Clearinghouse reviewed five qualifying Algebra I studies and rated their combined results as mixed." } }, "replication.conditions": { "state": "value_present", "value": { "kind": "text", "text": "Schools needed regular computer sessions, trained teachers, and time to implement the curriculum." }, "provenance": { "kind": "source_review", "basis": "C1 intervention and implementation discussion.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "Schools needed regular computer sessions, trained teachers, and time to implement the curriculum." } }, "revision_history[]": { "state": "value_present", "value": { "kind": "record_fields", "fields": { "revisions": [ { "date": "2026-10-20", "change": "Record drafted from the cited sources.", "editor": "AIBD" }, { "date": "2026-10-21", "change": "Added what the benefit is measured against, what happened since from the cited sources.", "source": "https://www.rand.org/pubs/research_briefs/RB9746.html; https://ies.ed.gov/ncee/wwc/Docs/InterventionReports/wwc_cognitivetutor_122116.pdf", "editor": "AIBD" }, { "date": "2026-10-21", "change": "Country of the developer added from the organization's own site or a registry.", "source": "https://www.carnegielearning.com/contact/", "editor": "AIBD" }, { "date": "2026-10-21", "change": "Prose pass: account, purpose, the system's part rewritten in shorter sentences with abbreviations expanded; facts, figures and quotations unchanged.", "editor": "AIBD" }, { "date": "2026-10-25", "change": "Reviewed the original evidence and supporting material; updated system details, methods, outcomes, limitations, costs, and source lineage.", "source": "https://journals.sagepub.com/doi/10.3102/0162373713507480", "editor": "AIBD" } ] } }, "provenance": { "kind": "legacy_record", "basis": "Preserved from the existing AIBD record; external sources were not rechecked.", "record_fields": [ "revisions" ], "source_refs": [] }, "review_state": "record_mapped", "coverage": "partial" }, "sector": { "state": "value_present", "value": { "kind": "text", "text": "Education." }, "provenance": { "kind": "source_review", "basis": "C1 intervention description.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "Education." } }, "sources[].access_version": { "state": "value_present", "value": { "kind": "text", "text": "The original manuscript, addendum, revised review, mechanism paper, and cost report were inspected. A later dataset paper supplies the school-year dates." }, "provenance": { "kind": "source_review", "basis": "Methods, results, and Tables 2–5.; pp. 1–6, Tables 1–4.; p. 6; pp. 23–24, Appendix A.4; p. 31, Table B.1, n. c.; pp. 249–52.; pp. ix–x, 5–11, 26–28; Tables 3.13–3.14.; §2 establishes the school years; §7 and the linked code repository describe available replication materials.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002", "AIBD-0214-source-0003", "AIBD-0214-source-0004", "AIBD-0214-source-0005", "AIBD-0214-source-0006" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "The original manuscript, addendum, revised review, mechanism paper, and cost report were inspected. A later dataset paper supplies the school-year dates." } }, "sources[].citation_locator": { "state": "value_present", "value": { "kind": "text", "text": "The original trial, its addendum, and the revised Clearinghouse review establish the result. Separate original papers describe the tutor and adoption costs." }, "provenance": { "kind": "source_review", "basis": "Methods, results, and Tables 2–5.; pp. 1–6, Tables 1–4.; p. 6; pp. 23–24, Appendix A.4; p. 31, Table B.1, n. c.; pp. 249–52.; pp. ix–x, 5–11, 26–28; Tables 3.13–3.14.; §2 establishes the school years; §7 and the linked code repository describe available replication materials.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002", "AIBD-0214-source-0003", "AIBD-0214-source-0004", "AIBD-0214-source-0005", "AIBD-0214-source-0006" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "The original trial, its addendum, and the revised Clearinghouse review establish the result. Separate original papers describe the tutor and adoption costs." } }, "sources[].id": { "state": "value_present", "value": { "kind": "identifiers", "identifiers": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002", "AIBD-0214-source-0003", "AIBD-0214-source-0004", "AIBD-0214-source-0005", "AIBD-0214-source-0006" ], "role": "retained evidence" }, "provenance": { "kind": "source_review", "basis": "Retained evidence in the reviewed order.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002", "AIBD-0214-source-0003", "AIBD-0214-source-0004", "AIBD-0214-source-0005", "AIBD-0214-source-0006" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "AIBD-0214-source-0001, AIBD-0214-source-0002, AIBD-0214-source-0003, AIBD-0214-source-0004, AIBD-0214-source-0005, AIBD-0214-source-0006" } }, "sources[].language": { "state": "value_present", "value": { "kind": "text", "text": "English." }, "provenance": { "kind": "source_review", "basis": "Methods, results, and Tables 2–5.; pp. 1–6, Tables 1–4.; p. 6; pp. 23–24, Appendix A.4; p. 31, Table B.1, n. c.; pp. 249–52.; pp. ix–x, 5–11, 26–28; Tables 3.13–3.14.; §2 establishes the school years; §7 and the linked code repository describe available replication materials.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002", "AIBD-0214-source-0003", "AIBD-0214-source-0004", "AIBD-0214-source-0005", "AIBD-0214-source-0006" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "English." } }, "sources[].relationship": { "state": "value_present", "value": { "kind": "text", "text": "RAND conducted the trial and cost analysis. The Clearinghouse independently reviewed the evidence. The technical account was written by developers." }, "provenance": { "kind": "source_review", "basis": "Methods, results, and Tables 2–5.; pp. 1–6, Tables 1–4.; p. 6; pp. 23–24, Appendix A.4; p. 31, Table B.1, n. c.; pp. 249–52.; pp. ix–x, 5–11, 26–28; Tables 3.13–3.14.; §2 establishes the school years; §7 and the linked code repository describe available replication materials.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002", "AIBD-0214-source-0003", "AIBD-0214-source-0004", "AIBD-0214-source-0005", "AIBD-0214-source-0006" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "RAND conducted the trial and cost analysis. The Clearinghouse independently reviewed the evidence. The technical account was written by developers." } }, "sources[].upstream_sources": { "state": "value_present", "value": { "kind": "structured", "data": { "reviewed_on": "2026-10-25", "lead_source_ref": "AIBD-0214-source-0001", "retained_source_order": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002", "AIBD-0214-source-0003", "AIBD-0214-source-0004", "AIBD-0214-source-0005", "AIBD-0214-source-0006" ], "origins": [ { "url": "https://journals.sagepub.com/doi/10.3102/0162373713507480", "title": "Effectiveness of Cognitive Tutor Algebra I at Scale", "locator": "Methods, results, and Tables 2–5.", "contribution": "Original school-randomized evaluation of the full curriculum.", "relationship": "Primary evaluation by RAND researchers, funded by the U.S. Department of Education.", "source_refs": [ "AIBD-0214-source-0001" ], "access": "The author manuscript was read through its indexed ResearchGate copy. Direct PDF retrieval returned HTTP 403. Use the journal’s 2014 date, not the repository’s inconsistent header.", "publication_date": "2014-06" }, { "url": "https://www.researchgate.net/publication/259639036_Effectiveness_of_Cognitive_Tutor_Algebra_I_at_Scale", "locator": "Author manuscript of the 2014 article.", "contribution": "Author manuscript of the 2014 article.", "relationship": "Access copy, supplement, or earlier version of the same work.", "source_refs": [ "AIBD-0214-source-0001" ] }, { "url": "https://www.rand.org/content/dam/rand/pubs/working_papers/WR1000/WR1050/RAND_WR1050.pdf", "title": "Addendum to Effectiveness of Cognitive Tutor Algebra I at Scale", "locator": "pp. 1–6, Tables 1–4.", "contribution": "Reanalysis excluding students without observed baseline scores, with revised samples and effects.", "relationship": "Original reanalysis by the trial’s authors. It is not a second trial.", "source_refs": [ "AIBD-0214-source-0002" ], "access": "The complete eight-page original PDF was read through the web tool; direct retrieval returned HTTP 403.", "publication_date": "2014-05-30" }, { "url": "https://ies.ed.gov/ncee/wwc/Docs/InterventionReports/wwc_cognitivetutor_122116.pdf", "title": "Cognitive Tutor® Algebra I", "locator": "p. 6; pp. 23–24, Appendix A.4; p. 31, Table B.1, n. c.", "contribution": "Independent evidence review and correction for multiple comparisons.", "relationship": "Independent review by the What Works Clearinghouse. It reuses the trial data rather than adding a new cohort.", "source_refs": [ "AIBD-0214-source-0003" ], "access": "Original revised PDF downloaded and relevant sections read.", "publication_date": "2016-12-21" }, { "url": "https://link.springer.com/content/pdf/10.3758/BF03194060.pdf", "title": "Cognitive Tutor: Applied Research in Mathematics Education", "locator": "pp. 249–52.", "contribution": "Original account of the cognitive model, step-by-step model tracing, and skill tracking.", "relationship": "Technical account by the system’s developers.", "source_refs": [ "AIBD-0214-source-0004" ], "access": "Original publisher PDF downloaded and mechanism sections read.", "publication_date": "2007-04" }, { "url": "https://www.rand.org/content/dam/rand/pubs/technical_reports/2012/RAND_TR1171-1.pdf", "title": "Analysis of Costs in an Algebra I Curriculum Effectiveness Study", "locator": "pp. ix–x, 5–11, 26–28; Tables 3.13–3.14.", "contribution": "Annualized adoption costs, cost assumptions, and the effect of including Houston.", "relationship": "Primary cost analysis by RAND using vendor prices and district information.", "source_refs": [ "AIBD-0214-source-0005" ], "access": "Relevant original PDF sections were read through the web tool. Direct retrieval returned HTTP 403.", "publication_date": "2012-07-12" }, { "url": "https://educationaldatamining.org/EDM2024/proceedings/2024.EDM-short-papers.52/index.html", "title": "Using Publicly Available Auxiliary Data to Improve Precision of Treatment Effect Estimation in a Randomized Efficacy Trial", "locator": "§2 establishes the school years; §7 and the linked code repository describe available replication materials.", "contribution": "The paper establishes the original trial’s school years and links public reanalysis code while explaining that school identities remain protected.", "relationship": "Later research using the trial dataset; retained only for the stated chronology.", "source_refs": [ "AIBD-0214-source-0006" ], "access": "The data-description and supporting-code sections were read, together with the linked aeis-aux-rct README. The full reanalysis was not reviewed.", "publication_date": "2024" }, { "url": "https://github.com/manncz/aeis-aux-rct", "title": "Data build and analysis for the CTAI reanalysis", "locator": "README, opening paragraphs and file inventory.", "contribution": "The authors’ original code repository for the later reanalysis. Its README states that protected school identities prevent a complete public rerun.", "relationship": "The authors’ original code repository for the later reanalysis. Its README states that protected school identities prevent a complete public rerun.", "source_refs": [ "AIBD-0214-source-0006" ] } ], "excluded_or_consolidated_sources": [ { "old_source_ref": "AIBD-0214-source-0001", "url": "https://www.rand.org/pubs/research_briefs/RB9746.html", "reason": "A summary or secondary account was replaced by the original evidence. Any unique earlier quotation remains in the audit artifact." }, { "old_source_ref": "AIBD-0214-source-0002", "url": "https://eric.ed.gov/?id=EJ1024233", "retained_source_ref": "AIBD-0214-source-0001", "reason": "Mirror or version consolidated into the original work." }, { "old_source_ref": "AIBD-0214-source-0003", "url": "https://www.rand.org/pubs/working_papers/WR1050.html", "retained_source_ref": "AIBD-0214-source-0002", "reason": "Mirror or version consolidated into the original work." }, { "old_source_ref": "AIBD-0214-source-0004", "url": "https://www.edweek.org/technology/educators-researchers-look-for-lessons-in-blended-learning-algebra-program/2014/01", "reason": "A summary or secondary account was replaced by the original evidence. Any unique earlier quotation remains in the audit artifact." } ], "review_scope": "Original evidence, systems, evaluation methods and results, costs, chronology, human roles, limitations, and follow-up.", "chronology_source": {}, "source_relationship_note": "Versions, supplements, reanalyses, and repeated developer statements are not independent replications.", "same_work_groups": [ [ "AIBD-0214-source-0001", "AIBD-0214-source-0002" ] ] } }, "provenance": { "kind": "source_review", "basis": "The original evidence and exact locators are preserved in this assessment.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002", "AIBD-0214-source-0003", "AIBD-0214-source-0004", "AIBD-0214-source-0005", "AIBD-0214-source-0006" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "The trial and addendum provide the estimates. The Clearinghouse review supplies the correction for multiple comparisons." } }, "strategic_assessments[]": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial", "family": "strategic assessments" }, "summary": { "state": "value_present", "value": { "kind": "text", "text": "Students using Cognitive Tutor Algebra I scored 0.21 standard deviations higher in the second year, with no statistically significant difference after correction for multiple comparisons." }, "provenance": { "kind": "source_review", "basis": "Methods, results, and Tables 2–5.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "Students using Cognitive Tutor Algebra I scored 0.21 standard deviations higher in the second year, with no statistically significant difference after correction for multiple comparisons." } }, "systems[].autonomy_permissions": { "state": "value_present", "value": { "kind": "text", "text": "The tutor selected exercises and gave feedback. Teachers remained responsible for instruction." }, "provenance": { "kind": "source_review", "basis": "C1 intervention description.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "The tutor selected exercises and gave feedback. Teachers remained responsible for instruction." } }, "systems[].component_role": { "state": "value_present", "value": { "kind": "text", "text": "The tutor checked students’ steps, provided hints, and selected practice for skills that needed work." }, "provenance": { "kind": "source_review", "basis": "C1 intervention description.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "The tutor checked students’ steps, provided hints, and selected practice for skills that needed work." } }, "systems[].deployment_configuration": { "state": "value_present", "value": { "kind": "text", "text": "Classes generally used the textbook and classroom activities three days a week and tutoring software two days a week." }, "provenance": { "kind": "source_review", "basis": "C1 intervention description.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "Classes generally used the textbook and classroom activities three days a week and tutoring software two days a week." } }, "systems[].inputs_outputs": { "state": "value_present", "value": { "kind": "text", "text": "Students entered algebra steps. The tutor returned feedback and selected further exercises." }, "provenance": { "kind": "source_review", "basis": "C1 intervention description.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "Students entered algebra steps. The tutor returned feedback and selected further exercises." } }, "systems[].model_name": { "state": "value_present", "value": { "kind": "text", "text": "An ACT-R-based cognitive model tracked students’ solution steps and mastery of individual skills." }, "provenance": { "kind": "source_review", "basis": "pp. 249–52.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0004" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "An ACT-R-based cognitive model tracked students’ solution steps and mastery of individual skills." } }, "systems[].model_version": { "state": "not_reported", "provenance": { "kind": "source_review", "basis": "Intervention and mechanism descriptions.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0004" ], "reviewed_on": "2026-10-25" }, "reason": "The study does not identify a numbered model or software release.", "review_state": "reviewed", "coverage": "complete", "display": { "answer": "The study does not identify a numbered model or software release." } }, "systems[].product_name": { "state": "value_present", "value": { "kind": "text", "text": "Cognitive Tutor Algebra I." }, "provenance": { "kind": "source_review", "basis": "C1 intervention description.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "Cognitive Tutor Algebra I." } }, "systems[].provider": { "state": "value_present", "value": { "kind": "text", "text": "Carnegie Learning." }, "provenance": { "kind": "source_review", "basis": "C1 intervention description.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "Carnegie Learning." } }, "taxonomy_crosswalks[]": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "title": { "state": "value_present", "value": { "kind": "text", "text": "Higher algebra scores in the trial’s second year" }, "provenance": { "kind": "source_review", "basis": "Methods, results, and Tables 2–5.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "Higher algebra scores in the trial’s second year" } }, "version": { "state": "value_present", "value": { "kind": "identifiers", "identifiers": [ "d0d3b347b993966f7ab31c89ecba4b49bddfe59ef866a45bb3f2774dab45228c" ], "role": "SHA-256 of normalized original case data" }, "provenance": { "kind": "curatorial", "basis": "Deterministic content version assigned by the local migration.", "record_fields": [], "source_refs": [] }, "review_state": "record_mapped", "coverage": "complete" }, "workflow": { "state": "value_present", "value": { "kind": "text", "text": "Teachers taught the curriculum and supervised students’ work with the software." }, "provenance": { "kind": "source_review", "basis": "C1 intervention description.", "record_fields": [], "source_refs": [ "AIBD-0214-source-0001", "AIBD-0214-source-0002" ], "reviewed_on": "2026-10-25" }, "review_state": "reviewed", "coverage": "complete", "display": { "answer": "Teachers taught the curriculum and supervised students’ work with the software." } } }, "summary": { "total": 91, "addressed": 91, "known": 73, "reviewed": 73, "record_mapped": 3, "not_assessed": 15, "complete": false, "by_state": { "value_present": 73, "not_reported": 3, "unresolved": 0, "not_assessed": 15, "not_applicable": 0, "withheld": 0, "source_unavailable": 0 } } } }