{ "id": "AIBD-0306", "title": "Two-week diary study: 16 blind and low-vision adults use a Cornell Tech scene-description app built on Microsoft Azure AI Vision for 354 everyday tasks; satisfaction averages 2.76/5", "summary": "Gonzalez Penuela et al. (CHI 2024) log 354 diary entries from 16 BLV participants using an Azure AI Vision app; uses include telling sunglasses from reading glasses without calling a spouse.", "date_observed": "2024-03-22", "date_basis": "unassessed", "date_precision": "unassessed", "benefit_status": null, "source_collection": null, "source_entry": null, "account": "Ricardo Gonzalez Penuela, Jazmin Collins, Cynthia Bennett and Shiri Azenkot (Cornell Tech and Google) ran a two-week diary study. It had 16 blind and low-vision adults in the United States (five men, 11 women, aged 24 to 67). Recruitment was via National Federation of the Blind mailing lists. They used an iPhone scene-description app the researchers built on Microsoft's Azure AI Vision image description application programming interface (API). It was chosen because it “exposes the latest deep learning algorithms”. Seeing AI also uses it. Participants submitted 354 entries (13 to 39 each). Documented uses include P1 photographing glasses. She “was not sure if this was my husband's sunglasses or my old pair of reading glasses from 4 years ago”. She got an answer without calling him. P5 photographed a hotel bed to check his roommate was not in it, and P15 photographed wine bottles in a shop rather than touching them. Participants scored descriptions 2.76 out of 5 for satisfaction and 2.43 out of 4 for trust on average. The authors say satisfaction did not track accuracy because users inferred information from partially correct descriptions. The paper appeared at CHI 2024 (doi 10.1145/3613904.3642211). The arXiv version was submitted on 22 March 2024. The study dates are not reported. The CHI EA 2025 follow-up used a custom app, VisionPal, and documented failures such as hallucinated braille labels.", "category": "benefit created", "unexpected": false, "purpose": "The study app was built to describe photographed scenes for blind and low-vision users, emulating Seeing AI's scene channel. The documented identifications were that purpose.", "system": "Cornell Tech study scene-description app (unnamed) built on the Microsoft Azure AI Vision image description API", "model_version": null, "developer": "Cornell Tech (app); Microsoft (Azure AI Vision image description API)", "deployer": "Study participants (16 blind and low-vision adults)", "location": "United States", "domain": "Accessibility / visual assistance", "segment": "Blind and low-vision users", "benefit_type": "Improved Accessibility", "taxonomy_note": null, "registry_ids": {}, "ratings": [ { "group": "personal", "depth": 1, "reason": "Participants completed small daily identifications without asking family. They rated descriptions 2.76/5 for satisfaction and 2.43/4 for trust on average. The app could not read text." } ], "ai_role": { "status": "assessed", "statement": "Azure AI Vision generated every description. The researchers' app only submitted photos and collected ratings. The paper documents individual successful uses and an overall judgment that descriptions “still need significant improvements”. Participants were allowed to use other tools during the study, so the diary covers only what they chose to photograph with this app." }, "evidence": [ { "kind": "peer-reviewed study", "citation": "Gonzalez Penuela RE, Collins J, Bennett CL, Azenkot S (2024). Investigating Use Cases of AI-Powered Scene Description Applications for Blind and Low Vision People. Proceedings of the CHI Conference on Human Factors in Computing Systems (CHI '24). doi:10.1145/3613904.3642211; arXiv:2403.15604.", "title": "Investigating Use Cases of AI-Powered Scene Description Applications for Blind and Low Vision People", "url": "https://arxiv.org/abs/2403.15604", "date": "2024-03-22", "written_by": "makers", "quote": "we conducted a two-week diary study where 16 BLV participants used an AI-powered scene description application we designed." }, { "kind": "peer-reviewed study", "citation": "Gonzalez Penuela et al. (2024), arXiv HTML version (P1 example).", "title": "Investigating Use Cases of AI-Powered Scene Description Applications for Blind and Low Vision People", "url": "https://arxiv.org/html/2403.15604", "date": "2024-03-22", "written_by": "makers", "quote": "She was happy to get an answer without having to call her husband." }, { "kind": "peer-reviewed study", "citation": "Gonzalez Penuela et al. (2024), arXiv HTML version (system and scores).", "title": "Investigating Use Cases of AI-Powered Scene Description Applications for Blind and Low Vision People", "url": "https://arxiv.org/html/2403.15604", "date": "2024-03-22", "written_by": "makers", "quote": "We collected a total of 354 diary entries from our 16 participants. On average, each participant submitted 22 entries." }, { "kind": "other", "citation": "Semantic Scholar record for doi 10.1145/3613904.3642211 (venue, authors and open-access link).", "title": "Investigating Use Cases of AI-Powered Scene Description Applications for Blind and Low Vision People", "url": "https://doi.org/10.1145/3613904.3642211", "date": "2024-03-22", "written_by": "independent", "quote": null }, { "kind": "preprint", "citation": "Gonzalez Penuela RE, Hu R, Lin S, Shende T, Azenkot S (2025). Towards Understanding the Use of MLLM-Enabled Applications for Visual Interpretation by Blind and Low Vision People. arXiv:2503.05899, submitted 7 March 2025; to appear at CHI 2025.", "title": "Towards Understanding the Use of MLLM-Enabled Applications for Visual Interpretation by Blind and Low Vision People", "url": "https://arxiv.org/abs/2503.05899", "date": "2025-03-07", "written_by": "makers", "quote": null } ], "replication": { "produced_by": [ "An iPhone app sending photos to the Azure AI Vision image description API", "Blind and low-vision users photographing objects they did not want to touch or ask others about", "A minimum of one photo per day for two weeks with a short questionnaire per entry" ], "must_hold": [ "The description is accurate enough for the user to infer what they need. Users sometimes drew answers from partially correct output", "The task does not require reading text, which the app could not do" ], "echoes": [ { "date": "2025-03-07", "setting": "Follow-up two-week diary study with 20 BLV people using an MLLM-enabled visual-interpretation app the authors developed (VisionPal per an earlier check); 553 entries, preliminary analysis of 60 from six participants: trust 3.75/5, satisfaction 4.15/5, with trust in high-stakes uses such as medical dosage advice (CHI EA 2025, arXiv 2503.05899)", "model_version": "multimodal large language model (unspecified)", "outcome": "recurred", "source": "https://arxiv.org/abs/2503.05899" } ] }, "paired_incident": null, "headline": "Azure AI Vision app helps blind, low-vision adults in 354 tasks", "metric": "2.76 out of 5 for satisfaction and 2.43 out of 4 for trust on average", "comparator": { "against": "asking a sighted family member (P1's husband)", "before": null, "after": "She was happy to get an answer without having to call her husband.", "basis": "measured", "quote": "For example, P1 submitted a photo of a pair of sunglasses when she “was not sure if this was my husband’s sunglasses or my old pair of reading glasses from 4 years ago.” She was happy to get an answer without having to call her husband.", "url": "https://arxiv.org/html/2403.15604" }, "later": { "status": "persisted", "date": "2025-03-07", "note": "The same Cornell Tech group's follow-up two-week diary study (CHI EA 2025), with 20 BLV people using a newer app they built on a multimodal large language model, collected 553 entries; a preliminary analysis of 60 entries found trust 3.75/5 and satisfaction 4.15/5, higher than the 2024 scores.", "quote": "We found that participants considered the application's visual interpretations trustworthy (mean 3.75 out of 5) and satisfying (mean 4.15 out of 5).", "url": "https://arxiv.org/abs/2503.05899" }, "value": null, "developer_country": "United States", "deployer_country": null, "unrated_because": null, "brief_notes": null, "brief": null, "revisions": [ { "date": "2026-10-20", "change": "Record drafted from the cited sources.", "source": null, "editor": "AIBD" }, { "date": "2026-10-21", "change": "Added what the benefit is measured against, what happened since from the cited sources.", "source": "https://arxiv.org/html/2403.15604; https://arxiv.org/abs/2503.05899", "editor": "AIBD" }, { "date": "2026-10-21", "change": "Country of the developer added from the organization's own site or a registry.", "source": "https://tech.cornell.edu/about/", "editor": "AIBD" }, { "date": "2026-10-21", "change": "Prose pass: a rating reason, account, purpose, the replication conditions, the system's part rewritten in shorter sentences with abbreviations expanded; facts, figures and quotations unchanged.", "source": null, "editor": "AIBD" }, { "date": "2026-10-21", "change": "Notes about which sources could be read were reworded for readers; facts, figures and quotations unchanged.", "source": null, "editor": "AIBD" }, { "date": "2026-10-22", "change": "Organization name standardized to Cornell Tech.", "source": null, "editor": "AIBD" } ], "confidence": "claimed", "shade": "low", "claim_status": "The makers' claim", "claim_basis": "Sources: one other independent source, two peer-reviewed studies from the makers and the makers' preprint. The cited evidence includes no independent study or report and no finding by a regulator or court.", "profile": { "case_id": "AIBD-0306", "catalog_version": "1.0.0", "source_fingerprint": "1b862e63ef09b0c2875bb7c10c509e9ce517d0f3c617c89e93f863609f0be69c", "collection_refs": [ { "collection_id": "aibd", "version": "snapshot-8bf7cfec9cb185aa" }, { "collection_id": "aibd", "version": "2026-10-25" } ], "source_ids": [ "AIBD-0306-source-0001", "AIBD-0306-source-0002", "AIBD-0306-source-0003", "AIBD-0306-source-0004", "AIBD-0306-source-0005" ], "families": { "strategic assessments": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial", "record_ids": [] }, "evaluations": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial", "record_ids": [] }, "monetary observations": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial", "record_ids": [] }, "institutional decisions": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial", "record_ids": [] } }, "fields": { "actors[]": { "state": "value_present", "value": { "kind": "record_fields", "fields": { "developer": "Cornell Tech (app); Microsoft (Azure AI Vision image description API)", "deployer": "Study participants (16 blind and low-vision adults)" } }, "provenance": { "kind": "legacy_record", "basis": "Preserved from the existing AIBD record; external sources were not rechecked.", "record_fields": [ "developer", "deployer" ], "source_refs": [] }, "review_state": "record_mapped", "coverage": "partial" }, "actors[].ownership_control": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "ai_contribution": { "state": "value_present", "value": { "kind": "record_fields", "fields": { "ai_role": { "status": "assessed", "statement": "Azure AI Vision generated every description. The researchers' app only submitted photos and collected ratings. The paper documents individual successful uses and an overall judgment that descriptions “still need significant improvements”. Participants were allowed to use other tools during the study, so the diary covers only what they chose to photograph with this app." } } }, "provenance": { "kind": "legacy_record", "basis": "Preserved from the existing AIBD record; external sources were not rechecked.", "record_fields": [ "ai_role" ], "source_refs": [] }, "review_state": "record_mapped", "coverage": "partial" }, "ai_relationship": { "state": "value_present", "value": { "kind": "record_fields", "fields": { "category": "benefit created", "ai_role": { "status": "assessed", "statement": "Azure AI Vision generated every description. The researchers' app only submitted photos and collected ratings. The paper documents individual successful uses and an overall judgment that descriptions “still need significant improvements”. Participants were allowed to use other tools during the study, so the diary covers only what they chose to photograph with this app." } } }, "provenance": { "kind": "legacy_record", "basis": "Preserved from the existing AIBD record; external sources were not rechecked.", "record_fields": [ "ai_role", "category" ], "source_refs": [] }, "review_state": "record_mapped", "coverage": "partial" }, "assessments[].alternatives": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "assessments[].assessor_date": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "assessments[].depth": { "state": "value_present", "value": { "kind": "record_fields", "fields": { "ratings": [ { "group": "personal", "depth": 1, "reason": "Participants completed small daily identifications without asking family. They rated descriptions 2.76/5 for satisfaction and 2.43/4 for trust on average. The app could not read text." } ] } }, "provenance": { "kind": "legacy_record", "basis": "Preserved from the existing AIBD record; external sources were not rechecked.", "record_fields": [ "ratings" ], "source_refs": [] }, "review_state": "record_mapped", "coverage": "partial" }, "assessments[].display_confidence": { "state": "value_present", "value": { "kind": "structured", "data": { "confidence": "low", "evidence_word": "claimed", "claim_status": "The makers' claim", "basis": "Sources: one other independent source, two peer-reviewed studies from the makers and the makers' preprint. The cited evidence includes no independent study or report and no finding by a regulator or court.", "interpretation": "Existing display rule preserved; the richer outcome-specific confidence question still needs review." } }, "provenance": { "kind": "curatorial", "basis": "Derived from the existing case display properties; not a new evidentiary judgment.", "record_fields": [], "source_refs": [] }, "review_state": "record_mapped", "coverage": "partial" }, "assessments[].rationale": { "state": "value_present", "value": { "kind": "record_fields", "fields": { "ratings": [ { "group": "personal", "depth": 1, "reason": "Participants completed small daily identifications without asking family. They rated descriptions 2.76/5 for satisfaction and 2.43/4 for trust on average. The app could not read text." } ] } }, "provenance": { "kind": "legacy_record", "basis": "Preserved from the existing AIBD record; external sources were not rechecked.", "record_fields": [ "ratings" ], "source_refs": [] }, "review_state": "record_mapped", "coverage": "partial" }, "assessments[].rubric_version": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "assessments[].status_basis": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "assessments[].target": { "state": "value_present", "value": { "kind": "record_fields", "fields": { "ratings": [ { "group": "personal", "depth": 1, "reason": "Participants completed small daily identifications without asking family. They rated descriptions 2.76/5 for satisfaction and 2.43/4 for trust on average. The app could not read text." } ] } }, "provenance": { "kind": "legacy_record", "basis": "Preserved from the existing AIBD record; external sources were not rechecked.", "record_fields": [ "ratings" ], "source_refs": [] }, "review_state": "record_mapped", "coverage": "partial" }, "availability": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "brief": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "case_kind": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "case_links[]": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "claims[].evidence_links": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "claims[].evidence_strength": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "claims[].missingness": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "claims[].proposition": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "cross_border_flows[]": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "decisions[].action_outcome": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial", "family": "institutional decisions" }, "decisions[].grounds": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial", "family": "institutional decisions" }, "decisions[].institution_task": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial", "family": "institutional decisions" }, "decisions[].success_criterion": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial", "family": "institutional decisions" }, "deployment_stage": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "economics[].amount": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial", "family": "monetary observations" }, "economics[].costs[]": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial", "family": "monetary observations" }, "economics[].counterfactual": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial", "family": "monetary observations" }, "economics[].currency_price_basis": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial", "family": "monetary observations" }, "economics[].gross_net": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial", "family": "monetary observations" }, "economics[].measure_type": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial", "family": "monetary observations" }, "economics[].period_unit": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial", "family": "monetary observations" }, "economics[].realization": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial", "family": "monetary observations" }, "economics[].recipient_payer": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial", "family": "monetary observations" }, "economics[].return_payback": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial", "family": "monetary observations" }, "economics[].valuation_method": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial", "family": "monetary observations" }, "evaluations[].ai_uplift": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial", "family": "evaluations" }, "evaluations[].comparator": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial", "family": "evaluations" }, "evaluations[].design": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial", "family": "evaluations" }, "evaluations[].effect": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial", "family": "evaluations" }, "evaluations[].exposure": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial", "family": "evaluations" }, "evaluations[].id": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial", "family": "evaluations" }, "evaluations[].measurement_method": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial", "family": "evaluations" }, "evaluations[].metric": { "state": "not_assessed", "provenance": { "kind": "legacy_record", "basis": "Mapped from the existing case description without a new external source review.", "record_fields": [ "metric" ], "source_refs": [] }, "review_state": "not_assessed", "coverage": "partial", "family": "evaluations", "display": { "answer": "2.76 out of 5 for satisfaction and 2.43 out of 4 for trust on average", "editor_note": "" } }, "evaluations[].reproduction_materials": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial", "family": "evaluations" }, "evaluations[].setting_sample": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial", "family": "evaluations" }, "evaluations[].uncertainty_limitations": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial", "family": "evaluations" }, "external_references[]": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "follow_up[]": { "state": "value_present", "value": { "kind": "record_fields", "fields": { "later": { "status": "persisted", "date": "2025-03-07", "note": "The same Cornell Tech group's follow-up two-week diary study (CHI EA 2025), with 20 BLV people using a newer app they built on a multimodal large language model, collected 553 entries; a preliminary analysis of 60 entries found trust 3.75/5 and satisfaction 4.15/5, higher than the 2024 scores.", "quote": "We found that participants considered the application's visual interpretations trustworthy (mean 3.75 out of 5) and satisfying (mean 4.15 out of 5).", "url": "https://arxiv.org/abs/2503.05899" } } }, "provenance": { "kind": "legacy_record", "basis": "Preserved from the existing AIBD record; external sources were not rechecked.", "record_fields": [ "later" ], "source_refs": [] }, "review_state": "record_mapped", "coverage": "partial" }, "human_oversight": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "id": { "state": "value_present", "value": { "kind": "record_fields", "fields": { "id": "AIBD-0306" } }, "provenance": { "kind": "legacy_record", "basis": "Preserved from the existing AIBD record; external sources were not rechecked.", "record_fields": [ "id" ], "source_refs": [] }, "review_state": "record_mapped", "coverage": "partial" }, "infrastructure_dependencies[]": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "outcomes[].beneficiaries": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "outcomes[].beneficiary_count": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "outcomes[].benefit_type": { "state": "value_present", "value": { "kind": "record_fields", "fields": { "benefit_type": "Improved Accessibility" } }, "provenance": { "kind": "legacy_record", "basis": "Preserved from the existing AIBD record; external sources were not rechecked.", "record_fields": [ "benefit_type" ], "source_refs": [] }, "review_state": "record_mapped", "coverage": "partial" }, "outcomes[].claim": { "state": "value_present", "value": { "kind": "record_fields", "fields": { "account": "Ricardo Gonzalez Penuela, Jazmin Collins, Cynthia Bennett and Shiri Azenkot (Cornell Tech and Google) ran a two-week diary study. It had 16 blind and low-vision adults in the United States (five men, 11 women, aged 24 to 67). Recruitment was via National Federation of the Blind mailing lists. They used an iPhone scene-description app the researchers built on Microsoft's Azure AI Vision image description application programming interface (API). It was chosen because it “exposes the latest deep learning algorithms”. Seeing AI also uses it. Participants submitted 354 entries (13 to 39 each). Documented uses include P1 photographing glasses. She “was not sure if this was my husband's sunglasses or my old pair of reading glasses from 4 years ago”. She got an answer without calling him. P5 photographed a hotel bed to check his roommate was not in it, and P15 photographed wine bottles in a shop rather than touching them. Participants scored descriptions 2.76 out of 5 for satisfaction and 2.43 out of 4 for trust on average. The authors say satisfaction did not track accuracy because users inferred information from partially correct descriptions. The paper appeared at CHI 2024 (doi 10.1145/3613904.3642211). The arXiv version was submitted on 22 March 2024. The study dates are not reported. The CHI EA 2025 follow-up used a custom app, VisionPal, and documented failures such as hallucinated braille labels.", "benefit_type": "Improved Accessibility" } }, "provenance": { "kind": "legacy_record", "basis": "Preserved from the existing AIBD record; external sources were not rechecked.", "record_fields": [ "account", "benefit_type" ], "source_refs": [] }, "review_state": "record_mapped", "coverage": "partial" }, "outcomes[].distribution": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "outcomes[].duration": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "outcomes[].id": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "outcomes[].stakeholder_group": { "state": "value_present", "value": { "kind": "record_fields", "fields": { "ratings": [ { "group": "personal", "depth": 1, "reason": "Participants completed small daily identifications without asking family. They rated descriptions 2.76/5 for satisfaction and 2.43/4 for trust on average. The app could not read text." } ] } }, "provenance": { "kind": "legacy_record", "basis": "Preserved from the existing AIBD record; external sources were not rechecked.", "record_fields": [ "ratings" ], "source_refs": [] }, "review_state": "record_mapped", "coverage": "partial" }, "outcomes[].status": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "outcomes[].tradeoffs": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "outcomes[].unexpected": { "state": "value_present", "value": { "kind": "record_fields", "fields": { "unexpected": false, "purpose": "The study app was built to describe photographed scenes for blind and low-vision users, emulating Seeing AI's scene channel. The documented identifications were that purpose." } }, "provenance": { "kind": "legacy_record", "basis": "Preserved from the existing AIBD record; external sources were not rechecked.", "record_fields": [ "unexpected", "purpose" ], "source_refs": [] }, "review_state": "record_mapped", "coverage": "partial" }, "places[]": { "state": "value_present", "value": { "kind": "record_fields", "fields": { "location": "United States", "developer_country": "United States" } }, "provenance": { "kind": "legacy_record", "basis": "Preserved from the existing AIBD record; external sources were not rechecked.", "record_fields": [ "location", "developer_country" ], "source_refs": [] }, "review_state": "record_mapped", "coverage": "partial" }, "purpose": { "state": "value_present", "value": { "kind": "record_fields", "fields": { "purpose": "The study app was built to describe photographed scenes for blind and low-vision users, emulating Seeing AI's scene channel. The documented identifications were that purpose." } }, "provenance": { "kind": "legacy_record", "basis": "Preserved from the existing AIBD record; external sources were not rechecked.", "record_fields": [ "purpose" ], "source_refs": [] }, "review_state": "record_mapped", "coverage": "partial" }, "record_dates": { "state": "value_present", "value": { "kind": "record_fields", "fields": { "date_observed": "2024-03-22", "revisions": [ { "date": "2026-10-20", "change": "Record drafted from the cited sources.", "editor": "AIBD" }, { "date": "2026-10-21", "change": "Added what the benefit is measured against, what happened since from the cited sources.", "source": "https://arxiv.org/html/2403.15604; https://arxiv.org/abs/2503.05899", "editor": "AIBD" }, { "date": "2026-10-21", "change": "Country of the developer added from the organization's own site or a registry.", "source": "https://tech.cornell.edu/about/", "editor": "AIBD" }, { "date": "2026-10-21", "change": "Prose pass: a rating reason, account, purpose, the replication conditions, the system's part rewritten in shorter sentences with abbreviations expanded; facts, figures and quotations unchanged.", "editor": "AIBD" }, { "date": "2026-10-21", "change": "Notes about which sources could be read were reworded for readers; facts, figures and quotations unchanged.", "editor": "AIBD" }, { "date": "2026-10-22", "change": "Organization name standardized to Cornell Tech.", "editor": "AIBD" } ] } }, "provenance": { "kind": "legacy_record", "basis": "Preserved from the existing AIBD record; external sources were not rechecked.", "record_fields": [ "date_observed", "revisions" ], "source_refs": [] }, "review_state": "record_mapped", "coverage": "partial" }, "replication.attempts[]": { "state": "value_present", "value": { "kind": "record_fields", "fields": { "replication": { "produced_by": [ "An iPhone app sending photos to the Azure AI Vision image description API", "Blind and low-vision users photographing objects they did not want to touch or ask others about", "A minimum of one photo per day for two weeks with a short questionnaire per entry" ], "must_hold": [ "The description is accurate enough for the user to infer what they need. Users sometimes drew answers from partially correct output", "The task does not require reading text, which the app could not do" ], "echoes": [ { "date": "2025-03-07", "setting": "Follow-up two-week diary study with 20 BLV people using an MLLM-enabled visual-interpretation app the authors developed (VisionPal per an earlier check); 553 entries, preliminary analysis of 60 from six participants: trust 3.75/5, satisfaction 4.15/5, with trust in high-stakes uses such as medical dosage advice (CHI EA 2025, arXiv 2503.05899)", "model_version": "multimodal large language model (unspecified)", "outcome": "recurred", "source": "https://arxiv.org/abs/2503.05899" } ] } } }, "provenance": { "kind": "legacy_record", "basis": "Preserved from the existing AIBD record; external sources were not rechecked.", "record_fields": [ "replication" ], "source_refs": [] }, "review_state": "record_mapped", "coverage": "partial" }, "replication.conditions": { "state": "value_present", "value": { "kind": "record_fields", "fields": { "replication": { "produced_by": [ "An iPhone app sending photos to the Azure AI Vision image description API", "Blind and low-vision users photographing objects they did not want to touch or ask others about", "A minimum of one photo per day for two weeks with a short questionnaire per entry" ], "must_hold": [ "The description is accurate enough for the user to infer what they need. Users sometimes drew answers from partially correct output", "The task does not require reading text, which the app could not do" ], "echoes": [ { "date": "2025-03-07", "setting": "Follow-up two-week diary study with 20 BLV people using an MLLM-enabled visual-interpretation app the authors developed (VisionPal per an earlier check); 553 entries, preliminary analysis of 60 from six participants: trust 3.75/5, satisfaction 4.15/5, with trust in high-stakes uses such as medical dosage advice (CHI EA 2025, arXiv 2503.05899)", "model_version": "multimodal large language model (unspecified)", "outcome": "recurred", "source": "https://arxiv.org/abs/2503.05899" } ] } } }, "provenance": { "kind": "legacy_record", "basis": "Preserved from the existing AIBD record; external sources were not rechecked.", "record_fields": [ "replication" ], "source_refs": [] }, "review_state": "record_mapped", "coverage": "partial" }, "revision_history[]": { "state": "value_present", "value": { "kind": "record_fields", "fields": { "revisions": [ { "date": "2026-10-20", "change": "Record drafted from the cited sources.", "editor": "AIBD" }, { "date": "2026-10-21", "change": "Added what the benefit is measured against, what happened since from the cited sources.", "source": "https://arxiv.org/html/2403.15604; https://arxiv.org/abs/2503.05899", "editor": "AIBD" }, { "date": "2026-10-21", "change": "Country of the developer added from the organization's own site or a registry.", "source": "https://tech.cornell.edu/about/", "editor": "AIBD" }, { "date": "2026-10-21", "change": "Prose pass: a rating reason, account, purpose, the replication conditions, the system's part rewritten in shorter sentences with abbreviations expanded; facts, figures and quotations unchanged.", "editor": "AIBD" }, { "date": "2026-10-21", "change": "Notes about which sources could be read were reworded for readers; facts, figures and quotations unchanged.", "editor": "AIBD" }, { "date": "2026-10-22", "change": "Organization name standardized to Cornell Tech.", "editor": "AIBD" } ] } }, "provenance": { "kind": "legacy_record", "basis": "Preserved from the existing AIBD record; external sources were not rechecked.", "record_fields": [ "revisions" ], "source_refs": [] }, "review_state": "record_mapped", "coverage": "partial" }, "sector": { "state": "value_present", "value": { "kind": "record_fields", "fields": { "domain": "Accessibility / visual assistance", "segment": "Blind and low-vision users" } }, "provenance": { "kind": "legacy_record", "basis": "Preserved from the existing AIBD record; external sources were not rechecked.", "record_fields": [ "domain", "segment" ], "source_refs": [] }, "review_state": "record_mapped", "coverage": "partial" }, "sources[].access_version": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "sources[].citation_locator": { "state": "value_present", "value": { "kind": "record_fields", "fields": { "evidence": [ { "kind": "peer-reviewed study", "citation": "Gonzalez Penuela RE, Collins J, Bennett CL, Azenkot S (2024). Investigating Use Cases of AI-Powered Scene Description Applications for Blind and Low Vision People. Proceedings of the CHI Conference on Human Factors in Computing Systems (CHI '24). doi:10.1145/3613904.3642211; arXiv:2403.15604.", "title": "Investigating Use Cases of AI-Powered Scene Description Applications for Blind and Low Vision People", "url": "https://arxiv.org/abs/2403.15604", "date": "2024-03-22", "written_by": "makers", "quote": "we conducted a two-week diary study where 16 BLV participants used an AI-powered scene description application we designed." }, { "kind": "peer-reviewed study", "citation": "Gonzalez Penuela et al. (2024), arXiv HTML version (P1 example).", "title": "Investigating Use Cases of AI-Powered Scene Description Applications for Blind and Low Vision People", "url": "https://arxiv.org/html/2403.15604", "date": "2024-03-22", "written_by": "makers", "quote": "She was happy to get an answer without having to call her husband." }, { "kind": "peer-reviewed study", "citation": "Gonzalez Penuela et al. (2024), arXiv HTML version (system and scores).", "title": "Investigating Use Cases of AI-Powered Scene Description Applications for Blind and Low Vision People", "url": "https://arxiv.org/html/2403.15604", "date": "2024-03-22", "written_by": "makers", "quote": "We collected a total of 354 diary entries from our 16 participants. On average, each participant submitted 22 entries." }, { "kind": "other", "citation": "Semantic Scholar record for doi 10.1145/3613904.3642211 (venue, authors and open-access link).", "title": "Investigating Use Cases of AI-Powered Scene Description Applications for Blind and Low Vision People", "url": "https://doi.org/10.1145/3613904.3642211", "date": "2024-03-22", "written_by": "independent" }, { "kind": "preprint", "citation": "Gonzalez Penuela RE, Hu R, Lin S, Shende T, Azenkot S (2025). Towards Understanding the Use of MLLM-Enabled Applications for Visual Interpretation by Blind and Low Vision People. arXiv:2503.05899, submitted 7 March 2025; to appear at CHI 2025.", "title": "Towards Understanding the Use of MLLM-Enabled Applications for Visual Interpretation by Blind and Low Vision People", "url": "https://arxiv.org/abs/2503.05899", "date": "2025-03-07", "written_by": "makers" } ] } }, "provenance": { "kind": "legacy_record", "basis": "Preserved from the existing AIBD record; external sources were not rechecked.", "record_fields": [ "evidence" ], "source_refs": [] }, "review_state": "record_mapped", "coverage": "partial" }, "sources[].id": { "state": "value_present", "value": { "kind": "identifiers", "identifiers": [ "AIBD-0306-source-0001", "AIBD-0306-source-0002", "AIBD-0306-source-0003", "AIBD-0306-source-0004", "AIBD-0306-source-0005" ], "role": "Existing evidence items, in their original order" }, "provenance": { "kind": "curatorial", "basis": "Stable within the fingerprinted record version; no source content or source review has been added.", "record_fields": [], "source_refs": [] }, "review_state": "record_mapped", "coverage": "complete" }, "sources[].language": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "sources[].relationship": { "state": "value_present", "value": { "kind": "record_fields", "fields": { "evidence": [ { "kind": "peer-reviewed study", "citation": "Gonzalez Penuela RE, Collins J, Bennett CL, Azenkot S (2024). Investigating Use Cases of AI-Powered Scene Description Applications for Blind and Low Vision People. Proceedings of the CHI Conference on Human Factors in Computing Systems (CHI '24). doi:10.1145/3613904.3642211; arXiv:2403.15604.", "title": "Investigating Use Cases of AI-Powered Scene Description Applications for Blind and Low Vision People", "url": "https://arxiv.org/abs/2403.15604", "date": "2024-03-22", "written_by": "makers", "quote": "we conducted a two-week diary study where 16 BLV participants used an AI-powered scene description application we designed." }, { "kind": "peer-reviewed study", "citation": "Gonzalez Penuela et al. (2024), arXiv HTML version (P1 example).", "title": "Investigating Use Cases of AI-Powered Scene Description Applications for Blind and Low Vision People", "url": "https://arxiv.org/html/2403.15604", "date": "2024-03-22", "written_by": "makers", "quote": "She was happy to get an answer without having to call her husband." }, { "kind": "peer-reviewed study", "citation": "Gonzalez Penuela et al. (2024), arXiv HTML version (system and scores).", "title": "Investigating Use Cases of AI-Powered Scene Description Applications for Blind and Low Vision People", "url": "https://arxiv.org/html/2403.15604", "date": "2024-03-22", "written_by": "makers", "quote": "We collected a total of 354 diary entries from our 16 participants. On average, each participant submitted 22 entries." }, { "kind": "other", "citation": "Semantic Scholar record for doi 10.1145/3613904.3642211 (venue, authors and open-access link).", "title": "Investigating Use Cases of AI-Powered Scene Description Applications for Blind and Low Vision People", "url": "https://doi.org/10.1145/3613904.3642211", "date": "2024-03-22", "written_by": "independent" }, { "kind": "preprint", "citation": "Gonzalez Penuela RE, Hu R, Lin S, Shende T, Azenkot S (2025). Towards Understanding the Use of MLLM-Enabled Applications for Visual Interpretation by Blind and Low Vision People. arXiv:2503.05899, submitted 7 March 2025; to appear at CHI 2025.", "title": "Towards Understanding the Use of MLLM-Enabled Applications for Visual Interpretation by Blind and Low Vision People", "url": "https://arxiv.org/abs/2503.05899", "date": "2025-03-07", "written_by": "makers" } ] } }, "provenance": { "kind": "legacy_record", "basis": "Preserved from the existing AIBD record; external sources were not rechecked.", "record_fields": [ "evidence" ], "source_refs": [] }, "review_state": "record_mapped", "coverage": "partial" }, "sources[].upstream_sources": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "strategic_assessments[]": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial", "family": "strategic assessments" }, "summary": { "state": "value_present", "value": { "kind": "record_fields", "fields": { "summary": "Gonzalez Penuela et al. (CHI 2024) log 354 diary entries from 16 BLV participants using an Azure AI Vision app; uses include telling sunglasses from reading glasses without calling a spouse.", "account": "Ricardo Gonzalez Penuela, Jazmin Collins, Cynthia Bennett and Shiri Azenkot (Cornell Tech and Google) ran a two-week diary study. It had 16 blind and low-vision adults in the United States (five men, 11 women, aged 24 to 67). Recruitment was via National Federation of the Blind mailing lists. They used an iPhone scene-description app the researchers built on Microsoft's Azure AI Vision image description application programming interface (API). It was chosen because it “exposes the latest deep learning algorithms”. Seeing AI also uses it. Participants submitted 354 entries (13 to 39 each). Documented uses include P1 photographing glasses. She “was not sure if this was my husband's sunglasses or my old pair of reading glasses from 4 years ago”. She got an answer without calling him. P5 photographed a hotel bed to check his roommate was not in it, and P15 photographed wine bottles in a shop rather than touching them. Participants scored descriptions 2.76 out of 5 for satisfaction and 2.43 out of 4 for trust on average. The authors say satisfaction did not track accuracy because users inferred information from partially correct descriptions. The paper appeared at CHI 2024 (doi 10.1145/3613904.3642211). The arXiv version was submitted on 22 March 2024. The study dates are not reported. The CHI EA 2025 follow-up used a custom app, VisionPal, and documented failures such as hallucinated braille labels." } }, "provenance": { "kind": "legacy_record", "basis": "Preserved from the existing AIBD record; external sources were not rechecked.", "record_fields": [ "summary", "account" ], "source_refs": [] }, "review_state": "record_mapped", "coverage": "partial" }, "systems[].autonomy_permissions": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "systems[].component_role": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "systems[].deployment_configuration": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "systems[].inputs_outputs": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "systems[].model_name": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "systems[].model_version": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "systems[].product_name": { "state": "value_present", "value": { "kind": "record_fields", "fields": { "system": "Cornell Tech study scene-description app (unnamed) built on the Microsoft Azure AI Vision image description API" } }, "provenance": { "kind": "legacy_record", "basis": "Preserved from the existing AIBD record; external sources were not rechecked.", "record_fields": [ "system" ], "source_refs": [] }, "review_state": "record_mapped", "coverage": "partial" }, "systems[].provider": { "state": "value_present", "value": { "kind": "structured", "data": { "application_developer": "Cornell Tech (app); Microsoft (Azure AI Vision image description API)" } }, "provenance": { "kind": "legacy_record", "basis": "Mapped from the existing case description without a new external source review.", "record_fields": [ "developer" ], "source_refs": [] }, "review_state": "record_mapped", "coverage": "partial" }, "taxonomy_crosswalks[]": { "state": "not_assessed", "review_state": "not_assessed", "coverage": "partial" }, "title": { "state": "value_present", "value": { "kind": "record_fields", "fields": { "title": "Two-week diary study: 16 blind and low-vision adults use a Cornell Tech scene-description app built on Microsoft Azure AI Vision for 354 everyday tasks; satisfaction averages 2.76/5", "headline": "Azure AI Vision app helps blind, low-vision adults in 354 tasks" } }, "provenance": { "kind": "legacy_record", "basis": "Preserved from the existing AIBD record; external sources were not rechecked.", "record_fields": [ "title", "headline" ], "source_refs": [] }, "review_state": "record_mapped", "coverage": "partial" }, "version": { "state": "value_present", "value": { "kind": "identifiers", "identifiers": [ "1b862e63ef09b0c2875bb7c10c509e9ce517d0f3c617c89e93f863609f0be69c" ], "role": "SHA-256 of normalized original case data" }, "provenance": { "kind": "curatorial", "basis": "Deterministic content version assigned by the local migration.", "record_fields": [], "source_refs": [] }, "review_state": "record_mapped", "coverage": "complete" }, "workflow": { "state": "value_present", "value": { "kind": "record_fields", "fields": { "account": "Ricardo Gonzalez Penuela, Jazmin Collins, Cynthia Bennett and Shiri Azenkot (Cornell Tech and Google) ran a two-week diary study. It had 16 blind and low-vision adults in the United States (five men, 11 women, aged 24 to 67). Recruitment was via National Federation of the Blind mailing lists. They used an iPhone scene-description app the researchers built on Microsoft's Azure AI Vision image description application programming interface (API). It was chosen because it “exposes the latest deep learning algorithms”. Seeing AI also uses it. Participants submitted 354 entries (13 to 39 each). Documented uses include P1 photographing glasses. She “was not sure if this was my husband's sunglasses or my old pair of reading glasses from 4 years ago”. She got an answer without calling him. P5 photographed a hotel bed to check his roommate was not in it, and P15 photographed wine bottles in a shop rather than touching them. Participants scored descriptions 2.76 out of 5 for satisfaction and 2.43 out of 4 for trust on average. The authors say satisfaction did not track accuracy because users inferred information from partially correct descriptions. The paper appeared at CHI 2024 (doi 10.1145/3613904.3642211). The arXiv version was submitted on 22 March 2024. The study dates are not reported. The CHI EA 2025 follow-up used a custom app, VisionPal, and documented failures such as hallucinated braille labels.", "replication": { "produced_by": [ "An iPhone app sending photos to the Azure AI Vision image description API", "Blind and low-vision users photographing objects they did not want to touch or ask others about", "A minimum of one photo per day for two weeks with a short questionnaire per entry" ], "must_hold": [ "The description is accurate enough for the user to infer what they need. Users sometimes drew answers from partially correct output", "The task does not require reading text, which the app could not do" ], "echoes": [ { "date": "2025-03-07", "setting": "Follow-up two-week diary study with 20 BLV people using an MLLM-enabled visual-interpretation app the authors developed (VisionPal per an earlier check); 553 entries, preliminary analysis of 60 from six participants: trust 3.75/5, satisfaction 4.15/5, with trust in high-stakes uses such as medical dosage advice (CHI EA 2025, arXiv 2503.05899)", "model_version": "multimodal large language model (unspecified)", "outcome": "recurred", "source": "https://arxiv.org/abs/2503.05899" } ] } } }, "provenance": { "kind": "legacy_record", "basis": "Preserved from the existing AIBD record; external sources were not rechecked.", "record_fields": [ "account", "replication" ], "source_refs": [] }, "review_state": "record_mapped", "coverage": "partial" } }, "summary": { "total": 91, "addressed": 91, "known": 29, "reviewed": 0, "record_mapped": 29, "not_assessed": 62, "complete": false, "by_state": { "value_present": 29, "not_reported": 0, "unresolved": 0, "not_assessed": 62, "not_applicable": 0, "withheld": 0, "source_unavailable": 0 } } } }