2024
Lin, Hongjin; Karusala, Naveena; Okolo, Chinasa T.; D’Ignazio, Catherine; Gajos, Krzysztof Z.
“Come to us first”: Centering Community Organizations in Artificial Intelligence for Social Good Partnerships Journal Article
In: Proc. ACM Hum.-Comput. Interact., vol. 8, no. CSCW2, 2024.
@article{lin2024come,
title = {"Come to us first": Centering Community Organizations in Artificial Intelligence for Social Good Partnerships},
author = {Hongjin Lin and Naveena Karusala and Chinasa T. Okolo and Catherine D'Ignazio and Krzysztof Z. Gajos},
url = {/papers/lin2024come.pdf},
doi = {10.1145/3687009},
year = {2024},
date = {2024-11-01},
journal = {Proc. ACM Hum.-Comput. Interact.},
volume = {8},
number = {CSCW2},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
abstract = {Artificial Intelligence for Social Good (AI4SG) has emerged as a growing body of research and practice exploring the potential of AI technologies to tackle social issues. This area emphasizes interdisciplinary partnerships with community organizations, such as non-profits and government agencies. However, amidst excitement about new advances in AI and their potential impact, the needs, expectations, and aspirations of these community organizations–and whether they are being met–are not well understood. Understanding these factors is important to ensure that the considerable efforts by AI teams and community organizations can actually achieve the positive social impact they strive for. Drawing on the Data Feminism framework, we explored the perspectives of community organization members on their partnerships with AI teams through 16 semi-structured interviews. Our study highlights the pervasive influence of funding agendas and the optimism surrounding AI's potential. Despite the significant intellectual contributions and labor provided by community organization members, their goals were frequently sidelined in favor of other stakeholders, including AI teams. While many community organization members expected tangible project deployment, only two out of 14 projects we studied reached the deployment stage. However, community organization members sustained their belief in the potential of the projects, still seeing diminished goals as valuable. To enhance the efficacy of future collaborations, our participants shared their aspirations for success, calling for co-leadership starting from the early stages of projects. We propose data co-liberation as a grounding principle for approaching AI4SG moving forward, positing that community organizations' co-leadership is essential for fostering more effective, sustainable, and ethical development of AI.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Artificial Intelligence for Social Good (AI4SG) has emerged as a growing body of research and practice exploring the potential of AI technologies to tackle social issues. This area emphasizes interdisciplinary partnerships with community organizations, such as non-profits and government agencies. However, amidst excitement about new advances in AI and their potential impact, the needs, expectations, and aspirations of these community organizations–and whether they are being met–are not well understood. Understanding these factors is important to ensure that the considerable efforts by AI teams and community organizations can actually achieve the positive social impact they strive for. Drawing on the Data Feminism framework, we explored the perspectives of community organization members on their partnerships with AI teams through 16 semi-structured interviews. Our study highlights the pervasive influence of funding agendas and the optimism surrounding AI’s potential. Despite the significant intellectual contributions and labor provided by community organization members, their goals were frequently sidelined in favor of other stakeholders, including AI teams. While many community organization members expected tangible project deployment, only two out of 14 projects we studied reached the deployment stage. However, community organization members sustained their belief in the potential of the projects, still seeing diminished goals as valuable. To enhance the efficacy of future collaborations, our participants shared their aspirations for success, calling for co-leadership starting from the early stages of projects. We propose data co-liberation as a grounding principle for approaching AI4SG moving forward, positing that community organizations’ co-leadership is essential for fostering more effective, sustainable, and ethical development of AI.
So, Jianna; Yang, Faye X.; Gupta, Anoopum S.; Gajos, Krzysztof Z.
“It’s Better to be Grounded in Reality”: a Speculative Exploration of Patient-Centered Digital Phenotyping for Neurological Conditions Proceedings Article
In: Proceedings of the 26th International ACM SIGACCESS Conference on Computers and Accessibility, Association for Computing Machinery, St. John’s, NL, Canada, 2024, ISBN: 9798400706776.
@inproceedings{so2024better,
title = {"It's Better to be Grounded in Reality": a Speculative Exploration of Patient-Centered Digital Phenotyping for Neurological Conditions},
author = {Jianna So and Faye X. Yang and Anoopum S. Gupta and Krzysztof Z. Gajos},
url = {/papers/so2024better.pdf},
doi = {10.1145/3663548.3688486},
isbn = {9798400706776},
year = {2024},
date = {2024-01-01},
booktitle = {Proceedings of the 26th International ACM SIGACCESS Conference on Computers and Accessibility},
publisher = {Association for Computing Machinery},
address = {St. John's, NL, Canada},
series = {ASSETS '24},
abstract = {Digital phenotyping in clinical research provides objective measures when evaluating neurological conditions, such as ataxias and Parkinson’s disease. While the clinical validity of digital phenotyping data is yet to be fully determined, individual research results are not reported back to participants due to apprehension about how complex data types should be represented, the manner in which results should be communicated to patients, and the possibility of uncertain results being misinterpreted. However, researchers are calling for individual results to be made available to participants, respecting participants’ ownership of their quantified selves and improving transparency of research practices. To investigate how patients with progressive conditions might value seeing their data, we are conducting an interview study with neurology patients who have participated in digital phenotyping. We report initial findings from four participants, who expressed interest in using digital phenotyping data to 1) motivate their care, 2) make perception of their condition concrete, 3) reduce labor in tracking and communicating their condition, and 4) perceive their contributions to clinical research. This work points to exciting potential of patient-centered digital phenotyping to benefit patients’ understanding of themselves, and push forward a paradigm of ethical data report-back.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Digital phenotyping in clinical research provides objective measures when evaluating neurological conditions, such as ataxias and Parkinson’s disease. While the clinical validity of digital phenotyping data is yet to be fully determined, individual research results are not reported back to participants due to apprehension about how complex data types should be represented, the manner in which results should be communicated to patients, and the possibility of uncertain results being misinterpreted. However, researchers are calling for individual results to be made available to participants, respecting participants’ ownership of their quantified selves and improving transparency of research practices. To investigate how patients with progressive conditions might value seeing their data, we are conducting an interview study with neurology patients who have participated in digital phenotyping. We report initial findings from four participants, who expressed interest in using digital phenotyping data to 1) motivate their care, 2) make perception of their condition concrete, 3) reduce labor in tracking and communicating their condition, and 4) perceive their contributions to clinical research. This work points to exciting potential of patient-centered digital phenotyping to benefit patients’ understanding of themselves, and push forward a paradigm of ethical data report-back.
Lin, Hongjin; Han, Tessa; Gajos, Krzysztof Z.; Gupta, Anoopum S.
Hevelius Report: Visualizing Web-Based Mobility Test Data For Clinical Decision and Learning Support Proceedings Article
In: Proceedings of the 26th International ACM SIGACCESS Conference on Computers and Accessibility, Association for Computing Machinery, St. John’s, NL, Canada, 2024, ISBN: 9798400706776.
@inproceedings{lin2024hevelius,
title = {Hevelius Report: Visualizing Web-Based Mobility Test Data For Clinical Decision and Learning Support},
author = {Hongjin Lin and Tessa Han and Krzysztof Z. Gajos and Anoopum S. Gupta},
url = {/papers/lin2024hevelius.pdf},
doi = {10.1145/3663548.3688490},
isbn = {9798400706776},
year = {2024},
date = {2024-01-01},
booktitle = {Proceedings of the 26th International ACM SIGACCESS Conference on Computers and Accessibility},
publisher = {Association for Computing Machinery},
address = {St. John's, NL, Canada},
series = {ASSETS '24},
abstract = {Hevelius, a web-based computer mouse test, measures arm movement and has been shown to accurately evaluate severity for patients with Parkinson’s disease and ataxias. A Hevelius session produces 32 numeric features, which may be hard to interpret, especially in time-constrained clinical settings. This work aims to support clinicians (and other stakeholders) in interpreting and connecting Hevelius features to clinical concepts. Through an iterative design process, we developed a visualization tool (Hevelius Report) that (1) abstracts six clinically relevant concepts from 32 features, (2) visualizes patient test results, and compares them to results from healthy controls and other patients, and (3) is an interactive app to meet the specific needs in different usage scenarios. Then, we conducted a preliminary user study through an online interview with three clinicians who were not involved in the project. They expressed interest in using Hevelius Report, especially for identifying subtle changes in their patients’ mobility that are hard to capture with existing clinical tests. Future work will integrate the visualization tool into the current clinical workflow of a neurology team and conduct systematic evaluations of the tool’s usefulness, usability, and effectiveness. Hevelius Report represents a promising solution for analyzing fine-motor test results and monitoring patients’ conditions and progressions.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Hevelius, a web-based computer mouse test, measures arm movement and has been shown to accurately evaluate severity for patients with Parkinson’s disease and ataxias. A Hevelius session produces 32 numeric features, which may be hard to interpret, especially in time-constrained clinical settings. This work aims to support clinicians (and other stakeholders) in interpreting and connecting Hevelius features to clinical concepts. Through an iterative design process, we developed a visualization tool (Hevelius Report) that (1) abstracts six clinically relevant concepts from 32 features, (2) visualizes patient test results, and compares them to results from healthy controls and other patients, and (3) is an interactive app to meet the specific needs in different usage scenarios. Then, we conducted a preliminary user study through an online interview with three clinicians who were not involved in the project. They expressed interest in using Hevelius Report, especially for identifying subtle changes in their patients’ mobility that are hard to capture with existing clinical tests. Future work will integrate the visualization tool into the current clinical workflow of a neurology team and conduct systematic evaluations of the tool’s usefulness, usability, and effectiveness. Hevelius Report represents a promising solution for analyzing fine-motor test results and monitoring patients’ conditions and progressions.
Karusala, Naveena; Upadhyay, Sohini; Veeraraghavan, Rajesh; Gajos, Krzysztof Z.
Understanding Contestability on the Margins: Implications for the Design of Algorithmic Decision-making in Public Services Proceedings Article
In: Proceedings of the 2024 CHI Conference on Human Factors in Computing Systems, Association for Computing Machinery, Honolulu, HI, USA, 2024, ISBN: 9798400703300.
@inproceedings{karusala2024understanding,
title = {Understanding Contestability on the Margins: Implications for the Design of Algorithmic Decision-making in Public Services},
author = {Naveena Karusala and Sohini Upadhyay and Rajesh Veeraraghavan and Krzysztof Z. Gajos},
url = {/papers/karusala2024understanding.pdf},
doi = {10.1145/3613904.3641898},
isbn = {9798400703300},
year = {2024},
date = {2024-01-01},
booktitle = {Proceedings of the 2024 CHI Conference on Human Factors in Computing Systems},
publisher = {Association for Computing Machinery},
address = {Honolulu, HI, USA},
series = {CHI '24},
abstract = {Policymakers have established that the ability to contest decisions made by or with algorithms is core to responsible artificial intelligence (AI). However, there has been a disconnect between research on contestability of algorithms, and what the situated practice of contestation looks like in contexts across the world, especially amongst communities on the margins. We address this gap through a qualitative study of follow-up and contestation in accessing public services for land ownership in rural India and affordable housing in the urban United States. We find there are significant barriers to exercising rights and contesting decisions, which intermediaries like NGO workers or lawyers work with communities to address. We draw on the notion of accompaniment in global health to highlight the open-ended work required to support people in navigating violent social systems. We discuss the implications of our findings for key aspects of contestability, including building capacity for contestation, human review, and the role of explanations. We also discuss how sociotechnical systems of algorithmic decision-making can embody accompaniment by taking on a higher burden of preventing denials and enabling contestation.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Policymakers have established that the ability to contest decisions made by or with algorithms is core to responsible artificial intelligence (AI). However, there has been a disconnect between research on contestability of algorithms, and what the situated practice of contestation looks like in contexts across the world, especially amongst communities on the margins. We address this gap through a qualitative study of follow-up and contestation in accessing public services for land ownership in rural India and affordable housing in the urban United States. We find there are significant barriers to exercising rights and contesting decisions, which intermediaries like NGO workers or lawyers work with communities to address. We draw on the notion of accompaniment in global health to highlight the open-ended work required to support people in navigating violent social systems. We discuss the implications of our findings for key aspects of contestability, including building capacity for contestation, human review, and the role of explanations. We also discuss how sociotechnical systems of algorithmic decision-making can embody accompaniment by taking on a higher burden of preventing denials and enabling contestation.
Ma, Zilin; Mei, Yiyang; Long, Yinru; Su, Zhaoyuan; Gajos, Krzysztof Z.
Evaluating the Experience of LGBTQ+ People Using Large Language Model Based Chatbots for Mental Health Support Proceedings Article
In: Proceedings of the 2024 CHI Conference on Human Factors in Computing Systems, Association for Computing Machinery, Honolulu, HI, USA, 2024, ISBN: 9798400703300.
@inproceedings{ma2024evaluating,
title = {Evaluating the Experience of LGBTQ+ People Using Large Language Model Based Chatbots for Mental Health Support},
author = {Zilin Ma and Yiyang Mei and Yinru Long and Zhaoyuan Su and Krzysztof Z. Gajos},
url = {/papers/ma2024evaluating.pdf},
doi = {10.1145/3613904.3642482},
isbn = {9798400703300},
year = {2024},
date = {2024-01-01},
booktitle = {Proceedings of the 2024 CHI Conference on Human Factors in Computing Systems},
publisher = {Association for Computing Machinery},
address = {Honolulu, HI, USA},
series = {CHI '24},
abstract = {LGBTQ+ individuals are increasingly turning to chatbots powered by large language models (LLMs) to meet their mental health needs. However, little research has explored whether these chatbots can adequately and safely provide tailored support for this demographic. We interviewed 18 LGBTQ+ and 13 non-LGBTQ+ participants about their experiences with LLM-based chatbots for mental health needs. LGBTQ+ participants relied on these chatbots for mental health support, likely due to an absence of support in real life. Notably, while LLMs offer prompt support, they frequently fall short in grasping the nuances of LGBTQ-specific challenges. Although fine-tuning LLMs to address LGBTQ+ needs can be a step in the right direction, it isn’t the panacea. The deeper issue is entrenched in societal discrimination. Consequently, we call on future researchers and designers to look beyond mere technical refinements and advocate for holistic strategies that confront and counteract the societal biases burdening the LGBTQ+ community.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
LGBTQ+ individuals are increasingly turning to chatbots powered by large language models (LLMs) to meet their mental health needs. However, little research has explored whether these chatbots can adequately and safely provide tailored support for this demographic. We interviewed 18 LGBTQ+ and 13 non-LGBTQ+ participants about their experiences with LLM-based chatbots for mental health needs. LGBTQ+ participants relied on these chatbots for mental health support, likely due to an absence of support in real life. Notably, while LLMs offer prompt support, they frequently fall short in grasping the nuances of LGBTQ-specific challenges. Although fine-tuning LLMs to address LGBTQ+ needs can be a step in the right direction, it isn’t the panacea. The deeper issue is entrenched in societal discrimination. Consequently, we call on future researchers and designers to look beyond mere technical refinements and advocate for holistic strategies that confront and counteract the societal biases burdening the LGBTQ+ community.
Swaroop, Siddharth; Buçinca, Zana; Gajos, Krzysztof Z.; Doshi-Velez, Finale
Accuracy-Time Tradeoffs in AI-Assisted Decision Making under Time Pressure Proceedings Article
In: Proceedings of the 29th International Conference on Intelligent User Interfaces, pp. 138–154, Association for Computing Machinery, Greenville, SC, USA, 2024, ISBN: 9798400705083.
@inproceedings{swaroop2024accuracy,
title = {Accuracy-Time Tradeoffs in AI-Assisted Decision Making under Time Pressure},
author = {Siddharth Swaroop and Zana Buçinca and Krzysztof Z. Gajos and Finale Doshi-Velez},
url = {/papers/swaroop2024accuracy.pdf},
doi = {10.1145/3640543.3645206},
isbn = {9798400705083},
year = {2024},
date = {2024-01-01},
booktitle = {Proceedings of the 29th International Conference on Intelligent User Interfaces},
pages = {138–154},
publisher = {Association for Computing Machinery},
address = {Greenville, SC, USA},
series = {IUI '24},
abstract = {In settings where users both need high accuracy and are time-pressured, such as doctors working in emergency rooms, we want to provide AI assistance that both increases decision accuracy and reduces decision-making time. Current literature focusses on how users interact with AI assistance when there is no time pressure, finding that different AI assistances have different benefits: some can reduce time taken while increasing overreliance on AI, while others do the opposite. The precise benefit can depend on both the user and task. In time-pressured scenarios, adapting when we show AI assistance is especially important: relying on the AI assistance can save time, and can therefore be beneficial when the AI is likely to be right. We would ideally adapt what AI assistance we show depending on various properties (of the task and of the user) in order to best trade off accuracy and time. We introduce a study where users have to answer a series of logic puzzles. We find that time pressure affects how users use different AI assistances, making some assistances more beneficial than others when compared to no-time-pressure settings. We also find that a user’s overreliance rate is a key predictor of their behaviour: overreliers and not-overreliers use different AI assistance types differently. We find marginal correlations between a user’s overreliance rate (which is related to the user’s trust in AI recommendations) and their personality traits (Big Five Personality traits). Overall, our work suggests that AI assistances have different accuracy-time tradeoffs when people are under time pressure compared to no time pressure, and we explore how we might adapt AI assistances in this setting.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
In settings where users both need high accuracy and are time-pressured, such as doctors working in emergency rooms, we want to provide AI assistance that both increases decision accuracy and reduces decision-making time. Current literature focusses on how users interact with AI assistance when there is no time pressure, finding that different AI assistances have different benefits: some can reduce time taken while increasing overreliance on AI, while others do the opposite. The precise benefit can depend on both the user and task. In time-pressured scenarios, adapting when we show AI assistance is especially important: relying on the AI assistance can save time, and can therefore be beneficial when the AI is likely to be right. We would ideally adapt what AI assistance we show depending on various properties (of the task and of the user) in order to best trade off accuracy and time. We introduce a study where users have to answer a series of logic puzzles. We find that time pressure affects how users use different AI assistances, making some assistances more beneficial than others when compared to no-time-pressure settings. We also find that a user’s overreliance rate is a key predictor of their behaviour: overreliers and not-overreliers use different AI assistance types differently. We find marginal correlations between a user’s overreliance rate (which is related to the user’s trust in AI recommendations) and their personality traits (Big Five Personality traits). Overall, our work suggests that AI assistances have different accuracy-time tradeoffs when people are under time pressure compared to no time pressure, and we explore how we might adapt AI assistances in this setting.
2023
Saksono, Herman; Morris, Vivien; Parker, Andrea G.; Gajos, Krzysztof Z.
Evaluating Similarity Variables for Peer Matching in Digital Health Storytelling Journal Article
In: Proc. ACM Hum.-Comput. Interact., vol. 7, no. CSCW2, 2023.
@article{saksono2023evaluating,
title = {Evaluating Similarity Variables for Peer Matching in Digital Health Storytelling},
author = {Herman Saksono and Vivien Morris and Andrea G. Parker and Krzysztof Z. Gajos},
url = {/papers/saksono2023evaluating.pdf},
doi = {10.1145/3610060},
year = {2023},
date = {2023-10-01},
journal = {Proc. ACM Hum.-Comput. Interact.},
volume = {7},
number = {CSCW2},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
abstract = {Peer matching can enhance the impact of social health technologies. By matching similar peers, online health communities can optimally facilitate social modeling that supports positive health attitudes and moods. However, little work has examined how to operationalize similarities in digital health tools, thus limiting our ability to perform optimal peer matching. To address this gap, we conducted a factorial experiment to examine how three categories of similarity variables (i.e., Demographic, Ability, Experiential) can be used to perform peer matching that supports the social modeling of physical activity. We focus this study on physical activity because it is a health behavior that reduces the risk of chronic diseases. We also prioritized this study for single-caregiver mothers who often face substantial barriers to being active because of immense employment and household responsibilities, especially Black single-caregiver mothers. We recruited 309 single-caregiver mothers (49% Black, 51% white), then we asked them to listen to peer audio storytelling about family physical activity. We randomly matched/mismatched the storyteller's profile using the three categories of similarity variables. Our analyses demonstrated that matching by Demographic variables led to a significantly higher Physical Activity Intention. Furthermore, our subgroup analyses indicated that Black single-caregiver mothers experienced a significant and immediate effect of peer matching in Physical Activity Intention, Self-efficacy, and mood. In contrast, white single-caregiver mothers did not report any significant immediate effect. Collectively, our data suggest that peer matching in health storytelling is potentially beneficial for racially minoritized groups; and that having diverse representations in health technology is required for promoting health equity.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Peer matching can enhance the impact of social health technologies. By matching similar peers, online health communities can optimally facilitate social modeling that supports positive health attitudes and moods. However, little work has examined how to operationalize similarities in digital health tools, thus limiting our ability to perform optimal peer matching. To address this gap, we conducted a factorial experiment to examine how three categories of similarity variables (i.e., Demographic, Ability, Experiential) can be used to perform peer matching that supports the social modeling of physical activity. We focus this study on physical activity because it is a health behavior that reduces the risk of chronic diseases. We also prioritized this study for single-caregiver mothers who often face substantial barriers to being active because of immense employment and household responsibilities, especially Black single-caregiver mothers. We recruited 309 single-caregiver mothers (49% Black, 51% white), then we asked them to listen to peer audio storytelling about family physical activity. We randomly matched/mismatched the storyteller’s profile using the three categories of similarity variables. Our analyses demonstrated that matching by Demographic variables led to a significantly higher Physical Activity Intention. Furthermore, our subgroup analyses indicated that Black single-caregiver mothers experienced a significant and immediate effect of peer matching in Physical Activity Intention, Self-efficacy, and mood. In contrast, white single-caregiver mothers did not report any significant immediate effect. Collectively, our data suggest that peer matching in health storytelling is potentially beneficial for racially minoritized groups; and that having diverse representations in health technology is required for promoting health equity.
Eklund, Nicole M; Ouillon, Jessey; Pandey, Vineet; Stephen, Christopher D; Schmahmann, Jeremy D; Edgerton, Jeremy; Gajos, Krzysztof Z; Gupta, Anoopum S
Real-life ankle submovements and computer mouse use reflect patient-reported function in adult ataxias Journal Article
In: Brain Communications, vol. 5, no. 2, 2023, ISSN: 2632-1297.
@article{eklund2023real,
title = {Real-life ankle submovements and computer mouse use reflect patient-reported function in adult ataxias},
author = {Nicole M Eklund and Jessey Ouillon and Vineet Pandey and Christopher D Stephen and Jeremy D Schmahmann and Jeremy Edgerton and Krzysztof Z Gajos and Anoopum S Gupta},
doi = {10.1093/braincomms/fcad064},
issn = {2632-1297},
year = {2023},
date = {2023-01-01},
journal = {Brain Communications},
volume = {5},
number = {2},
abstract = {Novel disease modifying therapies are being evaluated in spinocerebellar ataxias and multiple system atrophy. Clinician-performed disease rating scales are relatively insensitive for measuring disease change over time, resulting in large and long clinical trials. We tested the hypothesis that sensors worn continuously at home during natural behavior and a web-based computer mouse task performed at home could produce interpretable, meaningful, and reliable motor measures for potential use in clinical trials.Thirty-four individuals with degenerative ataxias (spinocerebellar ataxia types 1, 2, 3, and 6 and multiple system atrophy of the cerebellar type) and eight age-matched controls completed the cross-sectional study. Participants wore an ankle and wrist sensor continuously at-home for one week and completed the Hevelius computer mouse task eight times over four weeks. We examined properties of motor primitives called ``submovements'' derived from the continuous wearable sensors and properties of computer mouse clicks and trajectories in relationship to patient-reported measures of function (PROM-Ataxia) and ataxia rating scales (Scale for the Assessment and Rating of Ataxia and the Brief Ataxia Rating Scale). The test-retest reliability of digital measures and differences between ataxia and control participants were evaluated.Individuals with ataxia had smaller, slower, and less powerful ankle submovements during natural behavior at home. A composite measure based on ankle submovements strongly correlated with ataxia rating scale scores (Pearson's r = 0.82-0.88), strongly correlated with self-reported function (r = 0.81), had high test-retest reliability (intraclass correlation coefficient = 0.95), and distinguished ataxia and control participants, including preataxic individuals (N=4) from controls. A composite measure based on computer mouse movements and clicks strongly correlated with ataxia rating scale total (r = 0.86-0.88) and arm scores (r = 0.65-0.75), correlated well with self-reported function (r = 0.72-0.73), and had high test-retest reliability (intraclass correlation coefficient = 0.99).These data indicate that interpretable, meaningful, and highly reliable motor measures can be obtained from continuous measurement of natural movement, particularly at the ankle location, and from computer mouse movements during a simple point-and-click task performed at home. This study supports the use of these two inexpensive and easy-to-use technologies in longitudinal natural history studies in spinocerebellar ataxias and multiple system atrophy of the cerebellar type and shows promise as potential motor outcome measures in interventional trials.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Novel disease modifying therapies are being evaluated in spinocerebellar ataxias and multiple system atrophy. Clinician-performed disease rating scales are relatively insensitive for measuring disease change over time, resulting in large and long clinical trials. We tested the hypothesis that sensors worn continuously at home during natural behavior and a web-based computer mouse task performed at home could produce interpretable, meaningful, and reliable motor measures for potential use in clinical trials.Thirty-four individuals with degenerative ataxias (spinocerebellar ataxia types 1, 2, 3, and 6 and multiple system atrophy of the cerebellar type) and eight age-matched controls completed the cross-sectional study. Participants wore an ankle and wrist sensor continuously at-home for one week and completed the Hevelius computer mouse task eight times over four weeks. We examined properties of motor primitives called “submovements” derived from the continuous wearable sensors and properties of computer mouse clicks and trajectories in relationship to patient-reported measures of function (PROM-Ataxia) and ataxia rating scales (Scale for the Assessment and Rating of Ataxia and the Brief Ataxia Rating Scale). The test-retest reliability of digital measures and differences between ataxia and control participants were evaluated.Individuals with ataxia had smaller, slower, and less powerful ankle submovements during natural behavior at home. A composite measure based on ankle submovements strongly correlated with ataxia rating scale scores (Pearson’s r = 0.82-0.88), strongly correlated with self-reported function (r = 0.81), had high test-retest reliability (intraclass correlation coefficient = 0.95), and distinguished ataxia and control participants, including preataxic individuals (N=4) from controls. A composite measure based on computer mouse movements and clicks strongly correlated with ataxia rating scale total (r = 0.86-0.88) and arm scores (r = 0.65-0.75), correlated well with self-reported function (r = 0.72-0.73), and had high test-retest reliability (intraclass correlation coefficient = 0.99).These data indicate that interpretable, meaningful, and highly reliable motor measures can be obtained from continuous measurement of natural movement, particularly at the ankle location, and from computer mouse movements during a simple point-and-click task performed at home. This study supports the use of these two inexpensive and easy-to-use technologies in longitudinal natural history studies in spinocerebellar ataxias and multiple system atrophy of the cerebellar type and shows promise as potential motor outcome measures in interventional trials.
Pandey, Vineet; Khan, Nergis C.; Gupta, Anoopum S.; Gajos, Krzysztof Z.
In: ACM Transactions on Accessible Computing, vol. 16, no. 1, 2023, ISSN: 1936-7228.
@article{pandey2023accuracy,
title = {Accuracy and Reliability of At-home Quantification of Motor Impairments Using a Computer-based Pointing Task with Children with Ataxia-Telangiectasia},
author = {Vineet Pandey and Nergis C. Khan and Anoopum S. Gupta and Krzysztof Z. Gajos},
url = {/papers/pandey2023accuracy.pdf},
doi = {10.1145/3581790},
issn = {1936-7228},
year = {2023},
date = {2023-03-01},
journal = {ACM Transactions on Accessible Computing},
volume = {16},
number = {1},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
abstract = {Methods for obtaining accurate quantitative assessments of motor impairments are essential in accessibility research, design of adaptive ability-based assistive technologies, as well as in clinical care and medical research. Currently, such assessments are typically performed in controlled laboratory or clinical settings under professional supervision. Emerging approaches for collecting data in unsupervised settings have been shown to produce valid data when aggregated over large populations, but it is not yet established if in unsupervised settings measures of research or clinical significance can be collected accurately and reliably for individuals.
We conducted a study with 13 children with ataxia-telangiectasia and 9 healthy children to analyze the validity, test-retest reliability, and acceptability of at-home use of a recent active digital phenotyping system, called Hevelius. Hevelius produces 32 measures derived from the movement trajectories of the mouse cursor, and it produces a quantitative estimate of motor impairment in the dominant arm using the dominant arm component of the Brief Ataxia Rating Scale (BARS). The severity score estimates generated by Hevelius from single at-home sessions deviated from clinician-assigned BARS scores more than the severity score estimates generated from single sessions conducted under researcher supervision. However, taking a median of as few as 2 consecutive sessions produced severity score estimates that were as accurate or better than the estimates produced from single supervised sessions. Further, aggregating as few as 2 consecutive sessions resulted in good test-retest reliability (ICC = 0.81 for A-T participants).
This work demonstrated the feasibility of performing accurate and reliable quantitative assessments of individual motor impairments in the dominant arm through tasks performed at home without supervision by the researchers. Further work is needed, however, to assess how broadly these results generalize.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Methods for obtaining accurate quantitative assessments of motor impairments are essential in accessibility research, design of adaptive ability-based assistive technologies, as well as in clinical care and medical research. Currently, such assessments are typically performed in controlled laboratory or clinical settings under professional supervision. Emerging approaches for collecting data in unsupervised settings have been shown to produce valid data when aggregated over large populations, but it is not yet established if in unsupervised settings measures of research or clinical significance can be collected accurately and reliably for individuals.
We conducted a study with 13 children with ataxia-telangiectasia and 9 healthy children to analyze the validity, test-retest reliability, and acceptability of at-home use of a recent active digital phenotyping system, called Hevelius. Hevelius produces 32 measures derived from the movement trajectories of the mouse cursor, and it produces a quantitative estimate of motor impairment in the dominant arm using the dominant arm component of the Brief Ataxia Rating Scale (BARS). The severity score estimates generated by Hevelius from single at-home sessions deviated from clinician-assigned BARS scores more than the severity score estimates generated from single sessions conducted under researcher supervision. However, taking a median of as few as 2 consecutive sessions produced severity score estimates that were as accurate or better than the estimates produced from single supervised sessions. Further, aggregating as few as 2 consecutive sessions resulted in good test-retest reliability (ICC = 0.81 for A-T participants).
This work demonstrated the feasibility of performing accurate and reliable quantitative assessments of individual motor impairments in the dominant arm through tasks performed at home without supervision by the researchers. Further work is needed, however, to assess how broadly these results generalize.
We conducted a study with 13 children with ataxia-telangiectasia and 9 healthy children to analyze the validity, test-retest reliability, and acceptability of at-home use of a recent active digital phenotyping system, called Hevelius. Hevelius produces 32 measures derived from the movement trajectories of the mouse cursor, and it produces a quantitative estimate of motor impairment in the dominant arm using the dominant arm component of the Brief Ataxia Rating Scale (BARS). The severity score estimates generated by Hevelius from single at-home sessions deviated from clinician-assigned BARS scores more than the severity score estimates generated from single sessions conducted under researcher supervision. However, taking a median of as few as 2 consecutive sessions produced severity score estimates that were as accurate or better than the estimates produced from single supervised sessions. Further, aggregating as few as 2 consecutive sessions resulted in good test-retest reliability (ICC = 0.81 for A-T participants).
This work demonstrated the feasibility of performing accurate and reliable quantitative assessments of individual motor impairments in the dominant arm through tasks performed at home without supervision by the researchers. Further work is needed, however, to assess how broadly these results generalize.
Greenberg, David M.; Warrier, Varun; Abu-Akel, Ahmad; Allison, Carrie; Gajos, Krzysztof Z.; Reinecke, Katharina; Rentfrow, P. Jason; Radecki, Marcin A.; Baron-Cohen, Simon
Sex and age differences in “theory of mind” across 57 countries using the English version of the “Reading the Mind in the Eyes” Test Journal Article
In: Proceedings of the National Academy of Sciences, vol. 120, no. 1, pp. e2022385119, 2023.
@article{greenberg2023sex,
title = {Sex and age differences in “theory of mind” across 57 countries using the English version of the “Reading the Mind in the Eyes” Test},
author = {David M. Greenberg and Varun Warrier and Ahmad Abu-Akel and Carrie Allison and Krzysztof Z. Gajos and Katharina Reinecke and P. Jason Rentfrow and Marcin A. Radecki and Simon Baron-Cohen},
url = {https://www.pnas.org/doi/abs/10.1073/pnas.2022385119},
doi = {10.1073/pnas.2022385119},
year = {2023},
date = {2023-01-01},
journal = {Proceedings of the National Academy of Sciences},
volume = {120},
number = {1},
pages = {e2022385119},
abstract = {The “Reading the Mind in the Eyes” Test (Eyes Test) is a widely used assessment of “theory of mind.” The NIMH Research Domain Criteria recommends it as one of two tests for “understanding mental states.” Previous studies have demonstrated an on-average female advantage on the Eyes Test. However, it is unknown whether this female advantage exists across the lifespan and across a large number of countries. Thus, we tested sex and age differences using the English version of the Eyes Test in adolescents and adults across 57 countries. We also tested for associations with sociodemographic and cognitive/personality factors. We leveraged one discovery dataset (N = 305,726) and three validation datasets (Ns = 642; 5,284; and 1,087). The results show that: i) there is a replicable on-average female advantage in performance on the Eyes Test; ii) performance increases through adolescence and shallowly declines across adulthood; iii) the on-average female advantage is evident across the lifespan; iv) there is a significant on-average female advantage in 36 out of 57 countries; v) there is a significant on-average female advantage on translated (non-English) versions of the Eyes Test in 12 out of 16 countries, as confirmed by a systematic review; vi) D-scores, or empathizing-systemizing, predict Eyes Test performance above and beyond sex differences; and vii) the female advantage is negatively linked to “prosperity” and “autonomy,” and positively linked to “collectivism,” as confirmed by exploratory country-level analyses. We conclude that the on-average female advantage on the Eyes Test is observed across ages and most countries.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
The “Reading the Mind in the Eyes” Test (Eyes Test) is a widely used assessment of “theory of mind.” The NIMH Research Domain Criteria recommends it as one of two tests for “understanding mental states.” Previous studies have demonstrated an on-average female advantage on the Eyes Test. However, it is unknown whether this female advantage exists across the lifespan and across a large number of countries. Thus, we tested sex and age differences using the English version of the Eyes Test in adolescents and adults across 57 countries. We also tested for associations with sociodemographic and cognitive/personality factors. We leveraged one discovery dataset (N = 305,726) and three validation datasets (Ns = 642; 5,284; and 1,087). The results show that: i) there is a replicable on-average female advantage in performance on the Eyes Test; ii) performance increases through adolescence and shallowly declines across adulthood; iii) the on-average female advantage is evident across the lifespan; iv) there is a significant on-average female advantage in 36 out of 57 countries; v) there is a significant on-average female advantage on translated (non-English) versions of the Eyes Test in 12 out of 16 countries, as confirmed by a systematic review; vi) D-scores, or empathizing-systemizing, predict Eyes Test performance above and beyond sex differences; and vii) the female advantage is negatively linked to “prosperity” and “autonomy,” and positively linked to “collectivism,” as confirmed by exploratory country-level analyses. We conclude that the on-average female advantage on the Eyes Test is observed across ages and most countries.
2022
Lin, Jody L; Huber, Bernd; Amir, Ofra; Gehrmann, Sebastian; Ramirez, Kimberly S; Ochoa, Kimberly M; Asch, Steven M; Gajos, Krzysztof Z; Grosz, Barbara J; Sanders, Lee M
Barriers and Facilitators to the Implementation of Family-Centered Technology in Complex Care: Feasibility Study Journal Article
In: J Med Internet Res, vol. 24, no. 8, pp. e30902, 2022, ISSN: 1438-8871.
@article{lin2022barriers,
title = {Barriers and Facilitators to the Implementation of Family-Centered Technology in Complex Care: Feasibility Study},
author = {Jody L Lin and Bernd Huber and Ofra Amir and Sebastian Gehrmann and Kimberly S Ramirez and Kimberly M Ochoa and Steven M Asch and Krzysztof Z Gajos and Barbara J Grosz and Lee M Sanders},
url = {https://doi.org/10.2196/30902},
doi = {10.2196/30902},
issn = {1438-8871},
year = {2022},
date = {2022-08-23},
journal = {J Med Internet Res},
volume = {24},
number = {8},
pages = {e30902},
abstract = {Background: Care coordination is challenging but crucial for children with medical complexity (CMC). Technology-based solutions are increasingly prevalent but little is known about how to successfully deploy them in the care of CMC. Objective: The aim of this study was to assess the feasibility and acceptability of GoalKeeper (GK), an internet-based system for eliciting and monitoring family-centered goals for CMC, and to identify barriers and facilitators to implementation. Methods: We used the Consolidated Framework for Implementation Research (CFIR) to explore the barriers and facilitators to the implementation of GK as part of a clinical trial of GK in ambulatory clinics at a children's hospital (NCT03620071). The study was conducted in 3 phases: preimplementation, implementation (trial), and postimplementation. For the trial, we recruited providers at participating clinics and English-speaking parents of CMC<12 years of age with home internet access. All participants used GK during an initial clinic visit and for 3 months after. We conducted preimplementation focus groups and postimplementation semistructured exit interviews using the CFIR interview guide. Participant exit surveys assessed GK feasibility and acceptability on a 5-point Likert scale. For each interview, 3 independent coders used content analysis and serial coding reviews based on the CFIR qualitative analytic plan and assigned quantitative ratings to each CFIR construct (–2 strong barrier to +2 strong facilitator). Results: Preimplementation focus groups included 2 parents (1 male participant and 1 female participant) and 3 providers (1 in complex care, 1 in clinical informatics, and 1 in neurology). From focus groups, we developed 3 implementation strategies: education (parents: 5-minute demo; providers: 30-minute tutorial and 5-minute video on use in a clinic visit; both: instructional manual), tech support (in-person, virtual), and automated email reminders for parents. For implementation (April 1, 2019, to December 21, 2020), we enrolled 11 providers (7 female participants, 5 in complex care) and 35 parents (mean age 38.3, SD 7.8 years; n=28, 80% female; n=17, 49% Caucasian; n=16, 46% Hispanic; and n=30, 86% at least some college). One parent-provider pair did not use GK in the clinic visit, and few used GK after the visit. In 18 parent and 9 provider exit interviews, the key facilitators were shared goal setting, GK's internet accessibility and email reminders (parents), and GK's ability to set long-term goals and use at the end of visits (providers). A key barrier was GK's lack of integration into the electronic health record or patient portal. Most parents (13/19) and providers (6/9) would recommend GK to their peers. Conclusions: Family-centered technologies like GK are feasible and acceptable for the care of CMC, but sustained use depends on integration into electronic health records. Trial Registration: ClinicalTrials.gov NCT03620071; https://clinicaltrials.gov/ct2/show/NCT03620071},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Background: Care coordination is challenging but crucial for children with medical complexity (CMC). Technology-based solutions are increasingly prevalent but little is known about how to successfully deploy them in the care of CMC. Objective: The aim of this study was to assess the feasibility and acceptability of GoalKeeper (GK), an internet-based system for eliciting and monitoring family-centered goals for CMC, and to identify barriers and facilitators to implementation. Methods: We used the Consolidated Framework for Implementation Research (CFIR) to explore the barriers and facilitators to the implementation of GK as part of a clinical trial of GK in ambulatory clinics at a children’s hospital (NCT03620071). The study was conducted in 3 phases: preimplementation, implementation (trial), and postimplementation. For the trial, we recruited providers at participating clinics and English-speaking parents of CMC<12 years of age with home internet access. All participants used GK during an initial clinic visit and for 3 months after. We conducted preimplementation focus groups and postimplementation semistructured exit interviews using the CFIR interview guide. Participant exit surveys assessed GK feasibility and acceptability on a 5-point Likert scale. For each interview, 3 independent coders used content analysis and serial coding reviews based on the CFIR qualitative analytic plan and assigned quantitative ratings to each CFIR construct (–2 strong barrier to +2 strong facilitator). Results: Preimplementation focus groups included 2 parents (1 male participant and 1 female participant) and 3 providers (1 in complex care, 1 in clinical informatics, and 1 in neurology). From focus groups, we developed 3 implementation strategies: education (parents: 5-minute demo; providers: 30-minute tutorial and 5-minute video on use in a clinic visit; both: instructional manual), tech support (in-person, virtual), and automated email reminders for parents. For implementation (April 1, 2019, to December 21, 2020), we enrolled 11 providers (7 female participants, 5 in complex care) and 35 parents (mean age 38.3, SD 7.8 years; n=28, 80% female; n=17, 49% Caucasian; n=16, 46% Hispanic; and n=30, 86% at least some college). One parent-provider pair did not use GK in the clinic visit, and few used GK after the visit. In 18 parent and 9 provider exit interviews, the key facilitators were shared goal setting, GK’s internet accessibility and email reminders (parents), and GK’s ability to set long-term goals and use at the end of visits (providers). A key barrier was GK’s lack of integration into the electronic health record or patient portal. Most parents (13/19) and providers (6/9) would recommend GK to their peers. Conclusions: Family-centered technologies like GK are feasible and acceptable for the care of CMC, but sustained use depends on integration into electronic health records. Trial Registration: ClinicalTrials.gov NCT03620071; https://clinicaltrials.gov/ct2/show/NCT03620071
Ma, Zilin; Gajos, Krzysztof Z.
Not Just a Preference: Reducing Biased Decision-Making on Dating Websites Proceedings Article
In: CHI Conference on Human Factors in Computing Systems, Association for Computing Machinery, New Orleans, LA, USA, 2022, ISBN: 9781450391573.
@inproceedings{ma2022preference,
title = {Not Just a Preference: Reducing Biased Decision-Making on Dating Websites},
author = {Zilin Ma and Krzysztof Z. Gajos},
url = {/papers/ma2022preference.pdf},
doi = {10.1145/3491102.3517587},
isbn = {9781450391573},
year = {2022},
date = {2022-01-01},
booktitle = {CHI Conference on Human Factors in Computing Systems},
publisher = {Association for Computing Machinery},
address = {New Orleans, LA, USA},
series = {CHI '22},
abstract = {As dating websites are becoming an essential part of how people meet intimate and romantic partners, it is vital to design these systems to be resistant to, or at least do not amplify, bias and discrimination. Instead, the results of our online experiment with a simulated dating website, demonstrate that popular dating website design choices, such as the user of the swipe interface (swiping in one direction to indicate a like and in the other direction to express a dislike) and match scores, resulted in people racially biases choices even when they explicitly claimed not to have considered race in their decision-making. This bias was significantly reduced when the order of information presentation was reversed such that people first saw substantive profile information related to their explicitly-stated preferences before seeing the profile name and photo. These results indicate that currently-popular design choices amplify people’s implicit biases in their choices of potential romantic partners, but the effects of the implicit biases can be reduced by carefully redesigning the dating website interfaces.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
As dating websites are becoming an essential part of how people meet intimate and romantic partners, it is vital to design these systems to be resistant to, or at least do not amplify, bias and discrimination. Instead, the results of our online experiment with a simulated dating website, demonstrate that popular dating website design choices, such as the user of the swipe interface (swiping in one direction to indicate a like and in the other direction to express a dislike) and match scores, resulted in people racially biases choices even when they explicitly claimed not to have considered race in their decision-making. This bias was significantly reduced when the order of information presentation was reversed such that people first saw substantive profile information related to their explicitly-stated preferences before seeing the profile name and photo. These results indicate that currently-popular design choices amplify people’s implicit biases in their choices of potential romantic partners, but the effects of the implicit biases can be reduced by carefully redesigning the dating website interfaces.
Gajos, Krzysztof Z.; Mamykina, Lena
Do People Engage Cognitively with AI? Impact of AI Assistance on Incidental Learning Proceedings Article
In: pp. 794–806, Association for Computing Machinery, Helsinki, Finland, 2022, ISBN: 9781450391443.
@inproceedings{gajos2022people,
title = {Do People Engage Cognitively with AI? Impact of AI Assistance on Incidental Learning},
author = {Krzysztof Z. Gajos and Lena Mamykina},
url = {/papers/gajos2022people.pdf},
doi = {10.1145/3490099.3511138},
isbn = {9781450391443},
year = {2022},
date = {2022-01-01},
pages = {794–806},
publisher = {Association for Computing Machinery},
address = {Helsinki, Finland},
series = {IUI '22},
abstract = {When people receive advice while making difficult decisions, they often make better decisions in the moment and also increase their knowledge in the process. However, such incidental learning can only occur when people cognitively engage with the information they receive and process this information thoughtfully. How do people process the information and advice they receive from AI, and do they engage with it deeply enough to enable learning? To answer these questions, we conducted three experiments in which individuals were asked to make nutritional decisions and received simulated AI recommendations and explanations. In the first experiment, we found that when people were presented with both a recommendation and an explanation before making their choice, they made better decisions than they did when they received no such help, but they did not learn. In the second experiment, participants first made their own choice, and only then saw a recommendation and an explanation from AI; this condition also resulted in improved decisions, but no learning. However, in our third experiment, participants were presented with just an AI explanation but no recommendation and had to arrive at their own decision. This condition led to both more accurate decisions and learning gains. We hypothesize that learning gains in this condition were due to deeper engagement with explanations needed to arrive at the decisions. This work provides some of the most direct evidence to date that it may not be sufficient to provide people with AI-generated recommendations and explanations to ensure that people engage carefully with the AI-provided information. This work also presents one technique that enables incidental learning and, by implication, can help people process AI recommendations and explanations more carefully.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
When people receive advice while making difficult decisions, they often make better decisions in the moment and also increase their knowledge in the process. However, such incidental learning can only occur when people cognitively engage with the information they receive and process this information thoughtfully. How do people process the information and advice they receive from AI, and do they engage with it deeply enough to enable learning? To answer these questions, we conducted three experiments in which individuals were asked to make nutritional decisions and received simulated AI recommendations and explanations. In the first experiment, we found that when people were presented with both a recommendation and an explanation before making their choice, they made better decisions than they did when they received no such help, but they did not learn. In the second experiment, participants first made their own choice, and only then saw a recommendation and an explanation from AI; this condition also resulted in improved decisions, but no learning. However, in our third experiment, participants were presented with just an AI explanation but no recommendation and had to arrive at their own decision. This condition led to both more accurate decisions and learning gains. We hypothesize that learning gains in this condition were due to deeper engagement with explanations needed to arrive at the decisions. This work provides some of the most direct evidence to date that it may not be sufficient to provide people with AI-generated recommendations and explanations to ensure that people engage carefully with the AI-provided information. This work also presents one technique that enables incidental learning and, by implication, can help people process AI recommendations and explanations more carefully.
Ruotsalo, Tuukka; Weber, Sean; Gajos, Krzysztof Z.
Active tag recommendation for interactive entity search: Interaction effectiveness and retrieval performance Journal Article
In: Information Processing & Management, vol. 59, no. 2, pp. 102856, 2022, ISSN: 0306-4573.
@article{ruotsalo2022active,
title = {Active tag recommendation for interactive entity search: Interaction effectiveness and retrieval performance},
author = {Tuukka Ruotsalo and Sean Weber and Krzysztof Z. Gajos},
url = {https://www.sciencedirect.com/science/article/pii/S0306457321003277},
doi = {10.1016/j.ipm.2021.102856},
issn = {0306-4573},
year = {2022},
date = {2022-01-01},
journal = {Information Processing & Management},
volume = {59},
number = {2},
pages = {102856},
abstract = {We introduce active tag recommendation for interactive entity search, an approach that actively learns to suggest tags from preceding user interactions with the recommended tags. The approach utilizes an online reinforcement learning model and observes user interactions on the recommended tags to reward or penalize the model. Active tag recommendation is implemented as part of a realistic search engine indexing a large collection of movie data. The approach is evaluated in task-based user experiments comparing a complete search system enhanced with active tag recommendation to a control system in which active tag recommendation is not available. In the experiment, participants (N = 45) performed search tasks on the movie domain and the corresponding search interactions, information selections, and entity rankings were logged and analyzed. The results show that active tag recommendation (1) improves the ranking of entities compared to written-query interaction, (2) increases the amount of interaction and effectiveness of interactions to rank entities that end up being selected in a task, and (3) reduces, but does not substitute, the need for written-query interaction (4) without compromising task execution time. The results imply that active learning for search support can help users to interact with entity search systems by reducing the need for writing queries and improve search outcomes without compromising the time used for searching.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
We introduce active tag recommendation for interactive entity search, an approach that actively learns to suggest tags from preceding user interactions with the recommended tags. The approach utilizes an online reinforcement learning model and observes user interactions on the recommended tags to reward or penalize the model. Active tag recommendation is implemented as part of a realistic search engine indexing a large collection of movie data. The approach is evaluated in task-based user experiments comparing a complete search system enhanced with active tag recommendation to a control system in which active tag recommendation is not available. In the experiment, participants (N = 45) performed search tasks on the movie domain and the corresponding search interactions, information selections, and entity rankings were logged and analyzed. The results show that active tag recommendation (1) improves the ranking of entities compared to written-query interaction, (2) increases the amount of interaction and effectiveness of interactions to rank entities that end up being selected in a task, and (3) reduces, but does not substitute, the need for written-query interaction (4) without compromising task execution time. The results imply that active learning for search support can help users to interact with entity search systems by reducing the need for writing queries and improve search outcomes without compromising the time used for searching.
Khan, Nergis C; Pandey, Vineet; Gajos, Krzysztof Z; Gupta, Anoopum S
Free-Living Motor Activity Monitoring in Ataxia-Telangiectasia Journal Article
In: The Cerebellum, vol. 21, iss. 3, pp. 368–379, 2022.
@article{khan2022free,
title = {Free-Living Motor Activity Monitoring in Ataxia-Telangiectasia},
author = {Nergis C Khan and Vineet Pandey and Krzysztof Z Gajos and Anoopum S Gupta},
url = {https://doi.org/10.1007/s12311-021-01306-y},
doi = {10.1007/s12311-021-01306-y},
year = {2022},
date = {2022-01-01},
journal = {The Cerebellum},
volume = {21},
issue = {3},
pages = {368–379},
publisher = {Springer},
abstract = {With disease-modifying approaches under evaluation in ataxia-telangiectasia and other ataxias, there is a need for objective and reliable biomarkers of free-living motor function. In this study, we test the hypothesis that metrics derived from a single wrist sensor worn at home provide accurate, reliable, and interpretable information about neurological disease severity in children with A-T.<br/>A total of 15 children with A-T and 15 age- and sex-matched controls wore a sensor with a triaxial accelerometer on their dominant wrist for 1 week at home. Activity intensity measures, derived from the sensor data, were compared with in-person neurological evaluation on the Brief Ataxia Rating Scale (BARS) and performance on a validated computer mouse task.<br/>Children with A-T were inactive the same proportion of each day as controls but produced more low intensity movements (p < 0.01; Cohen's d = 1.48) and fewer high intensity movements (p < 0.001; Cohen's d = 1.71). The range of activity intensities was markedly reduced in A-T compared to controls (p < 0.0001; Cohen's d = 2.72). The activity metrics correlated strongly with arm, gait, and total clinical severity (r: 0.71–0.87; p < 0.0001), correlated with specific computer task motor features (r: 0.67–0.92; p < 0.01), demonstrated high reliability (r: 0.86–0.93; p < 0.00001), and were not significantly influenced by age in the healthy control group.<br/>Motor activity metrics from a single, inexpensive wrist sensor during free-living behavior provide accurate and reliable information about diagnosis, neurological disease severity, and motor performance. These low-burden measurements are applicable independent of ambulatory status and are potential digital behavioral biomarkers in A-T.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
With disease-modifying approaches under evaluation in ataxia-telangiectasia and other ataxias, there is a need for objective and reliable biomarkers of free-living motor function. In this study, we test the hypothesis that metrics derived from a single wrist sensor worn at home provide accurate, reliable, and interpretable information about neurological disease severity in children with A-T.<br/>A total of 15 children with A-T and 15 age- and sex-matched controls wore a sensor with a triaxial accelerometer on their dominant wrist for 1 week at home. Activity intensity measures, derived from the sensor data, were compared with in-person neurological evaluation on the Brief Ataxia Rating Scale (BARS) and performance on a validated computer mouse task.<br/>Children with A-T were inactive the same proportion of each day as controls but produced more low intensity movements (p < 0.01; Cohen’s d = 1.48) and fewer high intensity movements (p < 0.001; Cohen’s d = 1.71). The range of activity intensities was markedly reduced in A-T compared to controls (p < 0.0001; Cohen’s d = 2.72). The activity metrics correlated strongly with arm, gait, and total clinical severity (r: 0.71–0.87; p < 0.0001), correlated with specific computer task motor features (r: 0.67–0.92; p < 0.01), demonstrated high reliability (r: 0.86–0.93; p < 0.00001), and were not significantly influenced by age in the healthy control group.<br/>Motor activity metrics from a single, inexpensive wrist sensor during free-living behavior provide accurate and reliable information about diagnosis, neurological disease severity, and motor performance. These low-burden measurements are applicable independent of ambulatory status and are potential digital behavioral biomarkers in A-T.
2021
Brody, Julia Green; Cirillo, Piera M; Boronow, Katherine E; Havas, Laurie; Plumb, Marj; Susmann, Herbert P; Gajos, Krzysztof Z; Cohn, Barbara A
In: Environmental health perspectives, vol. 129, no. 11, pp. 117005, 2021.
@article{brody2021outcomes,
title = {Outcomes from Returning Individual versus Only Study-Wide Biomonitoring Results in an Environmental Exposure Study Using the Digital Exposure Report-Back Interface (DERBI)},
author = {Julia Green Brody and Piera M Cirillo and Katherine E Boronow and Laurie Havas and Marj Plumb and Herbert P Susmann and Krzysztof Z Gajos and Barbara A Cohn},
url = {https://doi.org/10.1289/EHP9072},
year = {2021},
date = {2021-01-01},
journal = {Environmental health perspectives},
volume = {129},
number = {11},
pages = {117005},
abstract = {\textbf{Backtround:} Study participants want to receive their biomonitoring results for environmental chemicals, and ethics guidelines encourage reporting back. However, few studies have quantitively assessed participants’ responses to individual exposure reports, and digital methods have not been evaluated.<br/>
\textbf{Objectives:}
We isolated effects of receiving personal results vs. only study-wide findings and investigated whether effects differed for Black participants.<br/>
\textbf{Methods:}
We randomly assigned a subset of 295 women from the Child Health and Development Studies, half of whom were Black, to receive a report with personal environmental chemical results or only study-wide (aggregate) findings. Reports included results for 42 chemicals and lipids and were prepared using the Digital Exposure Report-Back Interface (DERBI). Women were interviewed before and after viewing their report. We analyzed differences in website activity, emotional responses, and intentions to participate in future research by report type and race using Wilcoxon rank sum tests, Wilcoxon-Pratt signed ranks tests, and multiple regression.<br/>
\textbf{Results:} The personal report group spent approximately twice as much time on their reports as the aggregate group before the post-report-back interview. Among personal-report participants (n=93), 84% (78) viewed chemical group information for at least one personal result highlighted on their home page; among aggregate-report participants (n=94), 66% (62) viewed any chemical group page. Both groups reported strong positive feelings (curious, informed, interested, respected) about receiving results before and after report-back and mild negative feelings (helpless, scared, worried). Although most participants remained unworried after report-back, worry increased by a small amount in both groups. Among Black participants, higher post report-back worry was associated with having high levels of chemicals.<br/>
\textbf{Conclusions:}
Participants were motivated by their personal results to access online information about chemical sources and potential health effects. Report-back was associated with a small increase in worry, which could motivate appropriate action. Personal report-back increased engagement with exposure reports among Black participants.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Backtround: Study participants want to receive their biomonitoring results for environmental chemicals, and ethics guidelines encourage reporting back. However, few studies have quantitively assessed participants’ responses to individual exposure reports, and digital methods have not been evaluated.<br/>
Objectives:
We isolated effects of receiving personal results vs. only study-wide findings and investigated whether effects differed for Black participants.<br/>
Methods:
We randomly assigned a subset of 295 women from the Child Health and Development Studies, half of whom were Black, to receive a report with personal environmental chemical results or only study-wide (aggregate) findings. Reports included results for 42 chemicals and lipids and were prepared using the Digital Exposure Report-Back Interface (DERBI). Women were interviewed before and after viewing their report. We analyzed differences in website activity, emotional responses, and intentions to participate in future research by report type and race using Wilcoxon rank sum tests, Wilcoxon-Pratt signed ranks tests, and multiple regression.<br/>
Results: The personal report group spent approximately twice as much time on their reports as the aggregate group before the post-report-back interview. Among personal-report participants (n=93), 84% (78) viewed chemical group information for at least one personal result highlighted on their home page; among aggregate-report participants (n=94), 66% (62) viewed any chemical group page. Both groups reported strong positive feelings (curious, informed, interested, respected) about receiving results before and after report-back and mild negative feelings (helpless, scared, worried). Although most participants remained unworried after report-back, worry increased by a small amount in both groups. Among Black participants, higher post report-back worry was associated with having high levels of chemicals.<br/>
Conclusions:
Participants were motivated by their personal results to access online information about chemical sources and potential health effects. Report-back was associated with a small increase in worry, which could motivate appropriate action. Personal report-back increased engagement with exposure reports among Black participants.
Objectives:
We isolated effects of receiving personal results vs. only study-wide findings and investigated whether effects differed for Black participants.<br/>
Methods:
We randomly assigned a subset of 295 women from the Child Health and Development Studies, half of whom were Black, to receive a report with personal environmental chemical results or only study-wide (aggregate) findings. Reports included results for 42 chemicals and lipids and were prepared using the Digital Exposure Report-Back Interface (DERBI). Women were interviewed before and after viewing their report. We analyzed differences in website activity, emotional responses, and intentions to participate in future research by report type and race using Wilcoxon rank sum tests, Wilcoxon-Pratt signed ranks tests, and multiple regression.<br/>
Results: The personal report group spent approximately twice as much time on their reports as the aggregate group before the post-report-back interview. Among personal-report participants (n=93), 84% (78) viewed chemical group information for at least one personal result highlighted on their home page; among aggregate-report participants (n=94), 66% (62) viewed any chemical group page. Both groups reported strong positive feelings (curious, informed, interested, respected) about receiving results before and after report-back and mild negative feelings (helpless, scared, worried). Although most participants remained unworried after report-back, worry increased by a small amount in both groups. Among Black participants, higher post report-back worry was associated with having high levels of chemicals.<br/>
Conclusions:
Participants were motivated by their personal results to access online information about chemical sources and potential health effects. Report-back was associated with a small increase in worry, which could motivate appropriate action. Personal report-back increased engagement with exposure reports among Black participants.
Buçinca, Zana; Malaya, Maja Barbara; Gajos, Krzysztof Z.
To Trust or to Think: Cognitive Forcing Functions Can Reduce Overreliance on AI in AI-Assisted Decision-Making Journal Article
In: Proc. ACM Hum.-Comput. Interact., vol. 5, no. CSCW1, 2021.
@article{bucinca2021trust,
title = {To Trust or to Think: Cognitive Forcing Functions Can Reduce Overreliance on AI in AI-Assisted Decision-Making},
author = {Zana Buçinca and Maja Barbara Malaya and Krzysztof Z. Gajos},
url = {/papers/bucinca21trust.pdf},
doi = {10.1145/3449287},
year = {2021},
date = {2021-04-01},
journal = {Proc. ACM Hum.-Comput. Interact.},
volume = {5},
number = {CSCW1},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
abstract = {People supported by AI-powered decision support tools frequently overrely on the AI:
they accept an AI's suggestion even when that suggestion is wrong. Adding explanations
to the AI decisions does not appear to reduce the overreliance and some studies suggest
that it might even increase it. Informed by the dual-process theory of cognition,
we posit that people rarely engage analytically with each individual AI recommendation
and explanation, and instead develop general heuristics about whether and when to
follow the AI suggestions. Building on prior research on medical decision-making,
we designed three cognitive forcing interventions to compel people to engage more thoughtfully with the AI-generated explanations. We conducted an experiment (N=199),
in which we compared our three cognitive forcing designs to two simple explainable
AI approaches and to a no-AI baseline. The results demonstrate that cognitive forcing
significantly reduced overreliance compared to the simple explainable AI approaches.
However, there was a trade-off: people assigned the least favorable subjective ratings
to the designs that reduced the overreliance the most. To audit our work for intervention-generated
inequalities, we investigated whether our interventions benefited equally people with
different levels of Need for Cognition (i.e., motivation to engage in effortful mental
activities). Our results show that, on average, cognitive forcing interventions benefited
participants higher in Need for Cognition more. Our research suggests that human cognitive
motivation moderates the effectiveness of explainable AI solutions.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
People supported by AI-powered decision support tools frequently overrely on the AI:
they accept an AI’s suggestion even when that suggestion is wrong. Adding explanations
to the AI decisions does not appear to reduce the overreliance and some studies suggest
that it might even increase it. Informed by the dual-process theory of cognition,
we posit that people rarely engage analytically with each individual AI recommendation
and explanation, and instead develop general heuristics about whether and when to
follow the AI suggestions. Building on prior research on medical decision-making,
we designed three cognitive forcing interventions to compel people to engage more thoughtfully with the AI-generated explanations. We conducted an experiment (N=199),
in which we compared our three cognitive forcing designs to two simple explainable
AI approaches and to a no-AI baseline. The results demonstrate that cognitive forcing
significantly reduced overreliance compared to the simple explainable AI approaches.
However, there was a trade-off: people assigned the least favorable subjective ratings
to the designs that reduced the overreliance the most. To audit our work for intervention-generated
inequalities, we investigated whether our interventions benefited equally people with
different levels of Need for Cognition (i.e., motivation to engage in effortful mental
activities). Our results show that, on average, cognitive forcing interventions benefited
participants higher in Need for Cognition more. Our research suggests that human cognitive
motivation moderates the effectiveness of explainable AI solutions.
they accept an AI’s suggestion even when that suggestion is wrong. Adding explanations
to the AI decisions does not appear to reduce the overreliance and some studies suggest
that it might even increase it. Informed by the dual-process theory of cognition,
we posit that people rarely engage analytically with each individual AI recommendation
and explanation, and instead develop general heuristics about whether and when to
follow the AI suggestions. Building on prior research on medical decision-making,
we designed three cognitive forcing interventions to compel people to engage more thoughtfully with the AI-generated explanations. We conducted an experiment (N=199),
in which we compared our three cognitive forcing designs to two simple explainable
AI approaches and to a no-AI baseline. The results demonstrate that cognitive forcing
significantly reduced overreliance compared to the simple explainable AI approaches.
However, there was a trade-off: people assigned the least favorable subjective ratings
to the designs that reduced the overreliance the most. To audit our work for intervention-generated
inequalities, we investigated whether our interventions benefited equally people with
different levels of Need for Cognition (i.e., motivation to engage in effortful mental
activities). Our results show that, on average, cognitive forcing interventions benefited
participants higher in Need for Cognition more. Our research suggests that human cognitive
motivation moderates the effectiveness of explainable AI solutions.
Jacobs, Maia; He, Jeffrey; Pradier, Melanie F.; Lam, Barbara; Ahn, Andrew C.; McCoy, Thomas H.; Perlis, Roy H.; Doshi-Velez, Finale; Gajos, Krzysztof Z.
Designing AI for Trust and Collaboration in Time-Constrained Medical Decisions: A Sociotechnical Lens Proceedings Article
In: Proceedings of the 2021 CHI Conference on Human Factors in Computing Systems, Association for Computing Machinery, New York, NY, USA, 2021, ISBN: 9781450380966.
@inproceedings{jacobs2021designing,
title = {Designing AI for Trust and Collaboration in Time-Constrained Medical Decisions: A Sociotechnical Lens},
author = {Maia Jacobs and Jeffrey He and Melanie F. Pradier and Barbara Lam and Andrew C. Ahn and Thomas H. McCoy and Roy H. Perlis and Finale Doshi-Velez and Krzysztof Z. Gajos},
url = {/papers/jacobs21designing.pdf},
isbn = {9781450380966},
year = {2021},
date = {2021-01-01},
booktitle = {Proceedings of the 2021 CHI Conference on Human Factors in Computing Systems},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
abstract = {Major depressive disorder is a debilitating disease affecting 264 million people
worldwide. While many antidepressant medications are available, few clinical guidelines
support choosing among them. Decision support tools (DSTs) embodying machine learning
models may help improve the treatment selection process, but often fail in clinical
practice due to poor system integration. We use an iterative, co-design process to
investigate clinicians’ perceptions of using DSTs in antidepressant treatment decisions.
We identify ways in which DSTs need to engage with the healthcare sociotechnical system,
including clinical processes, patient preferences, resource constraints, and domain
knowledge. Our results suggest that clinical DSTs should be designed as multi-user
systems that support patient-provider collaboration and offer on-demand explanations
that address discrepancies between predictions and current standards of care. Through
this work, we demonstrate how current trends in explainable AI may be inappropriate
for clinical environments and consider paths towards designing these tools for real-world
medical systems.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Major depressive disorder is a debilitating disease affecting 264 million people
worldwide. While many antidepressant medications are available, few clinical guidelines
support choosing among them. Decision support tools (DSTs) embodying machine learning
models may help improve the treatment selection process, but often fail in clinical
practice due to poor system integration. We use an iterative, co-design process to
investigate clinicians’ perceptions of using DSTs in antidepressant treatment decisions.
We identify ways in which DSTs need to engage with the healthcare sociotechnical system,
including clinical processes, patient preferences, resource constraints, and domain
knowledge. Our results suggest that clinical DSTs should be designed as multi-user
systems that support patient-provider collaboration and offer on-demand explanations
that address discrepancies between predictions and current standards of care. Through
this work, we demonstrate how current trends in explainable AI may be inappropriate
for clinical environments and consider paths towards designing these tools for real-world
medical systems.
worldwide. While many antidepressant medications are available, few clinical guidelines
support choosing among them. Decision support tools (DSTs) embodying machine learning
models may help improve the treatment selection process, but often fail in clinical
practice due to poor system integration. We use an iterative, co-design process to
investigate clinicians’ perceptions of using DSTs in antidepressant treatment decisions.
We identify ways in which DSTs need to engage with the healthcare sociotechnical system,
including clinical processes, patient preferences, resource constraints, and domain
knowledge. Our results suggest that clinical DSTs should be designed as multi-user
systems that support patient-provider collaboration and offer on-demand explanations
that address discrepancies between predictions and current standards of care. Through
this work, we demonstrate how current trends in explainable AI may be inappropriate
for clinical environments and consider paths towards designing these tools for real-world
medical systems.
Lekschas, Fritz; Ampanavos, Spyridon; Siangliulue, Pao; Pfister, Hanspeter; Gajos, Krzysztof Z.
Ask Me or Tell Me? Enhancing the Effectiveness of Crowdsourced Design Feedback Proceedings Article
In: Proceedings of the 2021 CHI Conference on Human Factors in Computing Systems, Association for Computing Machinery, New York, NY, USA, 2021, ISBN: 9781450380966.
@inproceedings{lekschas2021ask,
title = {Ask Me or Tell Me? Enhancing the Effectiveness of Crowdsourced Design Feedback},
author = {Fritz Lekschas and Spyridon Ampanavos and Pao Siangliulue and Hanspeter Pfister and Krzysztof Z. Gajos},
url = {/papers/lekschas21ask.pdf},
isbn = {9781450380966},
year = {2021},
date = {2021-01-01},
booktitle = {Proceedings of the 2021 CHI Conference on Human Factors in Computing Systems},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
abstract = {Crowdsourced design feedback systems are emerging resources for getting large amounts of feedback in a short period of time. Traditionally, the feedback comes in the form of a declarative statement, which often contains positive or negative sentiment. Prior research has shown that overly negative or positive sentiment can strongly influence the perceived usefulness and acceptance of feedback and, subsequently, lead to ineffective design revisions. To enhance the effectiveness of crowdsourced design feedback, we investigate a new approach for mitigating the effects of negative or positive feedback by combining open-ended and thought-provoking questions with declarative feedback statements. We conducted two user studies to assess the effects of question-based feedback on the sentiment and quality of design revisions in the context of graphic design. We found that crowdsourced question-based feedback contains more neutral sentiment than statement-based feedback. Moreover, we provide evidence that presenting feedback as questions followed by statements leads to better design revisions than question- or statement-based feedback alone.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Crowdsourced design feedback systems are emerging resources for getting large amounts of feedback in a short period of time. Traditionally, the feedback comes in the form of a declarative statement, which often contains positive or negative sentiment. Prior research has shown that overly negative or positive sentiment can strongly influence the perceived usefulness and acceptance of feedback and, subsequently, lead to ineffective design revisions. To enhance the effectiveness of crowdsourced design feedback, we investigate a new approach for mitigating the effects of negative or positive feedback by combining open-ended and thought-provoking questions with declarative feedback statements. We conducted two user studies to assess the effects of question-based feedback on the sentiment and quality of design revisions in the context of graphic design. We found that crowdsourced question-based feedback contains more neutral sentiment than statement-based feedback. Moreover, we provide evidence that presenting feedback as questions followed by statements leads to better design revisions than question- or statement-based feedback alone.
Jacobs, Maia; Pradier, Melanie F.; Jr, Thomas H. McCoy; Perlis, Roy H.; Doshi-Velez, Finale; Gajos, Krzysztof Z.
How machine-learning recommendations influence clinician treatment selections: the example of antidepressant selection Journal Article
In: Translational Psychiatry, vol. 11, 2021.
@article{jacobs2021how,
title = {How machine-learning recommendations influence clinician treatment selections: the example of antidepressant selection},
author = {Maia Jacobs and Melanie F. Pradier and Thomas H. McCoy Jr and Roy H. Perlis and Finale Doshi-Velez and Krzysztof Z. Gajos},
url = {https://doi.org/10.1038/s41398-021-01224-x},
doi = {10.1038/s41398-021-01224-x},
year = {2021},
date = {2021-01-01},
journal = {Translational Psychiatry},
volume = {11},
abstract = {Decision support systems embodying machine learning models offer the promise of improved standard of care for major depressive disorder, but little is known about how clinicians' treatment decisions will be influenced by machine learning recommendations and explanations. We used a within-subject factorial experiment to present 220 clinicians with patient vignettes, each with or without a machine learning (ML) recommendation and one of multiple forms of explanation. We found that interacting with machine learning recommendations did not significantly improve clinicians' treatment selection accuracy, assessed as concordance with expert psychopharmacologist consensus, compared to baseline scenarios in which clinicians made treatment decisions independently. Interacting with incorrect recommendations paired with explanations that included limited but easily interpretable information did lead to a significant reduction in treatment selection accuracy compared to baseline questions. These results suggest that incorrect ML recommendations may adversely impact clinician treatment selections and that explanations are insufficient for addressing overreliance on imperfect ML algorithms. More generally, our findings challenge the common assumption that clinicians interacting with ML tools will perform better than either clinicians or ML algorithms individually.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Decision support systems embodying machine learning models offer the promise of improved standard of care for major depressive disorder, but little is known about how clinicians’ treatment decisions will be influenced by machine learning recommendations and explanations. We used a within-subject factorial experiment to present 220 clinicians with patient vignettes, each with or without a machine learning (ML) recommendation and one of multiple forms of explanation. We found that interacting with machine learning recommendations did not significantly improve clinicians’ treatment selection accuracy, assessed as concordance with expert psychopharmacologist consensus, compared to baseline scenarios in which clinicians made treatment decisions independently. Interacting with incorrect recommendations paired with explanations that included limited but easily interpretable information did lead to a significant reduction in treatment selection accuracy compared to baseline questions. These results suggest that incorrect ML recommendations may adversely impact clinician treatment selections and that explanations are insufficient for addressing overreliance on imperfect ML algorithms. More generally, our findings challenge the common assumption that clinicians interacting with ML tools will perform better than either clinicians or ML algorithms individually.
2020
Hu, Jingmei; Joung, Jiwon; Jacobs, Maia; Gajos, Krzysztof Z.; Seltzer, Margo I.
Improving Data Scientist Efficiency with Provenance Proceedings Article
In: Proceedings of the ACM/IEEE 42nd International Conference on Software Engineering, pp. 1086–1097, Association for Computing Machinery, Seoul, South Korea, 2020, ISBN: 9781450371216.
@inproceedings{hu20:improving,
title = {Improving Data Scientist Efficiency with Provenance},
author = {Jingmei Hu and Jiwon Joung and Maia Jacobs and Krzysztof Z. Gajos and Margo I. Seltzer},
url = {/papers/hu20improving.pdf},
doi = {10.1145/3377811.3380366},
isbn = {9781450371216},
year = {2020},
date = {2020-01-01},
booktitle = {Proceedings of the ACM/IEEE 42nd International Conference on Software Engineering},
pages = {1086–1097},
publisher = {Association for Computing Machinery},
address = {Seoul, South Korea},
series = {ICSE '20},
abstract = {Data scientists frequently analyze data by writing scripts. We conducted a contextual inquiry with interdisciplinary researchers, which revealed that parameter tuning is a highly iterative process and that debugging is time-consuming. As analysis scripts evolve and become more complex, analysts have difficulty conceptualizing their workflow. In particular, after editing a script, it becomes difficult to determine precisely which code blocks depend on the edit. Consequently, scientists frequently re-run entire scripts instead of re-running only the necessary parts. We present ProvBuild, a tool that leverages language-level provenance to streamline the debugging process by reducing programmer cognitive load and decreasing subsequent runtimes, leading to an overall reduction in elapsed debugging time. ProvBuild uses provenance to track dependencies in a script. When an analyst debugs a script, ProvBuild generates a simplifed script that contains only the information necessary to debug a particular problem. We demonstrate that debugging the simplified script lowers a programmer's cognitive load and permits faster re-execution when testing changes. The combination of reduced cognitive load and shorter runtime reduces the time necessary to debug a script. We quantitatively and qualitatively show that even though ProvBuild introduces overhead during a script's first execution, it is a more efficient way for users to debug and tune complex workflows. ProvBuild demonstrates a novel use of language-level provenance, in which it is used to proactively improve programmer productively rather than merely providing a way to retroactively gain insight into a body of code.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Data scientists frequently analyze data by writing scripts. We conducted a contextual inquiry with interdisciplinary researchers, which revealed that parameter tuning is a highly iterative process and that debugging is time-consuming. As analysis scripts evolve and become more complex, analysts have difficulty conceptualizing their workflow. In particular, after editing a script, it becomes difficult to determine precisely which code blocks depend on the edit. Consequently, scientists frequently re-run entire scripts instead of re-running only the necessary parts. We present ProvBuild, a tool that leverages language-level provenance to streamline the debugging process by reducing programmer cognitive load and decreasing subsequent runtimes, leading to an overall reduction in elapsed debugging time. ProvBuild uses provenance to track dependencies in a script. When an analyst debugs a script, ProvBuild generates a simplifed script that contains only the information necessary to debug a particular problem. We demonstrate that debugging the simplified script lowers a programmer’s cognitive load and permits faster re-execution when testing changes. The combination of reduced cognitive load and shorter runtime reduces the time necessary to debug a script. We quantitatively and qualitatively show that even though ProvBuild introduces overhead during a script’s first execution, it is a more efficient way for users to debug and tune complex workflows. ProvBuild demonstrates a novel use of language-level provenance, in which it is used to proactively improve programmer productively rather than merely providing a way to retroactively gain insight into a body of code.
Arnold, Kenneth C.; Chauncey, Krysta; Gajos, Krzysztof Z.
Predictive Text Encourages Predictable Writing Proceedings Article
In: Proceedings of the 25th International Conference on Intelligent User Interfaces, pp. 128–138, Association for Computing Machinery, Cagliari, Italy, 2020, ISBN: 9781450371186.
@inproceedings{arnold20:predictcive,
title = {Predictive Text Encourages Predictable Writing},
author = {Kenneth C. Arnold and Krysta Chauncey and Krzysztof Z. Gajos},
url = {/papers/arnold20predictive.pdf},
doi = {10.1145/3377325.3377523},
isbn = {9781450371186},
year = {2020},
date = {2020-01-01},
booktitle = {Proceedings of the 25th International Conference on Intelligent User Interfaces},
pages = {128–138},
publisher = {Association for Computing Machinery},
address = {Cagliari, Italy},
series = {IUI '20},
abstract = {Intelligent text entry systems, including the now-ubiquitous predictive keyboard, can make text entry more efficient, but little is known about how these systems affect the content that people write. To study how predictive text systems affect content, we compared image captions written with different kinds of predictive text suggestions. Our key findings were that captions written with suggestions were shorter and that they included fewer words that that the system did not predict. Suggestions also boosted text entry speed, but with diminishing benefit for faster typists. Our findings imply that text entry systems should be evaluated not just by speed and accuracy but also by their effect on the content written.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Intelligent text entry systems, including the now-ubiquitous predictive keyboard, can make text entry more efficient, but little is known about how these systems affect the content that people write. To study how predictive text systems affect content, we compared image captions written with different kinds of predictive text suggestions. Our key findings were that captions written with suggestions were shorter and that they included fewer words that that the system did not predict. Suggestions also boosted text entry speed, but with diminishing benefit for faster typists. Our findings imply that text entry systems should be evaluated not just by speed and accuracy but also by their effect on the content written.
Buçinca, Zana; Lin, Phoebe; Gajos, Krzysztof Z.; Glassman, Elena L.
Proxy Tasks and Subjective Measures Can Be Misleading in Evaluating Explainable AI Systems Proceedings Article
In: Proceedings of the 25th International Conference on Intelligent User Interfaces, pp. 454–464, Association for Computing Machinery, Cagliari, Italy, 2020, ISBN: 9781450371186.
@inproceedings{bucinca20:proxy,
title = {Proxy Tasks and Subjective Measures Can Be Misleading in Evaluating Explainable AI Systems},
author = {Zana Buçinca and Phoebe Lin and Krzysztof Z. Gajos and Elena L. Glassman},
url = {/papers/bucinca20proxy.pdf},
doi = {10.1145/3377325.3377498},
isbn = {9781450371186},
year = {2020},
date = {2020-01-01},
booktitle = {Proceedings of the 25th International Conference on Intelligent User Interfaces},
pages = {454–464},
publisher = {Association for Computing Machinery},
address = {Cagliari, Italy},
series = {IUI '20},
abstract = {Explainable artificially intelligent (XAI) systems form part of \textit{ sociotechnical systems, } e.g., human+AI teams tasked with making decisions. Yet, current XAI systems are rarely evaluated by measuring the performance of human+AI teams on actual decision-making tasks. We conducted two online experiments and one in-person think-aloud study to evaluate two currently common techniques for evaluating XAI systems: (1) using proxy, artificial tasks such as how well humans predict the AI's decision from the given explanations, and (2) using subjective measures of trust and preference as predictors of actual performance. The results of our experiments demonstrate that evaluations with proxy tasks did not predict the results of the evaluations with the actual decision-making tasks. Further, the subjective measures on evaluations with actual decision-making tasks did not predict the objective performance on those same tasks. Our results suggest that by employing misleading evaluation methods, our field may be inadvertently slowing its progress toward developing human+AI teams that can reliably perform better than humans or AIs alone.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Explainable artificially intelligent (XAI) systems form part of sociotechnical systems, e.g., human+AI teams tasked with making decisions. Yet, current XAI systems are rarely evaluated by measuring the performance of human+AI teams on actual decision-making tasks. We conducted two online experiments and one in-person think-aloud study to evaluate two currently common techniques for evaluating XAI systems: (1) using proxy, artificial tasks such as how well humans predict the AI’s decision from the given explanations, and (2) using subjective measures of trust and preference as predictors of actual performance. The results of our experiments demonstrate that evaluations with proxy tasks did not predict the results of the evaluations with the actual decision-making tasks. Further, the subjective measures on evaluations with actual decision-making tasks did not predict the objective performance on those same tasks. Our results suggest that by employing misleading evaluation methods, our field may be inadvertently slowing its progress toward developing human+AI teams that can reliably perform better than humans or AIs alone.
Huber, Bernd; Gajos, Krzysztof Z.
Conducting online virtual environment experiments with uncompensated, unsupervised samples Journal Article
In: PLOS ONE, vol. 15, no. 1, pp. 1–17, 2020.
@article{huber20:conducting,
title = {Conducting online virtual environment experiments with uncompensated, unsupervised samples},
author = {Bernd Huber and Krzysztof Z. Gajos},
url = {https://doi.org/10.1371/journal.pone.0227629},
doi = {10.1371/journal.pone.0227629},
year = {2020},
date = {2020-01-01},
journal = {PLOS ONE},
volume = {15},
number = {1},
pages = {1–17},
publisher = {Public Library of Science},
abstract = {Web-based experimentation with uncompensated and unsupervised samples allows for a larger and more diverse sample population, more generalizable results, and faster theory to experiment cycle. Given that participants are unsupervised, it is still unknown whether the data collected in such settings would be of sufficiently high quality to support robust conclusions. Therefore, we investigated the feasibility of conducting such experiments online using virtual environment technologies. We conducted a conceptual replication of two prior experiments that have been conducted in virtual environments. Our results replicate findings previously obtained in conventional laboratory settings. These results hold across different device types of participants (ranging from desktop, through mobile devices to immersive virtual reality headsets), suggesting that experiments can be conducted online with uncompensated samples in virtual environments.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Web-based experimentation with uncompensated and unsupervised samples allows for a larger and more diverse sample population, more generalizable results, and faster theory to experiment cycle. Given that participants are unsupervised, it is still unknown whether the data collected in such settings would be of sufficiently high quality to support robust conclusions. Therefore, we investigated the feasibility of conducting such experiments online using virtual environment technologies. We conducted a conceptual replication of two prior experiments that have been conducted in virtual environments. Our results replicate findings previously obtained in conventional laboratory settings. These results hold across different device types of participants (ranging from desktop, through mobile devices to immersive virtual reality headsets), suggesting that experiments can be conducted online with uncompensated samples in virtual environments.
Fischer, Shira H; Safran, Charles; Gajos, Krzysztof Z; Wright, Adam
Visualization of Electronic Health Record Data for Decision-Making in Diabetes and Congestive Heart Failure Journal Article
In: ACI Open, vol. 4, no. 01, pp. e35–e43, 2020.
@article{fischer20:visualization,
title = {Visualization of Electronic Health Record Data for Decision-Making in Diabetes and Congestive Heart Failure},
author = {Shira H Fischer and Charles Safran and Krzysztof Z Gajos and Adam Wright},
url = {/papers/fischer20visualization.pdf},
doi = {10.1055/s-0040-1702213},
year = {2020},
date = {2020-01-01},
journal = {ACI Open},
volume = {4},
number = {01},
pages = {e35–e43},
publisher = {Georg Thieme Verlag KG},
abstract = {Objective:<br/> The aim of this study is to study the impact of graphical representation of health record data on physician decision-making to inform the design of health information technology.
<br/><br/>Materials and Methods:<br/> We conducted a within participants crossover design study using a simulated electronic health record (EHR) in which we presented cases with and without visualized data designed to highlight important clinical trends or relationships, followed by assessment of the impact on decision-making about next steps for patients with chronic diseases. We then asked whether trends were observed and about usability and satisfaction using validated usability questions and asked open-ended questions as well. Time to answer questions was also collected.
<br/><br/>Results:<br/> Twenty-one primary care providers participated in the study, including five for testing only and sixteen for the full study. Questions about clinical assessment or next actions were answered correctly 55% of the time. Regarding objective trends in the data, participants described noticing the trends 85% of the time. Differences in noticing trends or difficulty level of questions were not statistically significant. Satisfaction with the tool was high and participants agreed strongly that it helped them make better decisions without adding to the time it took.
<br/><br/>Discussion:<br/> The simulation allowed us to test the impact of a visualization on clinician practice in a realistic setting. Designers of EHRs should consider the ways information presentation can affect decision-making.
<br/><br/>Conclusion:<br/> Testing visualization tools can be done in a clinically realistic context. Providers desire visualizations and believe that they help them make better and faster decisions.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Objective:<br/> The aim of this study is to study the impact of graphical representation of health record data on physician decision-making to inform the design of health information technology.
<br/><br/>Materials and Methods:<br/> We conducted a within participants crossover design study using a simulated electronic health record (EHR) in which we presented cases with and without visualized data designed to highlight important clinical trends or relationships, followed by assessment of the impact on decision-making about next steps for patients with chronic diseases. We then asked whether trends were observed and about usability and satisfaction using validated usability questions and asked open-ended questions as well. Time to answer questions was also collected.
<br/><br/>Results:<br/> Twenty-one primary care providers participated in the study, including five for testing only and sixteen for the full study. Questions about clinical assessment or next actions were answered correctly 55% of the time. Regarding objective trends in the data, participants described noticing the trends 85% of the time. Differences in noticing trends or difficulty level of questions were not statistically significant. Satisfaction with the tool was high and participants agreed strongly that it helped them make better decisions without adding to the time it took.
<br/><br/>Discussion:<br/> The simulation allowed us to test the impact of a visualization on clinician practice in a realistic setting. Designers of EHRs should consider the ways information presentation can affect decision-making.
<br/><br/>Conclusion:<br/> Testing visualization tools can be done in a clinically realistic context. Providers desire visualizations and believe that they help them make better and faster decisions.
<br/><br/>Materials and Methods:<br/> We conducted a within participants crossover design study using a simulated electronic health record (EHR) in which we presented cases with and without visualized data designed to highlight important clinical trends or relationships, followed by assessment of the impact on decision-making about next steps for patients with chronic diseases. We then asked whether trends were observed and about usability and satisfaction using validated usability questions and asked open-ended questions as well. Time to answer questions was also collected.
<br/><br/>Results:<br/> Twenty-one primary care providers participated in the study, including five for testing only and sixteen for the full study. Questions about clinical assessment or next actions were answered correctly 55% of the time. Regarding objective trends in the data, participants described noticing the trends 85% of the time. Differences in noticing trends or difficulty level of questions were not statistically significant. Satisfaction with the tool was high and participants agreed strongly that it helped them make better decisions without adding to the time it took.
<br/><br/>Discussion:<br/> The simulation allowed us to test the impact of a visualization on clinician practice in a realistic setting. Designers of EHRs should consider the ways information presentation can affect decision-making.
<br/><br/>Conclusion:<br/> Testing visualization tools can be done in a clinically realistic context. Providers desire visualizations and believe that they help them make better and faster decisions.
Gajos, Krzysztof Z.; Reinecke, Katharina; Donovan, Mary; Stephen, Christopher D.; Hung, Albert Y.; Schmahmann, Jeremy D.; Gupta, Anoopum S.
Computer Mouse Use Captures Ataxia and Parkinsonism, Enabling Accurate Measurement and Detection Journal Article
In: Movement Disorders, vol. 35, iss. 2, pp. 354–358, 2020.
@article{gajos20:computer,
title = {Computer Mouse Use Captures Ataxia and Parkinsonism, Enabling Accurate Measurement and Detection},
author = {Krzysztof Z. Gajos and Katharina Reinecke and Mary Donovan and Christopher D. Stephen and Albert Y. Hung and Jeremy D. Schmahmann and Anoopum S. Gupta},
url = {/papers/gajos19computer.pdf},
doi = {10.1002/mds.27915},
year = {2020},
date = {2020-02-01},
journal = {Movement Disorders},
volume = {35},
issue = {2},
pages = {354–358},
abstract = {Background:<br/>
Objective assessments of movement impairment are needed to support clinical trials and facilitate diagnosis. The objective of the current study was to determine if a rapid web‐based computer mouse test (Hevelius) could detect and accurately measure ataxia and parkinsonism.
<br/><br/>
Methods:<br/>
Ninety‐five ataxia, 46 parkinsonism, and 29 control participants and 229,017 online participants completed Hevelius. We trained machine‐learning models on age‐normalized Hevelius features to (1) measure severity and disease progression and (2) distinguish phenotypes from controls and from each other.
<br/><br/>
Results:<br/> Regression model estimates correlated strongly with clinical scores (from r = 0.66 for UPDRS dominant arm total to r = 0.83 for the Brief Ataxia Rating Scale). A disease change model identified ataxia progression with high sensitivity. Classification models distinguished ataxia or parkinsonism from healthy controls with high sensitivity (≥0.91) and specificity (≥0.90).
<br/><br/>
Conclusions:<br/>
Hevelius produces a granular and accurate motor assessment in a few minutes of mouse use and may be useful as an outcome measure and screening tool.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Background:<br/>
Objective assessments of movement impairment are needed to support clinical trials and facilitate diagnosis. The objective of the current study was to determine if a rapid web‐based computer mouse test (Hevelius) could detect and accurately measure ataxia and parkinsonism.
<br/><br/>
Methods:<br/>
Ninety‐five ataxia, 46 parkinsonism, and 29 control participants and 229,017 online participants completed Hevelius. We trained machine‐learning models on age‐normalized Hevelius features to (1) measure severity and disease progression and (2) distinguish phenotypes from controls and from each other.
<br/><br/>
Results:<br/> Regression model estimates correlated strongly with clinical scores (from r = 0.66 for UPDRS dominant arm total to r = 0.83 for the Brief Ataxia Rating Scale). A disease change model identified ataxia progression with high sensitivity. Classification models distinguished ataxia or parkinsonism from healthy controls with high sensitivity (≥0.91) and specificity (≥0.90).
<br/><br/>
Conclusions:<br/>
Hevelius produces a granular and accurate motor assessment in a few minutes of mouse use and may be useful as an outcome measure and screening tool.
Objective assessments of movement impairment are needed to support clinical trials and facilitate diagnosis. The objective of the current study was to determine if a rapid web‐based computer mouse test (Hevelius) could detect and accurately measure ataxia and parkinsonism.
<br/><br/>
Methods:<br/>
Ninety‐five ataxia, 46 parkinsonism, and 29 control participants and 229,017 online participants completed Hevelius. We trained machine‐learning models on age‐normalized Hevelius features to (1) measure severity and disease progression and (2) distinguish phenotypes from controls and from each other.
<br/><br/>
Results:<br/> Regression model estimates correlated strongly with clinical scores (from r = 0.66 for UPDRS dominant arm total to r = 0.83 for the Brief Ataxia Rating Scale). A disease change model identified ataxia progression with high sensitivity. Classification models distinguished ataxia or parkinsonism from healthy controls with high sensitivity (≥0.91) and specificity (≥0.90).
<br/><br/>
Conclusions:<br/>
Hevelius produces a granular and accurate motor assessment in a few minutes of mouse use and may be useful as an outcome measure and screening tool.
2019
Jacobs, Maia; Gheihman, Galina; Gajos, Krzysztof Z.; Gupta, Anoopum S.
“I Think We Know More Than Our Doctors”: How Primary Caregivers Manage Care Teams with Limited Disease-related Expertise Journal Article
In: Proc. ACM Hum.-Comput. Interact., vol. 3, no. CSCW, pp. 159:1–159:22, 2019, ISSN: 2573-0142.
@article{jacobs19:think,
title = {"I Think We Know More Than Our Doctors": How Primary Caregivers Manage Care Teams with Limited Disease-related Expertise},
author = {Maia Jacobs and Galina Gheihman and Krzysztof Z. Gajos and Anoopum S. Gupta},
url = {/papers/jacobs19think.pdf},
doi = {10.1145/3359261},
issn = {2573-0142},
year = {2019},
date = {2019-11-01},
journal = {Proc. ACM Hum.-Comput. Interact.},
volume = {3},
number = {CSCW},
pages = {159:1–159:22},
publisher = {ACM},
address = {New York, NY, USA},
abstract = {Current models of patient-provider relationships in chronic disease management assume that healthcare providers possess expertise in the patient's diagnosis and trajectory. When managing a rare disease, patients and caregivers often work with care teams who are unfamiliar with the disease or inexperienced in its treatment. A deeper understanding of these relationships is needed to identify opportunities for support under these circumstances. We conducted qualitative interviews with parental caregivers of children with a rare neurodegenerative disorder, ataxia-telangiectasia. We found that when working with care teams inexperienced with the disease, parents must take on additional responsibilities beyond daily health management, including educating their care teams, initiating changes to their child's healthcare, and preventing adverse clinical events. Online community support can serve as a valuable resource to support parents in assuming these responsibilities. However, the emotional consequences of participation and information overload have hindered parents' community engagement. Therefore, two important research agendas for supporting this community are through the development of tools that facilitate parents' care coordination roles and tools that increase access to online community support.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Current models of patient-provider relationships in chronic disease management assume that healthcare providers possess expertise in the patient’s diagnosis and trajectory. When managing a rare disease, patients and caregivers often work with care teams who are unfamiliar with the disease or inexperienced in its treatment. A deeper understanding of these relationships is needed to identify opportunities for support under these circumstances. We conducted qualitative interviews with parental caregivers of children with a rare neurodegenerative disorder, ataxia-telangiectasia. We found that when working with care teams inexperienced with the disease, parents must take on additional responsibilities beyond daily health management, including educating their care teams, initiating changes to their child’s healthcare, and preventing adverse clinical events. Online community support can serve as a valuable resource to support parents in assuming these responsibilities. However, the emotional consequences of participation and information overload have hindered parents’ community engagement. Therefore, two important research agendas for supporting this community are through the development of tools that facilitate parents’ care coordination roles and tools that increase access to online community support.
Huber, Bernd; Shieber, Stuart; Gajos, Krzysztof Z.
Automatically Analyzing Brainstorming Language Behavior with Meeter Journal Article
In: Proc. ACM Hum.-Comput. Interact., vol. 3, no. CSCW, pp. 30:1–30:17, 2019, ISSN: 2573-0142.
@article{huber19:automatically,
title = {Automatically Analyzing Brainstorming Language Behavior with Meeter},
author = {Bernd Huber and Stuart Shieber and Krzysztof Z. Gajos},
url = {/papers/huber19automatically.pdf},
doi = {10.1145/3359132},
issn = {2573-0142},
year = {2019},
date = {2019-11-01},
journal = {Proc. ACM Hum.-Comput. Interact.},
volume = {3},
number = {CSCW},
pages = {30:1–30:17},
publisher = {ACM},
address = {New York, NY, USA},
abstract = {Studying groups in such complex settings as group brainstorming would be much more informative if there were better tools to study them. Language both influences and indicates group behavior, and we need tools that let us study the content of what is communicated to understand how such dialogue acts as information sharing and shared understanding indicate group behavior. While one could annotate these spoken dialogue acts by hand, this is a tedious process that is not scalable. We present Meeter, a tool to more effectively study spoken group brainstorming interactions by automatically detecting information sharing, shared understanding, word counts, and group activation in spoken interactions. Our study shows that the measures computed by Meeter align with human-generated labels, and we present findings on the relationship between these measures and group outcomes, underlining the validity of the tool for studying groups. Our tool is valuable for researchers conducting group science, as well as designing groupware systems.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Studying groups in such complex settings as group brainstorming would be much more informative if there were better tools to study them. Language both influences and indicates group behavior, and we need tools that let us study the content of what is communicated to understand how such dialogue acts as information sharing and shared understanding indicate group behavior. While one could annotate these spoken dialogue acts by hand, this is a tedious process that is not scalable. We present Meeter, a tool to more effectively study spoken group brainstorming interactions by automatically detecting information sharing, shared understanding, word counts, and group activation in spoken interactions. Our study shows that the measures computed by Meeter align with human-generated labels, and we present findings on the relationship between these measures and group outcomes, underlining the validity of the tool for studying groups. Our tool is valuable for researchers conducting group science, as well as designing groupware systems.
Jacobs, Maia; Perlis, Roy H.; Pradier, Melanie F.; Doshi-Velez, Finale; Mynatt, Elizabeth; Gajos, Krzysztof Z.
Integrating AI Recommendations into the Pharmacologic Management of Major Depressive Disorder Proceedings Article
In: Proceedings of the workshop on Identifying Challenges and Opportunities in Human–AI Collaboration in Healthcare at CSCW 2019, 2019.
@inproceedings{jacobs19:integrating,
title = {Integrating AI Recommendations into the Pharmacologic Management of Major Depressive Disorder},
author = {Maia Jacobs and Roy H. Perlis and Melanie F. Pradier and Finale Doshi-Velez and Elizabeth Mynatt and Krzysztof Z. Gajos},
url = {/papers/jacobs19integrating.pdf},
year = {2019},
date = {2019-01-01},
booktitle = {Proceedings of the workshop on Identifying Challenges and Opportunities in Human–AI Collaboration in Healthcare at CSCW 2019},
abstract = {AI predictions provide an important opportunity to support clinicians during complex decision-making processes. One such process is selecting treatments for major depressive disorder (MDD). Towards the goal of implementing AI models that make MDD treatment recommendations, we have designed a factorial vignette study to assess how recommendations and explanations may influence clinician's treatment decisions. We report on our initial data analysis, evaluating the influence of incorrect predictions on antidepressant selection. We found that recommendation correctness had a significant effect on treatment selection accuracy.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
AI predictions provide an important opportunity to support clinicians during complex decision-making processes. One such process is selecting treatments for major depressive disorder (MDD). Towards the goal of implementing AI models that make MDD treatment recommendations, we have designed a factorial vignette study to assess how recommendations and explanations may influence clinician’s treatment decisions. We report on our initial data analysis, evaluating the influence of incorrect predictions on antidepressant selection. We found that recommendation correctness had a significant effect on treatment selection accuracy.
Han, Jung Min; Malkawi, Ali; Gajos, Krzysztof Z.
Eabbit 1.0: New Environmental Analysis Software for Solar Energy Representation Proceedings Article
In: Proceedings of Building Simulation, 2019.
@inproceedings{han19:eabbit,
title = {Eabbit 1.0: New Environmental Analysis Software for Solar Energy Representation},
author = {Jung Min Han and Ali Malkawi and Krzysztof Z. Gajos},
url = {/papers/han19eabbit.pdf},
year = {2019},
date = {2019-01-01},
booktitle = {Proceedings of Building Simulation},
abstract = {Given the challenges to designing high-performance buildings, the use of Building Performance Simulation (BPS) tools during the early design phase is indispensable. There are many tools for evaluating solar impact on buildings, ranging from energy use to daylighting and renewable energy. However, no tool accurately reflects architects' needs; all lack clear communication and proper visualization methods. This research addressed energy consumption and production measures impacted by exposure to the sun. The goal was to create a new type of early design decision support tool that functioned without running BPS optimization and parametric simulations. This was accomplished by developing important solar algorithms that were then used in a new method of solar representation for building design. It is important that usability assessments of this type of tool be conducted, especially with regards to its ability to satisfy architects' needs during the design process. However, such research is not yet common in the field of BPS. Thus, in the present research, user experiments were conducted to evaluate the effectiveness of Eabbit 1.0's key interface. The results of these experiments illustrate the impact of the proposed method on annual heating and cooling consumption. The results show that tool will assist users in making significantly better decisions regarding the installation of photovoltaic panels and other issues related to solar energy.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Given the challenges to designing high-performance buildings, the use of Building Performance Simulation (BPS) tools during the early design phase is indispensable. There are many tools for evaluating solar impact on buildings, ranging from energy use to daylighting and renewable energy. However, no tool accurately reflects architects’ needs; all lack clear communication and proper visualization methods. This research addressed energy consumption and production measures impacted by exposure to the sun. The goal was to create a new type of early design decision support tool that functioned without running BPS optimization and parametric simulations. This was accomplished by developing important solar algorithms that were then used in a new method of solar representation for building design. It is important that usability assessments of this type of tool be conducted, especially with regards to its ability to satisfy architects’ needs during the design process. However, such research is not yet common in the field of BPS. Thus, in the present research, user experiments were conducted to evaluate the effectiveness of Eabbit 1.0’s key interface. The results of these experiments illustrate the impact of the proposed method on annual heating and cooling consumption. The results show that tool will assist users in making significantly better decisions regarding the installation of photovoltaic panels and other issues related to solar energy.
Amir, Ofra; Grosz, Barbara J.; Gajos, Krzysztof Z.; Gultchin, Limor
Personalized change awareness: Reducing information overload in loosely-coupled teamwork Journal Article
In: Artificial Intelligence, vol. 275, pp. 204 – 233, 2019, ISSN: 0004-3702.
@article{amir19:personalized,
title = {Personalized change awareness: Reducing information overload in loosely-coupled teamwork},
author = {Ofra Amir and Barbara J. Grosz and Krzysztof Z. Gajos and Limor Gultchin},
url = {/papers/amir19personalized.pdf},
doi = {https://doi.org/10.1016/j.artint.2019.05.005},
issn = {0004-3702},
year = {2019},
date = {2019-01-01},
journal = {Artificial Intelligence},
volume = {275},
pages = {204 - 233},
abstract = {Complex tasks such as treating patients with chronic conditions and developing software products are typically accomplished by teams that collaborate over an extended time duration. To remain coordinated, team members need to be aware of others' activities if those activities are likely to affect their own actions. However, identifying such interactions and sharing information appropriately is challenging, especially when the activities of team members are loosely-coupled. In practice, team members often either lack important information about others' activities, or are overwhelmed by the need to review too much information. This paper presents Personalized Change Awareness, a new approach for supporting team coordination which aims to automatically identify and share the subset of information about others' activities that is most relevant to each of the team members. The paper formally defines the computational problem of information sharing in loosely-coupled teamwork, which underlies the personalized change awareness approach. It defines a new representation, Mutual Influence Potential Networks (MIP-Nets) and an algorithm, MIP-DOI, that uses this representation to determine the information that is most relevant to each team member. In contrast to existing information sharing algorithms in multi-agent teams, MIP-DOI does not assume the availability of a priori knowledge of a team's possible plans, because human teams rarely explicitly define detailed long-term plans in advance. We demonstrate the ability of MIP-DOI to identify relevant information using simulations of collaborative activities. We further evaluated the contribution of personalized change awareness to team performance in a controlled user study. To this end, we developed a personalized change awareness mechanism for collaborative writing, which used MIP-DOI to determine which changes to share with each author. This evaluation demonstrates that the Personalized Change Awareness approach resulted in higher productivity and lower perceived workload without any change in final quality compared to the currently prevalent approach of sharing all change information. Our results also demonstrate that merely reducing the amount of information shared with co-authors is not enough: sharing a random subset of changes resulted in significantly lower quality of work than sharing a personalized subset of changes.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Complex tasks such as treating patients with chronic conditions and developing software products are typically accomplished by teams that collaborate over an extended time duration. To remain coordinated, team members need to be aware of others’ activities if those activities are likely to affect their own actions. However, identifying such interactions and sharing information appropriately is challenging, especially when the activities of team members are loosely-coupled. In practice, team members often either lack important information about others’ activities, or are overwhelmed by the need to review too much information. This paper presents Personalized Change Awareness, a new approach for supporting team coordination which aims to automatically identify and share the subset of information about others’ activities that is most relevant to each of the team members. The paper formally defines the computational problem of information sharing in loosely-coupled teamwork, which underlies the personalized change awareness approach. It defines a new representation, Mutual Influence Potential Networks (MIP-Nets) and an algorithm, MIP-DOI, that uses this representation to determine the information that is most relevant to each team member. In contrast to existing information sharing algorithms in multi-agent teams, MIP-DOI does not assume the availability of a priori knowledge of a team’s possible plans, because human teams rarely explicitly define detailed long-term plans in advance. We demonstrate the ability of MIP-DOI to identify relevant information using simulations of collaborative activities. We further evaluated the contribution of personalized change awareness to team performance in a controlled user study. To this end, we developed a personalized change awareness mechanism for collaborative writing, which used MIP-DOI to determine which changes to share with each author. This evaluation demonstrates that the Personalized Change Awareness approach resulted in higher productivity and lower perceived workload without any change in final quality compared to the currently prevalent approach of sharing all change information. Our results also demonstrate that merely reducing the amount of information shared with co-authors is not enough: sharing a random subset of changes resulted in significantly lower quality of work than sharing a personalized subset of changes.
Huber, Bernd; Davis, Richard F. III; Cotter, Allison; Junkin, Emily; Yard, Mindy; Shieber, Stuart; Brestan-Knight, Elizabeth; Gajos, Krzysztof Z.
SpecialTime: Automatically Detecting Dialogue Acts from Speech to Support Parent-Child Interaction Therapy Proceedings Article
In: Proceedings of the 13th EAI International Conference on Pervasive Computing Technologies for Healthcare, pp. 139–148, ACM, Trento, Italy, 2019, ISBN: 978-1-4503-6126-2.
@inproceedings{huber19:specialtime,
title = {SpecialTime: Automatically Detecting Dialogue Acts from Speech to Support Parent-Child Interaction Therapy},
author = {Bernd Huber and Richard F. III Davis and Allison Cotter and Emily Junkin and Mindy Yard and Stuart Shieber and Elizabeth Brestan-Knight and Krzysztof Z. Gajos},
url = {/papers/huber19specialtime.pdf},
doi = {10.1145/3329189.3329203},
isbn = {978-1-4503-6126-2},
year = {2019},
date = {2019-01-01},
booktitle = {Proceedings of the 13th EAI International Conference on Pervasive Computing Technologies for Healthcare},
pages = {139–148},
publisher = {ACM},
address = {Trento, Italy},
series = {PervasiveHealth'19},
abstract = {Parent-child interaction therapy (PCIT) helps parents improve the quality of interaction with children who have behavior problems. The therapy trains parents to use effective dialogue acts when interacting with their children. Besides weekly coaching by therapists, the therapy relies on deliberate practice of skills by parents in their homes. We developed SpecialTime, a system that provides parents engaged in PCIT with automatic, real-time feedback on their dialogue act use. To do this, we first created a dataset of 6,022 parent dialogue acts, annotated by experts with dialogue act labels that therapists use to code parent speech. We then developed an algorithm that classifies the dialogue acts into 8 classes with an overall accuracy of 78%. To test the system in an actual clinical setting, we conducted a one month pilot study with four parents currently in therapy. The results suggest that automatic feedback on spoken dialogue acts is possible in PCIT, and that parents find the automatic feedback useful.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Parent-child interaction therapy (PCIT) helps parents improve the quality of interaction with children who have behavior problems. The therapy trains parents to use effective dialogue acts when interacting with their children. Besides weekly coaching by therapists, the therapy relies on deliberate practice of skills by parents in their homes. We developed SpecialTime, a system that provides parents engaged in PCIT with automatic, real-time feedback on their dialogue act use. To do this, we first created a dataset of 6,022 parent dialogue acts, annotated by experts with dialogue act labels that therapists use to code parent speech. We then developed an algorithm that classifies the dialogue acts into 8 classes with an overall accuracy of 78%. To test the system in an actual clinical setting, we conducted a one month pilot study with four parents currently in therapy. The results suggest that automatic feedback on spoken dialogue acts is possible in PCIT, and that parents find the automatic feedback useful.
Kim, Nam Wook; Im, Hyejin; Riche, Nathalie Henry; Wang, Alicia; Gajos, Krzysztof; Pfister, Hanspeter
DataSelfie: Empowering People to Design Personalized Visuals to Represent Their Data Proceedings Article
In: Proceedings of the 2019 CHI Conference on Human Factors in Computing Systems, pp. 79:1–79:12, ACM, Glasgow, Scotland Uk, 2019, ISBN: 978-1-4503-5970-2.
@inproceedings{kim19:dataselfie,
title = {DataSelfie: Empowering People to Design Personalized Visuals to Represent Their Data},
author = {Nam Wook Kim and Hyejin Im and Nathalie Henry Riche and Alicia Wang and Krzysztof Gajos and Hanspeter Pfister},
url = {/papers/kim19dataselfie.pdf},
doi = {10.1145/3290605.3300309},
isbn = {978-1-4503-5970-2},
year = {2019},
date = {2019-01-01},
booktitle = {Proceedings of the 2019 CHI Conference on Human Factors in Computing Systems},
pages = {79:1–79:12},
publisher = {ACM},
address = {Glasgow, Scotland Uk},
series = {CHI '19},
abstract = {Many personal informatics systems allow people to collect and manage personal data and reflect more deeply about themselves. However, these tools rarely offer ways to customize how the data is visualized. In this work, we investigate the question of how to enable people to determine the representation of their data. We analyzed the Dear Data project to gain insights into the design elements of personal visualizations. We developed DataSelfie, a novel system that allows individuals to gather personal data and design custom visuals to represent the collected data. We conducted a user study to evaluate the usability of the system as well as its potential for individual and collaborative sensemaking of the data.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Many personal informatics systems allow people to collect and manage personal data and reflect more deeply about themselves. However, these tools rarely offer ways to customize how the data is visualized. In this work, we investigate the question of how to enable people to determine the representation of their data. We analyzed the Dear Data project to gain insights into the design elements of personal visualizations. We developed DataSelfie, a novel system that allows individuals to gather personal data and design custom visuals to represent the collected data. We conducted a user study to evaluate the usability of the system as well as its potential for individual and collaborative sensemaking of the data.
2018
Li, Qisheng; Gajos, Krzysztof Z.; Reinecke, Katharina
Volunteer-Based Online Studies With Older Adults and People with Disabilities Proceedings Article
In: Proceedings of the 20th International ACM SIGACCESS Conference on Computers and Accessibility, pp. 229–241, ACM, Galway, Ireland, 2018, ISBN: 978-1-4503-5650-3.
@inproceedings{li18vounteer,
title = {Volunteer-Based Online Studies With Older Adults and People with Disabilities},
author = {Qisheng Li and Krzysztof Z. Gajos and Katharina Reinecke},
url = {/papers/li18volunteer.pdf},
doi = {10.1145/3234695.3236360},
isbn = {978-1-4503-5650-3},
year = {2018},
date = {2018-01-01},
booktitle = {Proceedings of the 20th International ACM SIGACCESS Conference on Computers and Accessibility},
pages = {229–241},
publisher = {ACM},
address = {Galway, Ireland},
series = {ASSETS '18},
abstract = {There are few large-scale empirical studies with people with disabilities or older adults, mainly because recruiting participants with specific characteristics is even harder than recruiting young and/or non-disabled populations. Analyzing four online experiments on LabintheWild with a total of 355,656 participants, we show that volunteer-based online experiments that provide personalized feedback attract large numbers of participants with diverse disabilities and ages and allow robust studies with these populations that replicate and extend the findings of prior laboratory studies. To find out what motivates people with disabilities to take part, we additionally analyzed participants' feedback and forum entries that discuss LabintheWild experiments. The results show that participants use the studies to diagnose themselves, compare their abilities to others, quantify potential impairments, self-experiment, and share their own stories - findings that we use to inform design guidelines for online experiment platforms that adequately support and engage people with disabilities.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
There are few large-scale empirical studies with people with disabilities or older adults, mainly because recruiting participants with specific characteristics is even harder than recruiting young and/or non-disabled populations. Analyzing four online experiments on LabintheWild with a total of 355,656 participants, we show that volunteer-based online experiments that provide personalized feedback attract large numbers of participants with diverse disabilities and ages and allow robust studies with these populations that replicate and extend the findings of prior laboratory studies. To find out what motivates people with disabilities to take part, we additionally analyzed participants’ feedback and forum entries that discuss LabintheWild experiments. The results show that participants use the studies to diagnose themselves, compare their abilities to others, quantify potential impairments, self-experiment, and share their own stories – findings that we use to inform design guidelines for online experiment platforms that adequately support and engage people with disabilities.
Jayatilaka, Lahiru; Sengeh, David M.; Herrmann, Charles; Bertuccelli, Luca; Antos, Dimitrios; Grosz, Barbara J.; Gajos, Krzysztof Z.
PETALS: Improving Learning of Expert Skill in Humanitarian Demining Proceedings Article
In: Proceedings of the 1st ACM SIGCAS Conference on Computing and Sustainable Societies, pp. 33:1–33:11, ACM, Menlo Park and San Jose, CA, USA, 2018, ISBN: 978-1-4503-5816-3.
@inproceedings{jayatilaka18:petals,
title = {PETALS: Improving Learning of Expert Skill in Humanitarian Demining},
author = {Lahiru Jayatilaka and David M. Sengeh and Charles Herrmann and Luca Bertuccelli and Dimitrios Antos and Barbara J. Grosz and Krzysztof Z. Gajos},
url = {/papers/jayatilaka18petals.pdf},
doi = {10.1145/3209811.3209871},
isbn = {978-1-4503-5816-3},
year = {2018},
date = {2018-01-01},
booktitle = {Proceedings of the 1st ACM SIGCAS Conference on Computing and Sustainable Societies},
pages = {33:1–33:11},
publisher = {ACM},
address = {Menlo Park and San Jose, CA, USA},
series = {COMPASS '18},
abstract = {To become proficient at landmine detection, novice deminers need to master several kinds of skills: the proper physical operation of the metal detector, the interpretation of the metal detector auditory feedback, and the abstract skill of constructing and interpreting mental representations of the "metallic signatures" produced by the buried objects. This last skill is particularly useful for safely dealing with mines laid out in cluster configurations, where their metallic signatures overlap and thus a danger exists that a deminer might either miss some of the mines or incorrectly assess their exact positions. However, some novice deminers find it challenging to learn how to properly reason about metallic signatures. We have developed PETALS, a system that explicitly visualizes a trainee's metal detector operation history on a training task as well as the edge points of the metallic signatures that the trainee collected. PETALS enables instructors to supervise multiple trainees at a time, to assess their performance at a glance, and to provide immediate and specific feedback both on the correctness of their final judgements about the number and positions of landmines, and on the process through which they arrived at their conclusions. The results of our field evaluations at the Humanitarian Demining Training Center showed that both the instructors and the trainees found the system a valuable addition to the training course. The results of a controlled study demonstrated that trainees who had access to PETALS during training made significantly fewer errors (6% error rate) on relevant tasks during the final exam (which was conducted without PETALS) than trainees who did not have access to PETALS during training (those participants had a 21% error rate).},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
To become proficient at landmine detection, novice deminers need to master several kinds of skills: the proper physical operation of the metal detector, the interpretation of the metal detector auditory feedback, and the abstract skill of constructing and interpreting mental representations of the “metallic signatures” produced by the buried objects. This last skill is particularly useful for safely dealing with mines laid out in cluster configurations, where their metallic signatures overlap and thus a danger exists that a deminer might either miss some of the mines or incorrectly assess their exact positions. However, some novice deminers find it challenging to learn how to properly reason about metallic signatures. We have developed PETALS, a system that explicitly visualizes a trainee’s metal detector operation history on a training task as well as the edge points of the metallic signatures that the trainee collected. PETALS enables instructors to supervise multiple trainees at a time, to assess their performance at a glance, and to provide immediate and specific feedback both on the correctness of their final judgements about the number and positions of landmines, and on the process through which they arrived at their conclusions. The results of our field evaluations at the Humanitarian Demining Training Center showed that both the instructors and the trainees found the system a valuable addition to the training course. The results of a controlled study demonstrated that trainees who had access to PETALS during training made significantly fewer errors (6% error rate) on relevant tasks during the final exam (which was conducted without PETALS) than trainees who did not have access to PETALS during training (those participants had a 21% error rate).
Arnold, Kenneth; Chauncey, Krysta; Gajos, Krzysztof
Sentiment Bias in Predictive Text Recommendations Results in Biased Writing Proceedings Article
In: Proceedings of Graphics Interface 2018, pp. 33 – 40, Canadian Human-Computer Communications Society / Societe canadienne du dialogue humain-machine, Toronto, Ontario, 2018, ISBN: 978-0-9947868-2-1.
@inproceedings{arnold18:sentiment,
title = {Sentiment Bias in Predictive Text Recommendations Results in Biased Writing},
author = {Kenneth Arnold and Krysta Chauncey and Krzysztof Gajos},
url = {/papers/arnold18sentiment.pdf},
doi = {10.20380/GI2018.05},
isbn = {978-0-9947868-2-1},
year = {2018},
date = {2018-01-01},
booktitle = {Proceedings of Graphics Interface 2018},
pages = {33 – 40},
publisher = {Canadian Human-Computer Communications Society / Societe canadienne du dialogue humain-machine},
address = {Toronto, Ontario},
series = {GI 2018},
abstract = {Prior research has demonstrated that intelligent systems make biased decisions because they are trained on biased data. As people increasingly leverage intelligent systems to enhance their productivity and creativity, could system biases affect what people create? We demonstrate that in at least one domain (writing restaurant reviews), biased system behavior leads to biased human behavior: People presented with phrasal text entry shortcuts that were skewed positive wrote more positive reviews than they did when presented with negative-skewed shortcuts. This result contributes to the pertinent debate about the role of intelligent systems in our society.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Prior research has demonstrated that intelligent systems make biased decisions because they are trained on biased data. As people increasingly leverage intelligent systems to enhance their productivity and creativity, could system biases affect what people create? We demonstrate that in at least one domain (writing restaurant reviews), biased system behavior leads to biased human behavior: People presented with phrasal text entry shortcuts that were skewed positive wrote more positive reviews than they did when presented with negative-skewed shortcuts. This result contributes to the pertinent debate about the role of intelligent systems in our society.
Moyne, M. M.; Herman, M.; Gajos, K. Z.; Walsh, C. J.; Holland, D. P.
The Development and Evaluation of DEFT, a Web-Based Tool for Engineering Design Education Journal Article
In: IEEE Transactions on Learning Technologies, vol. 11, no. 4, pp. 545-550, 2018, ISSN: 1939-1382.
@article{moyne18:development,
title = {The Development and Evaluation of DEFT, a Web-Based Tool for Engineering Design Education},
author = {M. M. Moyne and M. Herman and K. Z. Gajos and C. J. Walsh and D. P. Holland},
url = {/papers/moyne18development.pdf},
doi = {10.1109/TLT.2018.2810197},
issn = {1939-1382},
year = {2018},
date = {2018-10-01},
journal = {IEEE Transactions on Learning Technologies},
volume = {11},
number = {4},
pages = {545-550},
abstract = {This article describes the development of the Design Evaluation and Feedback Tool (DEFT), a custom-built web-based system that collects and reports data to support teaching, learning, and research in project-based engineering design education. The DEFT system collects data through short weekly questionnaires for students and instructors in engineering design classes, and uses these data to produce weekly reports for both types of user. The system is intended to engage students in reflective reporting on their experiential learning, to support educators in coaching student designers, and to serve as a data collection tool for education researchers. DEFT was developed through an iterative design and evaluation process, involving 185 students and 18 instructors at two universities. The system was evaluated using a combination of participation observation, user interviews, and anonymous questionnaires, and the results guided subsequent improvements to the system. This article describes the development and evaluation process, provides an overview of the resulting system, and ends by discussing the potential for DEFT to be used in evaluating and improving project-based design classes.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
This article describes the development of the Design Evaluation and Feedback Tool (DEFT), a custom-built web-based system that collects and reports data to support teaching, learning, and research in project-based engineering design education. The DEFT system collects data through short weekly questionnaires for students and instructors in engineering design classes, and uses these data to produce weekly reports for both types of user. The system is intended to engage students in reflective reporting on their experiential learning, to support educators in coaching student designers, and to serve as a data collection tool for education researchers. DEFT was developed through an iterative design and evaluation process, involving 185 students and 18 instructors at two universities. The system was evaluated using a combination of participation observation, user interviews, and anonymous questionnaires, and the results guided subsequent improvements to the system. This article describes the development and evaluation process, provides an overview of the resulting system, and ends by discussing the potential for DEFT to be used in evaluating and improving project-based design classes.
Wobbrock, Jacob O.; Gajos, Krzysztof Z.; Kane, Shaun K.; Vanderheiden, Gregg C.
Ability-based Design Journal Article
In: Commun. ACM, vol. 61, no. 6, pp. 62–71, 2018, ISSN: 0001-0782.
@article{wobbrock18:ability,
title = {Ability-based Design},
author = {Jacob O. Wobbrock and Krzysztof Z. Gajos and Shaun K. Kane and Gregg C. Vanderheiden},
url = {http://doi.acm.org/10.1145/3148051},
doi = {10.1145/3148051},
issn = {0001-0782},
year = {2018},
date = {2018-05-01},
journal = {Commun. ACM},
volume = {61},
number = {6},
pages = {62–71},
publisher = {ACM},
address = {New York, NY, USA},
abstract = {By focusing on users' abilities rather than disabilities, designers can create interactive systems better matched to those abilities.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
By focusing on users’ abilities rather than disabilities, designers can create interactive systems better matched to those abilities.
2017
Kim, Nam Wook; Bylinskii, Zoya; Borkin, Michelle A.; Gajos, Krzysztof Z.; Oliva, Aude; Durand, Fredo; Pfister, Hanspeter
BubbleView: An Interface for Crowdsourcing Image Importance Maps and Tracking Visual Attention Journal Article
In: ACM Trans. Comput.-Hum. Interact., vol. 24, no. 5, pp. 36:1–36:40, 2017, ISSN: 1073-0516.
@article{kim17:bubbleview,
title = {BubbleView: An Interface for Crowdsourcing Image Importance Maps and Tracking Visual Attention},
author = {Nam Wook Kim and Zoya Bylinskii and Michelle A. Borkin and Krzysztof Z. Gajos and Aude Oliva and Fredo Durand and Hanspeter Pfister},
url = {/papers/kim17bubbleview.pdf},
doi = {10.1145/3131275},
issn = {1073-0516},
year = {2017},
date = {2017-11-01},
journal = {ACM Trans. Comput.-Hum. Interact.},
volume = {24},
number = {5},
pages = {36:1–36:40},
publisher = {ACM},
address = {New York, NY, USA},
abstract = {In this article, we present BubbleView, an alternative methodology for eye tracking using discrete mouse clicks to measure which information people consciously choose to examine. BubbleView is a mouse-contingent, moving-window interface in which participants are presented with a series of blurred images and click to reveal "bubbles" – small, circular areas of the image at original resolution, similar to having a confined area of focus like the eye fovea. Across 10 experiments with 28 different parameter combinations, we evaluated BubbleView on a variety of image types: information visualizations, natural images, static webpages, and graphic designs, and compared the clicks to eye fixations collected with eye-trackers in controlled lab settings. We found that BubbleView clicks can both (i) successfully approximate eye fixations on different images, and (ii) be used to rank image and design elements by importance. BubbleView is designed to collect clicks on static images, and works best for defined tasks such as describing the content of an information visualization or measuring image importance. BubbleView data is cleaner and more consistent than related methodologies that use continuous mouse movements. Our analyses validate the use of mouse-contingent, moving-window methodologies as approximating eye fixations for different image and task types.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
In this article, we present BubbleView, an alternative methodology for eye tracking using discrete mouse clicks to measure which information people consciously choose to examine. BubbleView is a mouse-contingent, moving-window interface in which participants are presented with a series of blurred images and click to reveal “bubbles” – small, circular areas of the image at original resolution, similar to having a confined area of focus like the eye fovea. Across 10 experiments with 28 different parameter combinations, we evaluated BubbleView on a variety of image types: information visualizations, natural images, static webpages, and graphic designs, and compared the clicks to eye fixations collected with eye-trackers in controlled lab settings. We found that BubbleView clicks can both (i) successfully approximate eye fixations on different images, and (ii) be used to rank image and design elements by importance. BubbleView is designed to collect clicks on static images, and works best for defined tasks such as describing the content of an information visualization or measuring image importance. BubbleView data is cleaner and more consistent than related methodologies that use continuous mouse movements. Our analyses validate the use of mouse-contingent, moving-window methodologies as approximating eye fixations for different image and task types.
Chan, Joel; Siangliulue, Pao; McDonald, Denisa Qori; Liu, Ruixue; Moradinezhad, Reza; Aman, Safa; Solovey, Erin T.; Gajos, Krzysztof Z.; Dow, Steven P.
Semantically Far Inspirations Considered Harmful?: Accounting for Cognitive States in Collaborative Ideation Proceedings Article
In: Proceedings of the 2017 ACM SIGCHI Conference on Creativity and Cognition, pp. 93–105, ACM, Singapore, Singapore, 2017, ISBN: 978-1-4503-4403-6.
@inproceedings{chan17:semantically,
title = {Semantically Far Inspirations Considered Harmful?: Accounting for Cognitive States in Collaborative Ideation},
author = {Joel Chan and Pao Siangliulue and Denisa Qori McDonald and Ruixue Liu and Reza Moradinezhad and Safa Aman and Erin T. Solovey and Krzysztof Z. Gajos and Steven P. Dow},
url = {/papers/chan17semantically.pdf},
doi = {10.1145/3059454.3059455},
isbn = {978-1-4503-4403-6},
year = {2017},
date = {2017-01-01},
booktitle = {Proceedings of the 2017 ACM SIGCHI Conference on Creativity and Cognition},
pages = {93–105},
publisher = {ACM},
address = {Singapore, Singapore},
series = {C&C '17},
abstract = {Collaborative ideation systems can help people generate more creative ideas by exposing them to ideas different from their own. However, there are competing theoretical views on whether and when such exposure is helpful. Associationist theory suggests that exposing ideators to ideas that are semantically far from their own maximizes novel combinations of ideas. In contrast, SIAM theory cautions that systems should offer far ideas only when ideators reach an impasse (a cognitive state in which they have exhausted ideas within a particular category), and offer near ideas during productive ideation (a cognitive state in which they are actively exploring ideas within a category), which maximizes exploration within categories. Our research compares these theoretical recommendations. In an online experiment, 245 participants generated ideas for a themed wedding; we detected and validated participants' cognitive states using a combination of behavioral and neuroimaging data. Receiving far ideas during productive ideation resulted in slower ideation and less within-category exploration, without significant benefits for novelty, compared to receiving no inspirations. Participants were also more likely to hit an impasse when receiving far ideas during productive ideation. These findings suggest that far inspirational ideas can harm creativity if received during productive ideation.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Collaborative ideation systems can help people generate more creative ideas by exposing them to ideas different from their own. However, there are competing theoretical views on whether and when such exposure is helpful. Associationist theory suggests that exposing ideators to ideas that are semantically far from their own maximizes novel combinations of ideas. In contrast, SIAM theory cautions that systems should offer far ideas only when ideators reach an impasse (a cognitive state in which they have exhausted ideas within a particular category), and offer near ideas during productive ideation (a cognitive state in which they are actively exploring ideas within a category), which maximizes exploration within categories. Our research compares these theoretical recommendations. In an online experiment, 245 participants generated ideas for a themed wedding; we detected and validated participants’ cognitive states using a combination of behavioral and neuroimaging data. Receiving far ideas during productive ideation resulted in slower ideation and less within-category exploration, without significant benefits for novelty, compared to receiving no inspirations. Participants were also more likely to hit an impasse when receiving far ideas during productive ideation. These findings suggest that far inspirational ideas can harm creativity if received during productive ideation.
Burgermaster, Marissa; Gajos, Krzysztof Z.; Davidson, Patricia; Mamykina, Lena
The Role of Explanations in Casual Observational Learning About Nutrition Proceedings Article
In: Proceedings of the 2017 CHI Conference on Human Factors in Computing Systems, pp. 4097–4145, ACM, Denver, Colorado, USA, 2017, ISBN: 978-1-4503-4655-9.
@inproceedings{burgermaster17:role,
title = {The Role of Explanations in Casual Observational Learning About Nutrition},
author = {Marissa Burgermaster and Krzysztof Z. Gajos and Patricia Davidson and Lena Mamykina},
url = {/papers/burgermaster17role.pdf},
doi = {10.1145/3025453.3025874},
isbn = {978-1-4503-4655-9},
year = {2017},
date = {2017-01-01},
booktitle = {Proceedings of the 2017 CHI Conference on Human Factors in Computing Systems},
pages = {4097–4145},
publisher = {ACM},
address = {Denver, Colorado, USA},
series = {CHI '17},
abstract = {The ubiquity of internet-based nutrition information sharing indicates an opportunity to use social computing platforms to promote nutrition literacy and healthy nutritional choices. We conducted a series of experiments with unpaid volunteers using an online Nutrition Knowledge Test. The test asked participants to examine pairs of photographed meals and identify meals higher in a specific macronutrient (e.g., carbohydrate). After each answer, participants received no feedback on the accuracy of their answers, viewed proportions of peers choosing each response, received correctness feedback from an expert dietitian with or without expert-generated explanations, or received correctness feedback with crowd-generated explanations. The results showed that neither viewing peer responses nor correctness feedback alone improved learning. However, correctness feedback with explanations (i.e., modeling) led to significant learning gains, with no significant difference between explanations generated by experts or peers. This suggests the importance of explanations in social computing-based casual learning about nutrition and the potential for scaling this approach via crowdsourcing.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
The ubiquity of internet-based nutrition information sharing indicates an opportunity to use social computing platforms to promote nutrition literacy and healthy nutritional choices. We conducted a series of experiments with unpaid volunteers using an online Nutrition Knowledge Test. The test asked participants to examine pairs of photographed meals and identify meals higher in a specific macronutrient (e.g., carbohydrate). After each answer, participants received no feedback on the accuracy of their answers, viewed proportions of peers choosing each response, received correctness feedback from an expert dietitian with or without expert-generated explanations, or received correctness feedback with crowd-generated explanations. The results showed that neither viewing peer responses nor correctness feedback alone improved learning. However, correctness feedback with explanations (i.e., modeling) led to significant learning gains, with no significant difference between explanations generated by experts or peers. This suggests the importance of explanations in social computing-based casual learning about nutrition and the potential for scaling this approach via crowdsourcing.
Huber, Bernd; Reinecke, Katharina; Gajos, Krzysztof Z.
The Effect of Performance Feedback on Social Media Sharing at Volunteer-Based Online Experiment Platforms Proceedings Article
In: Proceedings of the 2017 CHI Conference on Human Factors in Computing Systems, pp. 1882–1886, ACM, Denver, Colorado, USA, 2017, ISBN: 978-1-4503-4655-9.
@inproceedings{huber17:effect,
title = {The Effect of Performance Feedback on Social Media Sharing at Volunteer-Based Online Experiment Platforms},
author = {Bernd Huber and Katharina Reinecke and Krzysztof Z. Gajos},
url = {/papers/huber17effect.pdf},
doi = {10.1145/3025453.3025553},
isbn = {978-1-4503-4655-9},
year = {2017},
date = {2017-01-01},
booktitle = {Proceedings of the 2017 CHI Conference on Human Factors in Computing Systems},
pages = {1882–1886},
publisher = {ACM},
address = {Denver, Colorado, USA},
series = {CHI '17},
abstract = {As an alternative to online labor markets, several platforms recruit unpaid online volunteers to participate in behavioral experiments that provide personalized feedback. These platforms rely on word-of-mouth sharing by previous participants for recruitment of new participants. We analyzed the impact of performance feedback provided at the end of an experiment on 81,131 participants' sharing behavior. We show that higher performing participants share significantly more. We also show that self-verification has a moderating effect: people who expected to do poorly are not affected by a high score, but people who expected to do as well as others or better, are. In a second experiment, we evaluate three distinct social comparison designs for the presentation of the results. As expected, the design that most emphasized participants' relative success led to most sharing. Contrary to our expectations, people who expected to do poorly benefited from the most optimistic social comparison more than participants who expected to do better than others.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
As an alternative to online labor markets, several platforms recruit unpaid online volunteers to participate in behavioral experiments that provide personalized feedback. These platforms rely on word-of-mouth sharing by previous participants for recruitment of new participants. We analyzed the impact of performance feedback provided at the end of an experiment on 81,131 participants’ sharing behavior. We show that higher performing participants share significantly more. We also show that self-verification has a moderating effect: people who expected to do poorly are not affected by a high score, but people who expected to do as well as others or better, are. In a second experiment, we evaluate three distinct social comparison designs for the presentation of the results. As expected, the design that most emphasized participants’ relative success led to most sharing. Contrary to our expectations, people who expected to do poorly benefited from the most optimistic social comparison more than participants who expected to do better than others.
Gajos, Krzysztof Z.; Chauncey, Krysta
The Influence of Personality Traits and Cognitive Load on the Use of Adaptive User Interfaces Proceedings Article
In: Proceedings of the 22Nd International Conference on Intelligent User Interfaces, pp. 301–306, ACM, Limassol, Cyprus, 2017, ISBN: 978-1-4503-4348-0.
@inproceedings{gajos17:personality,
title = {The Influence of Personality Traits and Cognitive Load on the Use of Adaptive User Interfaces},
author = {Krzysztof Z. Gajos and Krysta Chauncey},
url = {/papers/gajos17personality.pdf},
doi = {10.1145/3025171.3025192},
isbn = {978-1-4503-4348-0},
year = {2017},
date = {2017-01-01},
booktitle = {Proceedings of the 22Nd International Conference on Intelligent User Interfaces},
pages = {301–306},
publisher = {ACM},
address = {Limassol, Cyprus},
series = {IUI '17},
abstract = {One of the problems adaptive interfaces must solve is the issue of stability—users must be able to complete a familiar task reliably. Split Adaptive Interfaces, where a limited part of the screen contains copies of the interface elements predicted to be of immediate use, are one technique for resolving this difficulty. While prior work demonstrated that Split Adaptive Interfaces improve performance on average, the results of our study demonstrate systematic individual differences in the utilization of the adaptive features, which correlate with the stable user traits of Need for Cognition and Extraversion. Specifically, higher Need for Cognition (a willingness to undertake difficult mental activities) is correlated with increased utilization rates, while higher Extraversion (a general orientation towards seeking gratification from the external world) is negatively correlated with utilization rates. Our results also demonstrate a significant negative correlation between cognitive load induced by a secondary task and the utilization of the adaptive features. This effect, however, is very small (less than two percentage points). Together, these results provide additional evidence of the usefulness of the split adaptive interface approach and a negligible effect of additional cognitive load, but also demonstrate that the approach does not benefit all users equally.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
One of the problems adaptive interfaces must solve is the issue of stability—users must be able to complete a familiar task reliably. Split Adaptive Interfaces, where a limited part of the screen contains copies of the interface elements predicted to be of immediate use, are one technique for resolving this difficulty. While prior work demonstrated that Split Adaptive Interfaces improve performance on average, the results of our study demonstrate systematic individual differences in the utilization of the adaptive features, which correlate with the stable user traits of Need for Cognition and Extraversion. Specifically, higher Need for Cognition (a willingness to undertake difficult mental activities) is correlated with increased utilization rates, while higher Extraversion (a general orientation towards seeking gratification from the external world) is negatively correlated with utilization rates. Our results also demonstrate a significant negative correlation between cognitive load induced by a secondary task and the utilization of the adaptive features. This effect, however, is very small (less than two percentage points). Together, these results provide additional evidence of the usefulness of the split adaptive interface approach and a negligible effect of additional cognitive load, but also demonstrate that the approach does not benefit all users equally.
Booth, Serena; Tompkin, James; Pfister, Hanspeter; Waldo, Jim; Gajos, Krzysztof; Nagpal, Radhika
Piggybacking Robots: Human-Robot Overtrust in University Dormitory Security Proceedings Article
In: Proceedings of the 2017 ACM/IEEE International Conference on Human-Robot Interaction, pp. 426–434, ACM, Vienna, Austria, 2017, ISBN: 978-1-4503-4336-7.
@inproceedings{booth17:piggybacking,
title = {Piggybacking Robots: Human-Robot Overtrust in University Dormitory Security},
author = {Serena Booth and James Tompkin and Hanspeter Pfister and Jim Waldo and Krzysztof Gajos and Radhika Nagpal},
url = {/papers/booth17piggybacking.pdf},
doi = {10.1145/2909824.3020211},
isbn = {978-1-4503-4336-7},
year = {2017},
date = {2017-01-01},
booktitle = {Proceedings of the 2017 ACM/IEEE International Conference on Human-Robot Interaction},
pages = {426–434},
publisher = {ACM},
address = {Vienna, Austria},
series = {HRI '17},
abstract = {Can overtrust in robots compromise physical security? We conducted a series of experiments in which a robot positioned outside a secure-access student dormitory asked passersby to assist it to gain access. We found individual participants were comparably likely to assist the robot in exiting (40% assistance rate) as in entering (19%). When the robot was disguised as a food delivery agent for the fictional start-up Robot Grub, individuals were more likely to assist the robot in entering (76%). Groups of people were more likely than individuals to assist the robot in entering (71%). Lastly, we found participants who identified the robot as a bomb threat were just as likely to open the door (87%) as those who did not. Thus, we demonstrate that overtrust—the unfounded belief that the robot does not intend to deceive or carry risk—can represent a significant threat to physical security.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Can overtrust in robots compromise physical security? We conducted a series of experiments in which a robot positioned outside a secure-access student dormitory asked passersby to assist it to gain access. We found individual participants were comparably likely to assist the robot in exiting (40% assistance rate) as in entering (19%). When the robot was disguised as a food delivery agent for the fictional start-up Robot Grub, individuals were more likely to assist the robot in entering (76%). Groups of people were more likely than individuals to assist the robot in entering (71%). Lastly, we found participants who identified the robot as a bomb threat were just as likely to open the door (87%) as those who did not. Thus, we demonstrate that overtrust—the unfounded belief that the robot does not intend to deceive or carry risk—can represent a significant threat to physical security.
Oliveira, Nigini; Jun, Eunice; Croxson, Trevor; Gajos, Krzysztof Z.; Reinecke, Katharina
LabintheWild: How to Design Uncompensated, Feedback-driven Online Experiments Proceedings Article
In: Companion of the 2017 ACM Conference on Computer Supported Cooperative Work and Social Computing, pp. 25–28, ACM, Portland, Oregon, USA, 2017, ISBN: 978-1-4503-4688-7.
@inproceedings{oliveira17:labinthewild,
title = {LabintheWild: How to Design Uncompensated, Feedback-driven Online Experiments},
author = {Nigini Oliveira and Eunice Jun and Trevor Croxson and Krzysztof Z. Gajos and Katharina Reinecke},
url = {/papers/oliveira17labinthewild.pdf},
doi = {10.1145/3022198.3023267},
isbn = {978-1-4503-4688-7},
year = {2017},
date = {2017-01-01},
booktitle = {Companion of the 2017 ACM Conference on Computer Supported Cooperative Work and Social Computing},
pages = {25–28},
publisher = {ACM},
address = {Portland, Oregon, USA},
series = {CSCW '17 Companion},
abstract = {We present LabintheWild, an online experiment platform that provides participants with the opportunity to learn about themselves and compare themselves to others. In the past four years, LabintheWild has attracted approximately 3.5 million participants of diverse ages and education levels who came from more than 200 countries and regions. Leveraging our experience with LabintheWild, we show how researchers can design engaging and robust online experiments that provide personalized feedback. In particular, our interactive tutorial highlights challenges and best-practice guidelines for designing volunteer-based online experiments for diverse participant samples.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
We present LabintheWild, an online experiment platform that provides participants with the opportunity to learn about themselves and compare themselves to others. In the past four years, LabintheWild has attracted approximately 3.5 million participants of diverse ages and education levels who came from more than 200 countries and regions. Leveraging our experience with LabintheWild, we show how researchers can design engaging and robust online experiments that provide personalized feedback. In particular, our interactive tutorial highlights challenges and best-practice guidelines for designing volunteer-based online experiments for diverse participant samples.
Law, Edith; Gajos, Krzysztof Z.; Wiggins, Andrea; Gray, Mary L.; Williams, Alex
Crowdsourcing As a Tool for Research: Implications of Uncertainty Proceedings Article
In: Proceedings of the 2017 ACM Conference on Computer Supported Cooperative Work and Social Computing, pp. 1544–1561, ACM, Portland, Oregon, USA, 2017, ISBN: 978-1-4503-4335-0.
@inproceedings{law17:crowdsourcing,
title = {Crowdsourcing As a Tool for Research: Implications of Uncertainty},
author = {Edith Law and Krzysztof Z. Gajos and Andrea Wiggins and Mary L. Gray and Alex Williams},
url = {/papers/law17-crowdsourcing.pdf},
doi = {10.1145/2998181.2998197},
isbn = {978-1-4503-4335-0},
year = {2017},
date = {2017-01-01},
booktitle = {Proceedings of the 2017 ACM Conference on Computer Supported Cooperative Work and Social Computing},
pages = {1544–1561},
publisher = {ACM},
address = {Portland, Oregon, USA},
series = {CSCW '17},
abstract = {Numerous crowdsourcing platforms are now available to support research as well as commercial goals. However, crowdsourcing is not yet widely adopted by researchers for generating, processing or analyzing research data. This study develops a deeper understanding of the circumstances under which crowdsourcing is a useful, feasible or desirable tool for research, as well as the factors that may influence researchers' decisions around adopting crowdsourcing technology. We conducted semi-structured interviews with 18 researchers in diverse disciplines, spanning the humanities and sciences, to illuminate how research norms and practitioners' dispositions were related to uncertainties around research processes, data, knowledge, delegation and quality. The paper concludes with a discussion of the design implications for future crowdsourcing systems to support research.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Numerous crowdsourcing platforms are now available to support research as well as commercial goals. However, crowdsourcing is not yet widely adopted by researchers for generating, processing or analyzing research data. This study develops a deeper understanding of the circumstances under which crowdsourcing is a useful, feasible or desirable tool for research, as well as the factors that may influence researchers’ decisions around adopting crowdsourcing technology. We conducted semi-structured interviews with 18 researchers in diverse disciplines, spanning the humanities and sciences, to illuminate how research norms and practitioners’ dispositions were related to uncertainties around research processes, data, knowledge, delegation and quality. The paper concludes with a discussion of the design implications for future crowdsourcing systems to support research.
Boronow, Katherine E.; Susmann, Herbert P.; Gajos, Krzysztof Z.; Rudel, Ruthann A.; Arnold, Kenneth C.; Brown, Phil; Morello-Frosch, Rachel; Havas, Laurie; Brody, Julia Green
DERBI: A Digital Method to Help Researchers Offer “Right-to-Know” Personal Exposure Results Journal Article
In: Environmental Health Perspectives, vol. 125, no. 2, 2017.
@article{boronow17:derbi,
title = {DERBI: A Digital Method to Help Researchers Offer ``Right-to-Know'' Personal Exposure Results},
author = {Katherine E. Boronow and Herbert P. Susmann and Krzysztof Z. Gajos and Ruthann A. Rudel and Kenneth C. Arnold and Phil Brown and Rachel Morello-Frosch and Laurie Havas and Julia Green Brody},
url = {https://ehp.niehs.nih.gov/EHP702/},
year = {2017},
date = {2017-02-01},
journal = {Environmental Health Perspectives},
volume = {125},
number = {2},
abstract = {Researchers and clinicians in environmental health and medicine increasingly show respect for participants and patients by involving them in decision-making. In this context, the return of personal results to study participants is becoming ethical best practice, and many participants now expect to see their data. However, researchers often lack the time and expertise required for report-back, especially as studies measure greater numbers of analytes, including many without clear health guidelines. Our goal is to demonstrate how a prototype digital method, the Digital Exposure Report-Back Interface (DERBI), can reduce practical barriers to high-quality report-back. DERBI uses decision rules to automate the production of personalized summaries of notable results and generates individual results graphs with comparisons to the study group and benchmark populations. Reports discuss potential sources of chemical exposure, what is known and unknown about health effects, strategies for exposure reduction, and study-wide findings. Researcher tools promote discovery by drawing attention to patterns of high exposure and offer novel ways to increase participant engagement. DERBI reports have been field tested in two studies. Digital methods like DERBI reduce practical barriers to report-back, enabling researchers to meet their ethical obligations and participants to get knowledge they can use to make informed choices.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Researchers and clinicians in environmental health and medicine increasingly show respect for participants and patients by involving them in decision-making. In this context, the return of personal results to study participants is becoming ethical best practice, and many participants now expect to see their data. However, researchers often lack the time and expertise required for report-back, especially as studies measure greater numbers of analytes, including many without clear health guidelines. Our goal is to demonstrate how a prototype digital method, the Digital Exposure Report-Back Interface (DERBI), can reduce practical barriers to high-quality report-back. DERBI uses decision rules to automate the production of personalized summaries of notable results and generates individual results graphs with comparisons to the study group and benchmark populations. Reports discuss potential sources of chemical exposure, what is known and unknown about health effects, strategies for exposure reduction, and study-wide findings. Researcher tools promote discovery by drawing attention to patterns of high exposure and offer novel ways to increase participant engagement. DERBI reports have been field tested in two studies. Digital methods like DERBI reduce practical barriers to report-back, enabling researchers to meet their ethical obligations and participants to get knowledge they can use to make informed choices.
2016
Siangliulue, Pao; Chan, Joel; Dow, Steven P.; Gajos, Krzysztof Z.
IdeaHound: Improving Large-scale Collaborative Ideation with Crowd-Powered Real-time Semantic Modeling Proceedings Article
In: Proceedings of the 29th Annual Symposium on User Interface Software and Technology, pp. 609–624, ACM, Tokyo, Japan, 2016, ISBN: 978-1-4503-4189-9.
@inproceedings{siangliulue16:ideahound-uist,
title = {IdeaHound: Improving Large-scale Collaborative Ideation with Crowd-Powered Real-time Semantic Modeling},
author = {Pao Siangliulue and Joel Chan and Steven P. Dow and Krzysztof Z. Gajos},
url = {/papers/siangliulue16ideahound-uist.pdf},
doi = {10.1145/2984511.2984578},
isbn = {978-1-4503-4189-9},
year = {2016},
date = {2016-01-01},
booktitle = {Proceedings of the 29th Annual Symposium on User Interface Software and Technology},
pages = {609–624},
publisher = {ACM},
address = {Tokyo, Japan},
series = {UIST '16},
abstract = {Prior work on creativity support tools demonstrates how a computational semantic model of a solution space can enable interventions that substantially improve the number, quality and diversity of ideas. However, automated semantic modeling often falls short when people contribute short text snippets or sketches. Innovation platforms can employ humans to provide semantic judgments to construct a semantic model, but this relies on external workers completing a large number of tedious micro tasks. This requirement threatens both accuracy (external workers may lack expertise and context to make accurate semantic judgments) and scalability (external workers are costly). In this paper, we introduce IdeaHound, an ideation system that seamlessly integrates the task of defining semantic relationships among ideas into the primary task of idea generation. The system combines implicit human actions with machine learning to create a computational semantic model of the emerging solution space. The integrated nature of these judgments allows IdeaHound to leverage the expertise and efforts of participants who are already motivated to contribute to idea generation, overcoming the issues of scalability inherent to existing approaches. Our results show that participants were equally willing to use (and just as productive using) IdeaHound compared to a conventional platform that did not require organizing ideas. Our integrated crowdsourcing approach also creates a more accurate semantic model than an existing crowdsourced approach (performed by external crowds). We demonstrate how this model enables helpful creative interventions: providing diverse inspirational examples, providing similar ideas for a given idea and providing a visual overview of the solution space.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Prior work on creativity support tools demonstrates how a computational semantic model of a solution space can enable interventions that substantially improve the number, quality and diversity of ideas. However, automated semantic modeling often falls short when people contribute short text snippets or sketches. Innovation platforms can employ humans to provide semantic judgments to construct a semantic model, but this relies on external workers completing a large number of tedious micro tasks. This requirement threatens both accuracy (external workers may lack expertise and context to make accurate semantic judgments) and scalability (external workers are costly). In this paper, we introduce IdeaHound, an ideation system that seamlessly integrates the task of defining semantic relationships among ideas into the primary task of idea generation. The system combines implicit human actions with machine learning to create a computational semantic model of the emerging solution space. The integrated nature of these judgments allows IdeaHound to leverage the expertise and efforts of participants who are already motivated to contribute to idea generation, overcoming the issues of scalability inherent to existing approaches. Our results show that participants were equally willing to use (and just as productive using) IdeaHound compared to a conventional platform that did not require organizing ideas. Our integrated crowdsourcing approach also creates a more accurate semantic model than an existing crowdsourced approach (performed by external crowds). We demonstrate how this model enables helpful creative interventions: providing diverse inspirational examples, providing similar ideas for a given idea and providing a visual overview of the solution space.
Arnold, Kenneth C.; Gajos, Krzysztof Z.; Kalai, Adam T.
On Suggesting Phrases vs. Predicting Words for Mobile Text Composition Proceedings Article
In: Proceedings of the 29th Annual Symposium on User Interface Software and Technology, pp. 603–608, ACM, Tokyo, Japan, 2016, ISBN: 978-1-4503-4189-9.
@inproceedings{arnold16:suggesting,
title = {On Suggesting Phrases vs. Predicting Words for Mobile Text Composition},
author = {Kenneth C. Arnold and Krzysztof Z. Gajos and Adam T. Kalai},
url = {/papers/arnold16suggesting.pdf},
doi = {10.1145/2984511.2984584},
isbn = {978-1-4503-4189-9},
year = {2016},
date = {2016-01-01},
booktitle = {Proceedings of the 29th Annual Symposium on User Interface Software and Technology},
pages = {603–608},
publisher = {ACM},
address = {Tokyo, Japan},
series = {UIST '16},
abstract = {A system capable of suggesting multi-word phrases while someone is writing could supply ideas about content and phrasing and allow those ideas to be inserted efficiently. Meanwhile, statistical language modeling has provided various approaches to predicting phrases that users type. We introduce a simple extension to the familiar mobile keyboard suggestion interface that presents phrase suggestions that can be accepted by a repeated-tap gesture. In an extended composition task, we found that phrases were interpreted as suggestions that affected the content of what participants wrote more than conventional single-word suggestions, which were interpreted as predictions. We highlight a design challenge: how can a phrase suggestion system make valuable suggestions rather than just accurate predictions?},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
A system capable of suggesting multi-word phrases while someone is writing could supply ideas about content and phrasing and allow those ideas to be inserted efficiently. Meanwhile, statistical language modeling has provided various approaches to predicting phrases that users type. We introduce a simple extension to the familiar mobile keyboard suggestion interface that presents phrase suggestions that can be accepted by a repeated-tap gesture. In an extended composition task, we found that phrases were interpreted as suggestions that affected the content of what participants wrote more than conventional single-word suggestions, which were interpreted as predictions. We highlight a design challenge: how can a phrase suggestion system make valuable suggestions rather than just accurate predictions?
Kim, Sunyoung; Gajos, Krzysztof Z.; Muller, Michael; Grosz, Barbara J.
Acceptance of Mobile Technology by Older Adults: A Preliminary Study Proceedings Article
In: Proceedings of the 18th International Conference on Human-Computer Interaction with Mobile Devices and Services, pp. 147–157, ACM, Florence, Italy, 2016, ISBN: 978-1-4503-4408-1.
@inproceedings{kim16:acceptance,
title = {Acceptance of Mobile Technology by Older Adults: A Preliminary Study},
author = {Sunyoung Kim and Krzysztof Z. Gajos and Michael Muller and Barbara J. Grosz},
url = {/papers/skim16acceptance.pdf},
doi = {10.1145/2935334.2935380},
isbn = {978-1-4503-4408-1},
year = {2016},
date = {2016-01-01},
booktitle = {Proceedings of the 18th International Conference on Human-Computer Interaction with Mobile Devices and Services},
pages = {147–157},
publisher = {ACM},
address = {Florence, Italy},
series = {MobileHCI '16},
abstract = {Mobile technologies offer the potential for enhanced healthcare, especially by supporting self-management of chronic care. For these technologies to impact chronic care, they need to work for older adults, because the majority of people with chronic conditions are older. A major challenge remains: integrating the appropriate use of such technologies into the lives of older adults. We investigated how older adults would accept mobile technologies by interviewing two groups of older adults (technology adopters and non-adopters who aged 60+) about their experiences and perspectives to mobile technologies. Our preliminary results indicate that there is an additional phase, the intention to learn, and three relating factors, self-efficacy, conversion readiness, and peer support, that significantly influence the acceptance of mobile technologies among the participants, but are not represented in the existing models. With these findings, we propose a tentative theoretical model that extends the existing theories to explain the ways in which our participants came to accept mobile technologies. Future work should investigate the validity of the proposed model by testing our findings against younger people.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Mobile technologies offer the potential for enhanced healthcare, especially by supporting self-management of chronic care. For these technologies to impact chronic care, they need to work for older adults, because the majority of people with chronic conditions are older. A major challenge remains: integrating the appropriate use of such technologies into the lives of older adults. We investigated how older adults would accept mobile technologies by interviewing two groups of older adults (technology adopters and non-adopters who aged 60+) about their experiences and perspectives to mobile technologies. Our preliminary results indicate that there is an additional phase, the intention to learn, and three relating factors, self-efficacy, conversion readiness, and peer support, that significantly influence the acceptance of mobile technologies among the participants, but are not represented in the existing models. With these findings, we propose a tentative theoretical model that extends the existing theories to explain the ways in which our participants came to accept mobile technologies. Future work should investigate the validity of the proposed model by testing our findings against younger people.
Amir, Ofra; Grosz, Barbara; Gajos, Krzysztof Z.
Mutual Influence Potential Networks: Enabling Information Sharing in Loosely-Coupled Extended-Duration Teamwork Proceedings Article
In: Proceedings of IJCAI’16, 2016.
@inproceedings{amir16:mutual,
title = {Mutual Influence Potential Networks: Enabling Information Sharing in Loosely-Coupled Extended-Duration Teamwork},
author = {Ofra Amir and Barbara Grosz and Krzysztof Z. Gajos},
url = {/papers/amir16mutual.pdf},
year = {2016},
date = {2016-01-01},
booktitle = {Proceedings of IJCAI'16},
abstract = {Complex collaborative activities such as treating patients, co-authoring documents and developing software are often characterized by teamwork that is loosely coupled and extends in time. To remain coordinated and avoid conflicts, team members need to identify dependencies between their activities - which though loosely coupled may interact - and share information appropriately. The loose-coupling of tasks increases the difficulty of identifying dependencies, with the result that team members often lack important information or are overwhelmed by irrelevant information. This paper formalizes a new multi-agent systems problem, Information Sharing in Loosely-Coupled Extended-Duration Teamwork (ISLET). It defines a new representation, Mutual Influence Potential Networks (MIP-Nets) and an algorithm, MIP-DOI, that uses this representation to determine the information that is most relevant to each team member. Importantly, because the extended duration of the teamwork precludes team members developing complete plans in advance, the MIP-Nets approach, unlike prior work on information sharing, does not rely on a priori knowledge of a team's possible plans. Instead, it models collaboration patterns and dependencies among people and their activities based on team-members' interactions. Empirical evaluations show that this approach is able to learn collaboration patterns and identify relevant information to share with team members.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Complex collaborative activities such as treating patients, co-authoring documents and developing software are often characterized by teamwork that is loosely coupled and extends in time. To remain coordinated and avoid conflicts, team members need to identify dependencies between their activities – which though loosely coupled may interact – and share information appropriately. The loose-coupling of tasks increases the difficulty of identifying dependencies, with the result that team members often lack important information or are overwhelmed by irrelevant information. This paper formalizes a new multi-agent systems problem, Information Sharing in Loosely-Coupled Extended-Duration Teamwork (ISLET). It defines a new representation, Mutual Influence Potential Networks (MIP-Nets) and an algorithm, MIP-DOI, that uses this representation to determine the information that is most relevant to each team member. Importantly, because the extended duration of the teamwork precludes team members developing complete plans in advance, the MIP-Nets approach, unlike prior work on information sharing, does not rely on a priori knowledge of a team’s possible plans. Instead, it models collaboration patterns and dependencies among people and their activities based on team-members’ interactions. Empirical evaluations show that this approach is able to learn collaboration patterns and identify relevant information to share with team members.
Mamykina, Lena; Smyth, Thomas N.; Dimond, Jill P.; Gajos, Krzysztof Z.
Learning From the Crowd: Observational Learning in Crowdsourcing Communities Proceedings Article
In: Proceedings of the 2016 CHI Conference on Human Factors in Computing Systems, pp. 2635–2644, ACM, Santa Clara, California, USA, 2016, ISBN: 978-1-4503-3362-7.
@inproceedings{mamykina16:learning,
title = {Learning From the Crowd: Observational Learning in Crowdsourcing Communities},
author = {Lena Mamykina and Thomas N. Smyth and Jill P. Dimond and Krzysztof Z. Gajos},
url = {/papers/mamykina16learning.pdf},
doi = {10.1145/2858036.2858560},
isbn = {978-1-4503-3362-7},
year = {2016},
date = {2016-01-01},
booktitle = {Proceedings of the 2016 CHI Conference on Human Factors in Computing Systems},
pages = {2635–2644},
publisher = {ACM},
address = {Santa Clara, California, USA},
series = {CHI '16},
abstract = {Crowd work provides solutions to complex problems effectively, efficiently, and at low cost. Previous research showed that feedback, particularly correctness feedback can help crowd workers improve their performance; yet such feedback, particularly when generated by experts, is costly and difficult to scale. In our research we investigate approaches to facilitating continuous observational learning in crowdsourcing communities. In a study conducted with workers on Amazon Mechanical Turk, we asked workers to complete a set of tasks identifying nutritional composition of different meals. We examined workers' accuracy gains after being exposed to expert-generated feedback and to two types of peer-generated feedback: direct accuracy assessment with explanations of errors, and a comparison with solutions generated by other workers. The study further confirmed that expert-generated feedback is a powerful mechanism for facilitating learning and leads to significant gains in accuracy. However, the study also showed that comparing one's own solutions with a variety of solutions suggested by others and their comparative frequencies leads to significant gains in accuracy. This solution is particularly attractive because of its low cost, minimal impact on time and cost of job completion, and high potential for adoption by a variety of crowdsourcing platforms.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Crowd work provides solutions to complex problems effectively, efficiently, and at low cost. Previous research showed that feedback, particularly correctness feedback can help crowd workers improve their performance; yet such feedback, particularly when generated by experts, is costly and difficult to scale. In our research we investigate approaches to facilitating continuous observational learning in crowdsourcing communities. In a study conducted with workers on Amazon Mechanical Turk, we asked workers to complete a set of tasks identifying nutritional composition of different meals. We examined workers’ accuracy gains after being exposed to expert-generated feedback and to two types of peer-generated feedback: direct accuracy assessment with explanations of errors, and a comparison with solutions generated by other workers. The study further confirmed that expert-generated feedback is a powerful mechanism for facilitating learning and leads to significant gains in accuracy. However, the study also showed that comparing one’s own solutions with a variety of solutions suggested by others and their comparative frequencies leads to significant gains in accuracy. This solution is particularly attractive because of its low cost, minimal impact on time and cost of job completion, and high potential for adoption by a variety of crowdsourcing platforms.
Zhou, Sharon; Livingston, Ivy J.; Schiefsky, Mark; Shieber, Stuart M.; Gajos, Krzysztof Z.
Ingenium: Engaging Novice Students with Latin Grammar Proceedings Article
In: Proceedings of the 2016 CHI Conference on Human Factors in Computing Systems, pp. 944–956, ACM, Santa Clara, California, USA, 2016, ISBN: 978-1-4503-3362-7.
@inproceedings{zhou16:ingenium,
title = {Ingenium: Engaging Novice Students with Latin Grammar},
author = {Sharon Zhou and Ivy J. Livingston and Mark Schiefsky and Stuart M. Shieber and Krzysztof Z. Gajos},
url = {/papers/zhou16ingenium.pdf},
doi = {10.1145/2858036.2858239},
isbn = {978-1-4503-3362-7},
year = {2016},
date = {2016-01-01},
booktitle = {Proceedings of the 2016 CHI Conference on Human Factors in Computing Systems},
pages = {944–956},
publisher = {ACM},
address = {Santa Clara, California, USA},
series = {CHI '16},
abstract = {Reading Latin poses many difficulties for English speakers, because they are accustomed to relying on word order to determine the roles of words in a sentence. In Latin, the grammatical form of a word, and not its position, is responsible for determining the word's function in a sentence. It has proven challenging to develop pedagogical techniques that successfully draw students' attention to the grammar of Latin and that students find engaging enough to use. Building on some of the most promising prior work in Latin instruction–the Michigan Latin approach–and on the insights underlying block-based programming languages used to teach children the basics of computer science, we developed Ingenium. Ingenium uses abstract puzzle blocks to communicate grammatical concepts. Engaging students in grammatical reflection, Ingenium succeeds when students are able to effectively decipher the meaning of Latin sentences. We adapted Ingenium to be used for two standard classroom activities: sentence translations and fill-in-the-blank exercises. We evaluated Ingenium with 67 novice Latin students in universities across the United States. When using Ingenium, participants opted to perform more optional exercises, completed translation exercises with significantly fewer errors related to word order and errors overall, as well as reported higher levels of engagement and attention to grammar than when using a traditional text-based interface.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Reading Latin poses many difficulties for English speakers, because they are accustomed to relying on word order to determine the roles of words in a sentence. In Latin, the grammatical form of a word, and not its position, is responsible for determining the word’s function in a sentence. It has proven challenging to develop pedagogical techniques that successfully draw students’ attention to the grammar of Latin and that students find engaging enough to use. Building on some of the most promising prior work in Latin instruction–the Michigan Latin approach–and on the insights underlying block-based programming languages used to teach children the basics of computer science, we developed Ingenium. Ingenium uses abstract puzzle blocks to communicate grammatical concepts. Engaging students in grammatical reflection, Ingenium succeeds when students are able to effectively decipher the meaning of Latin sentences. We adapted Ingenium to be used for two standard classroom activities: sentence translations and fill-in-the-blank exercises. We evaluated Ingenium with 67 novice Latin students in universities across the United States. When using Ingenium, participants opted to perform more optional exercises, completed translation exercises with significantly fewer errors related to word order and errors overall, as well as reported higher levels of engagement and attention to grammar than when using a traditional text-based interface.
Law, Edith; Yin, Ming; Goh, Joslin; Chen, Kevin; Terry, Michael A.; Gajos, Krzysztof Z.
Curiosity Killed the Cat, but Makes Crowdwork Better Proceedings Article
In: Proceedings of the 2016 CHI Conference on Human Factors in Computing Systems, pp. 4098–4110, ACM, Santa Clara, California, USA, 2016, ISBN: 978-1-4503-3362-7.
@inproceedings{law16:curiosity,
title = {Curiosity Killed the Cat, but Makes Crowdwork Better},
author = {Edith Law and Ming Yin and Joslin Goh and Kevin Chen and Michael A. Terry and Krzysztof Z. Gajos},
url = {/papers/law16curiosity.pdf},
doi = {10.1145/2858036.2858144},
isbn = {978-1-4503-3362-7},
year = {2016},
date = {2016-01-01},
booktitle = {Proceedings of the 2016 CHI Conference on Human Factors in Computing Systems},
pages = {4098–4110},
publisher = {ACM},
address = {Santa Clara, California, USA},
series = {CHI '16},
abstract = {Crowdsourcing systems are designed to elicit help from humans to accomplish tasks that are still difficult for computers. How to motivate workers to stay longer and/or perform better in crowdsourcing systems is a critical question for designers. Previous work have explored different motivational frameworks, both extrinsic and intrinsic. In this work, we examine the potential for curiosity as a new type of intrinsic motivational driver to incentivize crowd workers. We design crowdsourcing task interfaces that explicitly incorporate mechanisms to induce curiosity and conduct a set of experiments on Amazon's Mechanical Turk. Our experiment results show that curiosity interventions improve worker retention without degrading performance, and the magnitude of the effects are influenced by both the personal characteristics of the worker and the nature of the task.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Crowdsourcing systems are designed to elicit help from humans to accomplish tasks that are still difficult for computers. How to motivate workers to stay longer and/or perform better in crowdsourcing systems is a critical question for designers. Previous work have explored different motivational frameworks, both extrinsic and intrinsic. In this work, we examine the potential for curiosity as a new type of intrinsic motivational driver to incentivize crowd workers. We design crowdsourcing task interfaces that explicitly incorporate mechanisms to induce curiosity and conduct a set of experiments on Amazon’s Mechanical Turk. Our experiment results show that curiosity interventions improve worker retention without degrading performance, and the magnitude of the effects are influenced by both the personal characteristics of the worker and the nature of the task.
Kim, Sunyoung; Iravantchi, Yasha; Gajos, Krzysztof Z.; Grosz, Barbara
SwellFit: a Wearable Sensor for Patients with Congestive Heart Failure Proceedings Article
In: Proceedings of the Workshop on Interactive Systems in Healthcare (WISH) 2016, 2016.
@inproceedings{kim16:SwellFit,
title = {SwellFit: a Wearable Sensor for Patients with Congestive Heart Failure},
author = {Sunyoung Kim and Yasha Iravantchi and Krzysztof Z. Gajos and Barbara Grosz},
url = {/papers/skim16swellfit.pdf},
year = {2016},
date = {2016-01-01},
booktitle = {Proceedings of the Workshop on Interactive Systems in Healthcare (WISH) 2016},
abstract = {Congestive heart failure (CHF) is the most common reason for hospitalization in people aged 65 years and older in the United States. Especially, a very high rate of unplanned readmissions within 30 days of discharge due to CHF occurs a heavy associated financial burden and degraded quality of care. To effectively prevent CHF readmission and improve overall care, this paper proposes SwellFit, a novel wearable sensor that helps outpatients to proactively monitor a physical symptom of worsening CHF, ankle swelling. Using a flex sensor, SwellFit monitors changes in ankle curvature as an indication of swelling. A pilot test with 4 adults showed that SwellFit successfully distinguish noise data and motion artifacts from ankle swelling. Collaborating with cardiac experts, we are currently planning to test SwellFit with hospitalized CHF patients who have swelling conditions to demonstrate the feasibility and reliability of SwellFit on detecting swelling from ankle curvature.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Congestive heart failure (CHF) is the most common reason for hospitalization in people aged 65 years and older in the United States. Especially, a very high rate of unplanned readmissions within 30 days of discharge due to CHF occurs a heavy associated financial burden and degraded quality of care. To effectively prevent CHF readmission and improve overall care, this paper proposes SwellFit, a novel wearable sensor that helps outpatients to proactively monitor a physical symptom of worsening CHF, ankle swelling. Using a flex sensor, SwellFit monitors changes in ankle curvature as an indication of swelling. A pilot test with 4 adults showed that SwellFit successfully distinguish noise data and motion artifacts from ankle swelling. Collaborating with cardiac experts, we are currently planning to test SwellFit with hospitalized CHF patients who have swelling conditions to demonstrate the feasibility and reliability of SwellFit on detecting swelling from ankle curvature.
Siangliulue, Pao; Chan, Joel; Huber, Bernd; Dow, Steven P.; Gajos, Krzysztof Z.
IdeaHound: Self-sustainable Idea Generation in Creative Online Communities Proceedings Article
In: Proceedings of the 19th ACM Conference on Computer Supported Cooperative Work and Social Computing Companion, pp. 98–101, ACM, San Francisco, California, USA, 2016, ISBN: 978-1-4503-3950-6.
@inproceedings{siangliulue16:ideahound,
title = {IdeaHound: Self-sustainable Idea Generation in Creative Online Communities},
author = {Pao Siangliulue and Joel Chan and Bernd Huber and Steven P. Dow and Krzysztof Z. Gajos},
url = {http://doi.acm.org/10.1145/2818052.2874335},
doi = {10.1145/2818052.2874335},
isbn = {978-1-4503-3950-6},
year = {2016},
date = {2016-01-01},
booktitle = {Proceedings of the 19th ACM Conference on Computer Supported Cooperative Work and Social Computing Companion},
pages = {98–101},
publisher = {ACM},
address = {San Francisco, California, USA},
series = {CSCW '16 Companion},
abstract = {One main challenge in large creative online communities is helping their members find inspirational ideas from a large pool of ideas. A high-level approach to address this challenge is to create a synthesis of emerging solution space that can be used to provide participants with creative and diverse inspirational ideas of others. Existing approaches to generate the synthesis of solution space either require community members to engage in tasks that detract from the main activity of generating ideas or depend on external crowd workers to help organize the ideas. We built IdeaHound a collaborative idea generation system that demonstrates an alternative "organic" human computation approach, where community members (rather than external crowds) contribute feedback about ideas as a byproduct of an activity that naturally integrates into the ideation process. This feedback in turn helps the community identify diverse inspirational ideas that can prompt community members to generate more high-quality and diverse ideas.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
One main challenge in large creative online communities is helping their members find inspirational ideas from a large pool of ideas. A high-level approach to address this challenge is to create a synthesis of emerging solution space that can be used to provide participants with creative and diverse inspirational ideas of others. Existing approaches to generate the synthesis of solution space either require community members to engage in tasks that detract from the main activity of generating ideas or depend on external crowd workers to help organize the ideas. We built IdeaHound a collaborative idea generation system that demonstrates an alternative “organic” human computation approach, where community members (rather than external crowds) contribute feedback about ideas as a byproduct of an activity that naturally integrates into the ideation process. This feedback in turn helps the community identify diverse inspirational ideas that can prompt community members to generate more high-quality and diverse ideas.
Huang, Cheng-Zhi Anna; Duvenaud, David; Gajos, Krzysztof Z.
ChordRipple: Recommending Chords to Help Novice Composers Go Beyond the Ordinary Proceedings Article
In: Proceedings of the 21st International Conference on Intelligent User Interfaces, pp. 241–250, ACM, Sonoma, California, USA, 2016, ISBN: 978-1-4503-4137-0.
@inproceedings{huang16chordripple,
title = {ChordRipple: Recommending Chords to Help Novice Composers Go Beyond the Ordinary},
author = {Cheng-Zhi Anna Huang and David Duvenaud and Krzysztof Z. Gajos},
url = {/papers/huang16chordripple.pdf},
doi = {10.1145/2856767.2856792},
isbn = {978-1-4503-4137-0},
year = {2016},
date = {2016-01-01},
booktitle = {Proceedings of the 21st International Conference on Intelligent User Interfaces},
pages = {241–250},
publisher = {ACM},
address = {Sonoma, California, USA},
series = {IUI '16},
abstract = {Novice composers often find it difficult to go beyond common chord progressions. To make it easier for composers to experiment with radical chord choices, we built a creativity support tool, CHORDRIPPLE, which makes chord recommendations that aim to be both diverse and appropriate to the current context. Composers can use it to help select the next chord, or to replace sequences of chords in an internally consistent manner.
To make such recommendations, we adapt a neural network model from natural language processing known as WORD2VEC to the music domain. This model learns chord embeddings from a corpus of chord sequences, placing chords nearby when they are used in similar contexts. The learned embeddings support creative substitutions between chords, and also exhibit topological properties that correspond to musical structure. For example, the major and minor chords are both arranged in the latent space in shapes corresponding to the circle-of-fifths.
Our structured observations with 14 music students show that the tool helped them explore a wider palette of chords, and to make "big jumps in just a few chords". It gave them "new ideas of ways to move forward in the piece", not just on a chord-to-chord level but also between phrases. Our controlled studies with 9 more music students show that more adventurous chords are adopted when composing with CHORDRIPPLE.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Novice composers often find it difficult to go beyond common chord progressions. To make it easier for composers to experiment with radical chord choices, we built a creativity support tool, CHORDRIPPLE, which makes chord recommendations that aim to be both diverse and appropriate to the current context. Composers can use it to help select the next chord, or to replace sequences of chords in an internally consistent manner.
To make such recommendations, we adapt a neural network model from natural language processing known as WORD2VEC to the music domain. This model learns chord embeddings from a corpus of chord sequences, placing chords nearby when they are used in similar contexts. The learned embeddings support creative substitutions between chords, and also exhibit topological properties that correspond to musical structure. For example, the major and minor chords are both arranged in the latent space in shapes corresponding to the circle-of-fifths.
Our structured observations with 14 music students show that the tool helped them explore a wider palette of chords, and to make “big jumps in just a few chords”. It gave them “new ideas of ways to move forward in the piece”, not just on a chord-to-chord level but also between phrases. Our controlled studies with 9 more music students show that more adventurous chords are adopted when composing with CHORDRIPPLE.
To make such recommendations, we adapt a neural network model from natural language processing known as WORD2VEC to the music domain. This model learns chord embeddings from a corpus of chord sequences, placing chords nearby when they are used in similar contexts. The learned embeddings support creative substitutions between chords, and also exhibit topological properties that correspond to musical structure. For example, the major and minor chords are both arranged in the latent space in shapes corresponding to the circle-of-fifths.
Our structured observations with 14 music students show that the tool helped them explore a wider palette of chords, and to make “big jumps in just a few chords”. It gave them “new ideas of ways to move forward in the piece”, not just on a chord-to-chord level but also between phrases. Our controlled studies with 9 more music students show that more adventurous chords are adopted when composing with CHORDRIPPLE.
Williams, Joseph Jay; Kim, Juho; Rafferty, Anna; Maldonado, Samuel; Gajos, Krzysztof Z.; Lasecki, Walter S.; Heffernan, Neil
AXIS: Generating Explanations at Scale with Learnersourcing and Machine Learning Proceedings Article
In: Proceedings of the Third (2016) ACM Conference on Learning @ Scale, pp. 379–388, ACM, Edinburgh, Scotland, UK, 2016, ISBN: 978-1-4503-3726-7.
@inproceedings{williams16:axis,
title = {AXIS: Generating Explanations at Scale with Learnersourcing and Machine Learning},
author = {Joseph Jay Williams and Juho Kim and Anna Rafferty and Samuel Maldonado and Krzysztof Z. Gajos and Walter S. Lasecki and Neil Heffernan},
url = {/papers/williams16axis.pdf},
doi = {10.1145/2876034.2876042},
isbn = {978-1-4503-3726-7},
year = {2016},
date = {2016-01-01},
booktitle = {Proceedings of the Third (2016) ACM Conference on Learning @ Scale},
pages = {379–388},
publisher = {ACM},
address = {Edinburgh, Scotland, UK},
series = {L@S '16},
abstract = {While explanations may help people learn by providing information about why an answer is correct, many problems on online platforms lack high-quality explanations. This paper presents AXIS (Adaptive eXplanation Improvement System), a system for obtaining explanations. AXIS asks learners to generate, revise, and evaluate explanations as they solve a problem, and then uses machine learning to dynamically determine which explanation to present to a future learner, based on previous learners' collective input. Results from a case study deployment and a randomized experiment demonstrate that AXIS elicits and identifies explanations that learners find helpful. Providing explanations from AXIS also objectively enhanced learning, when compared to the default practice where learners solved problems and received answers without explanations. The rated quality and learning benefit of AXIS explanations did not differ from explanations generated by an experienced instructor.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
While explanations may help people learn by providing information about why an answer is correct, many problems on online platforms lack high-quality explanations. This paper presents AXIS (Adaptive eXplanation Improvement System), a system for obtaining explanations. AXIS asks learners to generate, revise, and evaluate explanations as they solve a problem, and then uses machine learning to dynamically determine which explanation to present to a future learner, based on previous learners’ collective input. Results from a case study deployment and a randomized experiment demonstrate that AXIS elicits and identifies explanations that learners find helpful. Providing explanations from AXIS also objectively enhanced learning, when compared to the default practice where learners solved problems and received answers without explanations. The rated quality and learning benefit of AXIS explanations did not differ from explanations generated by an experienced instructor.
2015
Siangliulue, Pao; Chan, Joel; Gajos, Krzysztof Z.; Dow, Steven P.
Providing Timely Examples Improves the Quantity and Quality of Generated Ideas Proceedings Article
In: Proceedings of the 2015 ACM SIGCHI Conference on Creativity and Cognition, pp. 83–92, ACM, Glasgow, United Kingdom, 2015, ISBN: 978-1-4503-3598-0.
@inproceedings{siangliulue15:providing,
title = {Providing Timely Examples Improves the Quantity and Quality of Generated Ideas},
author = {Pao Siangliulue and Joel Chan and Krzysztof Z. Gajos and Steven P. Dow},
url = {/papers/siangliulue15providing.pdf},
doi = {10.1145/2757226.2757230},
isbn = {978-1-4503-3598-0},
year = {2015},
date = {2015-01-01},
booktitle = {Proceedings of the 2015 ACM SIGCHI Conference on Creativity and Cognition},
pages = {83–92},
publisher = {ACM},
address = {Glasgow, United Kingdom},
series = {C&C '15},
abstract = {Emerging online ideation platforms with thousands of example ideas provide an important resource for creative production. But how can ideators best use these examples to create new innovations? Recent work has suggested that not just the choice of examples, but also the timing of their delivery can impact creative outcomes. Building on existing cognitive theories of creative insight, we hypothesize that people are likely to benefit from examples when they run out of ideas. We explore two example delivery mechanisms that test this hypothesis: 1) a system that proactively provides examples when a user appears to have run out of ideas, and 2) a system that provides examples when a user explicitly requests them. Our online experiment (N=97) compared these two mechanisms against two baselines: providing no examples and automatically showing examples at a regular interval. Participants who requested examples themselves generated ideas that were rated the most novel by external evaluators. Participants who received ideas automatically when they appeared to be stuck produced the most ideas. Importantly, participants who received examples at a regular interval generated fewer ideas than participants who received no examples, suggesting that mere access to examples is not sufficient for creative inspiration. These results emphasize the importance of the timing of example delivery. Insights from this study can inform the design of collective ideation support systems that help people generate many high quality ideas.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Emerging online ideation platforms with thousands of example ideas provide an important resource for creative production. But how can ideators best use these examples to create new innovations? Recent work has suggested that not just the choice of examples, but also the timing of their delivery can impact creative outcomes. Building on existing cognitive theories of creative insight, we hypothesize that people are likely to benefit from examples when they run out of ideas. We explore two example delivery mechanisms that test this hypothesis: 1) a system that proactively provides examples when a user appears to have run out of ideas, and 2) a system that provides examples when a user explicitly requests them. Our online experiment (N=97) compared these two mechanisms against two baselines: providing no examples and automatically showing examples at a regular interval. Participants who requested examples themselves generated ideas that were rated the most novel by external evaluators. Participants who received ideas automatically when they appeared to be stuck produced the most ideas. Importantly, participants who received examples at a regular interval generated fewer ideas than participants who received no examples, suggesting that mere access to examples is not sufficient for creative inspiration. These results emphasize the importance of the timing of example delivery. Insights from this study can inform the design of collective ideation support systems that help people generate many high quality ideas.
Amir, Ofra; Grosz, Barbara J.; Gajos, Krzysztof Z.; Swenson, Sonja M.; Sanders, Lee M.
From Care Plans to Care Coordination: Opportunities for Computer Support of Teamwork in Complex Healthcare Proceedings Article
In: Proceedings of the 33rd Annual ACM Conference on Human Factors in Computing Systems, pp. 1419–1428, ACM, Seoul, Republic of Korea, 2015, ISBN: 978-1-4503-3145-6.
@inproceedings{amir15:care,
title = {From Care Plans to Care Coordination: Opportunities for Computer Support of Teamwork in Complex Healthcare},
author = {Ofra Amir and Barbara J. Grosz and Krzysztof Z. Gajos and Sonja M. Swenson and Lee M. Sanders},
url = {/papers/amir15care.pdf},
isbn = {978-1-4503-3145-6},
year = {2015},
date = {2015-01-01},
booktitle = {Proceedings of the 33rd Annual ACM Conference on Human Factors in Computing Systems},
pages = {1419–1428},
publisher = {ACM},
address = {Seoul, Republic of Korea},
series = {CHI '15},
abstract = {Children with complex health conditions require care from a large, diverse team of caregivers that includes multiple types of medical professionals, parents and community support organizations. Coordination of their outpatient care, essential for good outcomes, presents major challenges. Extensive healthcare research has shown that the use of integrated, team-based care plans improves care coordination, but such plans are rarely deployed in practice. This paper reports on a study of care teams treating children with complex conditions at a major university tertiary care center. This study investigated barriers to plan implementation and resultant care coordination problems. It revealed the complex nature of teamwork in complex care, which poses challenges to team coordination that extend beyond those identified in prior work and handled by existing coordination systems. The paper builds on a computational teamwork theory to identify opportunities for technology to support increased plan-based complex-care coordination and to propose design approaches for systems that enable and enhance such coordination.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Children with complex health conditions require care from a large, diverse team of caregivers that includes multiple types of medical professionals, parents and community support organizations. Coordination of their outpatient care, essential for good outcomes, presents major challenges. Extensive healthcare research has shown that the use of integrated, team-based care plans improves care coordination, but such plans are rarely deployed in practice. This paper reports on a study of care teams treating children with complex conditions at a major university tertiary care center. This study investigated barriers to plan implementation and resultant care coordination problems. It revealed the complex nature of teamwork in complex care, which poses challenges to team coordination that extend beyond those identified in prior work and handled by existing coordination systems. The paper builds on a computational teamwork theory to identify opportunities for technology to support increased plan-based complex-care coordination and to propose design approaches for systems that enable and enhance such coordination.
Kim, Nam Wook; Bylinskii, Zoya; Borkin, Michelle A.; Oliva, Aude; Gajos, Krzysztof Z.; Pfister, Hanspeter
A Crowdsourced Alternative to Eye-tracking for Visualization Understanding Proceedings Article
In: Proceedings of the 33rd Annual ACM Conference Extended Abstracts on Human Factors in Computing Systems, pp. 1349–1354, ACM, Seoul, Republic of Korea, 2015, ISBN: 978-1-4503-3146-3.
@inproceedings{kim15bubble,
title = {A Crowdsourced Alternative to Eye-tracking for Visualization Understanding},
author = {Nam Wook Kim and Zoya Bylinskii and Michelle A. Borkin and Aude Oliva and Krzysztof Z. Gajos and Hanspeter Pfister},
url = {/papers/nwkim15bubble-wip.pdf},
isbn = {978-1-4503-3146-3},
year = {2015},
date = {2015-01-01},
booktitle = {Proceedings of the 33rd Annual ACM Conference Extended Abstracts on Human Factors in Computing Systems},
pages = {1349–1354},
publisher = {ACM},
address = {Seoul, Republic of Korea},
series = {CHI EA '15},
abstract = {In this study we investigate the utility of using mouse clicks as an alternative for eye fixations in the context of understanding data visualizations. We developed a crowdsourced study online in which participants were presented with a series of images containing graphs and diagrams and asked to describe them. Each image was blurred so that the participant needed to click to reveal bubbles - small, circular areas of the image at normal resolution. This is similar to having a confined area of focus like the human eye fovea. We compared the bubble click data with the fixation data from a complementary eye-tracking experiment by calculating the similarity between the resulting heatmaps. A high similarity score suggests that our methodology may be a viable crowdsourced alternative to eye-tracking experiments, especially when little to no eye-tracking data is available. This methodology can also be used to complement eye-tracking studies with an additional behavioral measurement, since it is specifically designed to measure which information people consciously choose to examine for understanding visualizations.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
In this study we investigate the utility of using mouse clicks as an alternative for eye fixations in the context of understanding data visualizations. We developed a crowdsourced study online in which participants were presented with a series of images containing graphs and diagrams and asked to describe them. Each image was blurred so that the participant needed to click to reveal bubbles – small, circular areas of the image at normal resolution. This is similar to having a confined area of focus like the human eye fovea. We compared the bubble click data with the fixation data from a complementary eye-tracking experiment by calculating the similarity between the resulting heatmaps. A high similarity score suggests that our methodology may be a viable crowdsourced alternative to eye-tracking experiments, especially when little to no eye-tracking data is available. This methodology can also be used to complement eye-tracking studies with an additional behavioral measurement, since it is specifically designed to measure which information people consciously choose to examine for understanding visualizations.
Weir, Sarah; Kim, Juho; Gajos, Krzysztof Z.; Miller, Robert C.
Learnersourcing Subgoal Labels for How-to Videos Proceedings Article
In: Proceedings of CSCW’15, pp. 405–416, ACM, Vancouver, BC, Canada, 2015, ISBN: 978-1-4503-2922-4.
@inproceedings{weir15:learnersourcing,
title = {Learnersourcing Subgoal Labels for How-to Videos},
author = {Sarah Weir and Juho Kim and Krzysztof Z. Gajos and Robert C. Miller},
url = {/papers/weir15learnersourcing.pdf},
doi = {10.1145/2675133.2675219},
isbn = {978-1-4503-2922-4},
year = {2015},
date = {2015-01-01},
booktitle = {Proceedings of CSCW'15},
pages = {405–416},
publisher = {ACM},
address = {Vancouver, BC, Canada},
series = {CSCW '15},
abstract = {Websites like YouTube host millions of how-to videos, but their interfaces are not optimized for learning. Previous research suggests that people learn more from how-to videos when the videos are accompanied by outlines showing individual steps and labels for groups of steps (subgoals). We envision an alternative video player where the steps and subgoals are displayed alongside the video. To generate this information for existing videos, we introduce learnersourcing, an approach in which intrinsically motivated learners contribute to a human computation workflow as they naturally go about learning from the videos. To demonstrate this method, we deployed a live website with a workflow for constructing subgoal labels implemented on a set of introductory web programming videos. For the four videos with the highest participation, we found that a majority of learner-generated subgoals were comparable in quality to expert-generated ones. Learners commented that the system helped them grasp the material, suggesting that our workflow did not detract from the learning experience.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Websites like YouTube host millions of how-to videos, but their interfaces are not optimized for learning. Previous research suggests that people learn more from how-to videos when the videos are accompanied by outlines showing individual steps and labels for groups of steps (subgoals). We envision an alternative video player where the steps and subgoals are displayed alongside the video. To generate this information for existing videos, we introduce learnersourcing, an approach in which intrinsically motivated learners contribute to a human computation workflow as they naturally go about learning from the videos. To demonstrate this method, we deployed a live website with a workflow for constructing subgoal labels implemented on a set of introductory web programming videos. For the four videos with the highest participation, we found that a majority of learner-generated subgoals were comparable in quality to expert-generated ones. Learners commented that the system helped them grasp the material, suggesting that our workflow did not detract from the learning experience.
Reinecke, Katharina; Gajos, Krzysztof Z.
LabintheWild: Conducting Large-Scale Online Experiments With Uncompensated Samples Proceedings Article
In: Proceedings of the 18th ACM Conference on Computer Supported Cooperative Work & Social Computing, pp. 1364–1378, ACM, Vancouver, BC, Canada, 2015, ISBN: 978-1-4503-2922-4.
@inproceedings{reinecke15:labinthewild,
title = {LabintheWild: Conducting Large-Scale Online Experiments With Uncompensated Samples},
author = {Katharina Reinecke and Krzysztof Z. Gajos},
url = {/papers/reinecke15labinthewild.pdf},
doi = {10.1145/2675133.2675246},
isbn = {978-1-4503-2922-4},
year = {2015},
date = {2015-01-01},
booktitle = {Proceedings of the 18th ACM Conference on Computer Supported Cooperative Work & Social Computing},
pages = {1364–1378},
publisher = {ACM},
address = {Vancouver, BC, Canada},
series = {CSCW '15},
abstract = {Web-based experimentation with uncompensated and unsupervised samples has the potential to support the replication, verification, extension and generation of new results with larger and more diverse sample populations than previously seen. We introduce the experimental online platform LabintheWild, which provides participants with personalized feedback in exchange for participation in behavioral studies. In comparison to conventional in-lab studies, LabintheWild enables the recruitment of participants at larger scale and from more diverse demographic and geographic backgrounds. We analyze Google Analytics data, participants' comments, and tweets to discuss how participants hear about the platform, and why they might choose to participate. Analyzing three example experiments, we additionally show that these experiments replicate previous in-lab study results with comparable data quality.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Web-based experimentation with uncompensated and unsupervised samples has the potential to support the replication, verification, extension and generation of new results with larger and more diverse sample populations than previously seen. We introduce the experimental online platform LabintheWild, which provides participants with personalized feedback in exchange for participation in behavioral studies. In comparison to conventional in-lab studies, LabintheWild enables the recruitment of participants at larger scale and from more diverse demographic and geographic backgrounds. We analyze Google Analytics data, participants’ comments, and tweets to discuss how participants hear about the platform, and why they might choose to participate. Analyzing three example experiments, we additionally show that these experiments replicate previous in-lab study results with comparable data quality.
Siangliulue, Pao; Arnold, Kenneth C.; Gajos, Krzysztof Z.; Dow, Steven P.
Toward Collaborative Ideation at Scale: Leveraging Ideas from Others to Generate More Creative and Diverse Ideas Proceedings Article
In: Proceedings of the 18th ACM Conference on Computer Supported Cooperative Work & Social Computing, pp. 937–945, ACM, Vancouver, BC, Canada, 2015, ISBN: 978-1-4503-2922-4.
@inproceedings{siangliulue15:toward,
title = {Toward Collaborative Ideation at Scale: Leveraging Ideas from Others to Generate More Creative and Diverse Ideas},
author = {Pao Siangliulue and Kenneth C. Arnold and Krzysztof Z. Gajos and Steven P. Dow},
url = {/papers/siangliulue15ideation.pdf},
doi = {10.1145/2675133.2675239},
isbn = {978-1-4503-2922-4},
year = {2015},
date = {2015-01-01},
booktitle = {Proceedings of the 18th ACM Conference on Computer Supported Cooperative Work & Social Computing},
pages = {937–945},
publisher = {ACM},
address = {Vancouver, BC, Canada},
series = {CSCW '15},
abstract = {A growing number of large online collaborative idea generation platforms promise that by ideating together, people can create better ideas than any would have alone. But how might these platforms best leverage the number and diversity of contributors to help each contributor generate even better ideas? Prior research suggests that seeing examples of ideas generated by others can improve one's own ideation outcomes, but the benefit may depend on how creative and how diverse (i.e., different from each other) these examples are. There already exist scalable crowd-powered mechanisms to evaluate creativity of individual ideas, but few options exist for assessing the diversity of ideas. We contribute a new scalable crowd-powered method for evaluating the diversity of sets of ideas. The method relies on simple comparisons of similarity (is idea A more similar to B or C?) generated by a large number of non-expert contributors. We build on prior work on multidimensional scaling and active similarity learning techniques to create an abstract spatial idea map. Our validation study reveals that human raters agree with the estimates of dissimilarity derived from our idea map as much or more than they agree with each other. The results of our main experiment demonstrate that diverse sets of examples generated automatically using our idea map improve the diversity of ideas generated by participants compared to seeing randomly selected examples. Our results also corroborate findings from prior research showing that people presented with creative examples generated more creative ideas than those who saw a set of random examples. We see this work as a step toward building more effective online systems for supporting large scale collective ideation.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
A growing number of large online collaborative idea generation platforms promise that by ideating together, people can create better ideas than any would have alone. But how might these platforms best leverage the number and diversity of contributors to help each contributor generate even better ideas? Prior research suggests that seeing examples of ideas generated by others can improve one’s own ideation outcomes, but the benefit may depend on how creative and how diverse (i.e., different from each other) these examples are. There already exist scalable crowd-powered mechanisms to evaluate creativity of individual ideas, but few options exist for assessing the diversity of ideas. We contribute a new scalable crowd-powered method for evaluating the diversity of sets of ideas. The method relies on simple comparisons of similarity (is idea A more similar to B or C?) generated by a large number of non-expert contributors. We build on prior work on multidimensional scaling and active similarity learning techniques to create an abstract spatial idea map. Our validation study reveals that human raters agree with the estimates of dissimilarity derived from our idea map as much or more than they agree with each other. The results of our main experiment demonstrate that diverse sets of examples generated automatically using our idea map improve the diversity of ideas generated by participants compared to seeing randomly selected examples. Our results also corroborate findings from prior research showing that people presented with creative examples generated more creative ideas than those who saw a set of random examples. We see this work as a step toward building more effective online systems for supporting large scale collective ideation.
Li, Na; Gajos, Krzysztof Z.; Nakayama, Ken; Enos, Ryan
TELLab: An Experiential Learning Tool for Psychology Proceedings Article
In: Proceedings of the Second (2015) ACM Conference on Learning @ Scale, pp. 293–297, ACM, Vancouver, BC, Canada, 2015, ISBN: 978-1-4503-3411-2.
@inproceedings{li15:tellab,
title = {TELLab: An Experiential Learning Tool for Psychology},
author = {Na Li and Krzysztof Z. Gajos and Ken Nakayama and Ryan Enos},
url = {/papers/li15tellab-wip.pdf},
doi = {10.1145/2724660.2728678},
isbn = {978-1-4503-3411-2},
year = {2015},
date = {2015-01-01},
booktitle = {Proceedings of the Second (2015) ACM Conference on Learning @ Scale},
pages = {293–297},
publisher = {ACM},
address = {Vancouver, BC, Canada},
series = {L@S '15},
abstract = {In this paper, we discuss current practices and challenges of teaching psychology experiments. We review experiential learning and analogical learning pedagogies, which have informed the design of TELLab, an online platform for supporting effective experiential learning of psychology concepts.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
In this paper, we discuss current practices and challenges of teaching psychology experiments. We review experiential learning and analogical learning pedagogies, which have informed the design of TELLab, an online platform for supporting effective experiential learning of psychology concepts.
Kim, Sunyoung; Iravantchi, Yasha; Gajos, Krzysztof Z.; Grosz, Barbara
Exploring Opportunities for Social Infrastructure in Congestive Heart Failure Management Proceedings Article
In: Proceedings of the CSCW 2015 workshop on Moving Beyond e-Health and the Quantified Self, 2015.
@inproceedings{kim15:exploring,
title = {Exploring Opportunities for Social Infrastructure in Congestive Heart Failure Management},
author = {Sunyoung Kim and Yasha Iravantchi and Krzysztof Z. Gajos and Barbara Grosz},
url = {/papers/skim15exploring.pdf},
year = {2015},
date = {2015-01-01},
booktitle = {Proceedings of the CSCW 2015 workshop on Moving Beyond e-Health and the Quantified Self},
abstract = {As population ages and chronic disease increases, new models of health care delivery are inevitable. We propose to interweave the concept of chronic care management into social infrastructure to better support elderly individuals with chronic diseases. As an exemplar, we chose and investigated Congestive Heart Failure (CHF), since it is one of serious health threats among the elderly population in western society. From a preliminary investigation of literature and prior work, we suggest two implications for social care of chronic patients: increased social awareness, and social knowledge base. This is not an exhaustive list of implications, but a starting point to open up discussions towards social care for chronic diseases.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
As population ages and chronic disease increases, new models of health care delivery are inevitable. We propose to interweave the concept of chronic care management into social infrastructure to better support elderly individuals with chronic diseases. As an exemplar, we chose and investigated Congestive Heart Failure (CHF), since it is one of serious health threats among the elderly population in western society. From a preliminary investigation of literature and prior work, we suggest two implications for social care of chronic patients: increased social awareness, and social knowledge base. This is not an exhaustive list of implications, but a starting point to open up discussions towards social care for chronic diseases.
2014
Amir, Ofra; Grosz, Barbara J.; Gajos, Krzysztof Z.; Swenson, Sonja M.; Sanders, Lee M.
AI Support of Teamwork for Coordinated Care of Children with Complex Conditions Proceedings Article
In: AAAI Fall Symposium: Expanding the Boundaries of Health Informatics Using AI: Making Personalized and Participatory Medicine A Reality, 2014.
@inproceedings{amir14:ai,
title = {AI Support of Teamwork for Coordinated Care of Children with Complex Conditions},
author = {Ofra Amir and Barbara J. Grosz and Krzysztof Z. Gajos and Sonja M. Swenson and Lee M. Sanders},
url = {/papers/amir14AIsupport.pdf},
year = {2014},
date = {2014-01-01},
booktitle = {AAAI Fall Symposium: Expanding the Boundaries of Health Informatics Using AI: Making Personalized and Participatory Medicine A Reality},
abstract = {Children with complex health conditions require care from a large, diverse set of caregivers that includes parents and community support organizations as well as multiple types of medical professionals. Coordination of their care is essential for good outcomes, and extensive research has shown that the use of integrated, team-based care plans improves care coordination. Care plans, however, are rarely deployed in practice. This paper describes barriers to effective implementation of care plans in complex care revealed by a study of care providers treating such children. It draws on teamwork theories, identifying ways AI capabilities could enhance care plan use; describes the design of GoalKeeper, a system to support providers use of care plans; and describes initial work toward information sharing algorithms for such systems.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Children with complex health conditions require care from a large, diverse set of caregivers that includes parents and community support organizations as well as multiple types of medical professionals. Coordination of their care is essential for good outcomes, and extensive research has shown that the use of integrated, team-based care plans improves care coordination. Care plans, however, are rarely deployed in practice. This paper describes barriers to effective implementation of care plans in complex care revealed by a study of care providers treating such children. It draws on teamwork theories, identifying ways AI capabilities could enhance care plan use; describes the design of GoalKeeper, a system to support providers use of care plans; and describes initial work toward information sharing algorithms for such systems.
Kim, Juho; Guo, Philip J.; Cai, Carrie J.; Li, Shang-Wen (Daniel); Gajos, Krzysztof Z.; Miller, Robert C.
Data-driven Interaction Techniques for Improving Navigation of Educational Videos Proceedings Article
In: Proceedings of the 27th Annual ACM Symposium on User Interface Software and Technology, pp. 563–572, ACM, Honolulu, Hawaii, USA, 2014, ISBN: 978-1-4503-3069-5.
@inproceedings{kim14:datadriven,
title = {Data-driven Interaction Techniques for Improving Navigation of Educational Videos},
author = {Juho Kim and Philip J. Guo and Carrie J. Cai and Shang-Wen (Daniel) Li and Krzysztof Z. Gajos and Robert C. Miller},
url = {/papers/kim-uist14-lecturescape.pdf},
doi = {10.1145/2642918.2647389},
isbn = {978-1-4503-3069-5},
year = {2014},
date = {2014-01-01},
booktitle = {Proceedings of the 27th Annual ACM Symposium on User Interface Software and Technology},
pages = {563–572},
publisher = {ACM},
address = {Honolulu, Hawaii, USA},
series = {UIST '14},
abstract = {With an unprecedented scale of learners watching educational videos on online platforms such as MOOCs and YouTube, there is an opportunity to incorporate data generated from their interactions into the design of novel video interaction techniques. Interaction data has the potential to help not only instructors to improve their videos, but also to enrich the learning experience of educational video watchers. This paper explores the design space of data-driven interaction techniques for educational video navigation. We introduce a set of techniques that augment existing video interface widgets, including: a 2D video timeline with an embedded visualization of collective navigation traces; dynamic and non-linear timeline scrubbing; data-enhanced transcript search and keyword summary; automatic display of relevant still frames next to the video; and a visual summary representing points with high learner activity. To evaluate the feasibility of the techniques, we ran a laboratory user study with simulated learning tasks. Participants rated watching lecture videos with interaction data to be efficient and useful in completing the tasks. However, no significant differences were found in task performance, suggesting that interaction data may not always align with moment-by-moment information needs during the tasks.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
With an unprecedented scale of learners watching educational videos on online platforms such as MOOCs and YouTube, there is an opportunity to incorporate data generated from their interactions into the design of novel video interaction techniques. Interaction data has the potential to help not only instructors to improve their videos, but also to enrich the learning experience of educational video watchers. This paper explores the design space of data-driven interaction techniques for educational video navigation. We introduce a set of techniques that augment existing video interface widgets, including: a 2D video timeline with an embedded visualization of collective navigation traces; dynamic and non-linear timeline scrubbing; data-enhanced transcript search and keyword summary; automatic display of relevant still frames next to the video; and a visual summary representing points with high learner activity. To evaluate the feasibility of the techniques, we ran a laboratory user study with simulated learning tasks. Participants rated watching lecture videos with interaction data to be efficient and useful in completing the tasks. However, no significant differences were found in task performance, suggesting that interaction data may not always align with moment-by-moment information needs during the tasks.
Kim, Juho; Zhang, Amy X.; Kim, Jihee; Miller, Robert C.; Gajos, Krzysztof Z.
Content-aware Kinetic Scrolling for Supporting Web Page Navigation Proceedings Article
In: Proceedings of the 27th Annual ACM Symposium on User Interface Software and Technology, pp. 123–127, ACM, Honolulu, Hawaii, USA, 2014, ISBN: 978-1-4503-3069-5.
@inproceedings{kim14:contentaware,
title = {Content-aware Kinetic Scrolling for Supporting Web Page Navigation},
author = {Juho Kim and Amy X. Zhang and Jihee Kim and Robert C. Miller and Krzysztof Z. Gajos},
url = {/papers/kim-uist14-caks.pdf},
doi = {10.1145/2642918.2647401},
isbn = {978-1-4503-3069-5},
year = {2014},
date = {2014-01-01},
booktitle = {Proceedings of the 27th Annual ACM Symposium on User Interface Software and Technology},
pages = {123–127},
publisher = {ACM},
address = {Honolulu, Hawaii, USA},
series = {UIST '14},
abstract = {Long documents are abundant on the web today, and are accessed in increasing numbers from touchscreen devices such as mobile phones and tablets. Navigating long documents with small screens can be challenging both physically and cognitively because they compel the user to scroll a great deal and to mentally filter for important content. To support navigation of long documents on touchscreen devices, we introduce content-aware kinetic scrolling, a novel scrolling technique that dynamically applies pseudo-haptic feedback in the form of friction around points of high interest within the page. This allows users to quickly find interesting content while exploring without further cluttering the limited visual space. To model degrees of interest (DOI) for a variety of existing web pages, we introduce social wear, a method for capturing DOI based on social signals that indicate collective user interest. Our preliminary evaluation shows that users pay attention to items with kinetic scrolling feedback during search, recognition, and skimming tasks.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Long documents are abundant on the web today, and are accessed in increasing numbers from touchscreen devices such as mobile phones and tablets. Navigating long documents with small screens can be challenging both physically and cognitively because they compel the user to scroll a great deal and to mentally filter for important content. To support navigation of long documents on touchscreen devices, we introduce content-aware kinetic scrolling, a novel scrolling technique that dynamically applies pseudo-haptic feedback in the form of friction around points of high interest within the page. This allows users to quickly find interesting content while exploring without further cluttering the limited visual space. To model degrees of interest (DOI) for a variety of existing web pages, we introduce social wear, a method for capturing DOI based on social signals that indicate collective user interest. Our preliminary evaluation shows that users pay attention to items with kinetic scrolling feedback during search, recognition, and skimming tasks.
Kim, Juho; Guo, Philip J.; Seaton, Daniel T.; Mitros, Piotr; Gajos, Krzysztof Z.; Miller, Robert C.
Understanding In-video Dropouts and Interaction Peaks Inonline Lecture Videos Proceedings Article
In: Proceedings of the First ACM Conference on Learning @ Scale Conference, pp. 31–40, ACM, Atlanta, Georgia, USA, 2014, ISBN: 978-1-4503-2669-8.
@inproceedings{kim14:understanding,
title = {Understanding In-video Dropouts and Interaction Peaks Inonline Lecture Videos},
author = {Juho Kim and Philip J. Guo and Daniel T. Seaton and Piotr Mitros and Krzysztof Z. Gajos and Robert C. Miller},
url = {/papers/kim14video.pdf},
doi = {10.1145/2556325.2566237},
isbn = {978-1-4503-2669-8},
year = {2014},
date = {2014-01-01},
booktitle = {Proceedings of the First ACM Conference on Learning @ Scale Conference},
pages = {31–40},
publisher = {ACM},
address = {Atlanta, Georgia, USA},
series = {L@S '14},
abstract = {With thousands of learners watching the same online lec- ture videos, analyzing video watching patterns provides a unique opportunity to understand how students learn with videos. This paper reports a large-scale analysis of in-video dropout and peaks in viewership and student activity, using second-by-second user interaction data from 862 videos in four Massive Open Online Courses (MOOCs) on edX. We find higher dropout rates in longer videos, re-watching sessions (vs first-time), and tutorials (vs lectures). Peaks in rewatching sessions and play events indicate points of interest and confusion. Results show that tutorials (vs lectures) and re-watching sessions (vs first-time) lead to more frequent and sharper peaks. In attempting to reason why peaks occur by sampling 80 videos, we observe that 61% of the peaks accompany visual transitions in the video, e.g., a slide view to a classroom view. Based on this observation, we identify five student activity patterns that can explain peaks: starting from the beginning of a new material, returning to missed content, following a tutorial step, replaying a brief segment, and repeating a non-visual explanation. Our analysis has design implications for video authoring, editing, and interface design, providing a richer understanding of video learning on MOOCs.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
With thousands of learners watching the same online lec- ture videos, analyzing video watching patterns provides a unique opportunity to understand how students learn with videos. This paper reports a large-scale analysis of in-video dropout and peaks in viewership and student activity, using second-by-second user interaction data from 862 videos in four Massive Open Online Courses (MOOCs) on edX. We find higher dropout rates in longer videos, re-watching sessions (vs first-time), and tutorials (vs lectures). Peaks in rewatching sessions and play events indicate points of interest and confusion. Results show that tutorials (vs lectures) and re-watching sessions (vs first-time) lead to more frequent and sharper peaks. In attempting to reason why peaks occur by sampling 80 videos, we observe that 61% of the peaks accompany visual transitions in the video, e.g., a slide view to a classroom view. Based on this observation, we identify five student activity patterns that can explain peaks: starting from the beginning of a new material, returning to missed content, following a tutorial step, replaying a brief segment, and repeating a non-visual explanation. Our analysis has design implications for video authoring, editing, and interface design, providing a richer understanding of video learning on MOOCs.
Reinecke, Katharina; Gajos, Krzysztof Z.
Quantifying Visual Preferences Around the World Proceedings Article
In: Proceedings of the SIGCHI Conference on Human Factors in Computing Systems, pp. 11–20, ACM, Toronto, Ontario, Canada, 2014, ISBN: 978-1-4503-2473-1.
@inproceedings{reinecke14:visual,
title = {Quantifying Visual Preferences Around the World},
author = {Katharina Reinecke and Krzysztof Z. Gajos},
url = {/papers/reinecke14visual.pdf},
doi = {10.1145/2556288.2557052},
isbn = {978-1-4503-2473-1},
year = {2014},
date = {2014-01-01},
booktitle = {Proceedings of the SIGCHI Conference on Human Factors in Computing Systems},
pages = {11–20},
publisher = {ACM},
address = {Toronto, Ontario, Canada},
series = {CHI '14},
abstract = {Website aesthetics have been recognized as an influential moderator of people's behavior and perception. However, what users perceive as ``good design'' is subject to individual preferences, questioning the feasibility of universal design guidelines. To better understand how people's visual preferences differ, we collected 2.4 million ratings of the visual appeal of websites from nearly 40 thousand participants of diverse backgrounds. We address several gaps in the knowledge about design preferences of previously understudied groups. Among other findings, our results show that the level of colorfulness and visual complexity at which visual appeal is highest strongly varies: Females, for example, liked colorful websites more than males. A high education level generally lowers this preference for colorfulness. Russians preferred a lower visual complexity, and Macedonians liked highly colorful designs more than any other country in our dataset. We contribute a computational model and estimates of peak appeal that can be used to support rapid evaluations of website design prototypes for specific target groups.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Website aesthetics have been recognized as an influential moderator of people’s behavior and perception. However, what users perceive as “good design” is subject to individual preferences, questioning the feasibility of universal design guidelines. To better understand how people’s visual preferences differ, we collected 2.4 million ratings of the visual appeal of websites from nearly 40 thousand participants of diverse backgrounds. We address several gaps in the knowledge about design preferences of previously understudied groups. Among other findings, our results show that the level of colorfulness and visual complexity at which visual appeal is highest strongly varies: Females, for example, liked colorful websites more than males. A high education level generally lowers this preference for colorfulness. Russians preferred a lower visual complexity, and Macedonians liked highly colorful designs more than any other country in our dataset. We contribute a computational model and estimates of peak appeal that can be used to support rapid evaluations of website design prototypes for specific target groups.
Kim, Juho; Nguyen, Phu Tran; Weir, Sarah; Guo, Philip J.; Miller, Robert C.; Gajos, Krzysztof Z.
Crowdsourcing Step-by-step Information Extraction to Enhance Existing How-to Videos Proceedings Article
In: Proceedings of the 32Nd Annual ACM Conference on Human Factors in Computing Systems, pp. 4017–4026, ACM, Toronto, Ontario, Canada, 2014, ISBN: 978-1-4503-2473-1.
@inproceedings{kim14:crowdsourcing,
title = {Crowdsourcing Step-by-step Information Extraction to Enhance Existing How-to Videos},
author = {Juho Kim and Phu Tran Nguyen and Sarah Weir and Philip J. Guo and Robert C. Miller and Krzysztof Z. Gajos},
url = {/papers/kim14crowdsourcing.pdf},
doi = {10.1145/2556288.2556986},
isbn = {978-1-4503-2473-1},
year = {2014},
date = {2014-01-01},
booktitle = {Proceedings of the 32Nd Annual ACM Conference on Human Factors in Computing Systems},
pages = {4017–4026},
publisher = {ACM},
address = {Toronto, Ontario, Canada},
series = {CHI '14},
abstract = {<p>Millions of learners today use how-to videos to master new skills in a variety of domains. But browsing such videos is often tedious and inefficient because video player interfaces are not optimized for the unique step-by-step structure of such videos. This research aims to improve the learning experience of existing how-to videos with step-by-step annotations.</p><p>We first performed a formative study to verify that annotations are actually useful to learners. We created ToolScape, an interactive video player that displays step descriptions and intermediate result thumbnails in the video timeline. Learners in our study performed better and gained more self-efficacy using ToolScape versus a traditional video player.</p><p>To add the needed step annotations to existing how-to videos at scale, we introduce a novel crowdsourcing workflow. It extracts step-by-step structure from an existing video, including step times, descriptions, and before and after images. We introduce the Find-Verify-Expand design pattern for temporal and visual annotation, which applies clustering, text processing, and visual analysis algorithms to merge crowd output. The workflow does not rely on domain-specific customization, works on top of existing videos, and recruits untrained crowd workers. We evaluated the workflow with Mechanical Turk, using 75 cooking, makeup, and Photoshop videos on YouTube. Results show that our workflow can extract steps with a quality comparable to that of trained annotators across all three domains with 77% precision and 81% recall.</p>},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
<p>Millions of learners today use how-to videos to master new skills in a variety of domains. But browsing such videos is often tedious and inefficient because video player interfaces are not optimized for the unique step-by-step structure of such videos. This research aims to improve the learning experience of existing how-to videos with step-by-step annotations.</p><p>We first performed a formative study to verify that annotations are actually useful to learners. We created ToolScape, an interactive video player that displays step descriptions and intermediate result thumbnails in the video timeline. Learners in our study performed better and gained more self-efficacy using ToolScape versus a traditional video player.</p><p>To add the needed step annotations to existing how-to videos at scale, we introduce a novel crowdsourcing workflow. It extracts step-by-step structure from an existing video, including step times, descriptions, and before and after images. We introduce the Find-Verify-Expand design pattern for temporal and visual annotation, which applies clustering, text processing, and visual analysis algorithms to merge crowd output. The workflow does not rely on domain-specific customization, works on top of existing videos, and recruits untrained crowd workers. We evaluated the workflow with Mechanical Turk, using 75 cooking, makeup, and Photoshop videos on YouTube. Results show that our workflow can extract steps with a quality comparable to that of trained annotators across all three domains with 77% precision and 81% recall.</p>
Li, Louis; Gajos, Krzysztof Z.
Adaptive Click-and-cross: Adapting to Both Abilities and Task Improves Performance of Users with Impaired Dexterity Proceedings Article
In: Proceedings of the 19th International Conference on Intelligent User Interfaces, pp. 299–304, ACM, Haifa, Israel, 2014, ISBN: 978-1-4503-2184-6.
@inproceedings{li14:adaptive,
title = {Adaptive Click-and-cross: Adapting to Both Abilities and Task Improves Performance of Users with Impaired Dexterity},
author = {Louis Li and Krzysztof Z. Gajos},
url = {/papers/li14acnx.pdf},
doi = {10.1145/2557500.2557511},
isbn = {978-1-4503-2184-6},
year = {2014},
date = {2014-01-01},
booktitle = {Proceedings of the 19th International Conference on Intelligent User Interfaces},
pages = {299–304},
publisher = {ACM},
address = {Haifa, Israel},
series = {IUI '14},
abstract = {Computer users with impaired dexterity often have difficulty accessing small, densely packed user interface elements. Past research in software-based solutions has mainly employed two approaches: modifying the interface and modifying the interaction with the cursor. Each approach, however, has limitations. Modifying the user interface by enlarging interactive elements makes access efficient for simple interfaces but increases the cost of navigation for complex ones by displacing items to screens that require tabs or scrolling to reach. Modifying the interaction with the cursor makes access possible to unmodified interfaces but may perform poorly on densely packed targets or require the user to perform multiple steps. We developed a new approach that combines the strengths of the existing approaches while minimizing their shortcomings, introducing only minimal distortion to the original interface while making access to frequently used parts of the user interface efficient and access to all other parts possible. We instantiated this concept as Adaptive Click-and-Cross, a novel interaction technique. Our user study demonstrates that, for sufficiently complex interfaces, Adaptive Click-andCross slightly improves the performance of users with impaired dexterity compared to only modifying the interface or only modifying the cursor.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Computer users with impaired dexterity often have difficulty accessing small, densely packed user interface elements. Past research in software-based solutions has mainly employed two approaches: modifying the interface and modifying the interaction with the cursor. Each approach, however, has limitations. Modifying the user interface by enlarging interactive elements makes access efficient for simple interfaces but increases the cost of navigation for complex ones by displacing items to screens that require tabs or scrolling to reach. Modifying the interaction with the cursor makes access possible to unmodified interfaces but may perform poorly on densely packed targets or require the user to perform multiple steps. We developed a new approach that combines the strengths of the existing approaches while minimizing their shortcomings, introducing only minimal distortion to the original interface while making access to frequently used parts of the user interface efficient and access to all other parts possible. We instantiated this concept as Adaptive Click-and-Cross, a novel interaction technique. Our user study demonstrates that, for sufficiently complex interfaces, Adaptive Click-andCross slightly improves the performance of users with impaired dexterity compared to only modifying the interface or only modifying the cursor.
Huang, Cheng-Zhi Anna; Duvenaud, David; Arnold, Kenneth C.; Partridge, Brenton; Oberholtzer, Josiah W.; Gajos, Krzysztof Z.
Active Learning of Intuitive Control Knobs for Synthesizers Using Gaussian Processes Proceedings Article
In: Proceedings of the 19th International Conference on Intelligent User Interfaces, pp. 115–124, ACM, Haifa, Israel, 2014, ISBN: 978-1-4503-2184-6.
@inproceedings{huang14:active,
title = {Active Learning of Intuitive Control Knobs for Synthesizers Using Gaussian Processes},
author = {Cheng-Zhi Anna Huang and David Duvenaud and Kenneth C. Arnold and Brenton Partridge and Josiah W. Oberholtzer and Krzysztof Z. Gajos},
url = {/papers/huang14knobs.pdf},
doi = {10.1145/2557500.2557544},
isbn = {978-1-4503-2184-6},
year = {2014},
date = {2014-01-01},
booktitle = {Proceedings of the 19th International Conference on Intelligent User Interfaces},
pages = {115–124},
publisher = {ACM},
address = {Haifa, Israel},
series = {IUI '14},
abstract = {<p>Typical synthesizers only provide controls to the low-level parameters of sound synthesis, such as wave-shapes or filter envelopes. In contrast, composers often want to adjust and express higher-level qualities, such as how ``scary'' or ``steady'' sounds are perceived to be.</p><p>We develop a system which allows users to directly control abstract, high-level qualities of sounds. To do this, our system learns functions that map from synthesizer control settings to perceived levels of high-level qualities. Given these functions, our system can generate high-level knobs that directly adjust sounds to have more or less of those qualities. We model the functions mapping from control-parameters to the degree of each high-level quality using Gaussian processes, a nonparametric Bayesian model. These models can adjust to the complexity of the function being learned, account for nonlinear interaction between control-parameters, and allow us to characterize the uncertainty about the functions being learned.</p><p>By tracking uncertainty about the functions being learned, we can use active learning to quickly calibrate the tool, by querying the user about the sounds the system expects to most improve its performance. We show through simulations that this model-based active learning approach learns high-level knobs on certain classes of target concepts faster than several baselines, and give examples of the resulting automatically-constructed knobs which adjust levels of non-linear, high-level concepts.</p>},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
<p>Typical synthesizers only provide controls to the low-level parameters of sound synthesis, such as wave-shapes or filter envelopes. In contrast, composers often want to adjust and express higher-level qualities, such as how “scary” or “steady” sounds are perceived to be.</p><p>We develop a system which allows users to directly control abstract, high-level qualities of sounds. To do this, our system learns functions that map from synthesizer control settings to perceived levels of high-level qualities. Given these functions, our system can generate high-level knobs that directly adjust sounds to have more or less of those qualities. We model the functions mapping from control-parameters to the degree of each high-level quality using Gaussian processes, a nonparametric Bayesian model. These models can adjust to the complexity of the function being learned, account for nonlinear interaction between control-parameters, and allow us to characterize the uncertainty about the functions being learned.</p><p>By tracking uncertainty about the functions being learned, we can use active learning to quickly calibrate the tool, by querying the user about the sounds the system expects to most improve its performance. We show through simulations that this model-based active learning approach learns high-level knobs on certain classes of target concepts faster than several baselines, and give examples of the resulting automatically-constructed knobs which adjust levels of non-linear, high-level concepts.</p>
Kim, Juho; Li, Shang-Wen (Daniel); Cai, Carrie J.; Gajos, Krzysztof Z.; Miller, Robert C.
Leveraging Video Interaction Data and Content Analysis to Improve Video Learning Proceedings Article
In: Proceedings of the CHI 2014 Learning Innovation at Scale workshop, 2014.
@inproceedings{kim14:leveraging,
title = {Leveraging Video Interaction Data and Content Analysis to Improve Video Learning},
author = {Juho Kim and Shang-Wen (Daniel) Li and Carrie J. Cai and Krzysztof Z. Gajos and Robert C. Miller},
url = {/papers/kim14leveraging.pdf},
year = {2014},
date = {2014-01-01},
booktitle = {Proceedings of the CHI 2014 Learning Innovation at Scale workshop},
abstract = {Video has emerged as a dominant medium for online education, as witnessed by millions of students learning from educational videos on Massive Open Online Courses (MOOCs), Khan Academy, and YouTube. The large-scale data collected from students' interactions with video provide a unique opportunity to analyze and improve the video learning experience. We combine click-level interaction data, such as pausing, resuming, or navigating between points in the video, and video content analysis, such as visual, text, and speech, to analyze peaks in viewership and student activity. Such analysis can reveal points of interest or confusion in the video, and suggest production and editing improvements. Furthermore, we envision novel video interfaces and learning platforms that automatically adapt to learners' collective watching behaviors.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Video has emerged as a dominant medium for online education, as witnessed by millions of students learning from educational videos on Massive Open Online Courses (MOOCs), Khan Academy, and YouTube. The large-scale data collected from students’ interactions with video provide a unique opportunity to analyze and improve the video learning experience. We combine click-level interaction data, such as pausing, resuming, or navigating between points in the video, and video content analysis, such as visual, text, and speech, to analyze peaks in viewership and student activity. Such analysis can reveal points of interest or confusion in the video, and suggest production and editing improvements. Furthermore, we envision novel video interfaces and learning platforms that automatically adapt to learners’ collective watching behaviors.
Komarov, Steven; Gajos, Krzysztof Z.
Organic Peer Assessment Proceedings Article
In: Proceedings of the CHI 2014 Learning Innovation at Scale workshop, 2014.
@inproceedings{komarov14:organic,
title = {Organic Peer Assessment},
author = {Steven Komarov and Krzysztof Z. Gajos},
url = {/papers/komarov14organic.pdf},
year = {2014},
date = {2014-01-01},
booktitle = {Proceedings of the CHI 2014 Learning Innovation at Scale workshop},
abstract = {We present a web-based tool for a type of peer assessment we dubbed organic. In organic peer assessment there are no upper or lower limits on the number of assignments each peer has to review, avoiding the common issue of prematurely coercing students into activities they might fear and dislike. Instead, peer assessment occurs as a side effect of activities students find intrinsically motivating. We outline the basic set of functionality required for the implementation of our vision for peer assessment in an online environment and present the results of a preliminary study we conducted in a flipped classroom. We found that the quality of the summative assessment produced by the peers matched that of experts, and we encountered strong evidence that our peer assessment implementation had positive effects on achievement. We conclude with a discussion arguing that organic peer assessment is a valuable technique-distinct from formal peer assessment-for deployment in MOOCs.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
We present a web-based tool for a type of peer assessment we dubbed organic. In organic peer assessment there are no upper or lower limits on the number of assignments each peer has to review, avoiding the common issue of prematurely coercing students into activities they might fear and dislike. Instead, peer assessment occurs as a side effect of activities students find intrinsically motivating. We outline the basic set of functionality required for the implementation of our vision for peer assessment in an online environment and present the results of a preliminary study we conducted in a flipped classroom. We found that the quality of the summative assessment produced by the peers matched that of experts, and we encountered strong evidence that our peer assessment implementation had positive effects on achievement. We conclude with a discussion arguing that organic peer assessment is a valuable technique-distinct from formal peer assessment-for deployment in MOOCs.
2013
Borkin, Michelle A; Yeh, Chelsea S; Boyd, Madelaine; Macko, Peter; Gajos, KZ; Seltzer, M; Pfister, H
Evaluation of filesystem provenance visualization tools Journal Article
In: IEEE transactions on visualization and computer graphics, vol. 19, no. 12, pp. 2476–2485, 2013.
@article{borkin13:evaluation,
title = {Evaluation of filesystem provenance visualization tools},
author = {Michelle A Borkin and Chelsea S Yeh and Madelaine Boyd and Peter Macko and KZ Gajos and M Seltzer and H Pfister},
url = {/papers/borkin13inprov.pdf},
year = {2013},
date = {2013-01-01},
journal = {IEEE transactions on visualization and computer graphics},
volume = {19},
number = {12},
pages = {2476–2485},
abstract = {Having effective visualizations of filesystem provenance data is valuable for understanding its complex hierarchical struc- ture. The most common visual representation of provenance data is the node-link diagram. While effective for understanding local activity, the node-link diagram fails to offer a high-level summary of activity and inter-relationships within the data. We present a new tool, InProv, which displays filesystem provenance with an interactive radial-based tree layout. The tool also utilizes a new time-based hierarchical node grouping method for filesystem provenance data we developed to match the user's mental model and make data exploration more intuitive. We compared InProv to a conventional node-link based tool, Orbiter, in a quantitative evaluation with real users of filesystem provenance data including provenance data experts, IT professionals, and computational scientists. We also compared in the evaluation our new node grouping method to a conventional method. The results demonstrate that InProv results in higher accuracy in identifying system activity than Orbiter with large complex data sets. The results also show that our new time- based hierarchical node grouping method improves performance in both tools, and participants found both tools significantly easier to use with the new time-based node grouping method. Subjective measures show that participants found InProv to require less mental activity, less physical activity, less work, and is less stressful to use. Our study also reveals one of the first cases of gender differences in visualization; both genders had comparable performance with InProv, but women had a significantly lower average accuracy (56%) compared to men (70%) with Orbiter.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Having effective visualizations of filesystem provenance data is valuable for understanding its complex hierarchical struc- ture. The most common visual representation of provenance data is the node-link diagram. While effective for understanding local activity, the node-link diagram fails to offer a high-level summary of activity and inter-relationships within the data. We present a new tool, InProv, which displays filesystem provenance with an interactive radial-based tree layout. The tool also utilizes a new time-based hierarchical node grouping method for filesystem provenance data we developed to match the user’s mental model and make data exploration more intuitive. We compared InProv to a conventional node-link based tool, Orbiter, in a quantitative evaluation with real users of filesystem provenance data including provenance data experts, IT professionals, and computational scientists. We also compared in the evaluation our new node grouping method to a conventional method. The results demonstrate that InProv results in higher accuracy in identifying system activity than Orbiter with large complex data sets. The results also show that our new time- based hierarchical node grouping method improves performance in both tools, and participants found both tools significantly easier to use with the new time-based node grouping method. Subjective measures show that participants found InProv to require less mental activity, less physical activity, less work, and is less stressful to use. Our study also reveals one of the first cases of gender differences in visualization; both genders had comparable performance with InProv, but women had a significantly lower average accuracy (56%) compared to men (70%) with Orbiter.
Law, Edith; Dalton, Conner; Merrill, Nick; Young, Albert; Gajos, Krzysztof Z.
Curio: A Platform for Supporting Mixed-Expertise Crowdsourcing Proceedings Article
In: Proceedings of HCOMP 2013, AAAI Press, 2013.
@inproceedings{law13:curio,
title = {Curio: A Platform for Supporting Mixed-Expertise Crowdsourcing},
author = {Edith Law and Conner Dalton and Nick Merrill and Albert Young and Krzysztof Z. Gajos},
url = {/papers/law13curio.pdf},
year = {2013},
date = {2013-01-01},
booktitle = {Proceedings of HCOMP 2013},
publisher = {AAAI Press},
abstract = {Curio is a platform that enables researchers in the sciences and humanities, who are domain experts but not necessarily technically savvy, to create, monitor and control complex crowdsourcing projects with minimal effort. With Curio, we aim to contribute to the practice of citizen science and to make fundamental contributions to the study of human computation by developing new interfaces and algorithms for supporting mixed-expertise crowdsourcing, task decomposition, incentive design and quality control.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Curio is a platform that enables researchers in the sciences and humanities, who are domain experts but not necessarily technically savvy, to create, monitor and control complex crowdsourcing projects with minimal effort. With Curio, we aim to contribute to the practice of citizen science and to make fundamental contributions to the study of human computation by developing new interfaces and algorithms for supporting mixed-expertise crowdsourcing, task decomposition, incentive design and quality control.
Komarov, Steven; Reinecke, Katharina; Gajos, Krzysztof Z.
Crowdsourcing performance evaluations of user interfaces Proceedings Article
In: Proceedings of the SIGCHI Conference on Human Factors in Computing Systems, pp. 207–216, ACM, Paris, France, 2013, ISBN: 978-1-4503-1899-0.
@inproceedings{komarov13:crowdsourcing,
title = {Crowdsourcing performance evaluations of user interfaces},
author = {Steven Komarov and Katharina Reinecke and Krzysztof Z. Gajos},
url = {/papers/komarov13crowdsourcing.pdf},
doi = {10.1145/2470654.2470684},
isbn = {978-1-4503-1899-0},
year = {2013},
date = {2013-01-01},
booktitle = {Proceedings of the SIGCHI Conference on Human Factors in Computing Systems},
pages = {207–216},
publisher = {ACM},
address = {Paris, France},
series = {CHI '13},
abstract = {Online labor markets, such as Amazon's Mechanical Turk (MTurk), provide an attractive platform for conducting human subjects experiments because the relative ease of recruitment, low cost, and a diverse pool of potential participants enable larger-scale experimentation and faster experimental revision cycle compared to lab-based settings. However, because the experimenter gives up the direct control over the participants' environments and behavior, concerns about the quality of the data collected in online settings are pervasive. In this paper, we investigate the feasibility of conducting online performance evaluations of user interfaces with anonymous, unsupervised, paid participants recruited via MTurk. We implemented three performance experiments to re-evaluate three previously well-studied user interface designs. We conducted each experiment both in lab and online with participants recruited via MTurk. The analysis of our results did not yield any evidence of significant or substantial differences in the data collected in the two settings: All statistically significant differences detected in lab were also present on MTurk and the effect sizes were similar. In addition, there were no significant differences between the two settings in the raw task completion times, error rates, consistency, or the rates of utilization of the novel interaction mechanisms introduced in the experiments. These results suggest that MTurk may be a productive setting for conducting performance evaluations of user interfaces providing a complementary approach to existing methodologies.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Online labor markets, such as Amazon’s Mechanical Turk (MTurk), provide an attractive platform for conducting human subjects experiments because the relative ease of recruitment, low cost, and a diverse pool of potential participants enable larger-scale experimentation and faster experimental revision cycle compared to lab-based settings. However, because the experimenter gives up the direct control over the participants’ environments and behavior, concerns about the quality of the data collected in online settings are pervasive. In this paper, we investigate the feasibility of conducting online performance evaluations of user interfaces with anonymous, unsupervised, paid participants recruited via MTurk. We implemented three performance experiments to re-evaluate three previously well-studied user interface designs. We conducted each experiment both in lab and online with participants recruited via MTurk. The analysis of our results did not yield any evidence of significant or substantial differences in the data collected in the two settings: All statistically significant differences detected in lab were also present on MTurk and the effect sizes were similar. In addition, there were no significant differences between the two settings in the raw task completion times, error rates, consistency, or the rates of utilization of the novel interaction mechanisms introduced in the experiments. These results suggest that MTurk may be a productive setting for conducting performance evaluations of user interfaces providing a complementary approach to existing methodologies.
Reinecke, Katharina; Yeh, Tom; Miratrix, Luke; Mardiko, Rahmatri; Zhao, Yuechen; Liu, Jenny; Gajos, Krzysztof Z.
Predicting users’ first impressions of website aesthetics with a quantification of perceived visual complexity and colorfulness Proceedings Article
In: Proceedings of the SIGCHI Conference on Human Factors in Computing Systems, pp. 2049–2058, ACM, Paris, France, 2013, ISBN: 978-1-4503-1899-0.
@inproceedings{reinecke13:predicting,
title = {Predicting users' first impressions of website aesthetics with a quantification of perceived visual complexity and colorfulness},
author = {Katharina Reinecke and Tom Yeh and Luke Miratrix and Rahmatri Mardiko and Yuechen Zhao and Jenny Liu and Krzysztof Z. Gajos},
url = {/papers/reinecke13aesthetics.pdf},
doi = {10.1145/2470654.2481281},
isbn = {978-1-4503-1899-0},
year = {2013},
date = {2013-01-01},
booktitle = {Proceedings of the SIGCHI Conference on Human Factors in Computing Systems},
pages = {2049–2058},
publisher = {ACM},
address = {Paris, France},
series = {CHI '13},
abstract = {Users make lasting judgments about a website's appeal within a split second of seeing it for the first time. This first impression is influential enough to later affect their opinions of a site's usability and trustworthiness. In this paper, we demonstrate a means to predict the initial impression of aesthetics based on perceptual models of a website's colorfulness and visual complexity. In an online study, we collected ratings of colorfulness, visual complexity, and visual appeal of a set of 450 websites from 548 volunteers. Based on these data, we developed computational models that accurately measure the perceived visual complexity and colorfulness of website screenshots. In combination with demographic variables such as a user's education level and age, these models explain approximately half of the variance in the ratings of aesthetic appeal given after viewing a website for 500ms only.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Users make lasting judgments about a website’s appeal within a split second of seeing it for the first time. This first impression is influential enough to later affect their opinions of a site’s usability and trustworthiness. In this paper, we demonstrate a means to predict the initial impression of aesthetics based on perceptual models of a website’s colorfulness and visual complexity. In an online study, we collected ratings of colorfulness, visual complexity, and visual appeal of a set of 450 websites from 548 volunteers. Based on these data, we developed computational models that accurately measure the perceived visual complexity and colorfulness of website screenshots. In combination with demographic variables such as a user’s education level and age, these models explain approximately half of the variance in the ratings of aesthetic appeal given after viewing a website for 500ms only.
Flatla, David R.; Reinecke, Katharina; Gutwin, Carl; Gajos, Krzysztof Z.
SPRWeb: preserving subjective responses to website colour schemes through automatic recolouring Proceedings Article
In: Proceedings of the SIGCHI Conference on Human Factors in Computing Systems, pp. 2069–2078, ACM, Paris, France, 2013, ISBN: 978-1-4503-1899-0.
@inproceedings{flatla13:sprweb,
title = {SPRWeb: preserving subjective responses to website colour schemes through automatic recolouring},
author = {David R. Flatla and Katharina Reinecke and Carl Gutwin and Krzysztof Z. Gajos},
url = {/papers/flatla13sprweb.pdf},
doi = {10.1145/2470654.2481283},
isbn = {978-1-4503-1899-0},
year = {2013},
date = {2013-01-01},
booktitle = {Proceedings of the SIGCHI Conference on Human Factors in Computing Systems},
pages = {2069–2078},
publisher = {ACM},
address = {Paris, France},
series = {CHI '13},
abstract = {Colours are an important part of user experiences on the Web. Colour schemes influence the aesthetics, first impressions and long-term engagement with websites. However, five percent of people perceive a subset of all colours because they have colour vision deficiency (CVD), resulting in an unequal and less-rich user experience on the Web. Traditionally, people with CVD have been supported by recolouring tools that improve colour differentiability, but do not consider the subjective properties of colour schemes while recolouring. To address this, we developed SPRWeb, a tool that recolours websites to preserve subjective responses and improve colour differentiability - thus enabling users with CVD to have similar online experiences. To develop SPRWeb, we extended existing models of non-CVD subjective responses to CVD, then used this extended model to steer the recolouring process. In a lab study, we found that SPRWeb did significantly better than a standard recolouring tool at preserving the temperature and naturalness of websites, while achieving similar weight and differentiability preservation. We also found that recolouring did not preserve activity, and hypothesize that visual complexity influences activity more than colour. SPRWeb is the first tool to automatically preserve the subjective and perceptual properties of website colour schemes thereby equalizing the colour-based web experience for people with CVD.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Colours are an important part of user experiences on the Web. Colour schemes influence the aesthetics, first impressions and long-term engagement with websites. However, five percent of people perceive a subset of all colours because they have colour vision deficiency (CVD), resulting in an unequal and less-rich user experience on the Web. Traditionally, people with CVD have been supported by recolouring tools that improve colour differentiability, but do not consider the subjective properties of colour schemes while recolouring. To address this, we developed SPRWeb, a tool that recolours websites to preserve subjective responses and improve colour differentiability – thus enabling users with CVD to have similar online experiences. To develop SPRWeb, we extended existing models of non-CVD subjective responses to CVD, then used this extended model to steer the recolouring process. In a lab study, we found that SPRWeb did significantly better than a standard recolouring tool at preserving the temperature and naturalness of websites, while achieving similar weight and differentiability preservation. We also found that recolouring did not preserve activity, and hypothesize that visual complexity influences activity more than colour. SPRWeb is the first tool to automatically preserve the subjective and perceptual properties of website colour schemes thereby equalizing the colour-based web experience for people with CVD.
Reinecke, Katharina; Flatla, David R.; Solovey, Erin; Gutwin, Carl; Gajos, Krzysztof Z.; Heer, Jeffrey
Many people, many eyes: aggregating influences of visual perception on user interface design Proceedings Article
In: CHI ’13 Extended Abstracts on Human Factors in Computing Systems, pp. 3299–3302, ACM, Paris, France, 2013, ISBN: 978-1-4503-1952-2.
@inproceedings{reinecke13:many,
title = {Many people, many eyes: aggregating influences of visual perception on user interface design},
author = {Katharina Reinecke and David R. Flatla and Erin Solovey and Carl Gutwin and Krzysztof Z. Gajos and Jeffrey Heer},
url = {/papers/reinecke13manyPeople.pdf},
doi = {10.1145/2468356.2479671},
isbn = {978-1-4503-1952-2},
year = {2013},
date = {2013-01-01},
booktitle = {CHI '13 Extended Abstracts on Human Factors in Computing Systems},
pages = {3299–3302},
publisher = {ACM},
address = {Paris, France},
series = {CHI EA '13},
abstract = {Many factors influence a user's visual perception of an interface (e.g., culture, gender, visual impairment). In general, interface researchers and designers have considered these factors in isolation, without considering the combined effect of every factor influencing the visual perception of the user. As a result, interfaces have been optimized for single factors (e.g., improving accessibility for individuals with low vision), at the expense of optimizing for the individual's visual perception experience (e.g., considering cultural preferences and lighting conditions while assisting users with low vision). In this workshop, we will begin the process of combining the broad range of visual perception knowledge to create a holistic approach to understanding users' visual perception. The resulting knowledge pool will be used for generating interfaces better suited to the full range of users' visual perception abilities.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Many factors influence a user’s visual perception of an interface (e.g., culture, gender, visual impairment). In general, interface researchers and designers have considered these factors in isolation, without considering the combined effect of every factor influencing the visual perception of the user. As a result, interfaces have been optimized for single factors (e.g., improving accessibility for individuals with low vision), at the expense of optimizing for the individual’s visual perception experience (e.g., considering cultural preferences and lighting conditions while assisting users with low vision). In this workshop, we will begin the process of combining the broad range of visual perception knowledge to create a holistic approach to understanding users’ visual perception. The resulting knowledge pool will be used for generating interfaces better suited to the full range of users’ visual perception abilities.
Kim, Juho; Miller, Robert C.; Gajos, Krzysztof Z.
Learnersourcing subgoal labeling to support learning from how-to videos Proceedings Article
In: CHI ’13 Extended Abstracts on Human Factors in Computing Systems, pp. 685–690, ACM, Paris, France, 2013, ISBN: 978-1-4503-1952-2.
@inproceedings{kim13:learnersourcing,
title = {Learnersourcing subgoal labeling to support learning from how-to videos},
author = {Juho Kim and Robert C. Miller and Krzysztof Z. Gajos},
url = {/papers/kim13learnersourcing.pdf},
doi = {10.1145/2468356.2468477},
isbn = {978-1-4503-1952-2},
year = {2013},
date = {2013-01-01},
booktitle = {CHI '13 Extended Abstracts on Human Factors in Computing Systems},
pages = {685–690},
publisher = {ACM},
address = {Paris, France},
series = {CHI EA '13},
abstract = {Subgoal labeling is a technique known to support learning new knowledge by clustering a group of steps into a higher-level conceptual unit. It has been shown to improve learning by helping learners to form the right mental model. While many learners view video tutorials nowadays, subgoal labels are often not available unless manually provided at production
time. This work addresses the challenge of collecting and presenting subgoal labels to a large number of video tutorials. We introduce a mixed-initiative approach to collect subgoal labels in a scalable and efficient manner. The key component of this method is learnersourcing, which channels learners' activities using the video interface into useful input to the system. The presented method will contribute to the broader availability of subgoal labels in how-to videos.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Subgoal labeling is a technique known to support learning new knowledge by clustering a group of steps into a higher-level conceptual unit. It has been shown to improve learning by helping learners to form the right mental model. While many learners view video tutorials nowadays, subgoal labels are often not available unless manually provided at production
time. This work addresses the challenge of collecting and presenting subgoal labels to a large number of video tutorials. We introduce a mixed-initiative approach to collect subgoal labels in a scalable and efficient manner. The key component of this method is learnersourcing, which channels learners’ activities using the video interface into useful input to the system. The presented method will contribute to the broader availability of subgoal labels in how-to videos.
time. This work addresses the challenge of collecting and presenting subgoal labels to a large number of video tutorials. We introduce a mixed-initiative approach to collect subgoal labels in a scalable and efficient manner. The key component of this method is learnersourcing, which channels learners’ activities using the video interface into useful input to the system. The presented method will contribute to the broader availability of subgoal labels in how-to videos.
Reinecke, Katharina; Nguyen, Minh Khoa; Bernstein, Abraham; Näf, Michael; Gajos, Krzysztof Z.
Doodle around the world: online scheduling behavior reflects cultural differences in time perception and group decision-making Proceedings Article
In: Proceedings of the 2013 conference on Computer supported cooperative work, pp. 45–54, ACM, San Antonio, Texas, USA, 2013, ISBN: 978-1-4503-1331-5.
@inproceedings{reinecke13:doodle,
title = {Doodle around the world: online scheduling behavior reflects cultural differences in time perception and group decision-making},
author = {Katharina Reinecke and Minh Khoa Nguyen and Abraham Bernstein and Michael Näf and Krzysztof Z. Gajos},
url = {/papers/reinecke13doodle.pdf},
doi = {10.1145/2441776.2441784},
isbn = {978-1-4503-1331-5},
year = {2013},
date = {2013-01-01},
booktitle = {Proceedings of the 2013 conference on Computer supported cooperative work},
pages = {45–54},
publisher = {ACM},
address = {San Antonio, Texas, USA},
series = {CSCW '13},
abstract = {Event scheduling is a group decision-making process in which social dynamics influence people's choices and the overall outcome. As a result, scheduling is not simply a matter of finding a mutually agreeable time, but a process that is shaped by social norms and values, which can highly vary between countries. To investigate the influence of national culture on people's scheduling behavior we analyzed more than 1.5 million Doodle date/time polls from 211 countries. We found strong correlations between characteristics of national culture and several behavioral phenomena, such as that poll participants from collectivist countries respond earlier, agree to fewer options but find more consensus than predominantly individualist societies. Our study provides empirical evidence of behavioral differences in group decision-making and time perception with implications for cross-cultural collaborative work.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Event scheduling is a group decision-making process in which social dynamics influence people’s choices and the overall outcome. As a result, scheduling is not simply a matter of finding a mutually agreeable time, but a process that is shaped by social norms and values, which can highly vary between countries. To investigate the influence of national culture on people’s scheduling behavior we analyzed more than 1.5 million Doodle date/time polls from 211 countries. We found strong correlations between characteristics of national culture and several behavioral phenomena, such as that poll participants from collectivist countries respond earlier, agree to fewer options but find more consensus than predominantly individualist societies. Our study provides empirical evidence of behavioral differences in group decision-making and time perception with implications for cross-cultural collaborative work.
2012
Mao, A.; Chen, Y.; Gajos, K. Z.; Parkes, D. C.; Procaccia, A. D.; Zhang, H.
TurkServer: Enabling Synchronous and Longitudinal Online Experiments Proceedings Article
In: Proceedings of HCOMP’12, 2012.
@inproceedings{mao12:turkserver,
title = {TurkServer: Enabling Synchronous and Longitudinal Online Experiments},
author = {A. Mao and Y. Chen and K. Z. Gajos and D. C. Parkes and A. D. Procaccia and H. Zhang},
url = {/papers/mao12-turkserver.pdf},
year = {2012},
date = {2012-01-01},
booktitle = {Proceedings of HCOMP'12},
abstract = {With the proliferation of online labor markets and other social computing platforms, online experiments have become a low-cost and scalable way to empirically test hypotheses and mechanisms in both human computation and social science. Yet, despite the potential in designing more powerful and expressive on-line experiments using multiple subjects, researchers still face many technical and logistical difficulties. We see synchronous and longitudinal experiments involving real-time interaction between participants as a dual-use paradigm for both human computation and social science, and present TurkServer, a platform that facilitates these types of experiments on Amazon Mechanical Turk. Our work has the potential to make more fruitful online experiments accessible to researchers in many different fields.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
With the proliferation of online labor markets and other social computing platforms, online experiments have become a low-cost and scalable way to empirically test hypotheses and mechanisms in both human computation and social science. Yet, despite the potential in designing more powerful and expressive on-line experiments using multiple subjects, researchers still face many technical and logistical difficulties. We see synchronous and longitudinal experiments involving real-time interaction between participants as a dual-use paradigm for both human computation and social science, and present TurkServer, a platform that facilitates these types of experiments on Amazon Mechanical Turk. Our work has the potential to make more fruitful online experiments accessible to researchers in many different fields.
Gajos, Krzysztof Z.; Hurst, Amy; Findlater, Leah
Personalized dynamic accessibility Journal Article
In: interactions, vol. 19, no. 2, pp. 69–73, 2012, ISSN: 1072-5520.
@article{gajos12:personalized,
title = {Personalized dynamic accessibility},
author = {Krzysztof Z. Gajos and Amy Hurst and Leah Findlater},
url = {http://www.eecs.harvard.edu/~kgajos/papers/authorizer.html?url=http://dl.acm.org/authorize?6618349},
doi = {10.1145/2090150.2090167},
issn = {1072-5520},
year = {2012},
date = {2012-03-01},
journal = {interactions},
volume = {19},
number = {2},
pages = {69–73},
publisher = {ACM},
address = {New York, NY, USA},
abstract = {It is a cliche to point out that computers and the Internet have entered all parts of our lives. We need them at work; governments urge us to file our taxes online; students are required to use them in their classes; online businesses offer better deals than their brick-and-mortar counterparts; and it is becoming more difficult to maintain relationships without access to social networking and social media sites. Public discourse about accessibility focuses on the assertion that access to these technologies is essential for meaningful participation in today's society. Unfortunately, compliance with accessibility guidelines and standards is still not a part of mainstream software engineering and user interface design practice. As
a result, we must remind, beg, and threaten developers to make software accessible. But is this sufficient? Are we blinding ourselves to tomorrow's challenges as we fight yesterday's battles?<br/><br/>
We argue that it is both the possibility and the efficiency of access that are necessary for meaningful and equitable participation in society. As larger fractions of our personal and professional activities are conducted using computers, inefficient access limits what an individual can achieve. We propose a longterm vision of Personalized Dynamic Accessibility: We believe that user interfaces will enable more effective interaction if they reflect each person's unique abilities, devices, and environment.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
It is a cliche to point out that computers and the Internet have entered all parts of our lives. We need them at work; governments urge us to file our taxes online; students are required to use them in their classes; online businesses offer better deals than their brick-and-mortar counterparts; and it is becoming more difficult to maintain relationships without access to social networking and social media sites. Public discourse about accessibility focuses on the assertion that access to these technologies is essential for meaningful participation in today’s society. Unfortunately, compliance with accessibility guidelines and standards is still not a part of mainstream software engineering and user interface design practice. As
a result, we must remind, beg, and threaten developers to make software accessible. But is this sufficient? Are we blinding ourselves to tomorrow’s challenges as we fight yesterday’s battles?<br/><br/>
We argue that it is both the possibility and the efficiency of access that are necessary for meaningful and equitable participation in society. As larger fractions of our personal and professional activities are conducted using computers, inefficient access limits what an individual can achieve. We propose a longterm vision of Personalized Dynamic Accessibility: We believe that user interfaces will enable more effective interaction if they reflect each person’s unique abilities, devices, and environment.
a result, we must remind, beg, and threaten developers to make software accessible. But is this sufficient? Are we blinding ourselves to tomorrow’s challenges as we fight yesterday’s battles?<br/><br/>
We argue that it is both the possibility and the efficiency of access that are necessary for meaningful and equitable participation in society. As larger fractions of our personal and professional activities are conducted using computers, inefficient access limits what an individual can achieve. We propose a longterm vision of Personalized Dynamic Accessibility: We believe that user interfaces will enable more effective interaction if they reflect each person’s unique abilities, devices, and environment.
Jameson, Anthony; Gajos, Krzysztof Z.
Systems That Adapt to Their Users Book Section
In: Jacko, Julie A. (Ed.): Human-Computer Interaction Handbook: Fundamentals, Evolving Technologies, and Emerging Applications, CRC Press, 2012.
@incollection{jameson12:systems,
title = {Systems That Adapt to Their Users},
author = {Anthony Jameson and Krzysztof Z. Gajos},
editor = {Julie A. Jacko},
year = {2012},
date = {2012-01-01},
booktitle = {Human-Computer Interaction Handbook: Fundamentals, Evolving Technologies, and Emerging Applications},
publisher = {CRC Press},
edition = {Third Edition},
keywords = {},
pubstate = {published},
tppubtype = {incollection}
}
Zhang, Haoqi; Law, Edith; Miller, Rob; Gajos, Krzysztof; Parkes, David; Horvitz, Eric
Human computation tasks with global constraints Proceedings Article
In: Proceedings of the SIGCHI Conference on Human Factors in Computing Systems, pp. 217–226, ACM, Austin, Texas, USA, 2012, ISBN: 978-1-4503-1015-4.
@inproceedings{zhang12:human,
title = {Human computation tasks with global constraints},
author = {Haoqi Zhang and Edith Law and Rob Miller and Krzysztof Gajos and David Parkes and Eric Horvitz},
url = {/papers/zhang12-mobi.pdf},
doi = {10.1145/2207676.2207708},
isbn = {978-1-4503-1015-4},
year = {2012},
date = {2012-01-01},
booktitle = {Proceedings of the SIGCHI Conference on Human Factors in Computing Systems},
pages = {217–226},
publisher = {ACM},
address = {Austin, Texas, USA},
series = {CHI '12},
abstract = {An important class of tasks that are underexplored in current human computation systems are complex tasks with global constraints. One example of such a task is itinerary planning, where solutions consist of a sequence of activities that meet requirements specified by the requester. In this paper, we focus on the crowdsourcing of such plans as a case study of constraint-based human computation tasks and introduce a collaborative planning system called Mobi that illustrates a novel crowdware paradigm. Mobi presents a single interface that enables crowd participants to view the current solution context and make appropriate contributions based on current needs. We conduct experiments that explain how Mobi enables a crowd to effectively and collaboratively resolve global constraints, and discuss how the design princi- ples behind Mobi can more generally facilitate a crowd to tackle problems involving global constraints.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
An important class of tasks that are underexplored in current human computation systems are complex tasks with global constraints. One example of such a task is itinerary planning, where solutions consist of a sequence of activities that meet requirements specified by the requester. In this paper, we focus on the crowdsourcing of such plans as a case study of constraint-based human computation tasks and introduce a collaborative planning system called Mobi that illustrates a novel crowdware paradigm. Mobi presents a single interface that enables crowd participants to view the current solution context and make appropriate contributions based on current needs. We conduct experiments that explain how Mobi enables a crowd to effectively and collaboratively resolve global constraints, and discuss how the design princi- ples behind Mobi can more generally facilitate a crowd to tackle problems involving global constraints.
Gajos, Krzysztof; Reinecke, Katharina; Herrmann, Charles
Accurate measurements of pointing performance from in situ observations Proceedings Article
In: Proceedings of the 2012 ACM annual conference on Human Factors in Computing Systems, pp. 3157–3166, ACM, Austin, Texas, USA, 2012, ISBN: 978-1-4503-1015-4.
@inproceedings{gajos12:accurate,
title = {Accurate measurements of pointing performance from in situ observations},
author = {Krzysztof Gajos and Katharina Reinecke and Charles Herrmann},
url = {/papers/gajos12-accurateMeasurements.pdf},
doi = {10.1145/2208636.2208733},
isbn = {978-1-4503-1015-4},
year = {2012},
date = {2012-01-01},
booktitle = {Proceedings of the 2012 ACM annual conference on Human Factors in Computing Systems},
pages = {3157–3166},
publisher = {ACM},
address = {Austin, Texas, USA},
series = {CHI '12},
abstract = {We present a method for obtaining lab-quality measurements of pointing performance from unobtrusive observations of natural in situ interactions. Specifically, we have developed a set of user-independent classifiers for discriminating between deliberate, targeted mouse pointer movements and those movements that were affected by any extraneous factors. To develop and validate these classifiers, we developed logging software to unobtrusively record pointer trajectories as participants naturally interacted with their computers over the course of several weeks. Each participant also performed a set of pointing tasks in a formal study set-up. For each movement, we computed a set of measures capturing nuances of the trajectory and the speed, acceleration, and jerk profiles. Treating the observations from the formal study as positive examples of deliberate, targeted movements and the in situ observations as unlabeled data with an unknown mix of deliberate and distracted interactions, we used a recent advance in machine learning to develop the classifiers. Our results show that, on four distinct metrics, the data collected in-situ and filtered with our classifiers closely matches the results obtained from the formal experiment.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
We present a method for obtaining lab-quality measurements of pointing performance from unobtrusive observations of natural in situ interactions. Specifically, we have developed a set of user-independent classifiers for discriminating between deliberate, targeted mouse pointer movements and those movements that were affected by any extraneous factors. To develop and validate these classifiers, we developed logging software to unobtrusively record pointer trajectories as participants naturally interacted with their computers over the course of several weeks. Each participant also performed a set of pointing tasks in a formal study set-up. For each movement, we computed a set of measures capturing nuances of the trajectory and the speed, acceleration, and jerk profiles. Treating the observations from the formal study as positive examples of deliberate, targeted movements and the in situ observations as unlabeled data with an unknown mix of deliberate and distracted interactions, we used a recent advance in machine learning to develop the classifiers. Our results show that, on four distinct metrics, the data collected in-situ and filtered with our classifiers closely matches the results obtained from the formal experiment.
Kim, Juho; Malley, Benjamin; Brandt, Joel; Dontcheva, Mira; Joseph, Diana; Gajos, Krzysztof Z.; Miller, Robert C.
Photoshop with friends: a synchronous learning community for graphic design Proceedings Article
In: Proceedings of the ACM 2012 conference on Computer Supported Cooperative Work Companion, pp. 271–272, ACM, Seattle, Washington, USA, 2012, ISBN: 978-1-4503-1051-2.
@inproceedings{kim12:photoshop,
title = {Photoshop with friends: a synchronous learning community for graphic design},
author = {Juho Kim and Benjamin Malley and Joel Brandt and Mira Dontcheva and Diana Joseph and Krzysztof Z. Gajos and Robert C. Miller},
url = {/papers/kim12-photoshop.pdf},
doi = {10.1145/2141512.2141598},
isbn = {978-1-4503-1051-2},
year = {2012},
date = {2012-01-01},
booktitle = {Proceedings of the ACM 2012 conference on Computer Supported Cooperative Work Companion},
pages = {271–272},
publisher = {ACM},
address = {Seattle, Washington, USA},
series = {CSCW '12},
abstract = {Photoshop with Friends is an online community of learners exchanging just-in-time help on graphic design tasks. The system attempts to provide an interactive, visual, context-aware, and personalized mode of learning. Developed as a Facebook application, Photoshop with Friends allows users to help each other in live sessions, with built-in screen sharing, recording, and voice chat support. Major design decisions are guided by two laboratory studies that identified challenges in learning graphic design skills on the web.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Photoshop with Friends is an online community of learners exchanging just-in-time help on graphic design tasks. The system attempts to provide an interactive, visual, context-aware, and personalized mode of learning. Developed as a Facebook application, Photoshop with Friends allows users to help each other in live sessions, with built-in screen sharing, recording, and voice chat support. Major design decisions are guided by two laboratory studies that identified challenges in learning graphic design skills on the web.
2011
Hurst, Amy; Gajos, Krzysztof; Findlater, Leah; Wobbrock, Jacob; Sears, Andrew; Trewin, Shari
Dynamic accessibility: accommodating differences in ability and situation Proceedings Article
In: Proceedings of the 2011 annual conference extended abstracts on Human factors in computing systems, pp. 41–44, ACM, Vancouver, BC, Canada, 2011, ISBN: 978-1-4503-0268-5.
@inproceedings{hurst11:dynamic,
title = {Dynamic accessibility: accommodating differences in ability and situation},
author = {Amy Hurst and Krzysztof Gajos and Leah Findlater and Jacob Wobbrock and Andrew Sears and Shari Trewin},
url = {/papers/hurst-dynamic-chi11.pdf},
doi = {10.1145/1979742.1979589},
isbn = {978-1-4503-0268-5},
year = {2011},
date = {2011-01-01},
booktitle = {Proceedings of the 2011 annual conference extended abstracts on Human factors in computing systems},
pages = {41–44},
publisher = {ACM},
address = {Vancouver, BC, Canada},
series = {CHI EA '11},
abstract = {Human abilities are idiosyncratic and may change frequently. Static one-size-fits-many accessibility solutions miss the opportunities that arise from careful consideration of an individual's abilities and fail to address the sometimes dynamic aspect of those abilities, such as when a user's activity or context causes a situational impairment. The goal of this workshop is to bring together researchers and practitioners in accessibility, mobile HCI, and interactive intelligent systems who are pursuing agile, data-driven approaches that enable interactive systems to adapt or become adapted to the needs and abilities of a particular individual in a particular context.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Human abilities are idiosyncratic and may change frequently. Static one-size-fits-many accessibility solutions miss the opportunities that arise from careful consideration of an individual’s abilities and fail to address the sometimes dynamic aspect of those abilities, such as when a user’s activity or context causes a situational impairment. The goal of this workshop is to bring together researchers and practitioners in accessibility, mobile HCI, and interactive intelligent systems who are pursuing agile, data-driven approaches that enable interactive systems to adapt or become adapted to the needs and abilities of a particular individual in a particular context.
Noronha, Jon; Hysen, Eric; Zhang, Haoqi; Gajos, Krzysztof Z.
PlateMate: Crowdsourcing Nutrition Analysis from Food Photographs Proceedings Article
In: Proceedings of the 24th annual ACM symposium on User interface software and technology, pp. 1–12, ACM, Santa Barbara, California, USA, 2011, ISBN: 978-1-4503-0716-1.
@inproceedings{noronha11:platemate,
title = {PlateMate: Crowdsourcing Nutrition Analysis from Food Photographs},
author = {Jon Noronha and Eric Hysen and Haoqi Zhang and Krzysztof Z. Gajos},
url = {/papers/noronha-platemate-uist11.pdf},
doi = {10.1145/2047196.2047198},
isbn = {978-1-4503-0716-1},
year = {2011},
date = {2011-01-01},
booktitle = {Proceedings of the 24th annual ACM symposium on User interface software and technology},
pages = {1–12},
publisher = {ACM},
address = {Santa Barbara, California, USA},
series = {UIST '11},
abstract = {PlateMate allows users to take photos of their meals and receive estimates of food intake and composition. Accurate awareness of this information is considered a prerequisite to successful change of eating habits, but current methods for food logging via self-reporting, expert observation, or algorithmic analysis are time-consuming, expensive, or inaccurate. PlateMate crowdsources nutritional analysis from photographs using Amazon Mechanical Turk, automatically coordinating untrained workers to estimate a meal's calories, fat, carbohydrates, and protein. We present the Management framework for crowdsourcing complex tasks, which supports PlateMate's decomposition of the nutrition analysis workflow. Two evaluations show that the PlateMate system is nearly as accurate as a trained dietitian and easier to use for most users than traditional self-reporting, while remaining robust for general use across a wide variety of meal types.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
PlateMate allows users to take photos of their meals and receive estimates of food intake and composition. Accurate awareness of this information is considered a prerequisite to successful change of eating habits, but current methods for food logging via self-reporting, expert observation, or algorithmic analysis are time-consuming, expensive, or inaccurate. PlateMate crowdsources nutritional analysis from photographs using Amazon Mechanical Turk, automatically coordinating untrained workers to estimate a meal’s calories, fat, carbohydrates, and protein. We present the Management framework for crowdsourcing complex tasks, which supports PlateMate’s decomposition of the nutrition analysis workflow. Two evaluations show that the PlateMate system is nearly as accurate as a trained dietitian and easier to use for most users than traditional self-reporting, while remaining robust for general use across a wide variety of meal types.
Borkin, Michelle A.; Gajos, Krzysztof Z.; Peters, Amanda; Mitsouras, Dimitrios; Melchionna, Simone; Rybicki, Frank J.; Feldman, Charles L.; Pfister, Hanspeter
Evaluation of Artery Visualizations for Heart Disease Diagnosis Journal Article
In: IEEE Transactions on Visualization and Computer Graphics (Proceedings of Information Visualization 2011), vol. 17, no. 12, 2011.
@article{borkin11:evaluation,
title = {Evaluation of Artery Visualizations for Heart Disease Diagnosis},
author = {Michelle A. Borkin and Krzysztof Z. Gajos and Amanda Peters and Dimitrios Mitsouras and Simone Melchionna and Frank J. Rybicki and Charles L. Feldman and Hanspeter Pfister},
url = {/papers/borkin11-infoviz.pdf},
year = {2011},
date = {2011-12-01},
journal = {IEEE Transactions on Visualization and Computer Graphics (Proceedings of Information Visualization 2011)},
volume = {17},
number = {12},
abstract = {Heart disease is the number one killer in the United States, and finding indicators of the disease at an early stage is critical for treatment and prevention. In this paper we evaluate visualization techniques that enable the diagnosis of coronary artery disease. A key physical quantity of medical interest is endothelial shear stress (ESS). Low ESS has been associated with sites of lesion formation and rapid progression of disease in the coronary arteries. Having effective visualizations of a patient's ESS data is vital for the quick and thorough non-invasive evaluation by a cardiologist. We present a task taxonomy for hemodynamics based on a formative user study with domain experts. Based on the results of this study we developed HemoVis, an interactive visualization application for heart disease diagnosis that uses a novel 2D tree diagram representation of coronary artery trees. We present the results of a formal quantitative user study with domain experts that evaluates the effect of 2D versus 3D artery representations and of color maps on identifying regions of low ESS. We show statistically significant results demonstrating that our 2D visualizations are more accurate and efficient than 3D representations, and that a perceptually appropriate color map leads to fewer diagnostic mistakes than a rainbow color map.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Heart disease is the number one killer in the United States, and finding indicators of the disease at an early stage is critical for treatment and prevention. In this paper we evaluate visualization techniques that enable the diagnosis of coronary artery disease. A key physical quantity of medical interest is endothelial shear stress (ESS). Low ESS has been associated with sites of lesion formation and rapid progression of disease in the coronary arteries. Having effective visualizations of a patient’s ESS data is vital for the quick and thorough non-invasive evaluation by a cardiologist. We present a task taxonomy for hemodynamics based on a formative user study with domain experts. Based on the results of this study we developed HemoVis, an interactive visualization application for heart disease diagnosis that uses a novel 2D tree diagram representation of coronary artery trees. We present the results of a formal quantitative user study with domain experts that evaluates the effect of 2D versus 3D artery representations and of color maps on identifying regions of low ESS. We show statistically significant results demonstrating that our 2D visualizations are more accurate and efficient than 3D representations, and that a perceptually appropriate color map leads to fewer diagnostic mistakes than a rainbow color map.
Reinecke, Katharina; Gajos, Krzysztof Z.
One size fits many Westerners: How Cultural Abilities Challenge UI Design Proceedings Article
In: Proceedings of the Workshop on Dynamic Accessibility: Detecting and Accommodating Differences in Ability and Situation at CHI’11, 2011.
@inproceedings{reinecke11:culturalabilities,
title = {One size fits many Westerners: How Cultural Abilities Challenge UI Design},
author = {Katharina Reinecke and Krzysztof Z. Gajos},
url = {/papers/Reinecke_Gajos_CulturalAbilities.pdf},
year = {2011},
date = {2011-01-01},
booktitle = {Proceedings of the Workshop on Dynamic Accessibility: Detecting and Accommodating Differences in Ability and Situation at CHI'11},
abstract = {Cultural influences on our behavior are partly reflected in neuro-anatomical changes in our brains, altering our abilities to perceive and interpret information. This paper points out possible consequences of such ``cultural abilities'' for user interface design, and outlines challenges for systems that adapt to their users. Specifically, we describe how culture could influence our design decisions of when, how, what, why, and where to adapt.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Cultural influences on our behavior are partly reflected in neuro-anatomical changes in our brains, altering our abilities to perceive and interpret information. This paper points out possible consequences of such “cultural abilities” for user interface design, and outlines challenges for systems that adapt to their users. Specifically, we describe how culture could influence our design decisions of when, how, what, why, and where to adapt.
Wobbrock, Jacob O.; Kane, Shaun K.; Gajos, Krzysztof Z.; Harada, Susumu; Froehlich, Jon
Ability-Based Design: Concept, Principles and Examples Journal Article
In: ACM Trans. Access. Comput., vol. 3, iss. 3, pp. 9:1–9:27, 2011, ISSN: 1936-7228.
@article{wobbrock11:abd,
title = {Ability-Based Design: Concept, Principles and Examples},
author = {Jacob O. Wobbrock and Shaun K. Kane and Krzysztof Z. Gajos and Susumu Harada and Jon Froehlich},
url = {/papers/wobbrock11abd.pdf},
doi = {10.1145/1952383.1952384},
issn = {1936-7228},
year = {2011},
date = {2011-04-01},
journal = {ACM Trans. Access. Comput.},
volume = {3},
issue = {3},
pages = {9:1–9:27},
publisher = {ACM},
address = {New York, NY, USA},
abstract = {Current approaches to accessible computing share a common goal of making technology accessible to users with disabilities. Perhaps because of this goal, they may also share a tendency to centralize disability rather than ability. We present a refinement to these approaches called ability-based design that consists of focusing on ability throughout the design process in an effort to create systems that leverage the full range of human potential. Just as user-centered design shifted the focus of interactive system design from systems to users, ability-based design attempts to shift the focus of accessible design from disability to ability. Although prior approaches to accessible computing may consider users' abilities to some extent, ability-based design makes ability its central focus. We offer seven ability-based design principles and describe the projects that inspired their formulation. We also present a research agenda for ability-based design.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Current approaches to accessible computing share a common goal of making technology accessible to users with disabilities. Perhaps because of this goal, they may also share a tendency to centralize disability rather than ability. We present a refinement to these approaches called ability-based design that consists of focusing on ability throughout the design process in an effort to create systems that leverage the full range of human potential. Just as user-centered design shifted the focus of interactive system design from systems to users, ability-based design attempts to shift the focus of accessible design from disability to ability. Although prior approaches to accessible computing may consider users’ abilities to some extent, ability-based design makes ability its central focus. We offer seven ability-based design principles and describe the projects that inspired their formulation. We also present a research agenda for ability-based design.
Jayatilaka, Lahiru G.; Bertuccelli, Luca F.; Staszewski, James; Gajos, Krzysztof Z.
Evaluating a Pattern-Based Visual Support Approach for Humanitarian Landmine Clearance Proceedings Article
In: CHI ’11: Proceeding of the annual SIGCHI conference on Human factors in computing systems, ACM, New York, NY, USA, 2011.
@inproceedings{jayatilaka11:evaluating,
title = {Evaluating a Pattern-Based Visual Support Approach for Humanitarian Landmine Clearance},
author = {Lahiru G. Jayatilaka and Luca F. Bertuccelli and James Staszewski and Krzysztof Z. Gajos},
url = {/papers/jayatilaka-chi11.pdf},
doi = {10.1145/1978942.1979006},
year = {2011},
date = {2011-01-01},
booktitle = {CHI '11: Proceeding of the annual SIGCHI conference on Human factors in computing systems},
publisher = {ACM},
address = {New York, NY, USA},
abstract = {Unexploded landmines have severe post-conflict humanitarian repercussions: landmines cost lives, limbs and land. For deminers engaged in humanitarian landmine clearance, metal detectors remain the primary detection tool as more sophisticated technologies fail to get adopted due to restrictive cost, low reliability, and limited robustness. Metal detectors are, however, of limited effectiveness, as modern landmines contain only minimal amounts of metal, making them difficult to distinguish from the ubiquitous but harmless metallic clutter littering post-combat areas. We seek to improve the safety and efficiency of the demining process without introducing an inviable replacement for the metal detectors. Instead, we propose and evaluate a novel, pattern-based visual support approach inspired by the documented strategies employed by expert deminers. In our laboratory study, participants provided with a prototype of our support tool were 80% less likely to mistake a mine for harmless clutter. A follow-up study demonstrates the potential of our pattern-based approach to enable peer decision-making support during landmine clearance. Lastly, we identify several design opportunities for further improving deminers' decision making capabilities.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Unexploded landmines have severe post-conflict humanitarian repercussions: landmines cost lives, limbs and land. For deminers engaged in humanitarian landmine clearance, metal detectors remain the primary detection tool as more sophisticated technologies fail to get adopted due to restrictive cost, low reliability, and limited robustness. Metal detectors are, however, of limited effectiveness, as modern landmines contain only minimal amounts of metal, making them difficult to distinguish from the ubiquitous but harmless metallic clutter littering post-combat areas. We seek to improve the safety and efficiency of the demining process without introducing an inviable replacement for the metal detectors. Instead, we propose and evaluate a novel, pattern-based visual support approach inspired by the documented strategies employed by expert deminers. In our laboratory study, participants provided with a prototype of our support tool were 80% less likely to mistake a mine for harmless clutter. A follow-up study demonstrates the potential of our pattern-based approach to enable peer decision-making support during landmine clearance. Lastly, we identify several design opportunities for further improving deminers’ decision making capabilities.
2010
Gajos, Krzysztof Z.
Understanding How to Design Complex Brain-Controlled Applications Proceedings Article
In: Proceedings of the Workshop on Brain, Body and Bytes: Psychophysiological User Interaction at CHI’10, 2010.
@inproceedings{gajos10:understanding,
title = {Understanding How to Design Complex Brain-Controlled Applications},
author = {Krzysztof Z. Gajos},
url = {/papers/gajos-BrainBodyBytes-2010.pdf},
year = {2010},
date = {2010-01-01},
booktitle = {Proceedings of the Workshop on Brain, Body and Bytes: Psychophysiological User Interaction at CHI'10},
abstract = {<p>Most of the effort in brain-computer interface (BCI) research so far has been directed at developing better sensors and better ways of extracting useful information from the brain signal, while little effort has been directed at systematically understanding the unique strengths and limitations of this input modality and their implications for interaction design. Past proof of concept brain-controlled applications either involved very simple interaction or they augmented existing complex applications with external widgets to enable limited control.
</p><p>
This relative lack of research directed at developing applications and interaction methods specifically for brain-computer interfaces is a concern for several reasons. First, some BCI technologies are mature enough to be used soon by large numbers of paralyzed users. Lack of compelling brain-controlled applications or the tools and techniques for building such applications will significantly limit the impact of these technologies. Second, recent work on ability-based user interfaces has demonstrated that large gains in efficiency of interaction and user satisfaction can be achieved if user interfaces are designed with a user's specific abilities and devices in mind. The results of our studies showed that adapting user interfaces to the unique abilities of people with a range of motor impairments helped close the performance gap between those users and able-bodied people by over 60%. For BCI users, who need up to several tens of seconds to perform a single UI operation, efficiency of interaction will have an even larger impact and will likely determine whether an application is usable in practice. Lastly, a lot of the current research effort in BCI is directed at improving various aspects of the sensing and signal extraction technology. Good understanding of the user interaction requirements of brain-controlled applications will help inform and direct those efforts to maximize their impact on the user experience.
</p><p>
Motivated by these observations, we are starting a project to explore the properties and limitations of one particularly promising BCI paradigm as an input modality and to develop methods and tools for designing user interfaces for complex brain-controlled applications.</p>},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
<p>Most of the effort in brain-computer interface (BCI) research so far has been directed at developing better sensors and better ways of extracting useful information from the brain signal, while little effort has been directed at systematically understanding the unique strengths and limitations of this input modality and their implications for interaction design. Past proof of concept brain-controlled applications either involved very simple interaction or they augmented existing complex applications with external widgets to enable limited control.
</p><p>
This relative lack of research directed at developing applications and interaction methods specifically for brain-computer interfaces is a concern for several reasons. First, some BCI technologies are mature enough to be used soon by large numbers of paralyzed users. Lack of compelling brain-controlled applications or the tools and techniques for building such applications will significantly limit the impact of these technologies. Second, recent work on ability-based user interfaces has demonstrated that large gains in efficiency of interaction and user satisfaction can be achieved if user interfaces are designed with a user’s specific abilities and devices in mind. The results of our studies showed that adapting user interfaces to the unique abilities of people with a range of motor impairments helped close the performance gap between those users and able-bodied people by over 60%. For BCI users, who need up to several tens of seconds to perform a single UI operation, efficiency of interaction will have an even larger impact and will likely determine whether an application is usable in practice. Lastly, a lot of the current research effort in BCI is directed at improving various aspects of the sensing and signal extraction technology. Good understanding of the user interaction requirements of brain-controlled applications will help inform and direct those efforts to maximize their impact on the user experience.
</p><p>
Motivated by these observations, we are starting a project to explore the properties and limitations of one particularly promising BCI paradigm as an input modality and to develop methods and tools for designing user interfaces for complex brain-controlled applications.</p>
</p><p>
This relative lack of research directed at developing applications and interaction methods specifically for brain-computer interfaces is a concern for several reasons. First, some BCI technologies are mature enough to be used soon by large numbers of paralyzed users. Lack of compelling brain-controlled applications or the tools and techniques for building such applications will significantly limit the impact of these technologies. Second, recent work on ability-based user interfaces has demonstrated that large gains in efficiency of interaction and user satisfaction can be achieved if user interfaces are designed with a user’s specific abilities and devices in mind. The results of our studies showed that adapting user interfaces to the unique abilities of people with a range of motor impairments helped close the performance gap between those users and able-bodied people by over 60%. For BCI users, who need up to several tens of seconds to perform a single UI operation, efficiency of interaction will have an even larger impact and will likely determine whether an application is usable in practice. Lastly, a lot of the current research effort in BCI is directed at improving various aspects of the sensing and signal extraction technology. Good understanding of the user interaction requirements of brain-controlled applications will help inform and direct those efforts to maximize their impact on the user experience.
</p><p>
Motivated by these observations, we are starting a project to explore the properties and limitations of one particularly promising BCI paradigm as an input modality and to develop methods and tools for designing user interfaces for complex brain-controlled applications.</p>
Gajos, Krzysztof Z.; Weld, Daniel S.; Wobbrock, Jacob O.
Automatically generating personalized user interfaces with Supple Journal Article
In: Artificial Intelligence, vol. 174, iss. 12–13, pp. 910–950, 2010.
@article{gajos10:automatically,
title = {Automatically generating personalized user interfaces with Supple},
author = {Krzysztof Z. Gajos and Daniel S. Weld and Jacob O. Wobbrock},
url = {/papers/gajos10supple-aij.pdf},
doi = {10.1016/j.artint.2010.05.005},
year = {2010},
date = {2010-01-01},
journal = {Artificial Intelligence},
volume = {174},
issue = {12–13},
pages = {910–950},
publisher = {Elsevier},
abstract = {<p>Today's computer-human interfaces are typically designed with the
assumption that they are going to be used by an able-bodied person,
who is using a typical set of input and output devices, who has
typical perceptual and cognitive abilities, and who is sitting in a
stable, warm environment. Any deviation from these assumptions may
drastically hamper the person's effectiveness—not because of any
<it>inherent</it> barrier to interaction, but because of a mismatch
between the person's effective abilities and the assumptions underlying the
interface design.</p>
<p>We argue that automatic personalized interface generation is a feasible and
scalable solution to this challenge. We present our Supple system,
which can automatically generate interfaces adapted to a person's
devices, tasks, preferences, and abilities. In this paper we formally
define interface generation as an optimization problem and
demonstrate that, despite a large solution space (of up to 10^{17} possible interfaces), the problem is computationally feasible. In fact, for a particular class of cost functions, Supple produces exact solutions in under a second for most cases, and in
a little over a minute in the worst case encountered, thus enabling run-time generation of user interfaces. We further show
how several different design criteria can be expressed in the cost function,
enabling different kinds of personalization. We also demonstrate how
this approach enables extensive user- and system-initiated run-time
adaptations to the interfaces after they have been generated.</p>
<p>Supple is not intended to replace human user interface designers—instead, it
offers alternative user interfaces for those people whose devices,
tasks, preferences, and abilities are not sufficiently addressed by
the hand-crafted designs. Indeed, the results of our
study show that, compared to manufacturers' defaults, interfaces automatically generated by Supple significantly improve speed, accuracy and
satisfaction of people with motor impairments.</p>},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
<p>Today’s computer-human interfaces are typically designed with the
assumption that they are going to be used by an able-bodied person,
who is using a typical set of input and output devices, who has
typical perceptual and cognitive abilities, and who is sitting in a
stable, warm environment. Any deviation from these assumptions may
drastically hamper the person’s effectiveness—not because of any
<it>inherent</it> barrier to interaction, but because of a mismatch
between the person’s effective abilities and the assumptions underlying the
interface design.</p>
<p>We argue that automatic personalized interface generation is a feasible and
scalable solution to this challenge. We present our Supple system,
which can automatically generate interfaces adapted to a person’s
devices, tasks, preferences, and abilities. In this paper we formally
define interface generation as an optimization problem and
demonstrate that, despite a large solution space (of up to 1017 possible interfaces), the problem is computationally feasible. In fact, for a particular class of cost functions, Supple produces exact solutions in under a second for most cases, and in
a little over a minute in the worst case encountered, thus enabling run-time generation of user interfaces. We further show
how several different design criteria can be expressed in the cost function,
enabling different kinds of personalization. We also demonstrate how
this approach enables extensive user- and system-initiated run-time
adaptations to the interfaces after they have been generated.</p>
<p>Supple is not intended to replace human user interface designers—instead, it
offers alternative user interfaces for those people whose devices,
tasks, preferences, and abilities are not sufficiently addressed by
the hand-crafted designs. Indeed, the results of our
study show that, compared to manufacturers’ defaults, interfaces automatically generated by Supple significantly improve speed, accuracy and
satisfaction of people with motor impairments.</p>
assumption that they are going to be used by an able-bodied person,
who is using a typical set of input and output devices, who has
typical perceptual and cognitive abilities, and who is sitting in a
stable, warm environment. Any deviation from these assumptions may
drastically hamper the person’s effectiveness—not because of any
<it>inherent</it> barrier to interaction, but because of a mismatch
between the person’s effective abilities and the assumptions underlying the
interface design.</p>
<p>We argue that automatic personalized interface generation is a feasible and
scalable solution to this challenge. We present our Supple system,
which can automatically generate interfaces adapted to a person’s
devices, tasks, preferences, and abilities. In this paper we formally
define interface generation as an optimization problem and
demonstrate that, despite a large solution space (of up to 1017 possible interfaces), the problem is computationally feasible. In fact, for a particular class of cost functions, Supple produces exact solutions in under a second for most cases, and in
a little over a minute in the worst case encountered, thus enabling run-time generation of user interfaces. We further show
how several different design criteria can be expressed in the cost function,
enabling different kinds of personalization. We also demonstrate how
this approach enables extensive user- and system-initiated run-time
adaptations to the interfaces after they have been generated.</p>
<p>Supple is not intended to replace human user interface designers—instead, it
offers alternative user interfaces for those people whose devices,
tasks, preferences, and abilities are not sufficiently addressed by
the hand-crafted designs. Indeed, the results of our
study show that, compared to manufacturers’ defaults, interfaces automatically generated by Supple significantly improve speed, accuracy and
satisfaction of people with motor impairments.</p>
Huang, Eric; Zhang, Haoqi; Parkes, David C.; Gajos, Krzysztof Z.; Chen, Yiling
Toward automatic task design: a progress report Proceedings Article
In: Proceedings of the ACM SIGKDD Workshop on Human Computation, pp. 77–85, ACM, Washington DC, 2010, ISBN: 978-1-4503-0222-7.
@inproceedings{huang10:toward,
title = {Toward automatic task design: a progress report},
author = {Eric Huang and Haoqi Zhang and David C. Parkes and Krzysztof Z. Gajos and Yiling Chen},
url = {/papers/huang10hcomp.pdf},
doi = {10.1145/1837885.1837908},
isbn = {978-1-4503-0222-7},
year = {2010},
date = {2010-01-01},
booktitle = {Proceedings of the ACM SIGKDD Workshop on Human Computation},
pages = {77–85},
publisher = {ACM},
address = {Washington DC},
series = {HCOMP '10},
abstract = {A central challenge in human computation is in understanding how to design task environments that effectively attract participants and coordinate the problem solving process. In this paper, we consider a common problem that requesters face on Amazon Mechanical Turk: how should a task be designed so as to induce good output from workers? In posting a task, a requester decides how to break down the task into unit tasks, how much to pay for each unit task, and how many workers to assign to a unit task. These design decisions affect the rate at which workers complete unit tasks, as well as the quality of the work that results. Using image labeling as an example task, we consider the problem of designing the task to maximize the number of quality tags received within given time and budget constraints. We consider two different measures of work quality, and construct models for predicting the rate and quality of work based on observations of output to various designs. Preliminary results show that simple models can accurately predict the quality of output per unit task, but are less accurate in predicting the rate at which unit tasks complete. At a fixed rate of pay, our models generate different designs depending on the quality metric, and optimized designs obtain significantly more quality tags than baseline comparisons.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
A central challenge in human computation is in understanding how to design task environments that effectively attract participants and coordinate the problem solving process. In this paper, we consider a common problem that requesters face on Amazon Mechanical Turk: how should a task be designed so as to induce good output from workers? In posting a task, a requester decides how to break down the task into unit tasks, how much to pay for each unit task, and how many workers to assign to a unit task. These design decisions affect the rate at which workers complete unit tasks, as well as the quality of the work that results. Using image labeling as an example task, we consider the problem of designing the task to maximize the number of quality tags received within given time and budget constraints. We consider two different measures of work quality, and construct models for predicting the rate and quality of work based on observations of output to various designs. Preliminary results show that simple models can accurately predict the quality of output per unit task, but are less accurate in predicting the rate at which unit tasks complete. At a fixed rate of pay, our models generate different designs depending on the quality metric, and optimized designs obtain significantly more quality tags than baseline comparisons.
Jayatilaka, Lahiru G.; Bertuccelli, Luca F.; Staszewski, James; Gajos, Krzysztof Z.
PETALS: a visual interface for landmine detection Proceedings Article
In: Adjunct proceedings of the 23nd annual ACM symposium on User interface software and technology, pp. 427–428, ACM, New York, New York, USA, 2010, ISBN: 978-1-4503-0462-7.
@inproceedings{jayatilaka10:petals,
title = {PETALS: a visual interface for landmine detection},
author = {Lahiru G. Jayatilaka and Luca F. Bertuccelli and James Staszewski and Krzysztof Z. Gajos},
url = {/papers/jayatilaka10petals.pdf},
doi = {10.1145/1866218.1866254},
isbn = {978-1-4503-0462-7},
year = {2010},
date = {2010-01-01},
booktitle = {Adjunct proceedings of the 23nd annual ACM symposium on User interface software and technology},
pages = {427–428},
publisher = {ACM},
address = {New York, New York, USA},
series = {UIST '10},
abstract = {Post-conflict landmines have serious humanitarian repercussions:
landmines cost lives, limbs and land. The primary
method used to locate these buried devices relies on the inherently
dangerous and difficult task of a human listening to
audio feedback from a metal detector. Researchers have previously
hypothesized that expert operators respond to these
challenges by building mental patterns with metal detectors
through the identification of object-dependent spatially distributed
metallic fields. This paper presents the preliminary
stages of a novel interface— Pattern Enhancement Tool for
Assisting Landmine Sensing (PETALS)—that aims to assist
with building and visualizing these patterns, rather than relying
on memory alone. Simulated demining experiments
show that the experimental interface decreases classification
error from 23% to 5% and reduces localization error by 54%,
demonstrating the potential for PETALS to improve novice
deminer safety and efficiency.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Post-conflict landmines have serious humanitarian repercussions:
landmines cost lives, limbs and land. The primary
method used to locate these buried devices relies on the inherently
dangerous and difficult task of a human listening to
audio feedback from a metal detector. Researchers have previously
hypothesized that expert operators respond to these
challenges by building mental patterns with metal detectors
through the identification of object-dependent spatially distributed
metallic fields. This paper presents the preliminary
stages of a novel interface— Pattern Enhancement Tool for
Assisting Landmine Sensing (PETALS)—that aims to assist
with building and visualizing these patterns, rather than relying
on memory alone. Simulated demining experiments
show that the experimental interface decreases classification
error from 23% to 5% and reduces localization error by 54%,
demonstrating the potential for PETALS to improve novice
deminer safety and efficiency.
landmines cost lives, limbs and land. The primary
method used to locate these buried devices relies on the inherently
dangerous and difficult task of a human listening to
audio feedback from a metal detector. Researchers have previously
hypothesized that expert operators respond to these
challenges by building mental patterns with metal detectors
through the identification of object-dependent spatially distributed
metallic fields. This paper presents the preliminary
stages of a novel interface— Pattern Enhancement Tool for
Assisting Landmine Sensing (PETALS)—that aims to assist
with building and visualizing these patterns, rather than relying
on memory alone. Simulated demining experiments
show that the experimental interface decreases classification
error from 23% to 5% and reduces localization error by 54%,
demonstrating the potential for PETALS to improve novice
deminer safety and efficiency.
2009
Spaulding, Aaron; Gajos, Krzysztof Z.; Jameson, Anthony; Kristensson, Per Ola; Bunt, Andrea; Haines, Will
Usable intelligent interactive systems: CHI 2009 special interest group meeting Proceedings Article
In: CHI EA ’09: Proceedings of the 27th international conference extended abstracts on Human factors in computing systems, pp. 2743–2746, ACM, Boston, MA, USA, 2009, ISBN: 978-1-60558-247-4.
@inproceedings{spaulding09:usable,
title = {Usable intelligent interactive systems: CHI 2009 special interest group meeting},
author = {Aaron Spaulding and Krzysztof Z. Gajos and Anthony Jameson and Per Ola Kristensson and Andrea Bunt and Will Haines},
url = {/papers/chi09-SIG-usableiis.pdf},
doi = {10.1145/1520340.1520396},
isbn = {978-1-60558-247-4},
year = {2009},
date = {2009-01-01},
booktitle = {CHI EA '09: Proceedings of the 27th international conference extended abstracts on Human factors in computing systems},
pages = {2743–2746},
publisher = {ACM},
address = {Boston, MA, USA},
abstract = {``The AI and HCI communities have often been characterized as having opposing views of how humans and computers should interact'' observes Winograd in Shifting Viewpoints. It is time to narrow this gap. What was once considered the forefront of artificial intelligence (AI) research can now be found in commercial products. While some have failed, others, such as face detection in digital cameras or product recommendation systems, have become so mainstream they are no longer thought of as artificial intelligence. This special interest group provides a forum to examine the apparent gap between HCI and AI communities, to explore how intelligent technologies can enable novel interaction with computation, and to investigate the challenges associated with understanding human abilities, limitations, and preferences in order to drive the design of intelligent interactive systems.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
“The AI and HCI communities have often been characterized as having opposing views of how humans and computers should interact” observes Winograd in Shifting Viewpoints. It is time to narrow this gap. What was once considered the forefront of artificial intelligence (AI) research can now be found in commercial products. While some have failed, others, such as face detection in digital cameras or product recommendation systems, have become so mainstream they are no longer thought of as artificial intelligence. This special interest group provides a forum to examine the apparent gap between HCI and AI communities, to explore how intelligent technologies can enable novel interaction with computation, and to investigate the challenges associated with understanding human abilities, limitations, and preferences in order to drive the design of intelligent interactive systems.
Gajos, Krzysztof Z.
Beyond Feature Relevance: Incorporating Rich User Feedback Into Interactive Machine Learning Applications Proceedings Article
In: Proceedings of ADA-IML’09 Workshop at NIPS’09, 2009.
@inproceedings{gajos09:beyond,
title = {Beyond Feature Relevance: Incorporating Rich User Feedback Into Interactive Machine Learning Applications},
author = {Krzysztof Z. Gajos},
url = {/papers/adaiml09.pdf},
year = {2009},
date = {2009-01-01},
booktitle = {Proceedings of ADA-IML'09 Workshop at NIPS'09},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Findlater, Leah; Gajos, Krzysztof Z.
Design Space and Evaluation Challenges of Adaptive Graphical User Interfaces Journal Article
In: AI Magazine, vol. 30, no. 4, pp. 68–73, 2009.
@article{findlater09:design,
title = {Design Space and Evaluation Challenges of Adaptive Graphical User Interfaces},
author = {Leah Findlater and Krzysztof Z. Gajos},
url = {/papers/AIMag09-AUIs.pdf},
year = {2009},
date = {2009-01-01},
journal = {AI Magazine},
volume = {30},
number = {4},
pages = {68–73},
abstract = {Adaptive graphical user interfaces (GUIs) have the potential to improve performance and user satisfaction by automatically tailoring the presentation of functionality to each individual user. In practice, however, many challenges exist, and evaluation results of adaptive GUIs have been mixed. To guide researchers and designers in developing effective adaptive GUIs, we outline a design space and discuss three important aspects to consider when conducting user evaluations of these types of interfaces: the control and reporting of adaptive algorithm characteristics, the impact of task choice and user characteristics on the overall effectiveness of a design, and evaluation measures that are appropriate for adaptive interaction.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Adaptive graphical user interfaces (GUIs) have the potential to improve performance and user satisfaction by automatically tailoring the presentation of functionality to each individual user. In practice, however, many challenges exist, and evaluation results of adaptive GUIs have been mixed. To guide researchers and designers in developing effective adaptive GUIs, we outline a design space and discuss three important aspects to consider when conducting user evaluations of these types of interfaces: the control and reporting of adaptive algorithm characteristics, the impact of task choice and user characteristics on the overall effectiveness of a design, and evaluation measures that are appropriate for adaptive interaction.
2008
Wobbrock, Jacob O.; Gajos, Krzysztof Z.
Goal Crossing with Mice and Trackballs for People with Motor Impairments: Performance, Submovements, and Design Directions Journal Article
In: ACM Trans. Access. Comput., vol. 1, no. 1, pp. 1–37, 2008, ISSN: 1936-7228.
@article{wobbrock08:goal,
title = {Goal Crossing with Mice and Trackballs for People with Motor Impairments: Performance, Submovements, and Design Directions},
author = {Jacob O. Wobbrock and Krzysztof Z. Gajos},
url = {/papers/taccess08.pdf},
doi = {10.1145/1361203.1361207},
issn = {1936-7228},
year = {2008},
date = {2008-05-01},
journal = {ACM Trans. Access. Comput.},
volume = {1},
number = {1},
pages = {1–37},
publisher = {ACM},
address = {New York, NY, USA},
abstract = {Prior research shows that people withmotor impairments face considerable challenges when using
conventional mice and trackballs. One challenge is positioning the mouse cursor within confined
target areas; another is executing a precise click without slipping. These problems can make
mouse pointing in graphical user interfaces very difficult for some people. This article explores goal
crossing as an alternative strategy for more accessible target acquisition. In goal crossing, targets
are boundaries that are simply crossed by the mouse cursor. Thus, goal crossing avoids the two
aforementioned problems. To date, however, researchers have not examined the feasibility of goal
crossing for people with motor difficulties. We therefore present a study comparing area pointing
and goal crossing. Our performance results indicate that although Fitts¢ throughput for ablebodied
users is higher for area pointing than for goal crossing (4.72 vs. 3.61 bits/s), the opposite is
true for users with motor impairments (2.34 vs. 2.88 bits/s). However, error rates are higher for
goal crossing than for area pointing under a strict definition of crossing errors (6.23% vs. 1.94%).
We also present path analyses and an examination of submovement velocity, acceleration, and jerk
(the change in acceleration over time). These results show marked differences between crossing
and pointing and almost categorically favor crossing. An important finding is that crossing reduces
jerk for both participant groups, indicating more fluid, stable motion. To help realize the potential
of goal crossing for computer access, we offer design concepts for crossing widgets that address
the occlusion problem, which occurs when one crossing goal obscures another in persistent mousecursor
interfaces. This work provides the motivation and initial steps for further exploration of
goal crossing on the desktop, and may help researchers and designers to radically reshape user
interfaces to provide accessible goal crossing, thereby lowering barriers to access.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Prior research shows that people withmotor impairments face considerable challenges when using
conventional mice and trackballs. One challenge is positioning the mouse cursor within confined
target areas; another is executing a precise click without slipping. These problems can make
mouse pointing in graphical user interfaces very difficult for some people. This article explores goal
crossing as an alternative strategy for more accessible target acquisition. In goal crossing, targets
are boundaries that are simply crossed by the mouse cursor. Thus, goal crossing avoids the two
aforementioned problems. To date, however, researchers have not examined the feasibility of goal
crossing for people with motor difficulties. We therefore present a study comparing area pointing
and goal crossing. Our performance results indicate that although Fitts¢ throughput for ablebodied
users is higher for area pointing than for goal crossing (4.72 vs. 3.61 bits/s), the opposite is
true for users with motor impairments (2.34 vs. 2.88 bits/s). However, error rates are higher for
goal crossing than for area pointing under a strict definition of crossing errors (6.23% vs. 1.94%).
We also present path analyses and an examination of submovement velocity, acceleration, and jerk
(the change in acceleration over time). These results show marked differences between crossing
and pointing and almost categorically favor crossing. An important finding is that crossing reduces
jerk for both participant groups, indicating more fluid, stable motion. To help realize the potential
of goal crossing for computer access, we offer design concepts for crossing widgets that address
the occlusion problem, which occurs when one crossing goal obscures another in persistent mousecursor
interfaces. This work provides the motivation and initial steps for further exploration of
goal crossing on the desktop, and may help researchers and designers to radically reshape user
interfaces to provide accessible goal crossing, thereby lowering barriers to access.
conventional mice and trackballs. One challenge is positioning the mouse cursor within confined
target areas; another is executing a precise click without slipping. These problems can make
mouse pointing in graphical user interfaces very difficult for some people. This article explores goal
crossing as an alternative strategy for more accessible target acquisition. In goal crossing, targets
are boundaries that are simply crossed by the mouse cursor. Thus, goal crossing avoids the two
aforementioned problems. To date, however, researchers have not examined the feasibility of goal
crossing for people with motor difficulties. We therefore present a study comparing area pointing
and goal crossing. Our performance results indicate that although Fitts¢ throughput for ablebodied
users is higher for area pointing than for goal crossing (4.72 vs. 3.61 bits/s), the opposite is
true for users with motor impairments (2.34 vs. 2.88 bits/s). However, error rates are higher for
goal crossing than for area pointing under a strict definition of crossing errors (6.23% vs. 1.94%).
We also present path analyses and an examination of submovement velocity, acceleration, and jerk
(the change in acceleration over time). These results show marked differences between crossing
and pointing and almost categorically favor crossing. An important finding is that crossing reduces
jerk for both participant groups, indicating more fluid, stable motion. To help realize the potential
of goal crossing for computer access, we offer design concepts for crossing widgets that address
the occlusion problem, which occurs when one crossing goal obscures another in persistent mousecursor
interfaces. This work provides the motivation and initial steps for further exploration of
goal crossing on the desktop, and may help researchers and designers to radically reshape user
interfaces to provide accessible goal crossing, thereby lowering barriers to access.
Gajos, Krzysztof Z.; Everitt, Katherine; Tan, Desney S.; Czerwinski, Mary; Weld, Daniel S.
Predictability and accuracy in adaptive user interfaces Proceedings Article
In: CHI ’08: Proceeding of the twenty-sixth annual SIGCHI conference on Human factors in computing systems, pp. 1271–1274, ACM, New York, NY, USA, 2008, ISBN: 9781605580111.
@inproceedings{gajos08:predictability,
title = {Predictability and accuracy in adaptive user interfaces},
author = {Krzysztof Z. Gajos and Katherine Everitt and Desney S. Tan and Mary Czerwinski and Daniel S. Weld},
url = {/papers/kgajos-chi08-predictability.pdf},
doi = {10.1145/1357054.1357252},
isbn = {9781605580111},
year = {2008},
date = {2008-01-01},
booktitle = {CHI '08: Proceeding of the twenty-sixth annual SIGCHI conference on Human factors in computing systems},
pages = {1271–1274},
publisher = {ACM},
address = {New York, NY, USA},
abstract = {While proponents of adaptive user interfaces tout potential performance gains, critics argue that adaptation's unpredictability may disorient users, causing more harm than good. We present a study that examines the relative effects of predictability and accuracy on the usability of adaptive UIs. Our results show that increasing predictability and accuracy led to strongly improved satisfaction. Increasing accuracy also resulted in improved performance and higher utilization of the adaptive interface. Contrary to our expectations, improvement in accuracy had a stronger effect on performance, utilization and some satisfaction ratings than the improvement in predictability.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
While proponents of adaptive user interfaces tout potential performance gains, critics argue that adaptation’s unpredictability may disorient users, causing more harm than good. We present a study that examines the relative effects of predictability and accuracy on the usability of adaptive UIs. Our results show that increasing predictability and accuracy led to strongly improved satisfaction. Increasing accuracy also resulted in improved performance and higher utilization of the adaptive interface. Contrary to our expectations, improvement in accuracy had a stronger effect on performance, utilization and some satisfaction ratings than the improvement in predictability.
Gajos, Krzysztof Z.; Wobbrock, Jacob O.; Weld, Daniel S.
Improving the performance of motor-impaired users with automatically-generated, ability-based interfaces Proceedings Article
In: CHI ’08: Proceeding of the twenty-sixth annual SIGCHI conference on Human factors in computing systems, pp. 1257–1266, ACM, New York, NY, USA, 2008, ISBN: 9781605580111.
@inproceedings{gajos08:improving,
title = {Improving the performance of motor-impaired users with automatically-generated, ability-based interfaces},
author = {Krzysztof Z. Gajos and Jacob O. Wobbrock and Daniel S. Weld},
url = {/papers/skgajos-chi08-upple.pdf},
doi = {10.1145/1357054.1357250},
isbn = {9781605580111},
year = {2008},
date = {2008-01-01},
booktitle = {CHI '08: Proceeding of the twenty-sixth annual SIGCHI conference on Human factors in computing systems},
pages = {1257–1266},
publisher = {ACM},
address = {New York, NY, USA},
abstract = {We evaluate two methods for automatically generating personalized interfaces adapted to the individual motor capabilities of users with motor impairments. The first system, SUPPLE, adapts to user's capabilities indirectly by first using ARNAULD preference elicitation engine to model user's preferences regarding how he or she likes the interfaces to be created. The second, SUPPLE++, models a user's motor abilities directly through a set of one-time motor performance tests. In a study comparing these approaches to baseline interfaces, our results show that users with motor impairments were much faster and strongly preferred SUPPLE++ ability-based interfaces. Specifically, motor-impared participants were 26.4% faster using interfaces generated by SUPPLE++. They made 73% fewer errors, strongly preferred those interfaces to the manufacturers' defaults, and found them more efficient, easier to use, and much less physically tiring. These findings indicate that rather than requiring some users with motor impairments to adapt themselves to software using separate assistive technologies, software can now adapt itself to the capabilities of its users.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
We evaluate two methods for automatically generating personalized interfaces adapted to the individual motor capabilities of users with motor impairments. The first system, SUPPLE, adapts to user’s capabilities indirectly by first using ARNAULD preference elicitation engine to model user’s preferences regarding how he or she likes the interfaces to be created. The second, SUPPLE++, models a user’s motor abilities directly through a set of one-time motor performance tests. In a study comparing these approaches to baseline interfaces, our results show that users with motor impairments were much faster and strongly preferred SUPPLE++ ability-based interfaces. Specifically, motor-impared participants were 26.4% faster using interfaces generated by SUPPLE++. They made 73% fewer errors, strongly preferred those interfaces to the manufacturers’ defaults, and found them more efficient, easier to use, and much less physically tiring. These findings indicate that rather than requiring some users with motor impairments to adapt themselves to software using separate assistive technologies, software can now adapt itself to the capabilities of its users.
Gajos, Krzysztof Z.
Automatically Generating Personalized User Interfaces PhD Thesis
University of Washington, 2008.
@phdthesis{gajos08:automatically,
title = {Automatically Generating Personalized User Interfaces},
author = {Krzysztof Z. Gajos},
url = {/papers/kgajos-dissertation.pdf},
year = {2008},
date = {2008-01-01},
address = {Seattle, WA, USA},
school = {University of Washington},
abstract = {User Interfaces for today's software are usually created in a one-size-fits-all manner, making implicit assumptions about the needs, abilities, and preferences of the "average user" and the characteristics of the "average device." I argue that personalized user interfaces, which are adapted to a person's devices, tasks, preferences, and abilities, can improve user satisfaction and performance. I have developed three systems:
<ul><li>Supple, which uses decision-theoretic optimization to automatically generate user
interfaces adapted to a person's device and long-term usage;</li>
<li>Arnauld, which allows optimization-based systems to be adapted to users' preferences;
and</li>
<li>Ability Modeler and an extension of Supple that first performs a one-time assessment
of a person's motor abilities and then automatically generates user interfaces
predicted to be the fastest to use for that user.</li></ul>
My experiments show that these automatically generated, personalized user interfaces
significantly improve speed, accuracy, and satisfaction for users with motor impairments
compared to manufacturers' default interfaces. I also provide the first characterization of
the design space of adaptive graphical user interfaces, and demonstrate how such interfaces
can significantly improve the quality and efficiency of daily interactions for typical users.},
keywords = {},
pubstate = {published},
tppubtype = {phdthesis}
}
User Interfaces for today’s software are usually created in a one-size-fits-all manner, making implicit assumptions about the needs, abilities, and preferences of the "average user" and the characteristics of the "average device." I argue that personalized user interfaces, which are adapted to a person’s devices, tasks, preferences, and abilities, can improve user satisfaction and performance. I have developed three systems:
My experiments show that these automatically generated, personalized user interfaces
significantly improve speed, accuracy, and satisfaction for users with motor impairments
compared to manufacturers’ default interfaces. I also provide the first characterization of
the design space of adaptive graphical user interfaces, and demonstrate how such interfaces
can significantly improve the quality and efficiency of daily interactions for typical users.
- Supple, which uses decision-theoretic optimization to automatically generate user
interfaces adapted to a person’s device and long-term usage; - Arnauld, which allows optimization-based systems to be adapted to users’ preferences;
and - Ability Modeler and an extension of Supple that first performs a one-time assessment
of a person’s motor abilities and then automatically generates user interfaces
predicted to be the fastest to use for that user.
My experiments show that these automatically generated, personalized user interfaces
significantly improve speed, accuracy, and satisfaction for users with motor impairments
compared to manufacturers’ default interfaces. I also provide the first characterization of
the design space of adaptive graphical user interfaces, and demonstrate how such interfaces
can significantly improve the quality and efficiency of daily interactions for typical users.
Gajos, Krzysztof Z.; Weld, Daniel S.; Wobbrock, Jacob O.
Decision-Theoretic User Interface Generation Proceedings Article
In: AAAI’08, pp. 1532-1536, AAAI Press, 2008.
@inproceedings{gajos08:decision,
title = {Decision-Theoretic User Interface Generation},
author = {Krzysztof Z. Gajos and Daniel S. Weld and Jacob O. Wobbrock},
url = {/papers/kgajos-nectar08.pdf},
year = {2008},
date = {2008-01-01},
booktitle = {AAAI'08},
pages = {1532-1536},
publisher = {AAAI Press},
abstract = {For decades, researchers have debated the pros and cons of adaptive user interfaces with enthusiastic AI practitioners often confronting skeptical HCI experts. This paper summarizes the SUPPLE project's six years of work analyzing the characteristics of successful adaptive interfaces and developing efficient algorithms for their automatic generation. We conclude that personalized user interfaces, which are adapted to a person's devices, tasks, preferences and abilities, can improve user satisfaction and performance. Further, we demonstrate that automatic generation of these interfaces is computationally feasible.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
For decades, researchers have debated the pros and cons of adaptive user interfaces with enthusiastic AI practitioners often confronting skeptical HCI experts. This paper summarizes the SUPPLE project’s six years of work analyzing the characteristics of successful adaptive interfaces and developing efficient algorithms for their automatic generation. We conclude that personalized user interfaces, which are adapted to a person’s devices, tasks, preferences and abilities, can improve user satisfaction and performance. Further, we demonstrate that automatic generation of these interfaces is computationally feasible.
DeRenzi, Brian; Gajos, Krzysztof Z.; Parikh, Tapan S.; Lesh, Neal; Mitchell, Marc; Borriello, Baetano
Opportunities for Intelligent Interfaces Aiding Healthcare in Low-Income Countries Proceedings Article
In: Workshop on Intelligent User Interfaces for Developing Regions (at IUI’08), 2008.
@inproceedings{derenzi08:opportunities,
title = {Opportunities for Intelligent Interfaces Aiding Healthcare in Low-Income Countries},
author = {Brian DeRenzi and Krzysztof Z. Gajos and Tapan S. Parikh and Neal Lesh and Marc Mitchell and Baetano Borriello},
url = {/papers/derenzi-iui4dr08.pdf},
year = {2008},
date = {2008-01-01},
booktitle = {Workshop on Intelligent User Interfaces for Developing Regions (at IUI'08)},
abstract = {Child mortality is one of the most pressing health concerns almost 10 million children die worldwide each year before reaching their fifth birthday, mostly in low-income countries. To aid overburdened and undertrained health workers the World Health Organization (WHO) and United Nations Children's Fund (UNICEF) have developed clinical guidelines, such as the Integrated Management of Childhood Illness (IMCI) to help with the classification and treatment of common childhood illness. To help with deployment, we have developed an electronic version (e-IMCI) that runs on a PDA. From July to September 2007, we ran a pilot of e-IMCI in southern Tanzania. The system guides health workers step-by-step through the treatment algorithms and automatically calculates drug doses. Our results suggest that electronic implementations of protocols such as IMCI can reduce training time and improve adherence to the protocol. They also highlight several important challenges including varying levels of education, language and expertise, which could be most adequately addressed by implementing novel intelligent user interfaces and systems.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Child mortality is one of the most pressing health concerns almost 10 million children die worldwide each year before reaching their fifth birthday, mostly in low-income countries. To aid overburdened and undertrained health workers the World Health Organization (WHO) and United Nations Children’s Fund (UNICEF) have developed clinical guidelines, such as the Integrated Management of Childhood Illness (IMCI) to help with the classification and treatment of common childhood illness. To help with deployment, we have developed an electronic version (e-IMCI) that runs on a PDA. From July to September 2007, we ran a pilot of e-IMCI in southern Tanzania. The system guides health workers step-by-step through the treatment algorithms and automatically calculates drug doses. Our results suggest that electronic implementations of protocols such as IMCI can reduce training time and improve adherence to the protocol. They also highlight several important challenges including varying levels of education, language and expertise, which could be most adequately addressed by implementing novel intelligent user interfaces and systems.
Gajos, Krzysztof Z.; Weld, Daniel S.
Usable AI: Experience and Reflections Proceedings Article
In: Proceedings of the workshop on Usable AI at CHI’08, 2008.
@inproceedings{gajos08:usable,
title = {Usable AI: Experience and Reflections},
author = {Krzysztof Z. Gajos and Daniel S. Weld},
url = {/papers/kgajos-UsableAI08.pdf},
year = {2008},
date = {2008-01-01},
booktitle = {Proceedings of the workshop on Usable AI at CHI'08},
abstract = {We believe that AI has much to offer HCI, in particular allowing for the quick construction of personalized and personalizable interfaces. In this position paper, we report on our experience from four recent investigations of automatic personalization. We then step back and comment on the overall enterprise of making AI usable.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
We believe that AI has much to offer HCI, in particular allowing for the quick construction of personalized and personalizable interfaces. In this position paper, we report on our experience from four recent investigations of automatic personalization. We then step back and comment on the overall enterprise of making AI usable.
2007
Wobbrock, Jacob O.; Gajos, Krzysztof Z.
A comparison of area pointing and goal crossing for people with and without motor impairments Proceedings Article
In: Assets ’07: Proceedings of the 9th international ACM SIGACCESS conference on Computers and accessibility, pp. 3–10, ACM, New York, NY, USA, 2007, ISBN: 9781595935731.
@inproceedings{Wobbrock2007Comparison,
title = {A comparison of area pointing and goal crossing for people with and without motor impairments},
author = {Jacob O. Wobbrock and Krzysztof Z. Gajos},
url = {/papers/assets-07.pdf},
doi = {10.1145/1296843.1296847},
isbn = {9781595935731},
year = {2007},
date = {2007-01-01},
booktitle = {Assets '07: Proceedings of the 9th international ACM SIGACCESS conference on Computers and accessibility},
pages = {3–10},
publisher = {ACM},
address = {New York, NY, USA},
abstract = {Prior work has highlighted the challenges faced by people with motor impairments when trying to acquire on-screen targets using a mouse or trackball. Two reasons for this are the difficulty of positioning the mouse cursor within a confined area, and the challenge of accurately executing a click. We hypothesize that both of these difficulties with area pointing may be alleviated in a different target acquisition paradigm called goal crossing. In goal crossing, users do not acquire a confined area, but instead pass over a target line. Although goal crossing has been studied for able-bodied users, its suitability for people with motor impairments is unknown. We present a study of 16 people, 8 of whom had motor impairments, using mice and trackballs to do area pointing and goal crossing. Our results indicate that Fitts' law models both techniques for both user groups. Furthermore, although throughput for able-bodied users was higher for area pointing than for goal crossing (4.72 vs. 3.61 bits/s), the opposite was true for users with motor impairments (2.34 vs. 2.88 bits/s), suggesting that goal crossing may be viable for them. However, error rates were higher for goal crossing than for area pointing under a strict definition of crossing errors (6.23% vs. 1.94%). Subjective results indicate a preference for goal crossing among motor-impaired users. This work provides the empirical foundation from which to pursue the design of crossing-based interfaces as accessible alternatives to pointing-based interfaces.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Prior work has highlighted the challenges faced by people with motor impairments when trying to acquire on-screen targets using a mouse or trackball. Two reasons for this are the difficulty of positioning the mouse cursor within a confined area, and the challenge of accurately executing a click. We hypothesize that both of these difficulties with area pointing may be alleviated in a different target acquisition paradigm called goal crossing. In goal crossing, users do not acquire a confined area, but instead pass over a target line. Although goal crossing has been studied for able-bodied users, its suitability for people with motor impairments is unknown. We present a study of 16 people, 8 of whom had motor impairments, using mice and trackballs to do area pointing and goal crossing. Our results indicate that Fitts’ law models both techniques for both user groups. Furthermore, although throughput for able-bodied users was higher for area pointing than for goal crossing (4.72 vs. 3.61 bits/s), the opposite was true for users with motor impairments (2.34 vs. 2.88 bits/s), suggesting that goal crossing may be viable for them. However, error rates were higher for goal crossing than for area pointing under a strict definition of crossing errors (6.23% vs. 1.94%). Subjective results indicate a preference for goal crossing among motor-impaired users. This work provides the empirical foundation from which to pursue the design of crossing-based interfaces as accessible alternatives to pointing-based interfaces.
Gajos, Krzysztof Z.; Wobbrock, Jacob O.; Weld, Daniel S.
Automatically generating user interfaces adapted to users’ motor and vision capabilities Proceedings Article
In: UIST ’07: Proceedings of the 20th annual ACM symposium on User interface software and technology, pp. 231–240, ACM Press, New York, NY, USA, 2007, ISBN: 9781595936792.
@inproceedings{gajos07:automatically,
title = {Automatically generating user interfaces adapted to users' motor and vision capabilities},
author = {Krzysztof Z. Gajos and Jacob O. Wobbrock and Daniel S. Weld},
url = {/papers/kgajos-uist07.pdf},
doi = {10.1145/1294211.1294253},
isbn = {9781595936792},
year = {2007},
date = {2007-01-01},
booktitle = {UIST '07: Proceedings of the 20th annual ACM symposium on User interface software and technology},
pages = {231–240},
publisher = {ACM Press},
address = {New York, NY, USA},
abstract = {Most of today's GUIs are designed for the typical, able-bodied user; atypical users are, for the most part, left to adapt as best they can, perhaps using specialized assistive technologies as an aid. In this paper, we present an alternative approach: SUPPLE++ automatically generates interfaces which are tailored to an individual's motor capabilities and can be easily adjusted to accommodate varying vision capabilities. SUPPLE++ models users' motor capabilities based on a onetime motor performance test and uses this model in an optimization process, generating a personalized interface. A preliminary study indicates that while there is still room for improvement, SUPPLE++ allowed one user to complete tasks that she could not perform using a standard interface, while for the remaining users it resulted in an average time savings of 20%, ranging from an slowdown of 3% to a speedup of 43%.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Most of today’s GUIs are designed for the typical, able-bodied user; atypical users are, for the most part, left to adapt as best they can, perhaps using specialized assistive technologies as an aid. In this paper, we present an alternative approach: SUPPLE++ automatically generates interfaces which are tailored to an individual’s motor capabilities and can be easily adjusted to accommodate varying vision capabilities. SUPPLE++ models users’ motor capabilities based on a onetime motor performance test and uses this model in an optimization process, generating a personalized interface. A preliminary study indicates that while there is still room for improvement, SUPPLE++ allowed one user to complete tasks that she could not perform using a standard interface, while for the remaining users it resulted in an average time savings of 20%, ranging from an slowdown of 3% to a speedup of 43%.
2006
Richter, Kai; Nichols, Jeffrey; Gajos, Krzysztof; Seffah, Ahmed
The many faces of consistency in cross-platform design Proceedings Article
In: CHI ’06: CHI ’06 extended abstracts on Human factors in computing systems, pp. 1639–1642, ACM Press, New York, NY, USA, 2006, ISBN: 1595932984.
@inproceedings{richter06:many,
title = {The many faces of consistency in cross-platform design},
author = {Kai Richter and Jeffrey Nichols and Krzysztof Gajos and Ahmed Seffah},
url = {/papers/consistency06.pdf},
doi = {10.1145/1125451.1125751},
isbn = {1595932984},
year = {2006},
date = {2006-01-01},
booktitle = {CHI '06: CHI '06 extended abstracts on Human factors in computing systems},
pages = {1639–1642},
publisher = {ACM Press},
address = {New York, NY, USA},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Gajos, Krzysztof Z.; Czerwinski, Mary; Tan, Desney S.; Weld, Daniel S.
Exploring the design space for adaptive graphical user interfaces Proceedings Article
In: AVI ’06: Proceedings of the working conference on Advanced visual interfaces, pp. 201–208, ACM Press, New York, NY, USA, 2006, ISBN: 1595933530.
@inproceedings{gajos06:exploring,
title = {Exploring the design space for adaptive graphical user interfaces},
author = {Krzysztof Z. Gajos and Mary Czerwinski and Desney S. Tan and Daniel S. Weld},
url = {/papers/kgajos-avi06.pdf},
isbn = {1595933530},
year = {2006},
date = {2006-01-01},
booktitle = {AVI '06: Proceedings of the working conference on Advanced visual interfaces},
pages = {201–208},
publisher = {ACM Press},
address = {New York, NY, USA},
abstract = {For decades, researchers have presented different adaptive user interfaces and discussed the pros and cons of adaptation on task performance and satisfaction. Little research, however, has been directed at isolating and understanding those aspects of adaptive interfaces which make some of them successful and others not. We have designed and implemented three adaptive graphical interfaces and evaluated them in two experiments along with a nonadaptive baseline. In this paper we synthesize our results with previous work and discuss how different design choices and interactions affect the success of adaptive graphical user interfaces.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
For decades, researchers have presented different adaptive user interfaces and discussed the pros and cons of adaptation on task performance and satisfaction. Little research, however, has been directed at isolating and understanding those aspects of adaptive interfaces which make some of them successful and others not. We have designed and implemented three adaptive graphical interfaces and evaluated them in two experiments along with a nonadaptive baseline. In this paper we synthesize our results with previous work and discuss how different design choices and interactions affect the success of adaptive graphical user interfaces.
Gajos, Krzysztof Z.; Long, Jing J.; Weld, Daniel S.
Automatically generating custom user interfaces for users with physical disabilities Proceedings Article
In: Assets ’06: Proceedings of the 8th international ACM SIGACCESS conference on Computers and accessibility, pp. 243–244, ACM Press, New York, NY, USA, 2006, ISBN: 1595932909.
@inproceedings{gajos06:automatically,
title = {Automatically generating custom user interfaces for users with physical disabilities},
author = {Krzysztof Z. Gajos and Jing J. Long and Daniel S. Weld},
url = {/papers/kgajos-assets06.pdf},
doi = {10.1145/1168987.1169036},
isbn = {1595932909},
year = {2006},
date = {2006-01-01},
booktitle = {Assets '06: Proceedings of the 8th international ACM SIGACCESS conference on Computers and accessibility},
pages = {243–244},
publisher = {ACM Press},
address = {New York, NY, USA},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Richter, Kai; Nichols, Jeffrey; Gajos, Krzysztof; Seffah, Ahmed (Ed.)
Proceedings of the CHI’06 Workshop on The Many Faces of Consistency in Cross Platform Design Proceedings
2006.
@proceedings{richter06:proceedings,
title = {Proceedings of the CHI'06 Workshop on The Many Faces of Consistency in Cross Platform Design},
editor = {Kai Richter and Jeffrey Nichols and Krzysztof Gajos and Ahmed Seffah},
url = {http://sunsite.informatik.rwth-aachen.de/Publications/CEUR-WS/Vol-198/},
year = {2006},
date = {2006-01-01},
keywords = {},
pubstate = {published},
tppubtype = {proceedings}
}
2005
Gajos, Krzysztof; Christianson, David; Hoffmann, Raphael; Shaked, Tal; Henning, Kiera; Long, Jing J.; Weld, Daniel S.
Fast and Robust Interface Generation for Ubiquitous Applications Proceedings Article
In: UbiComp 2005: Ubiquitous Computing, pp. 37–55, Springer, Berlin / Heidelberg, 2005.
@inproceedings{gajos05:fast,
title = {Fast and Robust Interface Generation for Ubiquitous Applications},
author = {Krzysztof Gajos and David Christianson and Raphael Hoffmann and Tal Shaked and Kiera Henning and Jing J. Long and Daniel S. Weld},
url = {/papers/kgajos-ubicomp05.pdf},
doi = {10.1007/11551201_3},
year = {2005},
date = {2005-01-01},
booktitle = {UbiComp 2005: Ubiquitous Computing},
journal = {Lecture Notes in Computer Science : UbiComp 2005: Ubiquitous Computing},
volume = {3660},
pages = {37–55},
publisher = {Springer},
address = {Berlin / Heidelberg},
series = {Lecture Notes in Computer Science},
abstract = {We present Supple, a novel toolkit which automatically generates interfaces for ubiquitous applications. Designers need only specify declarative models of the interface and desired hardware device and Supple uses decision-theoretic optimization to automatically generate a concrete rendering for that device. This paper provides an overview of our system and describes key extensions that barred the previous version from practical application. Specifically, we describe a functional modeling language capable of representing complex applications. We propose a new adaptation strategy, split interfaces, which speeds access to common interface features without disorienting the user. We present a customization facility that allows designers and end users to override Supple's automatic rendering decisions. We describe a distributed architecture which enables computationally-impoverished devices to benefit from Supple interfaces. Finally, we present experiments and a preliminary user-study that demonstrate the practicality of our approach.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
We present Supple, a novel toolkit which automatically generates interfaces for ubiquitous applications. Designers need only specify declarative models of the interface and desired hardware device and Supple uses decision-theoretic optimization to automatically generate a concrete rendering for that device. This paper provides an overview of our system and describes key extensions that barred the previous version from practical application. Specifically, we describe a functional modeling language capable of representing complex applications. We propose a new adaptation strategy, split interfaces, which speeds access to common interface features without disorienting the user. We present a customization facility that allows designers and end users to override Supple’s automatic rendering decisions. We describe a distributed architecture which enables computationally-impoverished devices to benefit from Supple interfaces. Finally, we present experiments and a preliminary user-study that demonstrate the practicality of our approach.
Gajos, Krzysztof; Weld, Daniel S.
Preference elicitation for interface optimization Proceedings Article
In: UIST ’05: Proceedings of the 18th annual ACM symposium on User interface software and technology, pp. 173–182, ACM Press, New York, NY, USA, 2005, ISBN: 1595932712.
@inproceedings{gajos05:preference,
title = {Preference elicitation for interface optimization},
author = {Krzysztof Gajos and Daniel S. Weld},
url = {/papers/kgajos-uist05.pdf},
doi = {10.1145/1095034.1095063},
isbn = {1595932712},
year = {2005},
date = {2005-01-01},
booktitle = {UIST '05: Proceedings of the 18th annual ACM symposium on User interface software and technology},
pages = {173–182},
publisher = {ACM Press},
address = {New York, NY, USA},
abstract = {Decision-theoretic optimization is becoming a popular tool in the user interface community, but creating accurate cost (or utility) functions has become a bottleneck - in most cases the numerous parameters of these functions are chosen manually, which is a tedious and error-prone process. This paper describes ARNAULD, a general interactive tool for eliciting user preferences concerning concrete outcomes and using this feedback to automatically learn a factored cost function. We empirically evaluate our machine learning algorithm and two automatic query generation approaches and report on an informal user study.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Decision-theoretic optimization is becoming a popular tool in the user interface community, but creating accurate cost (or utility) functions has become a bottleneck – in most cases the numerous parameters of these functions are chosen manually, which is a tedious and error-prone process. This paper describes ARNAULD, a general interactive tool for eliciting user preferences concerning concrete outcomes and using this feedback to automatically learn a factored cost function. We empirically evaluate our machine learning algorithm and two automatic query generation approaches and report on an informal user study.
Gajos, Krzysztof; Wu, Anthony; Weld, Daniel S.
Cross-Device Consistency in Automatically Generated User Interfaces Proceedings Article
In: Proceedings of Workshop on Multi-User and Ubiquitous User Interfaces (MU3I’05), 2005.
@inproceedings{gajos05:consistency,
title = {Cross-Device Consistency in Automatically Generated User Interfaces},
author = {Krzysztof Gajos and Anthony Wu and Daniel S. Weld},
url = {/papers/kgajos-mu3i05.pdf},
year = {2005},
date = {2005-01-01},
booktitle = {Proceedings of Workshop on Multi-User and Ubiquitous User Interfaces (MU3I'05)},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
2004
Patterson, Donald J.; Liao, Lin; Gajos, Krzysztof; Collier, Michael; Livic, Nik; Olson, Katherine; Wang, Shiaokai; Fox, Dieter; Kautz, Henry
Opportunity Knocks: A System to Provide Cognitive Assistance with Transportation Services Proceedings Article
In: UbiComp 2004: Ubiquitous Computing, pp. 433–450, Springer, Berlin / Heidelberg, 2004.
@inproceedings{patterson04:ok,
title = {Opportunity Knocks: A System to Provide Cognitive Assistance with Transportation Services},
author = {Donald J. Patterson and Lin Liao and Krzysztof Gajos and Michael Collier and Nik Livic and Katherine Olson and Shiaokai Wang and Dieter Fox and Henry Kautz},
url = {/papers/ok-ubicomp04.pdf},
year = {2004},
date = {2004-01-01},
booktitle = {UbiComp 2004: Ubiquitous Computing},
journal = {UbiComp 2004: Ubiquitous Computing},
volume = {3205},
pages = {433–450},
publisher = {Springer},
address = {Berlin / Heidelberg},
series = {Lecture Notes in Computer Science},
abstract = {We present an automated transportation routing system, called “Opportunity Knocks,†whose goal is to improve the efficiency, safety and independence of individuals with mild cognitive disabilities. Our system is implemented on a combination of a Bluetooth sensor beacon that broadcasts GPS data, a GPRS-enabled cell-phone, and remote activity inference software. The system uses a novel inference engine that does not require users to explicitly provide information about the start or ending points of their journeys; instead this information is learned from users’ past behavior. Futhermore, we demonstrate how route errors can be detected and how the system helps to correct the errors with real-time transit information. In addition we present a novel solution to the problem of labeling positions with place names.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
We present an automated transportation routing system, called “Opportunity Knocks,†whose goal is to improve the efficiency, safety and independence of individuals with mild cognitive disabilities. Our system is implemented on a combination of a Bluetooth sensor beacon that broadcasts GPS data, a GPRS-enabled cell-phone, and remote activity inference software. The system uses a novel inference engine that does not require users to explicitly provide information about the start or ending points of their journeys; instead this information is learned from users’ past behavior. Futhermore, we demonstrate how route errors can be detected and how the system helps to correct the errors with real-time transit information. In addition we present a novel solution to the problem of labeling positions with place names.
Gajos, Krzysztof; Weld, Daniel S.
SUPPLE: automatically generating user interfaces Proceedings Article
In: IUI ’04: Proceedings of the 9th international conference on Intelligent user interface, pp. 93–100, ACM Press, New York, NY, USA, 2004, ISBN: 1581138156.
@inproceedings{gajos04:supple,
title = {SUPPLE: automatically generating user interfaces},
author = {Krzysztof Gajos and Daniel S. Weld},
url = {/papers/supple-iui04.pdf},
doi = {10.1145/964442.964461},
isbn = {1581138156},
year = {2004},
date = {2004-01-01},
booktitle = {IUI '04: Proceedings of the 9th international conference on Intelligent user interface},
pages = {93–100},
publisher = {ACM Press},
address = {New York, NY, USA},
abstract = {In order to give people ubiquitous access to software applications, device controllers, and Internet services, it will be necessary to automatically adapt user interfaces to the computational devices at hand (e.g., cell phones, PDAs, touch panels, etc.). While previous researchers have proposed solutions to this problem, each has limitations. This paper proposes a novel solution based on treating interface adaptation as an optimization problem. When asked to render an interface on a specific device, our Supple system searches for the rendition that meets the device's constraints and minimizes the estimated effort for the user's expected interface actions. We make several contributions: 1) precisely defining the interface rendition problem, 2) demonstrating how user traces can be used to customize interface rendering to particular user's usage pattern, 3) presenting an efficient interface rendering algorithm, 4) performing experiments that demonstrate the utility of our approach.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
In order to give people ubiquitous access to software applications, device controllers, and Internet services, it will be necessary to automatically adapt user interfaces to the computational devices at hand (e.g., cell phones, PDAs, touch panels, etc.). While previous researchers have proposed solutions to this problem, each has limitations. This paper proposes a novel solution based on treating interface adaptation as an optimization problem. When asked to render an interface on a specific device, our Supple system searches for the rendition that meets the device’s constraints and minimizes the estimated effort for the user’s expected interface actions. We make several contributions: 1) precisely defining the interface rendition problem, 2) demonstrating how user traces can be used to customize interface rendering to particular user’s usage pattern, 3) presenting an efficient interface rendering algorithm, 4) performing experiments that demonstrate the utility of our approach.
Gajos, Krzysztof; Hoffmann, Raphael; Weld, Daniel S.
Improving User Interface Personalization Proceedings Article
In: Supplementary Proceedings of UIST’04, Santa Fe, NM, 2004.
@inproceedings{gajos04:improving,
title = {Improving User Interface Personalization},
author = {Krzysztof Gajos and Raphael Hoffmann and Daniel S. Weld},
url = {/papers/kgajos-uist04.pdf},
year = {2004},
date = {2004-01-01},
booktitle = {Supplementary Proceedings of UIST'04},
address = {Santa Fe, NM},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
2003
Weld, Daniel S.; Anderson, Corin; Domingos, Pedro; Etzioni, Oren; Gajos, Krzysztof; Lau, Tessa; Wolfman, Steve
Automatically Personalizing User Interfaces Proceedings Article
In: IJCAI03, Acapulco, Mexico, 2003, (Invited paper).
@inproceedings{weld03:automatically,
title = {Automatically Personalizing User Interfaces},
author = {Daniel S. Weld and Corin Anderson and Pedro Domingos and Oren Etzioni and Krzysztof Gajos and Tessa Lau and Steve Wolfman},
url = {/papers/weld-ijcai03.pdf},
year = {2003},
date = {2003-08-01},
booktitle = {IJCAI03},
address = {Acapulco, Mexico},
abstract = {Todays computer interfaces are one-size-fits-all.},
note = {Invited paper},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Todays computer interfaces are one-size-fits-all.
2002
Gajos, Krzysztof; Fox, Harold; Shrobe, Howard
End User Empowerment in Human Centered Pervasive Computing Proceedings Article
In: Proceedings of the First International Conference on Pervasive Computing, Pervasive 2002 – companion volume, pp. 134–140, 2002.
@inproceedings{gajos02:end,
title = {End User Empowerment in Human Centered Pervasive Computing},
author = {Krzysztof Gajos and Harold Fox and Howard Shrobe},
url = {/papers/pervasive02.pdf},
year = {2002},
date = {2002-01-01},
booktitle = {Proceedings of the First International Conference on Pervasive Computing, Pervasive 2002 - companion volume},
pages = {134–140},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Oh, Alice; Fox, Harold; Kleek, Max Van; Adler, Aaron; Gajos, Krzysztof; Morency, Louis-Philippe; Darrell, Trevor
Evaluating look-to-talk: a gaze-aware interface in a collaborative environment Proceedings Article
In: CHI ’02: CHI ’02 extended abstracts on Human factors in computing systems, pp. 650–651, ACM Press, New York, NY, USA, 2002, ISBN: 1581134541.
@inproceedings{oh02:evaluating,
title = {Evaluating look-to-talk: a gaze-aware interface in a collaborative environment},
author = {Alice Oh and Harold Fox and Max Van Kleek and Aaron Adler and Krzysztof Gajos and Louis-Philippe Morency and Trevor Darrell},
url = {/papers/ltt-chi2002.pdf},
doi = {10.1145/506443.506528},
isbn = {1581134541},
year = {2002},
date = {2002-01-01},
booktitle = {CHI '02: CHI '02 extended abstracts on Human factors in computing systems},
pages = {650–651},
publisher = {ACM Press},
address = {New York, NY, USA},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
2001
Gajos, Krzysztof; Weisman, Luke; Shrobe, Howard
Design Principles for Resource Management Systems for Intelligent Spaces Book Section
In: Self-Adaptive Software: Applications, vol. 2614, pp. 143–158, Springer, 2001.
@incollection{gajos01:design,
title = {Design Principles for Resource Management Systems for Intelligent Spaces},
author = {Krzysztof Gajos and Luke Weisman and Howard Shrobe},
url = {/papers/iwsas01.pdf},
year = {2001},
date = {2001-01-01},
booktitle = {Self-Adaptive Software: Applications},
journal = {Self-Adaptive Software: Applications},
volume = {2614},
pages = {143–158},
publisher = {Springer},
series = {Lecture Notes in Computer Science},
abstract = {The idea of ubiquitous computing and smart environments is no longer a dream and has long become a serious area of research and soon this technology will start entering our every day lives. There are two major obstacles that prevent this technology from spreading. First, different smart spaces are equipped with very different kinds of devices (e.g. a projector vs. a computer monitor, vs. a TV set). Second, multiple applications running in a space at the same time inevitably contend for those devices and other scarce resources. The underlying software in a smart space needs to provide tools for self-adaptivity in that it shields the rest of the software from the physical constraints of the space, and that it dynamically adjusts the allocation of scarce resources as the number and priorities of active tasks change. We argue that a resource manager can provide the necessary functionality. This paper presents a set of guiding principles for building high-level resource management tools for smart spaces. We present conclusions we arrived at after two years of exploring the topic in the Intelligent Room Project at the MIT AI Lab. The paper is based on a number of implemented and tested tools.},
keywords = {},
pubstate = {published},
tppubtype = {incollection}
}
The idea of ubiquitous computing and smart environments is no longer a dream and has long become a serious area of research and soon this technology will start entering our every day lives. There are two major obstacles that prevent this technology from spreading. First, different smart spaces are equipped with very different kinds of devices (e.g. a projector vs. a computer monitor, vs. a TV set). Second, multiple applications running in a space at the same time inevitably contend for those devices and other scarce resources. The underlying software in a smart space needs to provide tools for self-adaptivity in that it shields the rest of the software from the physical constraints of the space, and that it dynamically adjusts the allocation of scarce resources as the number and priorities of active tasks change. We argue that a resource manager can provide the necessary functionality. This paper presents a set of guiding principles for building high-level resource management tools for smart spaces. We present conclusions we arrived at after two years of exploring the topic in the Intelligent Room Project at the MIT AI Lab. The paper is based on a number of implemented and tested tools.
Gajos, Krzysztof
Rascal – A Resource Manager for Multi Agent Systems in Smart Spaces Proceedings Article
In: CEEMAS ’01: Revised Papers from the Second International Workshop of Central and Eastern Europe on Multi-Agent Systems, pp. 111–120, Springer-Verlag, London, UK, 2001, ISBN: 3540433708.
@inproceedings{gajos01:rascal,
title = {Rascal - A Resource Manager for Multi Agent Systems in Smart Spaces},
author = {Krzysztof Gajos},
url = {/papers/ceemas01.pdf},
isbn = {3540433708},
year = {2001},
date = {2001-01-01},
booktitle = {CEEMAS '01: Revised Papers from the Second International Workshop of Central and Eastern Europe on Multi-Agent Systems},
pages = {111–120},
publisher = {Springer-Verlag},
address = {London, UK},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Gajos, Krzysztof; Kulkarni, Ajay
FIRE: An Information Retrieval Interface For Intelligent Environments Proceedings Article
In: Proceedings of International Workshop on Information Presentation and Natural Multimodal Dialogue IPNMD 2001, Verona, Italy, 2001.
@inproceedings{gajos01:fire,
title = {FIRE: An Information Retrieval Interface For Intelligent Environments},
author = {Krzysztof Gajos and Ajay Kulkarni},
url = {/papers/ipnmd01-fire.pdf},
year = {2001},
date = {2001-12-01},
booktitle = {Proceedings of International Workshop on Information Presentation and Natural Multimodal Dialogue IPNMD 2001},
address = {Verona, Italy},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}