All Cohorts Combined (101 Students)
This group of 101 students is made up 5 cohorts who completed our materials between fall of 2016 and spring of 2018. These cohorts include: 14 community college students from fall 2016, 25 high school students from fall 2016, 19 high school students from summer 2017, 4 high school students from fall 2017, and 39 community college students from fall 2017 - spring 2018. The data from these cohorts can be combined because all 5 cohorts completed the same materials (with the exception of a few minor changes that will be documented in the manuscript currently in preparation) within similar time frames. All cohorts except the 39 community college students from fall 2017 - spring 2018 completed an immediate intervention study with no comparison group. Combining the data from all cohorts allows us to look at questions that don’t depend on the immediate vs. delayed intervention framework with a higher degree of power.
The code in the following analyses uses the following data frames: pretest.rds, lesson.rds, posttestAll.rds. The raw data can be downloaded from the “Materials” tab, and the process used to get the raw data into the pretest.rds, lesson.rds, posttestAll.rds format can be found in the “Data Process” tab.
Which TI problems were most difficult?
We looked at the pattern of performance on each of the 20 types of trigonometric identity problems, broken down into 3 groups - bottom, middle, top - based on performance on TI1.
library(ggplot2) #v 3.2.1
library(dplyr) #v 0.8.3
library(data.table) #v 1.12.2
library(DT) #v 0.8
library(jsonlite) #v 1.6
library(tidyr) #v 1.0.0
library(reshape2) #v 1.4.3
allTrigID <- readRDS("posttestAll.rds")
sin <- function(degree){
round(base::sin(degree/180*pi),digits=4)
}
cos <- function(degree){
round(base::cos(degree/180*pi),digits=4)
}
conjunctionJunction <- function(stim, theta) {
evalStim <- eval(parse(text=stim))
for(func in c('sin', 'cos')){
for( sign in c('', '-')) {
testAnswer = eval(parse(text=paste0(sign, func, "(", theta, ")")))
if(isTRUE(all.equal(evalStim, testAnswer, tolerance = 0.01))) {
return(paste0(sign, func))
}
}
}
return("")
}
allTrigID$correctAnswer <- mapply(conjunctionJunction, allTrigID$stimulus, allTrigID$theta)
allTrigID2 <- separate(allTrigID, response, c("res.sign", "res.func"), sep="(?<=-)", remove = FALSE)
allTrigID2$res.func <- ifelse(allTrigID2$res.sign=="sin" | allTrigID2$res.sign=="cos",allTrigID2$res.sign,allTrigID2$res.func)
allTrigID2$res.sign <- ifelse(allTrigID2$res.sign=="sin" | allTrigID2$res.sign=="cos","+",allTrigID2$res.sign)
allTrigID2 <- separate(allTrigID2, correctAnswer, c("ans.sign", "ans.func"), sep="(?<=-)", remove = FALSE)
allTrigID2$ans.func <- ifelse(allTrigID2$ans.sign=="sin" | allTrigID2$ans.sign=="cos",allTrigID2$ans.sign,allTrigID2$ans.func)
allTrigID2$ans.sign <- ifelse(allTrigID2$ans.sign=="sin" | allTrigID2$ans.sign=="cos","+",allTrigID2$ans.sign)
allTrigID2$error <- ifelse(allTrigID2$res.sign!=allTrigID2$ans.sign & allTrigID2$res.func!=allTrigID2$ans.func, "Both wrong",
ifelse(allTrigID2$res.sign!=allTrigID2$ans.sign, "Wrong sign",
ifelse(allTrigID2$res.func!=allTrigID2$ans.func, "Wrong function",
"Correct")))
d <- allTrigID2 %>%
group_by(func,xsign,delta) %>%
summarize(acc=mean(correct)) %>%
as.data.frame
d2 <- subset(allTrigID2, select = c(user, func, xsign, delta, correct, error, rt))
d2a <-
d2 %>%
group_by(user) %>%
summarise(Average = round(mean(correct)*100,1))
d2a$third <- ntile(d2a$Average, 3)
#create three data frames, one for each third of the data based on performance
bottom <- d2a[(d2a$third == 1),]
bottom <- bottom$user
middle <- d2a[(d2a$third == 2),]
middle <- middle$user
top <- d2a[(d2a$third == 3),]
top <- top$user
d2bottom <- d2 [d2$user %in% bottom,]
d2middle <- d2 [d2$user %in% middle,]
d2top <- d2 [d2$user %in% top,]
allTrigID2$signOrCorrect = (allTrigID2$error == "Correct") | (allTrigID2$error == "Wrong sign")
allTrigID2$funcOrCorrect = (allTrigID2$error == "Correct") | (allTrigID2$error == "Wrong function")
dbottom2 <- allTrigID2 [allTrigID2$user %in% bottom,]
dbottom = dbottom2 %>%
group_by(func,xsign,delta) %>%
summarize(acc=mean(correct)) %>%
as.data.frame
dbottomFuncCorrect <- dbottom2 %>%
group_by(func,xsign,delta) %>%
summarise(acc=mean(signOrCorrect), count=n())
dbottomSignCorrect <- dbottom2 %>%
group_by(func,xsign,delta) %>%
summarise(acc=mean(funcOrCorrect), count=n())
dmiddle2 <- allTrigID2 [allTrigID2$user %in% middle,]
dmiddle = dmiddle2 %>%
group_by(func,xsign,delta) %>%
summarize(acc=mean(correct)) %>%
as.data.frame
dmiddleFuncCorrect <- dmiddle2 %>%
group_by(func,xsign,delta) %>%
summarise(acc=mean(signOrCorrect), count=n())
dmiddleSignCorrect <- dmiddle2 %>%
group_by(func,xsign,delta) %>%
summarise(acc=mean(funcOrCorrect), count=n())
dtop2 <- allTrigID2 [allTrigID2$user %in% top,]
dtop = dtop2 %>%
group_by(func,xsign,delta) %>%
summarize(acc=mean(correct)) %>%
as.data.frame
dtopFuncCorrect <- dtop2 %>%
group_by(func,xsign,delta) %>%
summarise(acc=mean(signOrCorrect), count=n())
dtopSignCorrect <- dtop2 %>%
group_by(func,xsign,delta) %>%
summarise(acc=mean(funcOrCorrect), count=n())
errorType <- function(i) {
if(i=="bottom") {d2 <- d2bottom
d <- dbottom}
if(i=="middle") {d2 <- d2middle
d <- dmiddle}
if(i=="top") {d2 <- d2top
d <- dtop}
if(i=="bottomFuncCorrect") {d2 <- d2bottom
d <- dbottomFuncCorrect}
if(i=="bottomSignCorrect") {d2 <- d2bottom
d <- dbottomSignCorrect}
if(i=="middleFuncCorrect") {d2 <- d2middle
d <- dmiddleFuncCorrect}
if(i=="middleSignCorrect") {d2 <- d2middle
d <- dmiddleSignCorrect}
if(i=="topFuncCorrect") {d2 <- d2top
d <- dtopFuncCorrect}
if(i=="topSignCorrect") {d2 <- d2top
d <- dtopSignCorrect}
d3 <- d2 %>%
count(func,xsign,delta,error) %>%
spread(error, n, fill = 0)
d4 <- merge(d, d3, by=c("func","xsign","delta"))
d4$delta = factor(d4$delta,levels=c(0,90,-90,180,-180))
d4$xsign = factor(d4$xsign,levels=c("+","-"))
#d$rule = factor(c(0,0,2,5,0, 4,0,1,0,4, 0,0,3,5,0, 4,0,1,0,4))
d4$rule = factor(c(0,0,3,0,5, 4,0,1,4,0, 0,0,2,0,5, 4,0,1,4,0))
d4$rule = factor(d4$rule,levels=c(1:5,0),labels=c('x±0=x','sin(–x)=–sin(x)','cos(–x)=cos(x)','func(x±180)=–func(x)','func(90–x)=opp(x)','Multiple/Other'))
#d$rule = factor(d$rule,levels=c(1:5,0),labels=c('$x\\pm0=x$','$\\sin(-x)=-\\sin(x)$','$\\cos(-x)=\\cos(x)$','$\\textrm{func}(x\\pm180)=-\\textrm{func}(x)$','$\\textrm{func}(90-x)=\\textrm{opp}(x)$','Multiple/Other'))
d4$signfunc = factor(interaction(d4$xsign,d4$func),levels=c('+.sin','+.cos','-.sin','-.cos'),labels=c('sin(+θ +/- Δ)','cos(+θ +/- Δ)','sin(-θ +/- Δ)','cos(-θ +/- Δ)'))
#d$signfunc = factor(interaction(d$xsign,d$func),levels=c('+.sin','+.cos','-.sin','-.cos'),labels=c('$\\sin(+\\theta \\pm \\Delta)$','$\\cos(+\\theta \\pm \\Delta)$','$\\sin(-\\theta \\pm \\Delta)$','$\\cos(-\\theta \\pm \\Delta)$'))
d4$delta = factor(d4$delta,levels=c(-180,-90,0,90,180))
d4 <- gather(d4, ErrorType, ErrorCount, -func, -xsign, -delta, -acc, -rule, -signfunc)
d4$ErrorType <- factor(d4$ErrorType, levels=c("Both wrong","Wrong function","Wrong sign","Correct"))
return(d4)
}
d4bottom <- errorType("bottom")
d4bottom$third <- "bottom"
d4middle <- errorType("middle")
d4middle$third <- "middle"
d4top <- errorType("top")
d4top$third <- "top"
##########################################
d4bottomFuncCorrect <- errorType("bottomFuncCorrect")
d4bottomFuncCorrect$third <- "bottom"
d4middleFuncCorrect <- errorType("middleFuncCorrect")
d4middleFuncCorrect$third <- "middle"
d4topFuncCorrect <- errorType("topFuncCorrect")
d4topFuncCorrect$third <- "top"
d4bottomSignCorrect <- errorType("bottomSignCorrect")
d4bottomSignCorrect$third <- "bottom"
d4middleSignCorrect <- errorType("middleSignCorrect")
d4middleSignCorrect$third <- "middle"
d4topSignCorrect <- errorType("topSignCorrect")
d4topSignCorrect$third <- "top"
##########################################
all <- rbind(d4bottom,d4middle,d4top)
ggplot(all,aes(delta,acc,fill=rule))+
stat_summary(fun.y=mean,geom='bar') +
facet_grid(third~signfunc)+theme_bw(20)+
labs(x='Shift Δ',y='Accuracy',fill='Rule')+
scale_fill_manual(values = c("#fb9a99","#a6cee3","#1f78b4", "#b2df8a","#F2ED2B","#999999")) +
theme(panel.grid.minor=element_blank(), panel.grid.major=element_blank(), strip.background=element_blank())+scale_y_continuous(limits=c(0,1),labels = c(0,25,50,75,100))
The following graph presents the same results with the error responses broken down by whether the participant chose the alternative with the correct function but wrong signum, the incorrect function but correct signum, or neither the correct function nor signum.
ggplot(all,aes(delta,ErrorCount,fill=ErrorType))+
geom_bar(stat="identity")+
facet_grid(third~signfunc)+theme_bw(20)+
labs(x='Shift Δ',y='Count',fill='Error Type') 
Which lesson problems are most difficult?
###find most frequently missed problems
lessonData <- readRDS("lesson.rds")
userIDs <- read.csv("combinedUserIDs.csv")
#exclude those who didn't meet threshold
noThreshold <- c(1719,1653,1735,1998,2038)
userIDs <- userIDs [! userIDs$TrigID %in% noThreshold,]
userIDs <- userIDs$TrigID
##only look at first attempt data
lessonData <- lessonData[lessonData$trialNumber==1,]
chs14 <- c(1:2,5:13,15:22,23:29,30:36)
ch56 <- c(37:45,47:57)
ch14Data <- lessonData [ lessonData$page_id %in% chs14,]
ch56Data <- lessonData [ lessonData$page_id %in% ch56,]
lessonDataAverageAll <-
lessonData %>%
group_by(user) %>%
summarise(PercentCorrect = round(mean(firstCorrect)*100,1))
lessonDataAverageCh14 <-
ch14Data %>%
group_by(user) %>%
summarise(PercentCorrect = round(mean(firstCorrect)*100,1))
lessonDataAverageCh56 <-
ch56Data %>%
group_by(user) %>%
summarise(PercentCorrect = round(mean(firstCorrect)*100,1))
LessonDataByProblem <-
lessonData %>%
group_by(order) %>%
summarise(PercentCorrect = round(mean(firstCorrect)*100,1),
MedianRT = round(median(rt)/1000,1))
##reduce dataset to problems less than 60% of students got correct
LessonDataLess60 <- LessonDataByProblem[LessonDataByProblem$PercentCorrect<60.1,]
lessthan60 <- LessonDataLess60$order
library(kableExtra) #v 1.1.0
mostMissedProblems <- read.csv("lessthan60.csv")
kable(mostMissedProblems) %>%
kable_styling(bootstrap_options = c("striped", "hover", "condensed", "responsive"))| Problem | Chapter_No | Chapter | Page_No | Page | ProblemText |
|---|---|---|---|---|---|
| 12 | 1 | XY Positions | 2 | Mirror Images | Is this statement true or false? The position at x = -(-3) is the mirror image of the mirror image of the position at x = 3. True (correct) False |
| 97 | 2 | Circle Positions | 16 | Theta and Degrees | Perform a rotation of 360 degrees, observe where it ends up, and think about what it means. Do rotations of different amounts result in different positions on the circle? yes sometimes but not always (correct) Never |
| 153 | 3 | Circle Rotations | 26 | Rotations to Same Positions - continued | What is the value of theta, between -360 degrees and 0 degrees, that corresponds to this same position? Enter this value now! (correct answer = -15) |
| 156 | 3 | Circle Rotations | 27 | Moving on a Line | Many mathematical ideas have their corresponding movements in the numerical space. This is true not just on the unit circle, but also on the number line at the very beginning of this course. As you know, the origin of a horizontal axis is at position x = 0. Now, imagine a point 3 positive units away from the origin. Starting from that position, perform the following movements: move 2 units in the positive direction move 3 units in the negative direction move 4 units in the negative direction. Where would the point end up? Mark that final position. (correct answer = -2) |
| 159 | 3 | Circle Rotations | 27 | Moving on a Line | Now let’s think about motions a bit more generally. For every motion in the positive direction, is there a motion in the negative direction that cancels that motion completely? always (correct) sometimes Never |
| 256 | 4 | Linking the Circle with the XY Plane | 36 | Seeing the Invisible Sine and Cosine | Now, mark 250 degrees on the circle, and directly enter the value of cos(250 degrees). |
| 322 | 5 | Rotations, Reflections and Positions | 39 | Sine and Cosine of Up-Down Reflections | Let’s look at another example. Mark a circle position theta in between 90 degrees and 180 degrees, such that sin(theta) = 0.3. (correct answer = 162.54) |
| 350 | 5 | Rotations, Reflections and Positions | 40 | Diametrically Opposite Circle Positions | We asked Mario, Luigi, and Princess Peach to give an expression that corresponds to the diametrical opposite of -theta. Here are the expressions they wrote: Mario: (-180 degrees-theta) Luigi: (theta-180 degrees) Princess Peach: (-theta+180 degrees) Which one of them got it wrong? Mario Luigi (correct) Princess Peach |
| 351 | 5 | Rotations, Reflections and Positions | 40 | Diametrically Opposite Circle Positions | The second expression, (180 degrees-theta), is very different though. In this case, we start at 180 degrees and rotate theta degrees in the negative direction. This is not the diametrical opposite of ? (unless theta is 0 degrees). Again, suppose that theta = 20 degrees. The canvas is currently displaying its diametrical opposite position of theta = 20 degrees as an open dot. However, it is not the same position as (180 degrees-theta), which is numerically (180 degrees-20 degrees) = 160 degrees and can be reached by starting at 180 degrees and then rotating 20 degrees in the negative direction. Now, mark the position (180 degrees-theta) on the diagram, and then click Submit. (correct answer = 160) |
| 437 | 5 | Rotations, Reflections and Positions | 43 | Quarter Turn Rotations | At the end of this section, we asked Mario, Luigi, and Princess Peach to give an expression that represents a position on the circle corresponding to the quarter-turn rotation of -theta. One correct answer is (-theta+90 degrees), but none of them chose that answer. Here are the expressions they wrote: Mario: (theta-90 degrees) Luigi: (90 degrees-theta) Princess Peach: (theta+90 degrees) Which one of them got it correct? Mario Luigi (correct) Princess Peach |
| 449 | 5 | Rotations, Reflections and Positions | 44 | Quarter-Turn Rotations: Horizontal and Vertical Aspects | If you look at the vertical position of 20 degrees, you will notice that it is a little bit above the origin. You can read off its vertical position (i.e., its y-coordinate) and see that it is about 0.34. Now, consider the open point’s horizontal position. As you can see, it is a bit to the left of the origin. Try to read off the open point’s horizontal, x-coordinate. Is it the same or a different from the vertical, y-coordinate of 20 degrees? Same Different (correct) |
| 452 | 5 | Rotations, Reflections and Positions | 44 | Quarter-Turn Rotations: Horizontal and Vertical Aspects | Luigi also agrees that the two values are different, but he is trying to give a new reason to explain this difference. He says, “The vertical position of 20 degrees is in the vertical direction, while the horizontal position of (90 degrees+20 degrees) is in the horizontal direction. That’s why they don’t have the same value.” What do you think of Luigi’s explanation on the different values? Is it reasonable? Yes No (correct) |
| 466 | 5 | Rotations, Reflections and Positions | 44 | Quarter-Turn Rotations: Horizontal and Vertical Aspects | Are the horizontal position of your point and the vertical position of its quarter-turn rotation both positive? Yes (correct) No |
| 467 | 5 | Rotations, Reflections and Positions | 44 | Quarter-Turn Rotations: Horizontal and Vertical Aspects | Given what you have observed, you should be able to appreciate the relationship between the x-coordinate of your point with the y-coordinate of its 90 degrees-rotation. Do they have the same value, the opposite value, or neither? Same (correct) Opposite Neither |
| 472 | 5 | Rotations, Reflections and Positions | 44 | Quarter-Turn Rotations: Horizontal and Vertical Aspects | How about the x-coordinate of 10 degrees and the x-coordinate of (90 degrees+10 degrees)? Are they the same, opposite, or neither? Same Opposite Neither (correct) |
| 575 | 6 | Comparing Values | 55 | Extending to Less Familiar Problems | Which expression has the same numerical value as sin(50 degrees-90 degrees)? sin(50 degrees) -sin(50 degrees) cos(50 degrees) -cos(50 degrees) (correct) |
| 590 | 6 | Comparing Values | 56 | Practice - I | Which expression has the same numerical value as sin(180 degrees-10 degrees)? sin(10 degrees) (correct) -sin(10 degrees) cos(10 degrees) -cos(10 degrees) |
What are the patterns of performance for the most frequently missed problems?
Below we show the accuracy results for the problems listed above broken down by the bottom, middle, and top thirds of the group as measured by TI1 performance.
lessthan60Data <- lessonData [ lessonData$order %in% lessthan60,]
lessonBottom <- lessthan60Data [lessthan60Data$user %in% bottom,]
lessonMiddle <- lessthan60Data [lessthan60Data$user %in% middle,]
lessonTop <- lessthan60Data [lessthan60Data$user %in% top,]
lessthan60Bottom <-
lessonBottom%>%
group_by(order) %>%
summarise(Bottom = round(mean(firstCorrect)*100,1))
lessthan60Middle <-
lessonMiddle%>%
group_by(order) %>%
summarise(Middle = round(mean(firstCorrect)*100,1))
lessthan60Top <-
lessonTop%>%
group_by(order) %>%
summarise(Top = round(mean(firstCorrect)*100,1))
lessonMerged <- merge(lessthan60Bottom,lessthan60Middle,by="order")
lessonMerged <- merge(lessonMerged,lessthan60Top,by="order")
lessonMergedLong <- gather(lessonMerged, third, percentCorrect, Bottom:Top, factor_key=TRUE)
lessonMergedLong$third <- factor(lessonMergedLong$third, levels=c("Bottom","Middle","Top"))
# ggplot(lessonMergedLong,aes(as.factor(problem),percentCorrect,fill=third))+
# geom_bar(stat="identity",position="dodge") +
# labs(x='Problem #',y='Percent Correct',fill='Third')
ggplot(lessonMergedLong,aes(x=third,y=percentCorrect,fill=third))+
geom_bar(stat="identity") +
facet_wrap(~as.factor(order),ncol=6) +
labs(x='Third',y='Percent Correct',fill='Third') +
theme(strip.text.x = element_text(size = 14),axis.text.x = element_text(size = 10),legend.text=element_text(size=12))