germanet
2025-07-07
library(ggplot2)
library(lme4)## Loading required package: Matrixlibrary(lmerTest)##
## Attaching package: 'lmerTest'## The following object is masked from 'package:lme4':
##
## lmer## The following object is masked from 'package:stats':
##
## steplibrary(car)## Loading required package: carDatalibrary(sjPlot)## Install package "strengejacke" from GitHub (`devtools::install_github("strengejacke/strengejacke")`) to load all sj-packages at once!library(emmeans)## Warning: package 'emmeans' was built under R version 4.4.3## Welcome to emmeans.
## Caution: You lose important information if you filter this package's results.
## See '? untidy'library(tidyr)##
## Attaching package: 'tidyr'## The following objects are masked from 'package:Matrix':
##
## expand, pack, unpacklibrary(lattice)
library(irr)## Loading required package: lpSolvelibrary(cvms)
library(epiR)## Loading required package: survival## Package epiR 2.0.84 is loaded## Type help(epi.about) for summary information## Type browseVignettes(package = 'epiR') to learn how to use epiR for applied epidemiological analyses## library(dplyr)##
## Attaching package: 'dplyr'## The following object is masked from 'package:car':
##
## recode## The following objects are masked from 'package:stats':
##
## filter, lag## The following objects are masked from 'package:base':
##
## intersect, setdiff, setequal, unionlibrary(psych)##
## Attaching package: 'psych'## The following object is masked from 'package:car':
##
## logit## The following objects are masked from 'package:ggplot2':
##
## %+%, alphalibrary(multcomp)## Loading required package: mvtnorm## Loading required package: TH.data## Warning: package 'TH.data' was built under R version 4.4.3## Loading required package: MASS##
## Attaching package: 'MASS'## The following object is masked from 'package:dplyr':
##
## select##
## Attaching package: 'TH.data'## The following object is masked from 'package:MASS':
##
## geyserlibrary(multcompView)
library(effects)## Use the command
## lattice::trellis.par.set(effectsTheme())
## to customize lattice options for effects plots.
## See ?effectsTheme for details.library(ggeffects) # install.packages("ggeffects")
library(summarytools)
library(report)
library(performance)## Warning: package 'performance' was built under R version 4.4.3##
## Attaching package: 'performance'## The following object is masked from 'package:irr':
##
## iccPolyseme case:
path="/Users/zinn/Projects/EKUT/GermaNet/GermaNet_LLM/R_analyses/annotations/"
data=read.table( paste0(path, "polysemes_corrected.csv"), header=T, sep=";")
#data=read.table( paste0(path, "polysemes.csv"), header=T, sep=";")
nrow(data)## [1] 5400# for Table 5. LLMs and their error types
counts <- data %>%
group_by(errorType, model) %>%
summarise(count = n(), .groups = "drop")
print(counts)## # A tibble: 27 × 3
## errorType model count
## <chr> <chr> <int>
## 1 0 claude-3-5-haiku-20241022 1049
## 2 0 deepseek-chat 1256
## 3 0 gpt-4o 1220
## 4 FA claude-3-5-haiku-20241022 258
## 5 FA deepseek-chat 194
## 6 FA gpt-4o 220
## 7 FG claude-3-5-haiku-20241022 20
## 8 FG deepseek-chat 14
## 9 FG gpt-4o 12
## 10 FL claude-3-5-haiku-20241022 252
## # ℹ 17 more rowsdata$logfreq=log(data$frequency)
data$scalelogfreq=scale(data$logfreq)
data$loghyper=log(data$frequencyHypernym+1)
data$scaleloghyper=scale(data$loghyper)
data$lengthLemma=nchar(data$lemma)
data$lengthHypernym=nchar(data$hypernym)
mean(data$sentenceLength)## [1] 11.60926median(data$sentenceLength)## [1] 11mean(data$sentenceStringLength)## [1] 81.56815median(data$sentenceLength)## [1] 11claudedata=data[data$model=="claude-3-5-haiku-20241022",]
nrow(claudedata)## [1] 1800mean(claudedata$sentenceStringLength)## [1] 78.65556mean(claudedata$sentenceLength)## [1] 10.71111chatdata=data[data$model=="gpt-4o",]
nrow(chatdata)## [1] 1800mean(chatdata$sentenceStringLength)## [1] 91.52778mean(chatdata$sentenceLength)## [1] 13.22778deepseekdata=data[data$model=="deepseek-chat",]
nrow(deepseekdata)## [1] 1800mean(deepseekdata$sentenceStringLength)## [1] 74.52111mean(deepseekdata$sentenceLength)## [1] 10.88889# 5.2.1 Length of example sentences and sentence quality
# ---------------------------------
sl = lmer(sentenceLength ~ model + (1 | lemma), data = data)
Anova(sl, type="III")## Analysis of Deviance Table (Type III Wald chisquare tests)
##
## Response: sentenceLength
## Chisq Df Pr(>Chisq)
## (Intercept) 7470.9 1 < 2.2e-16 ***
## model 1558.7 2 < 2.2e-16 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1summary(sl)## Linear mixed model fit by REML. t-tests use Satterthwaite's method [
## lmerModLmerTest]
## Formula: sentenceLength ~ model + (1 | lemma)
## Data: data
##
## REML criterion at convergence: 23858.6
##
## Scaled residuals:
## Min 1Q Median 3Q Max
## -3.1635 -0.7237 -0.0458 0.6153 4.1496
##
## Random effects:
## Groups Name Variance Std.Dev.
## lemma (Intercept) 1.636 1.279
## Residual 4.556 2.134
## Number of obs: 5400, groups: lemma, 122
##
## Fixed effects:
## Estimate Std. Error df t value Pr(>|t|)
## (Intercept) 1.094e+01 1.266e-01 1.514e+02 86.435 <2e-16 ***
## modeldeepseek-chat 1.778e-01 7.115e-02 5.276e+03 2.499 0.0125 *
## modelgpt-4o 2.517e+00 7.115e-02 5.276e+03 35.372 <2e-16 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Correlation of Fixed Effects:
## (Intr) mdldp-
## mdldpsk-cht -0.281
## modelgpt-4o -0.281 0.500# sentenceLength (not significant)
goodness.glmer=glmer(as.factor(goodSentence) ~ sentenceLength +(1|lemma), data=data,family="binomial",control=glmerControl(optimizer="bobyqa"))
Anova(goodness.glmer, type="III")## Analysis of Deviance Table (Type III Wald chisquare tests)
##
## Response: as.factor(goodSentence)
## Chisq Df Pr(>Chisq)
## (Intercept) 0.200 1 0.6547
## sentenceLength 25.044 1 5.604e-07 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1summary(goodness.glmer)## Generalized linear mixed model fit by maximum likelihood (Laplace
## Approximation) [glmerMod]
## Family: binomial ( logit )
## Formula: as.factor(goodSentence) ~ sentenceLength + (1 | lemma)
## Data: data
## Control: glmerControl(optimizer = "bobyqa")
##
## AIC BIC logLik -2*log(L) df.resid
## 6434.2 6454.0 -3214.1 6428.2 5397
##
## Scaled residuals:
## Min 1Q Median 3Q Max
## -4.2298 -0.9958 0.4336 0.7379 2.4309
##
## Random effects:
## Groups Name Variance Std.Dev.
## lemma (Intercept) 1.004 1.002
## Number of obs: 5400, groups: lemma, 122
##
## Fixed effects:
## Estimate Std. Error z value Pr(>|z|)
## (Intercept) 0.07993 0.17872 0.447 0.655
## sentenceLength 0.06403 0.01280 5.004 5.6e-07 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Correlation of Fixed Effects:
## (Intr)
## sentncLngth -0.837plot(density(data$scalelogfreq))
# 1. Create contingency table
tbl <- table(data$errorType, data$numberWordSenses)
# 2. Run chi-square test
chisq.test(tbl)## Warning in chisq.test(tbl): Chi-squared approximation may be incorrect##
## Pearson's Chi-squared test
##
## data: tbl
## X-squared = 316.74, df = 48, p-value < 2.2e-16chisq.test(tbl, simulate.p.value = TRUE, B = 10000)##
## Pearson's Chi-squared test with simulated p-value (based on 10000
## replicates)
##
## data: tbl
## X-squared = 316.74, df = NA, p-value = 9.999e-05# => the two categorical variables in your table are highly dependent.
# Open a PNG device
png(paste0(path, "./results/polysemes_errorType_by_numSenses.png"), width = 1200, height = 800, res = 150)
mosaicplot(tbl, color = TRUE, las = 2,
main = "Mosaic plot: Error Type by Number of Senses",
xlab = "Error Type", ylab = "Number of Word Senses")
# Close the device to save the file
dev.off()## quartz_off_screen
## 2png(paste0(path, "./results/polysemes_wordSenseDistribution.png"), width = 1200, height = 800, res = 150)
hist(data$numberWordSenses,
main = "Distribution of Number of Word Senses",
xlab = "Number of Senses",
ylab = "Number of Sentences",
col = "skyblue", border = "white")
dev.off()## quartz_off_screen
## 2data=read.table( paste0(path, "polysemes_corrected.csv"), header=T, sep=";")
# this is the long format, not the wide one (!)
nrow(data)## [1] 5400data$logfreq=log(data$frequency)
data$scalelogfreq=scale(data$logfreq)
table(data$numberWordSenses)##
## 2 3 4 5 6 7 8
## 3348 756 648 270 108 126 144range(data$numberWordSenses)## [1] 2 8# for polysemes, center for model fit (intercept at zero)
data$cNumberWordSenses = data$numberWordSenses - 2
range(data$cNumberWordSenses)## [1] 0 6table(data$cNumberWordSenses)##
## 0 1 2 3 4 5 6
## 3348 756 648 270 108 126 144# less skew:
data$numberWordSenses_fourBins <- ifelse(data$cNumberWordSenses > 2, 3, data$cNumberWordSenses)
table(data$numberWordSenses_fourBins)##
## 0 1 2 3
## 3348 756 648 648# 5.2.2 No frequency data for word senses; what can be said about lemma frequency and its level of polysemy?
# ----------------------------------------------------------------------------------------------------------
cor.test(data$logfreq, data$numberWordSenses, method = c("kendall")) ##
## Kendall's rank correlation tau
##
## data: data$logfreq and data$numberWordSenses
## z = 16.742, p-value < 2.2e-16
## alternative hypothesis: true tau is not equal to 0
## sample estimates:
## tau
## 0.1747551cor.test(data$logfreq, data$numberWordSenses_fourBins, method = c("kendall")) ##
## Kendall's rank correlation tau
##
## data: data$logfreq and data$numberWordSenses_fourBins
## z = 17.353, p-value < 2.2e-16
## alternative hypothesis: true tau is not equal to 0
## sample estimates:
## tau
## 0.1825823# just checking: we get identical values for scalelogfreq
cor.test(data$scalelogfreq, data$numberWordSenses, method = c("kendall")) ##
## Kendall's rank correlation tau
##
## data: data$scalelogfreq and data$numberWordSenses
## z = 16.742, p-value < 2.2e-16
## alternative hypothesis: true tau is not equal to 0
## sample estimates:
## tau
## 0.1747551cor.test(data$scalelogfreq, data$numberWordSenses_fourBins, method = c("kendall")) ##
## Kendall's rank correlation tau
##
## data: data$scalelogfreq and data$numberWordSenses_fourBins
## z = 17.353, p-value < 2.2e-16
## alternative hypothesis: true tau is not equal to 0
## sample estimates:
## tau
## 0.1825823cor.test(data$logfreq, data$numberWordSenses, method = c("spearman")) ## Warning in cor.test.default(data$logfreq, data$numberWordSenses, method =
## c("spearman")): Cannot compute exact p-value with ties##
## Spearman's rank correlation rho
##
## data: data$logfreq and data$numberWordSenses
## S = 2.0181e+10, p-value < 2.2e-16
## alternative hypothesis: true rho is not equal to 0
## sample estimates:
## rho
## 0.2310385cor.test(data$logfreq, data$numberWordSenses_fourBins, method = c("spearman")) ## Warning in cor.test.default(data$logfreq, data$numberWordSenses_fourBins, :
## Cannot compute exact p-value with ties##
## Spearman's rank correlation rho
##
## data: data$logfreq and data$numberWordSenses_fourBins
## S = 2.0045e+10, p-value < 2.2e-16
## alternative hypothesis: true rho is not equal to 0
## sample estimates:
## rho
## 0.2362137cor.test(data$logfreq, data$numberWordSenses, method = c("pearson")) ##
## Pearson's product-moment correlation
##
## data: data$logfreq and data$numberWordSenses
## t = 17.324, df = 5398, p-value < 2.2e-16
## alternative hypothesis: true correlation is not equal to 0
## 95 percent confidence interval:
## 0.2040703 0.2546082
## sample estimates:
## cor
## 0.2294939cor.test(data$logfreq, data$numberWordSenses_fourBins, method = c("pearson")) ##
## Pearson's product-moment correlation
##
## data: data$logfreq and data$numberWordSenses_fourBins
## t = 17.776, df = 5398, p-value < 2.2e-16
## alternative hypothesis: true correlation is not equal to 0
## 95 percent confidence interval:
## 0.2098081 0.2602056
## sample estimates:
## cor
## 0.2351649# logfreq ~ numberWordSenses
# ---------------------------
# semantic field impact on log frequency
logfreq_lm_sf=lm(logfreq ~ semanticField , data=data)
summary(logfreq_lm_sf)##
## Call:
## lm(formula = logfreq ~ semanticField, data = data)
##
## Residuals:
## Min 1Q Median 3Q Max
## -8.2384 -0.6828 0.3191 1.5748 3.2699
##
## Coefficients:
## Estimate Std. Error t value Pr(>|t|)
## (Intercept) 12.57170 0.05710 220.189 < 2e-16 ***
## semanticFieldAttribut 0.31915 0.13457 2.372 0.0177 *
## semanticFieldBesitz -0.69502 0.14951 -4.649 3.42e-06 ***
## semanticFieldForm 1.37557 0.26474 5.196 2.11e-07 ***
## semanticFieldGefuehl -1.28888 0.21866 -5.895 3.99e-09 ***
## semanticFieldGeschehen -0.13807 0.09971 -1.385 0.1662
## semanticFieldGruppe -1.68216 0.13778 -12.209 < 2e-16 ***
## semanticFieldKoerper -1.61382 0.21866 -7.381 1.82e-13 ***
## semanticFieldKognition -0.98517 0.14130 -6.972 3.50e-12 ***
## semanticFieldKommunikation -0.83238 0.15980 -5.209 1.97e-07 ***
## semanticFieldMenge -2.84429 0.26474 -10.744 < 2e-16 ***
## semanticFieldMensch -1.19776 0.12414 -9.649 < 2e-16 ***
## semanticFieldNahrung -1.86272 0.19150 -9.727 < 2e-16 ***
## semanticFieldnatGegenstand -2.39137 0.37002 -6.463 1.12e-10 ***
## semanticFieldnatPhaenomen -5.05876 0.21866 -23.136 < 2e-16 ***
## semanticFieldOrt 0.66550 0.12199 5.455 5.11e-08 ***
## semanticFieldPflanze -2.21259 0.37002 -5.980 2.38e-09 ***
## semanticFieldRelation 0.15494 0.23816 0.651 0.5154
## semanticFieldSubstanz -5.39391 0.37002 -14.577 < 2e-16 ***
## semanticFieldTier -3.05300 0.18155 -16.816 < 2e-16 ***
## semanticFieldZeit 1.74418 0.37002 4.714 2.49e-06 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Residual standard error: 2.194 on 5379 degrees of freedom
## Multiple R-squared: 0.2274, Adjusted R-squared: 0.2245
## F-statistic: 79.17 on 20 and 5379 DF, p-value: < 2.2e-16report(logfreq_lm_sf)## We fitted a linear model (estimated using OLS) to predict logfreq with
## semanticField (formula: logfreq ~ semanticField). The model explains a
## statistically significant and moderate proportion of variance (R2 = 0.23, F(20,
## 5379) = 79.17, p < .001, adj. R2 = 0.22). The model's intercept, corresponding
## to semanticField = Artefakt, is at 12.57 (95% CI [12.46, 12.68], t(5379) =
## 220.19, p < .001). Within this model:
##
## - The effect of semanticField [Attribut] is statistically significant and
## positive (beta = 0.32, 95% CI [0.06, 0.58], t(5379) = 2.37, p = 0.018; Std.
## beta = 0.13, 95% CI [0.02, 0.23])
## - The effect of semanticField [Besitz] is statistically significant and
## negative (beta = -0.70, 95% CI [-0.99, -0.40], t(5379) = -4.65, p < .001; Std.
## beta = -0.28, 95% CI [-0.40, -0.16])
## - The effect of semanticField [Form] is statistically significant and positive
## (beta = 1.38, 95% CI [0.86, 1.89], t(5379) = 5.20, p < .001; Std. beta = 0.55,
## 95% CI [0.34, 0.76])
## - The effect of semanticField [Gefuehl] is statistically significant and
## negative (beta = -1.29, 95% CI [-1.72, -0.86], t(5379) = -5.89, p < .001; Std.
## beta = -0.52, 95% CI [-0.69, -0.35])
## - The effect of semanticField [Geschehen] is statistically non-significant and
## negative (beta = -0.14, 95% CI [-0.33, 0.06], t(5379) = -1.38, p = 0.166; Std.
## beta = -0.06, 95% CI [-0.13, 0.02])
## - The effect of semanticField [Gruppe] is statistically significant and
## negative (beta = -1.68, 95% CI [-1.95, -1.41], t(5379) = -12.21, p < .001; Std.
## beta = -0.68, 95% CI [-0.78, -0.57])
## - The effect of semanticField [Koerper] is statistically significant and
## negative (beta = -1.61, 95% CI [-2.04, -1.19], t(5379) = -7.38, p < .001; Std.
## beta = -0.65, 95% CI [-0.82, -0.48])
## - The effect of semanticField [Kognition] is statistically significant and
## negative (beta = -0.99, 95% CI [-1.26, -0.71], t(5379) = -6.97, p < .001; Std.
## beta = -0.40, 95% CI [-0.51, -0.28])
## - The effect of semanticField [Kommunikation] is statistically significant and
## negative (beta = -0.83, 95% CI [-1.15, -0.52], t(5379) = -5.21, p < .001; Std.
## beta = -0.33, 95% CI [-0.46, -0.21])
## - The effect of semanticField [Menge] is statistically significant and negative
## (beta = -2.84, 95% CI [-3.36, -2.33], t(5379) = -10.74, p < .001; Std. beta =
## -1.14, 95% CI [-1.35, -0.93])
## - The effect of semanticField [Mensch] is statistically significant and
## negative (beta = -1.20, 95% CI [-1.44, -0.95], t(5379) = -9.65, p < .001; Std.
## beta = -0.48, 95% CI [-0.58, -0.38])
## - The effect of semanticField [Nahrung] is statistically significant and
## negative (beta = -1.86, 95% CI [-2.24, -1.49], t(5379) = -9.73, p < .001; Std.
## beta = -0.75, 95% CI [-0.90, -0.60])
## - The effect of semanticField [natGegenstand] is statistically significant and
## negative (beta = -2.39, 95% CI [-3.12, -1.67], t(5379) = -6.46, p < .001; Std.
## beta = -0.96, 95% CI [-1.25, -0.67])
## - The effect of semanticField [natPhaenomen] is statistically significant and
## negative (beta = -5.06, 95% CI [-5.49, -4.63], t(5379) = -23.14, p < .001; Std.
## beta = -2.03, 95% CI [-2.20, -1.86])
## - The effect of semanticField [Ort] is statistically significant and positive
## (beta = 0.67, 95% CI [0.43, 0.90], t(5379) = 5.46, p < .001; Std. beta = 0.27,
## 95% CI [0.17, 0.36])
## - The effect of semanticField [Pflanze] is statistically significant and
## negative (beta = -2.21, 95% CI [-2.94, -1.49], t(5379) = -5.98, p < .001; Std.
## beta = -0.89, 95% CI [-1.18, -0.60])
## - The effect of semanticField [Relation] is statistically non-significant and
## positive (beta = 0.15, 95% CI [-0.31, 0.62], t(5379) = 0.65, p = 0.515; Std.
## beta = 0.06, 95% CI [-0.13, 0.25])
## - The effect of semanticField [Substanz] is statistically significant and
## negative (beta = -5.39, 95% CI [-6.12, -4.67], t(5379) = -14.58, p < .001; Std.
## beta = -2.17, 95% CI [-2.46, -1.87])
## - The effect of semanticField [Tier] is statistically significant and negative
## (beta = -3.05, 95% CI [-3.41, -2.70], t(5379) = -16.82, p < .001; Std. beta =
## -1.23, 95% CI [-1.37, -1.08])
## - The effect of semanticField [Zeit] is statistically significant and positive
## (beta = 1.74, 95% CI [1.02, 2.47], t(5379) = 4.71, p < .001; Std. beta = 0.70,
## 95% CI [0.41, 0.99])
##
## Standardized parameters were obtained by fitting the model on a standardized
## version of the dataset. 95% Confidence Intervals (CIs) and p-values were
## computed using a Wald t-distribution approximation.emm_sf <- emmeans(logfreq_lm_sf, ~ semanticField)
pairs(emm_sf, adjust = "tukey")## contrast estimate SE df t.ratio p.value
## Artefakt - Attribut -0.3191 0.1350 5379 -2.372 0.7069
## Artefakt - Besitz 0.6950 0.1500 5379 4.649 0.0007
## Artefakt - Form -1.3756 0.2650 5379 -5.196 <0.0001
## Artefakt - Gefuehl 1.2889 0.2190 5379 5.895 <0.0001
## Artefakt - Geschehen 0.1381 0.0997 5379 1.385 0.9988
## Artefakt - Gruppe 1.6822 0.1380 5379 12.209 <0.0001
## Artefakt - Koerper 1.6138 0.2190 5379 7.381 <0.0001
## Artefakt - Kognition 0.9852 0.1410 5379 6.972 <0.0001
## Artefakt - Kommunikation 0.8324 0.1600 5379 5.209 <0.0001
## Artefakt - Menge 2.8443 0.2650 5379 10.744 <0.0001
## Artefakt - Mensch 1.1978 0.1240 5379 9.649 <0.0001
## Artefakt - Nahrung 1.8627 0.1920 5379 9.727 <0.0001
## Artefakt - natGegenstand 2.3914 0.3700 5379 6.463 <0.0001
## Artefakt - natPhaenomen 5.0588 0.2190 5379 23.136 <0.0001
## Artefakt - Ort -0.6655 0.1220 5379 -5.455 <0.0001
## Artefakt - Pflanze 2.2126 0.3700 5379 5.980 <0.0001
## Artefakt - Relation -0.1549 0.2380 5379 -0.651 1.0000
## Artefakt - Substanz 5.3939 0.3700 5379 14.577 <0.0001
## Artefakt - Tier 3.0530 0.1820 5379 16.816 <0.0001
## Artefakt - Zeit -1.7442 0.3700 5379 -4.714 0.0005
## Attribut - Besitz 1.0142 0.1840 5379 5.505 <0.0001
## Attribut - Form -1.0564 0.2860 5379 -3.696 0.0327
## Attribut - Gefuehl 1.6080 0.2440 5379 6.598 <0.0001
## Attribut - Geschehen 0.4572 0.1470 5379 3.116 0.1874
## Attribut - Gruppe 2.0013 0.1750 5379 11.446 <0.0001
## Attribut - Koerper 1.9330 0.2440 5379 7.931 <0.0001
## Attribut - Kognition 1.3043 0.1780 5379 7.342 <0.0001
## Attribut - Kommunikation 1.1515 0.1930 5379 5.976 <0.0001
## Attribut - Menge 3.1634 0.2860 5379 11.069 <0.0001
## Attribut - Mensch 1.5169 0.1640 5379 9.232 <0.0001
## Attribut - Nahrung 2.1819 0.2200 5379 9.932 <0.0001
## Attribut - natGegenstand 2.7105 0.3850 5379 7.034 <0.0001
## Attribut - natPhaenomen 5.3779 0.2440 5379 22.066 <0.0001
## Attribut - Ort -0.3464 0.1630 5379 -2.129 0.8600
## Attribut - Pflanze 2.5317 0.3850 5379 6.570 <0.0001
## Attribut - Relation 0.1642 0.2610 5379 0.628 1.0000
## Attribut - Substanz 5.7131 0.3850 5379 14.825 <0.0001
## Attribut - Tier 3.3721 0.2110 5379 15.976 <0.0001
## Attribut - Zeit -1.4250 0.3850 5379 -3.698 0.0326
## Besitz - Form -2.0706 0.2930 5379 -7.064 <0.0001
## Besitz - Gefuehl 0.5939 0.2520 5379 2.354 0.7197
## Besitz - Geschehen -0.5569 0.1610 5379 -3.469 0.0692
## Besitz - Gruppe 0.9871 0.1870 5379 5.290 <0.0001
## Besitz - Koerper 0.9188 0.2520 5379 3.642 0.0395
## Besitz - Kognition 0.2901 0.1890 5379 1.533 0.9953
## Besitz - Kommunikation 0.1374 0.2030 5379 0.675 1.0000
## Besitz - Menge 2.1493 0.2930 5379 7.332 <0.0001
## Besitz - Mensch 0.5027 0.1770 5379 2.844 0.3456
## Besitz - Nahrung 1.1677 0.2290 5379 5.096 <0.0001
## Besitz - natGegenstand 1.6964 0.3910 5379 4.340 0.0026
## Besitz - natPhaenomen 4.3637 0.2520 5379 17.297 <0.0001
## Besitz - Ort -1.3605 0.1750 5379 -7.763 <0.0001
## Besitz - Pflanze 1.5176 0.3910 5379 3.883 0.0167
## Besitz - Relation -0.8500 0.2690 5379 -3.155 0.1694
## Besitz - Substanz 4.6989 0.3910 5379 12.023 <0.0001
## Besitz - Tier 2.3580 0.2210 5379 10.675 <0.0001
## Besitz - Zeit -2.4392 0.3910 5379 -6.241 <0.0001
## Form - Gefuehl 2.6644 0.3340 5379 7.984 <0.0001
## Form - Geschehen 1.5136 0.2710 5379 5.583 <0.0001
## Form - Gruppe 3.0577 0.2870 5379 10.642 <0.0001
## Form - Koerper 2.9894 0.3340 5379 8.957 <0.0001
## Form - Kognition 2.3607 0.2890 5379 8.168 <0.0001
## Form - Kommunikation 2.2079 0.2980 5379 7.397 <0.0001
## Form - Menge 4.2199 0.3660 5379 11.543 <0.0001
## Form - Mensch 2.5733 0.2810 5379 9.157 <0.0001
## Form - Nahrung 3.2383 0.3170 5379 10.228 <0.0001
## Form - natGegenstand 3.7669 0.4480 5379 8.413 <0.0001
## Form - natPhaenomen 6.4343 0.3340 5379 19.280 <0.0001
## Form - Ort 0.7101 0.2800 5379 2.535 0.5810
## Form - Pflanze 3.5882 0.4480 5379 8.014 <0.0001
## Form - Relation 1.2206 0.3470 5379 3.519 0.0590
## Form - Substanz 6.7695 0.4480 5379 15.119 <0.0001
## Form - Tier 4.4286 0.3110 5379 14.254 <0.0001
## Form - Zeit -0.3686 0.4480 5379 -0.823 1.0000
## Gefuehl - Geschehen -1.1508 0.2260 5379 -5.084 <0.0001
## Gefuehl - Gruppe 0.3933 0.2460 5379 1.602 0.9919
## Gefuehl - Koerper 0.3249 0.2980 5379 1.089 1.0000
## Gefuehl - Kognition -0.3037 0.2480 5379 -1.227 0.9998
## Gefuehl - Kommunikation -0.4565 0.2590 5379 -1.766 0.9753
## Gefuehl - Menge 1.5554 0.3340 5379 4.661 0.0006
## Gefuehl - Mensch -0.0911 0.2380 5379 -0.383 1.0000
## Gefuehl - Nahrung 0.5738 0.2790 5379 2.055 0.8946
## Gefuehl - natGegenstand 1.1025 0.4220 5379 2.612 0.5203
## Gefuehl - natPhaenomen 3.7699 0.2980 5379 12.629 <0.0001
## Gefuehl - Ort -1.9544 0.2370 5379 -8.246 <0.0001
## Gefuehl - Pflanze 0.9237 0.4220 5379 2.188 0.8278
## Gefuehl - Relation -1.4438 0.3130 5379 -4.612 0.0008
## Gefuehl - Substanz 4.1050 0.4220 5379 9.724 <0.0001
## Gefuehl - Tier 1.7641 0.2720 5379 6.474 <0.0001
## Gefuehl - Zeit -3.0331 0.4220 5379 -7.185 <0.0001
## Geschehen - Gruppe 1.5441 0.1500 5379 10.315 <0.0001
## Geschehen - Koerper 1.4757 0.2260 5379 6.520 <0.0001
## Geschehen - Kognition 0.8471 0.1530 5379 5.539 <0.0001
## Geschehen - Kommunikation 0.6943 0.1700 5379 4.080 0.0078
## Geschehen - Menge 2.7062 0.2710 5379 9.981 <0.0001
## Geschehen - Mensch 1.0597 0.1370 5379 7.722 <0.0001
## Geschehen - Nahrung 1.7246 0.2000 5379 8.613 <0.0001
## Geschehen - natGegenstand 2.2533 0.3750 5379 6.015 <0.0001
## Geschehen - natPhaenomen 4.9207 0.2260 5379 21.739 <0.0001
## Geschehen - Ort -0.8036 0.1350 5379 -5.939 <0.0001
## Geschehen - Pflanze 2.0745 0.3750 5379 5.538 <0.0001
## Geschehen - Relation -0.2930 0.2450 5379 -1.195 0.9999
## Geschehen - Substanz 5.2558 0.3750 5379 14.030 <0.0001
## Geschehen - Tier 2.9149 0.1910 5379 15.282 <0.0001
## Geschehen - Zeit -1.8823 0.3750 5379 -5.025 0.0001
## Gruppe - Koerper -0.0683 0.2460 5379 -0.278 1.0000
## Gruppe - Kognition -0.6970 0.1800 5379 -3.870 0.0175
## Gruppe - Kommunikation -0.8498 0.1950 5379 -4.359 0.0024
## Gruppe - Menge 1.1621 0.2870 5379 4.045 0.0090
## Gruppe - Mensch -0.4844 0.1670 5379 -2.901 0.3075
## Gruppe - Nahrung 0.1806 0.2220 5379 0.815 1.0000
## Gruppe - natGegenstand 0.7092 0.3860 5379 1.835 0.9631
## Gruppe - natPhaenomen 3.3766 0.2460 5379 13.753 <0.0001
## Gruppe - Ort -2.3477 0.1650 5379 -14.197 <0.0001
## Gruppe - Pflanze 0.5304 0.3860 5379 1.372 0.9989
## Gruppe - Relation -1.8371 0.2630 5379 -6.984 <0.0001
## Gruppe - Substanz 3.7117 0.3860 5379 9.604 <0.0001
## Gruppe - Tier 1.3708 0.2130 5379 6.432 <0.0001
## Gruppe - Zeit -3.4263 0.3860 5379 -8.865 <0.0001
## Koerper - Kognition -0.6287 0.2480 5379 -2.540 0.5772
## Koerper - Kommunikation -0.7814 0.2590 5379 -3.023 0.2347
## Koerper - Menge 1.2305 0.3340 5379 3.687 0.0338
## Koerper - Mensch -0.4161 0.2380 5379 -1.747 0.9779
## Koerper - Nahrung 0.2489 0.2790 5379 0.891 1.0000
## Koerper - natGegenstand 0.7776 0.4220 5379 1.842 0.9616
## Koerper - natPhaenomen 3.4449 0.2980 5379 11.541 <0.0001
## Koerper - Ort -2.2793 0.2370 5379 -9.617 <0.0001
## Koerper - Pflanze 0.5988 0.4220 5379 1.418 0.9983
## Koerper - Relation -1.7688 0.3130 5379 -5.650 <0.0001
## Koerper - Substanz 3.7801 0.4220 5379 8.955 <0.0001
## Koerper - Tier 1.4392 0.2720 5379 5.282 <0.0001
## Koerper - Zeit -3.3580 0.4220 5379 -7.955 <0.0001
## Kognition - Kommunikation -0.1528 0.1970 5379 -0.774 1.0000
## Kognition - Menge 1.8591 0.2890 5379 6.432 <0.0001
## Kognition - Mensch 0.2126 0.1700 5379 1.252 0.9997
## Kognition - Nahrung 0.8776 0.2240 5379 3.920 0.0146
## Kognition - natGegenstand 1.4062 0.3880 5379 3.626 0.0416
## Kognition - natPhaenomen 4.0736 0.2480 5379 16.459 <0.0001
## Kognition - Ort -1.6507 0.1680 5379 -9.807 <0.0001
## Kognition - Pflanze 1.2274 0.3880 5379 3.165 0.1651
## Kognition - Relation -1.1401 0.2650 5379 -4.304 0.0031
## Kognition - Substanz 4.4087 0.3880 5379 11.370 <0.0001
## Kognition - Tier 2.0678 0.2150 5379 9.599 <0.0001
## Kognition - Zeit -2.7293 0.3880 5379 -7.039 <0.0001
## Kommunikation - Menge 2.0119 0.2980 5379 6.740 <0.0001
## Kommunikation - Mensch 0.3654 0.1860 5379 1.969 0.9273
## Kommunikation - Nahrung 1.0303 0.2360 5379 4.366 0.0024
## Kommunikation - natGegenstand 1.5590 0.3950 5379 3.948 0.0131
## Kommunikation - natPhaenomen 4.2264 0.2590 5379 16.349 <0.0001
## Kommunikation - Ort -1.4979 0.1840 5379 -8.136 <0.0001
## Kommunikation - Pflanze 1.3802 0.3950 5379 3.495 0.0638
## Kommunikation - Relation -0.9873 0.2750 5379 -3.588 0.0473
## Kommunikation - Substanz 4.5615 0.3950 5379 11.552 <0.0001
## Kommunikation - Tier 2.2206 0.2280 5379 9.740 <0.0001
## Kommunikation - Zeit -2.5766 0.3950 5379 -6.525 <0.0001
## Menge - Mensch -1.6465 0.2810 5379 -5.859 <0.0001
## Menge - Nahrung -0.9816 0.3170 5379 -3.100 0.1948
## Menge - natGegenstand -0.4529 0.4480 5379 -1.012 1.0000
## Menge - natPhaenomen 2.2145 0.3340 5379 6.635 <0.0001
## Menge - Ort -3.5098 0.2800 5379 -12.531 <0.0001
## Menge - Pflanze -0.6317 0.4480 5379 -1.411 0.9984
## Menge - Relation -2.9992 0.3470 5379 -8.648 <0.0001
## Menge - Substanz 2.5496 0.4480 5379 5.694 <0.0001
## Menge - Tier 0.2087 0.3110 5379 0.672 1.0000
## Menge - Zeit -4.5885 0.4480 5379 -10.248 <0.0001
## Mensch - Nahrung 0.6650 0.2130 5379 3.115 0.1877
## Mensch - natGegenstand 1.1936 0.3820 5379 3.126 0.1827
## Mensch - natPhaenomen 3.8610 0.2380 5379 16.214 <0.0001
## Mensch - Ort -1.8633 0.1540 5379 -12.085 <0.0001
## Mensch - Pflanze 1.0148 0.3820 5379 2.658 0.4840
## Mensch - Relation -1.3527 0.2560 5379 -5.281 <0.0001
## Mensch - Substanz 4.1961 0.3820 5379 10.989 <0.0001
## Mensch - Tier 1.8552 0.2050 5379 9.069 <0.0001
## Mensch - Zeit -2.9419 0.3820 5379 -7.705 <0.0001
## Nahrung - natGegenstand 0.5287 0.4090 5379 1.293 0.9995
## Nahrung - natPhaenomen 3.1960 0.2790 5379 11.446 <0.0001
## Nahrung - Ort -2.5282 0.2120 5379 -11.913 <0.0001
## Nahrung - Pflanze 0.3499 0.4090 5379 0.856 1.0000
## Nahrung - Relation -2.0177 0.2950 5379 -6.845 <0.0001
## Nahrung - Substanz 3.5312 0.4090 5379 8.639 <0.0001
## Nahrung - Tier 1.1903 0.2510 5379 4.738 0.0004
## Nahrung - Zeit -3.6069 0.4090 5379 -8.824 <0.0001
## natGegenstand - natPhaenomen 2.6674 0.4220 5379 6.319 <0.0001
## natGegenstand - Ort -3.0569 0.3810 5379 -8.020 <0.0001
## natGegenstand - Pflanze -0.1788 0.5170 5379 -0.346 1.0000
## natGegenstand - Relation -2.5463 0.4330 5379 -5.887 <0.0001
## natGegenstand - Substanz 3.0025 0.5170 5379 5.807 <0.0001
## natGegenstand - Tier 0.6616 0.4040 5379 1.637 0.9895
## natGegenstand - Zeit -4.1356 0.5170 5379 -7.999 <0.0001
## natPhaenomen - Ort -5.7243 0.2370 5379 -24.152 <0.0001
## natPhaenomen - Pflanze -2.8462 0.4220 5379 -6.742 <0.0001
## natPhaenomen - Relation -5.2137 0.3130 5379 -16.653 <0.0001
## natPhaenomen - Substanz 0.3352 0.4220 5379 0.794 1.0000
## natPhaenomen - Tier -2.0058 0.2720 5379 -7.361 <0.0001
## natPhaenomen - Zeit -6.8029 0.4220 5379 -16.115 <0.0001
## Ort - Pflanze 2.8781 0.3810 5379 7.551 <0.0001
## Ort - Relation 0.5106 0.2550 5379 2.001 0.9160
## Ort - Substanz 6.0594 0.3810 5379 15.898 <0.0001
## Ort - Tier 3.7185 0.2030 5379 18.292 <0.0001
## Ort - Zeit -1.0787 0.3810 5379 -2.830 0.3555
## Pflanze - Relation -2.3675 0.4330 5379 -5.473 <0.0001
## Pflanze - Substanz 3.1813 0.5170 5379 6.153 <0.0001
## Pflanze - Tier 0.8404 0.4040 5379 2.079 0.8839
## Pflanze - Zeit -3.9568 0.5170 5379 -7.653 <0.0001
## Relation - Substanz 5.5489 0.4330 5379 12.828 <0.0001
## Relation - Tier 3.2079 0.2880 5379 11.124 <0.0001
## Relation - Zeit -1.5892 0.4330 5379 -3.674 0.0354
## Substanz - Tier -2.3409 0.4040 5379 -5.792 <0.0001
## Substanz - Zeit -7.1381 0.5170 5379 -13.806 <0.0001
## Tier - Zeit -4.7972 0.4040 5379 -11.869 <0.0001
##
## P value adjustment: tukey method for comparing a family of 21 estimatesttt <- as.data.frame(multcomp::cld(emm_sf, adjust = "tukey"))## Note: adjust = "tukey" was changed to "sidak"
## because "tukey" is only appropriate for one set of pairwise comparisonsView(ttt)
contrast_sf <- pairs(emm_sf, adjust = "tukey")
contrast_table <- summary(contrast_sf)
contrast_table## contrast estimate SE df t.ratio p.value
## Artefakt - Attribut -0.3191 0.1350 5379 -2.372 0.7069
## Artefakt - Besitz 0.6950 0.1500 5379 4.649 0.0007
## Artefakt - Form -1.3756 0.2650 5379 -5.196 <0.0001
## Artefakt - Gefuehl 1.2889 0.2190 5379 5.895 <0.0001
## Artefakt - Geschehen 0.1381 0.0997 5379 1.385 0.9988
## Artefakt - Gruppe 1.6822 0.1380 5379 12.209 <0.0001
## Artefakt - Koerper 1.6138 0.2190 5379 7.381 <0.0001
## Artefakt - Kognition 0.9852 0.1410 5379 6.972 <0.0001
## Artefakt - Kommunikation 0.8324 0.1600 5379 5.209 <0.0001
## Artefakt - Menge 2.8443 0.2650 5379 10.744 <0.0001
## Artefakt - Mensch 1.1978 0.1240 5379 9.649 <0.0001
## Artefakt - Nahrung 1.8627 0.1920 5379 9.727 <0.0001
## Artefakt - natGegenstand 2.3914 0.3700 5379 6.463 <0.0001
## Artefakt - natPhaenomen 5.0588 0.2190 5379 23.136 <0.0001
## Artefakt - Ort -0.6655 0.1220 5379 -5.455 <0.0001
## Artefakt - Pflanze 2.2126 0.3700 5379 5.980 <0.0001
## Artefakt - Relation -0.1549 0.2380 5379 -0.651 1.0000
## Artefakt - Substanz 5.3939 0.3700 5379 14.577 <0.0001
## Artefakt - Tier 3.0530 0.1820 5379 16.816 <0.0001
## Artefakt - Zeit -1.7442 0.3700 5379 -4.714 0.0005
## Attribut - Besitz 1.0142 0.1840 5379 5.505 <0.0001
## Attribut - Form -1.0564 0.2860 5379 -3.696 0.0327
## Attribut - Gefuehl 1.6080 0.2440 5379 6.598 <0.0001
## Attribut - Geschehen 0.4572 0.1470 5379 3.116 0.1874
## Attribut - Gruppe 2.0013 0.1750 5379 11.446 <0.0001
## Attribut - Koerper 1.9330 0.2440 5379 7.931 <0.0001
## Attribut - Kognition 1.3043 0.1780 5379 7.342 <0.0001
## Attribut - Kommunikation 1.1515 0.1930 5379 5.976 <0.0001
## Attribut - Menge 3.1634 0.2860 5379 11.069 <0.0001
## Attribut - Mensch 1.5169 0.1640 5379 9.232 <0.0001
## Attribut - Nahrung 2.1819 0.2200 5379 9.932 <0.0001
## Attribut - natGegenstand 2.7105 0.3850 5379 7.034 <0.0001
## Attribut - natPhaenomen 5.3779 0.2440 5379 22.066 <0.0001
## Attribut - Ort -0.3464 0.1630 5379 -2.129 0.8600
## Attribut - Pflanze 2.5317 0.3850 5379 6.570 <0.0001
## Attribut - Relation 0.1642 0.2610 5379 0.628 1.0000
## Attribut - Substanz 5.7131 0.3850 5379 14.825 <0.0001
## Attribut - Tier 3.3721 0.2110 5379 15.976 <0.0001
## Attribut - Zeit -1.4250 0.3850 5379 -3.698 0.0326
## Besitz - Form -2.0706 0.2930 5379 -7.064 <0.0001
## Besitz - Gefuehl 0.5939 0.2520 5379 2.354 0.7197
## Besitz - Geschehen -0.5569 0.1610 5379 -3.469 0.0692
## Besitz - Gruppe 0.9871 0.1870 5379 5.290 <0.0001
## Besitz - Koerper 0.9188 0.2520 5379 3.642 0.0395
## Besitz - Kognition 0.2901 0.1890 5379 1.533 0.9953
## Besitz - Kommunikation 0.1374 0.2030 5379 0.675 1.0000
## Besitz - Menge 2.1493 0.2930 5379 7.332 <0.0001
## Besitz - Mensch 0.5027 0.1770 5379 2.844 0.3456
## Besitz - Nahrung 1.1677 0.2290 5379 5.096 <0.0001
## Besitz - natGegenstand 1.6964 0.3910 5379 4.340 0.0026
## Besitz - natPhaenomen 4.3637 0.2520 5379 17.297 <0.0001
## Besitz - Ort -1.3605 0.1750 5379 -7.763 <0.0001
## Besitz - Pflanze 1.5176 0.3910 5379 3.883 0.0167
## Besitz - Relation -0.8500 0.2690 5379 -3.155 0.1694
## Besitz - Substanz 4.6989 0.3910 5379 12.023 <0.0001
## Besitz - Tier 2.3580 0.2210 5379 10.675 <0.0001
## Besitz - Zeit -2.4392 0.3910 5379 -6.241 <0.0001
## Form - Gefuehl 2.6644 0.3340 5379 7.984 <0.0001
## Form - Geschehen 1.5136 0.2710 5379 5.583 <0.0001
## Form - Gruppe 3.0577 0.2870 5379 10.642 <0.0001
## Form - Koerper 2.9894 0.3340 5379 8.957 <0.0001
## Form - Kognition 2.3607 0.2890 5379 8.168 <0.0001
## Form - Kommunikation 2.2079 0.2980 5379 7.397 <0.0001
## Form - Menge 4.2199 0.3660 5379 11.543 <0.0001
## Form - Mensch 2.5733 0.2810 5379 9.157 <0.0001
## Form - Nahrung 3.2383 0.3170 5379 10.228 <0.0001
## Form - natGegenstand 3.7669 0.4480 5379 8.413 <0.0001
## Form - natPhaenomen 6.4343 0.3340 5379 19.280 <0.0001
## Form - Ort 0.7101 0.2800 5379 2.535 0.5810
## Form - Pflanze 3.5882 0.4480 5379 8.014 <0.0001
## Form - Relation 1.2206 0.3470 5379 3.519 0.0590
## Form - Substanz 6.7695 0.4480 5379 15.119 <0.0001
## Form - Tier 4.4286 0.3110 5379 14.254 <0.0001
## Form - Zeit -0.3686 0.4480 5379 -0.823 1.0000
## Gefuehl - Geschehen -1.1508 0.2260 5379 -5.084 <0.0001
## Gefuehl - Gruppe 0.3933 0.2460 5379 1.602 0.9919
## Gefuehl - Koerper 0.3249 0.2980 5379 1.089 1.0000
## Gefuehl - Kognition -0.3037 0.2480 5379 -1.227 0.9998
## Gefuehl - Kommunikation -0.4565 0.2590 5379 -1.766 0.9753
## Gefuehl - Menge 1.5554 0.3340 5379 4.661 0.0006
## Gefuehl - Mensch -0.0911 0.2380 5379 -0.383 1.0000
## Gefuehl - Nahrung 0.5738 0.2790 5379 2.055 0.8946
## Gefuehl - natGegenstand 1.1025 0.4220 5379 2.612 0.5203
## Gefuehl - natPhaenomen 3.7699 0.2980 5379 12.629 <0.0001
## Gefuehl - Ort -1.9544 0.2370 5379 -8.246 <0.0001
## Gefuehl - Pflanze 0.9237 0.4220 5379 2.188 0.8278
## Gefuehl - Relation -1.4438 0.3130 5379 -4.612 0.0008
## Gefuehl - Substanz 4.1050 0.4220 5379 9.724 <0.0001
## Gefuehl - Tier 1.7641 0.2720 5379 6.474 <0.0001
## Gefuehl - Zeit -3.0331 0.4220 5379 -7.185 <0.0001
## Geschehen - Gruppe 1.5441 0.1500 5379 10.315 <0.0001
## Geschehen - Koerper 1.4757 0.2260 5379 6.520 <0.0001
## Geschehen - Kognition 0.8471 0.1530 5379 5.539 <0.0001
## Geschehen - Kommunikation 0.6943 0.1700 5379 4.080 0.0078
## Geschehen - Menge 2.7062 0.2710 5379 9.981 <0.0001
## Geschehen - Mensch 1.0597 0.1370 5379 7.722 <0.0001
## Geschehen - Nahrung 1.7246 0.2000 5379 8.613 <0.0001
## Geschehen - natGegenstand 2.2533 0.3750 5379 6.015 <0.0001
## Geschehen - natPhaenomen 4.9207 0.2260 5379 21.739 <0.0001
## Geschehen - Ort -0.8036 0.1350 5379 -5.939 <0.0001
## Geschehen - Pflanze 2.0745 0.3750 5379 5.538 <0.0001
## Geschehen - Relation -0.2930 0.2450 5379 -1.195 0.9999
## Geschehen - Substanz 5.2558 0.3750 5379 14.030 <0.0001
## Geschehen - Tier 2.9149 0.1910 5379 15.282 <0.0001
## Geschehen - Zeit -1.8823 0.3750 5379 -5.025 0.0001
## Gruppe - Koerper -0.0683 0.2460 5379 -0.278 1.0000
## Gruppe - Kognition -0.6970 0.1800 5379 -3.870 0.0175
## Gruppe - Kommunikation -0.8498 0.1950 5379 -4.359 0.0024
## Gruppe - Menge 1.1621 0.2870 5379 4.045 0.0090
## Gruppe - Mensch -0.4844 0.1670 5379 -2.901 0.3075
## Gruppe - Nahrung 0.1806 0.2220 5379 0.815 1.0000
## Gruppe - natGegenstand 0.7092 0.3860 5379 1.835 0.9631
## Gruppe - natPhaenomen 3.3766 0.2460 5379 13.753 <0.0001
## Gruppe - Ort -2.3477 0.1650 5379 -14.197 <0.0001
## Gruppe - Pflanze 0.5304 0.3860 5379 1.372 0.9989
## Gruppe - Relation -1.8371 0.2630 5379 -6.984 <0.0001
## Gruppe - Substanz 3.7117 0.3860 5379 9.604 <0.0001
## Gruppe - Tier 1.3708 0.2130 5379 6.432 <0.0001
## Gruppe - Zeit -3.4263 0.3860 5379 -8.865 <0.0001
## Koerper - Kognition -0.6287 0.2480 5379 -2.540 0.5772
## Koerper - Kommunikation -0.7814 0.2590 5379 -3.023 0.2347
## Koerper - Menge 1.2305 0.3340 5379 3.687 0.0338
## Koerper - Mensch -0.4161 0.2380 5379 -1.747 0.9779
## Koerper - Nahrung 0.2489 0.2790 5379 0.891 1.0000
## Koerper - natGegenstand 0.7776 0.4220 5379 1.842 0.9616
## Koerper - natPhaenomen 3.4449 0.2980 5379 11.541 <0.0001
## Koerper - Ort -2.2793 0.2370 5379 -9.617 <0.0001
## Koerper - Pflanze 0.5988 0.4220 5379 1.418 0.9983
## Koerper - Relation -1.7688 0.3130 5379 -5.650 <0.0001
## Koerper - Substanz 3.7801 0.4220 5379 8.955 <0.0001
## Koerper - Tier 1.4392 0.2720 5379 5.282 <0.0001
## Koerper - Zeit -3.3580 0.4220 5379 -7.955 <0.0001
## Kognition - Kommunikation -0.1528 0.1970 5379 -0.774 1.0000
## Kognition - Menge 1.8591 0.2890 5379 6.432 <0.0001
## Kognition - Mensch 0.2126 0.1700 5379 1.252 0.9997
## Kognition - Nahrung 0.8776 0.2240 5379 3.920 0.0146
## Kognition - natGegenstand 1.4062 0.3880 5379 3.626 0.0416
## Kognition - natPhaenomen 4.0736 0.2480 5379 16.459 <0.0001
## Kognition - Ort -1.6507 0.1680 5379 -9.807 <0.0001
## Kognition - Pflanze 1.2274 0.3880 5379 3.165 0.1651
## Kognition - Relation -1.1401 0.2650 5379 -4.304 0.0031
## Kognition - Substanz 4.4087 0.3880 5379 11.370 <0.0001
## Kognition - Tier 2.0678 0.2150 5379 9.599 <0.0001
## Kognition - Zeit -2.7293 0.3880 5379 -7.039 <0.0001
## Kommunikation - Menge 2.0119 0.2980 5379 6.740 <0.0001
## Kommunikation - Mensch 0.3654 0.1860 5379 1.969 0.9273
## Kommunikation - Nahrung 1.0303 0.2360 5379 4.366 0.0024
## Kommunikation - natGegenstand 1.5590 0.3950 5379 3.948 0.0131
## Kommunikation - natPhaenomen 4.2264 0.2590 5379 16.349 <0.0001
## Kommunikation - Ort -1.4979 0.1840 5379 -8.136 <0.0001
## Kommunikation - Pflanze 1.3802 0.3950 5379 3.495 0.0638
## Kommunikation - Relation -0.9873 0.2750 5379 -3.588 0.0473
## Kommunikation - Substanz 4.5615 0.3950 5379 11.552 <0.0001
## Kommunikation - Tier 2.2206 0.2280 5379 9.740 <0.0001
## Kommunikation - Zeit -2.5766 0.3950 5379 -6.525 <0.0001
## Menge - Mensch -1.6465 0.2810 5379 -5.859 <0.0001
## Menge - Nahrung -0.9816 0.3170 5379 -3.100 0.1948
## Menge - natGegenstand -0.4529 0.4480 5379 -1.012 1.0000
## Menge - natPhaenomen 2.2145 0.3340 5379 6.635 <0.0001
## Menge - Ort -3.5098 0.2800 5379 -12.531 <0.0001
## Menge - Pflanze -0.6317 0.4480 5379 -1.411 0.9984
## Menge - Relation -2.9992 0.3470 5379 -8.648 <0.0001
## Menge - Substanz 2.5496 0.4480 5379 5.694 <0.0001
## Menge - Tier 0.2087 0.3110 5379 0.672 1.0000
## Menge - Zeit -4.5885 0.4480 5379 -10.248 <0.0001
## Mensch - Nahrung 0.6650 0.2130 5379 3.115 0.1877
## Mensch - natGegenstand 1.1936 0.3820 5379 3.126 0.1827
## Mensch - natPhaenomen 3.8610 0.2380 5379 16.214 <0.0001
## Mensch - Ort -1.8633 0.1540 5379 -12.085 <0.0001
## Mensch - Pflanze 1.0148 0.3820 5379 2.658 0.4840
## Mensch - Relation -1.3527 0.2560 5379 -5.281 <0.0001
## Mensch - Substanz 4.1961 0.3820 5379 10.989 <0.0001
## Mensch - Tier 1.8552 0.2050 5379 9.069 <0.0001
## Mensch - Zeit -2.9419 0.3820 5379 -7.705 <0.0001
## Nahrung - natGegenstand 0.5287 0.4090 5379 1.293 0.9995
## Nahrung - natPhaenomen 3.1960 0.2790 5379 11.446 <0.0001
## Nahrung - Ort -2.5282 0.2120 5379 -11.913 <0.0001
## Nahrung - Pflanze 0.3499 0.4090 5379 0.856 1.0000
## Nahrung - Relation -2.0177 0.2950 5379 -6.845 <0.0001
## Nahrung - Substanz 3.5312 0.4090 5379 8.639 <0.0001
## Nahrung - Tier 1.1903 0.2510 5379 4.738 0.0004
## Nahrung - Zeit -3.6069 0.4090 5379 -8.824 <0.0001
## natGegenstand - natPhaenomen 2.6674 0.4220 5379 6.319 <0.0001
## natGegenstand - Ort -3.0569 0.3810 5379 -8.020 <0.0001
## natGegenstand - Pflanze -0.1788 0.5170 5379 -0.346 1.0000
## natGegenstand - Relation -2.5463 0.4330 5379 -5.887 <0.0001
## natGegenstand - Substanz 3.0025 0.5170 5379 5.807 <0.0001
## natGegenstand - Tier 0.6616 0.4040 5379 1.637 0.9895
## natGegenstand - Zeit -4.1356 0.5170 5379 -7.999 <0.0001
## natPhaenomen - Ort -5.7243 0.2370 5379 -24.152 <0.0001
## natPhaenomen - Pflanze -2.8462 0.4220 5379 -6.742 <0.0001
## natPhaenomen - Relation -5.2137 0.3130 5379 -16.653 <0.0001
## natPhaenomen - Substanz 0.3352 0.4220 5379 0.794 1.0000
## natPhaenomen - Tier -2.0058 0.2720 5379 -7.361 <0.0001
## natPhaenomen - Zeit -6.8029 0.4220 5379 -16.115 <0.0001
## Ort - Pflanze 2.8781 0.3810 5379 7.551 <0.0001
## Ort - Relation 0.5106 0.2550 5379 2.001 0.9160
## Ort - Substanz 6.0594 0.3810 5379 15.898 <0.0001
## Ort - Tier 3.7185 0.2030 5379 18.292 <0.0001
## Ort - Zeit -1.0787 0.3810 5379 -2.830 0.3555
## Pflanze - Relation -2.3675 0.4330 5379 -5.473 <0.0001
## Pflanze - Substanz 3.1813 0.5170 5379 6.153 <0.0001
## Pflanze - Tier 0.8404 0.4040 5379 2.079 0.8839
## Pflanze - Zeit -3.9568 0.5170 5379 -7.653 <0.0001
## Relation - Substanz 5.5489 0.4330 5379 12.828 <0.0001
## Relation - Tier 3.2079 0.2880 5379 11.124 <0.0001
## Relation - Zeit -1.5892 0.4330 5379 -3.674 0.0354
## Substanz - Tier -2.3409 0.4040 5379 -5.792 <0.0001
## Substanz - Zeit -7.1381 0.5170 5379 -13.806 <0.0001
## Tier - Zeit -4.7972 0.4040 5379 -11.869 <0.0001
##
## P value adjustment: tukey method for comparing a family of 21 estimatescld(emm_sf, adjust = "tukey")## Note: adjust = "tukey" was changed to "sidak"
## because "tukey" is only appropriate for one set of pairwise comparisons## semanticField emmean SE df lower.CL upper.CL .group
## Substanz 7.18 0.3660 5379 6.07 8.29 1
## natPhaenomen 7.51 0.2110 5379 6.87 8.15 1
## Tier 9.52 0.1720 5379 9.00 10.04 2
## Menge 9.73 0.2590 5379 8.94 10.51 23
## natGegenstand 10.18 0.3660 5379 9.07 11.29 234
## Pflanze 10.36 0.3660 5379 9.25 11.47 2345
## Nahrung 10.71 0.1830 5379 10.15 11.26 34
## Gruppe 10.89 0.1250 5379 10.51 11.27 4
## Koerper 10.96 0.2110 5379 10.32 11.60 45
## Gefuehl 11.28 0.2110 5379 10.64 11.92 456
## Mensch 11.37 0.1100 5379 11.04 11.71 456
## Kognition 11.59 0.1290 5379 11.19 11.98 56
## Kommunikation 11.74 0.1490 5379 11.29 12.19 56
## Besitz 11.88 0.1380 5379 11.46 12.30 67
## Geschehen 12.43 0.0817 5379 12.19 12.68 78
## Artefakt 12.57 0.0571 5379 12.40 12.74 8
## Relation 12.73 0.2310 5379 12.03 13.43 7890
## Attribut 12.89 0.1220 5379 12.52 13.26 89
## Ort 13.24 0.1080 5379 12.91 13.56 90A
## Form 13.95 0.2590 5379 13.16 14.73 0A
## Zeit 14.32 0.3660 5379 13.21 15.42 A
##
## Confidence level used: 0.95
## Conf-level adjustment: sidak method for 21 estimates
## P value adjustment: tukey method for comparing a family of 21 estimates
## significance level used: alpha = 0.05
## NOTE: If two or more means share the same grouping symbol,
## then we cannot show them to be different.
## But we also did not show them to be the same.logfreq_lm=lm(logfreq ~ numberWordSenses, data=data)
logfreq_lm4=lm(logfreq ~ numberWordSenses_fourBins, data=data)
AIC(logfreq_lm, logfreq_lm4)## df AIC
## logfreq_lm 3 24894.12
## logfreq_lm4 3 24879.08# => In a simple LM, numberWordSenses_fourBins fits the data substantially better
summary(logfreq_lm4)##
## Call:
## lm(formula = logfreq ~ numberWordSenses_fourBins, data = data)
##
## Residuals:
## Min 1Q Median 3Q Max
## -8.4103 -1.6330 0.7481 1.8870 4.0222
##
## Coefficients:
## Estimate Std. Error t value Pr(>|t|)
## (Intercept) 11.54575 0.04002 288.50 <2e-16 ***
## numberWordSenses_fourBins 0.54563 0.03069 17.78 <2e-16 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Residual standard error: 2.421 on 5398 degrees of freedom
## Multiple R-squared: 0.0553, Adjusted R-squared: 0.05513
## F-statistic: 316 on 1 and 5398 DF, p-value: < 2.2e-16report(logfreq_lm4)## We fitted a linear model (estimated using OLS) to predict logfreq with
## numberWordSenses_fourBins (formula: logfreq ~ numberWordSenses_fourBins). The
## model explains a statistically significant and weak proportion of variance (R2
## = 0.06, F(1, 5398) = 316.00, p < .001, adj. R2 = 0.06). The model's intercept,
## corresponding to numberWordSenses_fourBins = 0, is at 11.55 (95% CI [11.47,
## 11.62], t(5398) = 288.50, p < .001). Within this model:
##
## - The effect of numberWordSenses fourBins is statistically significant and
## positive (beta = 0.55, 95% CI [0.49, 0.61], t(5398) = 17.78, p < .001; Std.
## beta = 0.24, 95% CI [0.21, 0.26])
##
## Standardized parameters were obtained by fitting the model on a standardized
## version of the dataset. 95% Confidence Intervals (CIs) and p-values were
## computed using a Wald t-distribution approximation.plot_model(logfreq_lm4, type="pred", terms=c("numberWordSenses_fourBins"))
ggsave(paste0(path, "./results/polysemes_logfreq_numberWordSenses.png"), width= 7, height=7)
# now with semantic fields as random effect
logfreq_lmer_fourBins_sf=lmer(logfreq ~ numberWordSenses_fourBins +(1|semanticField), data=data)
Anova(logfreq_lmer_fourBins_sf, type="III")## Analysis of Deviance Table (Type III Wald chisquare tests)
##
## Response: logfreq
## Chisq Df Pr(>Chisq)
## (Intercept) 718.55 1 < 2.2e-16 ***
## numberWordSenses_fourBins 436.69 1 < 2.2e-16 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1summary(logfreq_lmer_fourBins_sf)## Linear mixed model fit by REML. t-tests use Satterthwaite's method [
## lmerModLmerTest]
## Formula: logfreq ~ numberWordSenses_fourBins + (1 | semanticField)
## Data: data
##
## REML criterion at convergence: 23490.1
##
## Scaled residuals:
## Min 1Q Median 3Q Max
## -3.7939 -0.5239 0.1976 0.6633 1.6811
##
## Random effects:
## Groups Name Variance Std.Dev.
## semanticField (Intercept) 3.427 1.851
## Residual 4.450 2.110
## Number of obs: 5400, groups: semanticField, 21
##
## Fixed effects:
## Estimate Std. Error df t value Pr(>|t|)
## (Intercept) 1.091e+01 4.070e-01 1.979e+01 26.81 <2e-16 ***
## numberWordSenses_fourBins 6.434e-01 3.079e-02 5.394e+03 20.90 <2e-16 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Correlation of Fixed Effects:
## (Intr)
## nmbrWrdSn_B -0.045# goodSentence ~ semanticField
# -------------------------------
# sparse data in some semantic fields
table(data$semanticField)##
## Artefakt Attribut Besitz Form Gefuehl
## 1476 324 252 72 108
## Geschehen Gruppe Koerper Kognition Kommunikation
## 720 306 108 288 216
## Menge Mensch Nahrung natGegenstand natPhaenomen
## 72 396 144 36 108
## Ort Pflanze Relation Substanz Tier
## 414 36 90 36 162
## Zeit
## 36goodness.glmer_sq_sf=glmer(as.factor(goodSentence) ~ semanticField +(1|sentenceLength), data=data,family="binomial",control=glmerControl(optimizer="bobyqa"))
Anova(goodness.glmer_sq_sf, type="III")## Analysis of Deviance Table (Type III Wald chisquare tests)
##
## Response: as.factor(goodSentence)
## Chisq Df Pr(>Chisq)
## (Intercept) 47.018 1 7.034e-12 ***
## semanticField 161.881 20 < 2.2e-16 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1summary(goodness.glmer_sq_sf, type="III")## Warning in summary.merMod(goodness.glmer_sq_sf, type = "III"): additional
## arguments ignored## Generalized linear mixed model fit by maximum likelihood (Laplace
## Approximation) [glmerMod]
## Family: binomial ( logit )
## Formula: as.factor(goodSentence) ~ semanticField + (1 | sentenceLength)
## Data: data
## Control: glmerControl(optimizer = "bobyqa")
##
## AIC BIC logLik -2*log(L) df.resid
## 6827.2 6972.3 -3391.6 6783.2 5378
##
## Scaled residuals:
## Min 1Q Median 3Q Max
## -2.3747 -1.1525 0.5811 0.7359 1.5655
##
## Random effects:
## Groups Name Variance Std.Dev.
## sentenceLength (Intercept) 0.04323 0.2079
## Number of obs: 5400, groups: sentenceLength, 19
##
## Fixed effects:
## Estimate Std. Error z value Pr(>|z|)
## (Intercept) 0.55802 0.08138 6.857 7.03e-12 ***
## semanticFieldAttribut 0.20095 0.13124 1.531 0.125727
## semanticFieldBesitz 0.43736 0.15169 2.883 0.003937 **
## semanticFieldForm -0.48227 0.24323 -1.983 0.047396 *
## semanticFieldGefuehl -0.23532 0.20360 -1.156 0.247762
## semanticFieldGeschehen 0.52714 0.10225 5.156 2.53e-07 ***
## semanticFieldGruppe -0.24434 0.12846 -1.902 0.057166 .
## semanticFieldKoerper 0.81703 0.25081 3.258 0.001124 **
## semanticFieldKognition 0.47235 0.14540 3.249 0.001160 **
## semanticFieldKommunikation -0.08733 0.15072 -0.579 0.562306
## semanticFieldMenge 0.52038 0.27505 1.892 0.058502 .
## semanticFieldMensch -0.26861 0.11631 -2.309 0.020920 *
## semanticFieldNahrung 0.58172 0.20152 2.887 0.003893 **
## semanticFieldnatGegenstand 0.56031 0.39012 1.436 0.150932
## semanticFieldnatPhaenomen -0.74663 0.20193 -3.697 0.000218 ***
## semanticFieldOrt -0.40321 0.11386 -3.541 0.000398 ***
## semanticFieldPflanze -1.15468 0.35293 -3.272 0.001069 **
## semanticFieldRelation 0.77278 0.26470 2.919 0.003506 **
## semanticFieldSubstanz -0.29605 0.34382 -0.861 0.389204
## semanticFieldTier 0.26595 0.18124 1.467 0.142273
## semanticFieldZeit 1.05509 0.45065 2.341 0.019220 *
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1##
## Correlation matrix not shown by default, as p = 21 > 12.
## Use print(x, correlation=TRUE) or
## vcov(x) if you need itemm_sf <- emmeans(goodness.glmer_sq_sf, ~ semanticField, type = "response")
sf_contrasts <- contrast(
emm_sf,
method = "pairwise",
adjust = "tukey"
)
# summary(sf_contrasts)
pairs_sf <- pairs(emm_sf, adjust = "tukey")
#summary(pairs_sf)
emm_sf_prob <- emmeans(
goodness.glmer_sq_sf,
~ semanticField,
type = "response"
)
# goodSentence ~ numberWordSenses
# -------------------------------
goodness.glmer_nws=glmer(as.factor(goodSentence) ~ numberWordSenses +(1|sentenceLength), data=data,family="binomial",control=glmerControl(optimizer="bobyqa"))
Anova(goodness.glmer_nws, type="III")## Analysis of Deviance Table (Type III Wald chisquare tests)
##
## Response: as.factor(goodSentence)
## Chisq Df Pr(>Chisq)
## (Intercept) 123.711 1 < 2.2e-16 ***
## numberWordSenses 22.655 1 1.939e-06 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1summary(goodness.glmer_nws)## Generalized linear mixed model fit by maximum likelihood (Laplace
## Approximation) [glmerMod]
## Family: binomial ( logit )
## Formula: as.factor(goodSentence) ~ numberWordSenses + (1 | sentenceLength)
## Data: data
## Control: glmerControl(optimizer = "bobyqa")
##
## AIC BIC logLik -2*log(L) df.resid
## 6935.7 6955.5 -3464.9 6929.7 5397
##
## Scaled residuals:
## Min 1Q Median 3Q Max
## -1.5703 -1.2879 0.6673 0.7152 1.0510
##
## Random effects:
## Groups Name Variance Std.Dev.
## sentenceLength (Intercept) 0.02763 0.1662
## Number of obs: 5400, groups: sentenceLength, 19
##
## Fixed effects:
## Estimate Std. Error z value Pr(>|z|)
## (Intercept) 0.90407 0.08128 11.12 < 2e-16 ***
## numberWordSenses -0.09504 0.01997 -4.76 1.94e-06 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Correlation of Fixed Effects:
## (Intr)
## nmbrWrdSnss -0.717goodness.glmer_nws4=glmer(as.factor(goodSentence) ~ numberWordSenses_fourBins +(1|sentenceLength), data=data,family="binomial",control=glmerControl(optimizer="bobyqa"))
# fourBins fits better
anova(goodness.glmer_nws, goodness.glmer_nws4)## Data: data
## Models:
## goodness.glmer_nws: as.factor(goodSentence) ~ numberWordSenses + (1 | sentenceLength)
## goodness.glmer_nws4: as.factor(goodSentence) ~ numberWordSenses_fourBins + (1 | sentenceLength)
## npar AIC BIC logLik -2*log(L) Chisq Df Pr(>Chisq)
## goodness.glmer_nws 3 6935.7 6955.5 -3464.9 6929.7
## goodness.glmer_nws4 3 6922.7 6942.4 -3458.3 6916.7 13.047 0Anova(goodness.glmer_nws4, type="III")## Analysis of Deviance Table (Type III Wald chisquare tests)
##
## Response: as.factor(goodSentence)
## Chisq Df Pr(>Chisq)
## (Intercept) 171.03 1 < 2.2e-16 ***
## numberWordSenses_fourBins 35.88 1 2.099e-09 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1summary(goodness.glmer_nws4)## Generalized linear mixed model fit by maximum likelihood (Laplace
## Approximation) [glmerMod]
## Family: binomial ( logit )
## Formula:
## as.factor(goodSentence) ~ numberWordSenses_fourBins + (1 | sentenceLength)
## Data: data
## Control: glmerControl(optimizer = "bobyqa")
##
## AIC BIC logLik -2*log(L) df.resid
## 6922.7 6942.4 -3458.3 6916.7 5397
##
## Scaled residuals:
## Min 1Q Median 3Q Max
## -1.5834 -1.2557 0.6557 0.7281 0.9799
##
## Random effects:
## Groups Name Variance Std.Dev.
## sentenceLength (Intercept) 0.02286 0.1512
## Number of obs: 5400, groups: sentenceLength, 19
##
## Fixed effects:
## Estimate Std. Error z value Pr(>|z|)
## (Intercept) 0.75124 0.05744 13.08 < 2e-16 ***
## numberWordSenses_fourBins -0.16414 0.02740 -5.99 2.1e-09 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Correlation of Fixed Effects:
## (Intr)
## nmbrWrdSn_B -0.370# but is there a better model?
goodness.glmer_nws4_rs=glmer(as.factor(goodSentence) ~ numberWordSenses_fourBins +(1+numberWordSenses_fourBins|sentenceLength), data=data,family="binomial",control=glmerControl(optimizer="bobyqa"))
# rs is better
anova(goodness.glmer_nws4, goodness.glmer_nws4_rs)## Data: data
## Models:
## goodness.glmer_nws4: as.factor(goodSentence) ~ numberWordSenses_fourBins + (1 | sentenceLength)
## goodness.glmer_nws4_rs: as.factor(goodSentence) ~ numberWordSenses_fourBins + (1 + numberWordSenses_fourBins | sentenceLength)
## npar AIC BIC logLik -2*log(L) Chisq Df
## goodness.glmer_nws4 3 6922.7 6942.4 -3458.3 6916.7
## goodness.glmer_nws4_rs 5 6920.1 6953.0 -3455.0 6910.1 6.5941 2
## Pr(>Chisq)
## goodness.glmer_nws4
## goodness.glmer_nws4_rs 0.03699 *
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1Anova(goodness.glmer_nws4_rs, type="III")## Analysis of Deviance Table (Type III Wald chisquare tests)
##
## Response: as.factor(goodSentence)
## Chisq Df Pr(>Chisq)
## (Intercept) 267.719 1 < 2.2e-16 ***
## numberWordSenses_fourBins 16.047 1 6.18e-05 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1summary(goodness.glmer_nws4_rs)## Generalized linear mixed model fit by maximum likelihood (Laplace
## Approximation) [glmerMod]
## Family: binomial ( logit )
## Formula:
## as.factor(goodSentence) ~ numberWordSenses_fourBins + (1 + numberWordSenses_fourBins |
## sentenceLength)
## Data: data
## Control: glmerControl(optimizer = "bobyqa")
##
## AIC BIC logLik -2*log(L) df.resid
## 6920.1 6953.0 -3455.0 6910.1 5395
##
## Scaled residuals:
## Min 1Q Median 3Q Max
## -1.5279 -1.3247 0.6688 0.6998 1.0970
##
## Random effects:
## Groups Name Variance Std.Dev. Corr
## sentenceLength (Intercept) 0.008436 0.09185
## numberWordSenses_fourBins 0.008502 0.09221 0.33
## Number of obs: 5400, groups: sentenceLength, 19
##
## Fixed effects:
## Estimate Std. Error z value Pr(>|z|)
## (Intercept) 0.75789 0.04632 16.362 < 2e-16 ***
## numberWordSenses_fourBins -0.16607 0.04146 -4.006 6.18e-05 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Correlation of Fixed Effects:
## (Intr)
## nmbrWrdSn_B -0.229plot_model(goodness.glmer_nws4_rs, type="pred", terms=c("numberWordSenses_fourBins"))## You are calculating adjusted predictions on the population-level (i.e.
## `type = "fixed"`) for a *generalized* linear mixed model.
## This may produce biased estimates due to Jensen's inequality. Consider
## setting `bias_correction = TRUE` to correct for this bias.
## See also the documentation of the `bias_correction` argument.
ggsave(paste0(path, "./results/polysemes_numberWordSenses.png"), width= 7, height=7)
# goodSentence ~ numberWordSenses*model
# -------------------------------------
# fixed effect: model, removed annotator as ranef because of singularFit
goodness.glmer_model=glmer(as.factor(goodSentence) ~ model +(1|sentenceLength), data=data,family="binomial",control=glmerControl(optimizer="bobyqa"))
# isSingular
# goodness.glmer_model_rs=glmer(as.factor(goodSentence) ~ model +(1+model|sentenceLength), data=data,family="binomial",control=glmerControl(optimizer="bobyqa"))
Anova(goodness.glmer_model, type="III")## Analysis of Deviance Table (Type III Wald chisquare tests)
##
## Response: as.factor(goodSentence)
## Chisq Df Pr(>Chisq)
## (Intercept) 21.709 1 3.174e-06 ***
## model 52.965 2 3.153e-12 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1summary(goodness.glmer_model)## Generalized linear mixed model fit by maximum likelihood (Laplace
## Approximation) [glmerMod]
## Family: binomial ( logit )
## Formula: as.factor(goodSentence) ~ model + (1 | sentenceLength)
## Data: data
## Control: glmerControl(optimizer = "bobyqa")
##
## AIC BIC logLik -2*log(L) df.resid
## 6907.1 6933.4 -3449.5 6899.1 5396
##
## Scaled residuals:
## Min 1Q Median 3Q Max
## -1.7119 -1.2294 0.6492 0.7147 0.9746
##
## Random effects:
## Groups Name Variance Std.Dev.
## sentenceLength (Intercept) 0.0381 0.1952
## Number of obs: 5400, groups: sentenceLength, 19
##
## Fixed effects:
## Estimate Std. Error z value Pr(>|z|)
## (Intercept) 0.35413 0.07601 4.659 3.17e-06 ***
## modeldeepseek-chat 0.50330 0.07055 7.134 9.73e-13 ***
## modelgpt-4o 0.33526 0.07518 4.459 8.22e-06 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Correlation of Fixed Effects:
## (Intr) mdldp-
## mdldpsk-cht -0.426
## modelgpt-4o -0.507 0.445plot_model(goodness.glmer_model, type="pred", terms=c("model"))
ggsave(paste0(path, "./results/polysemes_model.png"), width= 7, height=7)
# todo: but is this the best model?
ranef(goodness.glmer_model)## $sentenceLength
## (Intercept)
## 5 -0.08634936
## 6 -0.08739568
## 7 -0.30105687
## 8 -0.30262116
## 9 -0.16691143
## 10 -0.01757621
## 11 0.14138609
## 12 0.10461593
## 13 0.05897266
## 14 0.01636458
## 15 0.21776391
## 16 0.17464111
## 17 0.08598805
## 18 -0.01476090
## 19 -0.00909159
## 20 -0.02833764
## 21 0.09757870
## 22 0.06959268
## 23 0.03085803
##
## with conditional variances for "sentenceLength"# goodSentence ~ numberWordSenses * model
# ---------------------------------------
# fixed effects: number word senses and model, but no interaction
goodness.glmer_poly_nws_model=glmer(as.factor(goodSentence) ~ numberWordSenses_fourBins*model +(1|sentenceLength), data=data,family="binomial",control=glmerControl(optimizer="bobyqa"))
# factoring in model also improves model fit
anova(goodness.glmer_nws4_rs, goodness.glmer_poly_nws_model)## Data: data
## Models:
## goodness.glmer_nws4_rs: as.factor(goodSentence) ~ numberWordSenses_fourBins + (1 + numberWordSenses_fourBins | sentenceLength)
## goodness.glmer_poly_nws_model: as.factor(goodSentence) ~ numberWordSenses_fourBins * model + (1 | sentenceLength)
## npar AIC BIC logLik -2*log(L) Chisq Df
## goodness.glmer_nws4_rs 5 6920.1 6953.0 -3455.0 6910.1
## goodness.glmer_poly_nws_model 7 6869.5 6915.7 -3427.8 6855.5 54.554 2
## Pr(>Chisq)
## goodness.glmer_nws4_rs
## goodness.glmer_poly_nws_model 1.425e-12 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1Anova(goodness.glmer_poly_nws_model, type="III")## Analysis of Deviance Table (Type III Wald chisquare tests)
##
## Response: as.factor(goodSentence)
## Chisq Df Pr(>Chisq)
## (Intercept) 55.245 1 1.064e-13 ***
## numberWordSenses_fourBins 24.664 1 6.825e-07 ***
## model 31.550 2 1.410e-07 ***
## numberWordSenses_fourBins:model 4.156 2 0.1252
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1summary(goodness.glmer_poly_nws_model)## Generalized linear mixed model fit by maximum likelihood (Laplace
## Approximation) [glmerMod]
## Family: binomial ( logit )
## Formula: as.factor(goodSentence) ~ numberWordSenses_fourBins * model +
## (1 | sentenceLength)
## Data: data
## Control: glmerControl(optimizer = "bobyqa")
##
## AIC BIC logLik -2*log(L) df.resid
## 6869.5 6915.7 -3427.8 6855.5 5393
##
## Scaled residuals:
## Min 1Q Median 3Q Max
## -1.7222 -1.2748 0.6402 0.7402 1.1753
##
## Random effects:
## Groups Name Variance Std.Dev.
## sentenceLength (Intercept) 0.01061 0.103
## Number of obs: 5400, groups: sentenceLength, 19
##
## Fixed effects:
## Estimate Std. Error z value
## (Intercept) 0.50912 0.06850 7.433
## numberWordSenses_fourBins -0.23041 0.04639 -4.966
## modeldeepseek-chat 0.48542 0.08748 5.549
## modelgpt-4o 0.29044 0.08799 3.301
## numberWordSenses_fourBins:modeldeepseek-chat 0.03526 0.06454 0.546
## numberWordSenses_fourBins:modelgpt-4o 0.13003 0.06559 1.983
## Pr(>|z|)
## (Intercept) 1.06e-13 ***
## numberWordSenses_fourBins 6.82e-07 ***
## modeldeepseek-chat 2.87e-08 ***
## modelgpt-4o 0.000964 ***
## numberWordSenses_fourBins:modeldeepseek-chat 0.584830
## numberWordSenses_fourBins:modelgpt-4o 0.047407 *
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Correlation of Fixed Effects:
## (Intr) nmWS_B mdldp- mdlg-4 nmWS_B:-
## nmbrWrdSn_B -0.506
## mdldpsk-cht -0.589 0.385
## modelgpt-4o -0.626 0.346 0.455
## nmbrWrS_B:- 0.357 -0.685 -0.587 -0.264
## nmbrWS_B:-4 0.380 -0.704 -0.274 -0.550 0.485plot_model(goodness.glmer_poly_nws_model, type="pred", terms=c("numberWordSenses_fourBins", "model"))
ggsave(paste0(path, "./results/polysemes_numberWordSenses*model.png"), width= 7, height=7)
plot_model(goodness.glmer_poly_nws_model, type="pred", terms=c("model","numberWordSenses_fourBins"))
ggsave(paste0(path, "./results/polysemes_numberWordSenses*model_inv.png"), width= 7, height=7)data=read.table( paste0(path, "polysemes_corrected_hyper_pol.csv"), header=T, sep=";")
# this is the long format, not the wide one (!)
nrow(data)## [1] 5400data$logfreq=log(data$frequency)
data$scalelogfreq=scale(data$logfreq)
table(data$numberWordSenses)##
## 2 3 4 5 6 7 8
## 3348 756 648 270 108 126 144range(data$numberWordSenses)## [1] 2 8# for polysemes, center for model fit (intercept at zero)
data$cNumberWordSenses = data$numberWordSenses - 2
range(data$cNumberWordSenses)## [1] 0 6table(data$cNumberWordSenses)##
## 0 1 2 3 4 5 6
## 3348 756 648 270 108 126 144# less skew:
data$numberWordSenses_fourBins <- ifelse(data$cNumberWordSenses > 2, 3, data$cNumberWordSenses)
table(data$numberWordSenses_fourBins)##
## 0 1 2 3
## 3348 756 648 648# hypernyms that are monosemes
# ----------------------------
# we are only interested in data where hypernymPolysemy equals 1 and artifical equals no
data_monosemous_hyps_only = data[data$hypernymPolysemy=="1" & data$artificial=="no" & data$frequencyHypernym>0,]
nrow(data_monosemous_hyps_only)## [1] 2322# 2322 entries
View(data_monosemous_hyps_only)
data_monosemous_hyps_only$loghyper=log(data_monosemous_hyps_only$frequencyHypernym)
data_monosemous_hyps_only$scaleloghyper=scale(data_monosemous_hyps_only$loghyper)
range(data_monosemous_hyps_only$scaleloghyper)## [1] -3.569634 2.085835# 5.2.3 The frequency of the hypernyn
# ----------------------------------
# scaleloghyper only
goodness.glmer_scaleloghyper=glmer(as.factor(goodSentence) ~ scaleloghyper + (1|sentenceLength), data=data_monosemous_hyps_only,family="binomial",control=glmerControl(optimizer="bobyqa"))
#goodness.glmer_scaleloghyper_rs=glmer(as.factor(goodSentence) ~ scaleloghyper +(1+scaleloghyper|sentenceLength), data=data_monosemous_hyps_only,family="binomial",control=glmerControl(optimizer="bobyqa"))
# rs does not yield better model fit
#anova(goodness.glmer_scaleloghyper, goodness.glmer_scaleloghyper_rs)
Anova(goodness.glmer_scaleloghyper, type="III")## Analysis of Deviance Table (Type III Wald chisquare tests)
##
## Response: as.factor(goodSentence)
## Chisq Df Pr(>Chisq)
## (Intercept) 66.7861 1 3.026e-16 ***
## scaleloghyper 7.1291 1 0.007584 **
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1summary(goodness.glmer_scaleloghyper)## Generalized linear mixed model fit by maximum likelihood (Laplace
## Approximation) [glmerMod]
## Family: binomial ( logit )
## Formula: as.factor(goodSentence) ~ scaleloghyper + (1 | sentenceLength)
## Data: data_monosemous_hyps_only
## Control: glmerControl(optimizer = "bobyqa")
##
## AIC BIC logLik -2*log(L) df.resid
## 2851.0 2868.2 -1422.5 2845.0 2319
##
## Scaled residuals:
## Min 1Q Median 3Q Max
## -2.0144 -1.2694 0.5853 0.6949 0.9618
##
## Random effects:
## Groups Name Variance Std.Dev.
## sentenceLength (Intercept) 0.09194 0.3032
## Number of obs: 2322, groups: sentenceLength, 18
##
## Fixed effects:
## Estimate Std. Error z value Pr(>|z|)
## (Intercept) 0.80009 0.09790 8.172 3.03e-16 ***
## scaleloghyper 0.12184 0.04563 2.670 0.00758 **
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Correlation of Fixed Effects:
## (Intr)
## scalelghypr 0.041report(goodness.glmer_scaleloghyper)## We fitted a logistic mixed model (estimated using ML and BOBYQA optimizer) to
## predict goodSentence with scaleloghyper (formula: as.factor(goodSentence) ~
## scaleloghyper). The model included sentenceLength as random effect (formula: ~1
## | sentenceLength). The model's total explanatory power is weak (conditional R2
## = 0.03) and the part related to the fixed effects alone (marginal R2) is of
## 4.37e-03. The model's intercept, corresponding to scaleloghyper = 0, is at 0.80
## (95% CI [0.61, 0.99], p < .001). Within this model:
##
## - The effect of scaleloghyper is statistically significant and positive (beta =
## 0.12, 95% CI [0.03, 0.21], p = 0.008; Std. beta = 0.12, 95% CI [0.03, 0.21])
##
## Standardized parameters were obtained by fitting the model on a standardized
## version of the dataset. 95% Confidence Intervals (CIs) and p-values were
## computed using a Wald z-distribution approximation.# scaleloghyper interacting with model
#-------------------------------------
goodness.glmer_scaleloghyper_model=glmer(as.factor(goodSentence) ~ scaleloghyper*model +(1|sentenceLength), data=data_monosemous_hyps_only,family="binomial",control=glmerControl(optimizer="bobyqa"))
Anova(goodness.glmer_scaleloghyper_model, type="III")## Analysis of Deviance Table (Type III Wald chisquare tests)
##
## Response: as.factor(goodSentence)
## Chisq Df Pr(>Chisq)
## (Intercept) 14.3971 1 0.000148 ***
## scaleloghyper 4.7633 1 0.029073 *
## model 34.5563 2 3.135e-08 ***
## scaleloghyper:model 0.9896 2 0.609683
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1summary(goodness.glmer_scaleloghyper_model)## Generalized linear mixed model fit by maximum likelihood (Laplace
## Approximation) [glmerMod]
## Family: binomial ( logit )
## Formula: as.factor(goodSentence) ~ scaleloghyper * model + (1 | sentenceLength)
## Data: data_monosemous_hyps_only
## Control: glmerControl(optimizer = "bobyqa")
##
## AIC BIC logLik -2*log(L) df.resid
## 2823.3 2863.6 -1404.7 2809.3 2315
##
## Scaled residuals:
## Min 1Q Median 3Q Max
## -2.2433 -1.1797 0.5675 0.6753 1.1527
##
## Random effects:
## Groups Name Variance Std.Dev.
## sentenceLength (Intercept) 0.07864 0.2804
## Number of obs: 2322, groups: sentenceLength, 18
##
## Fixed effects:
## Estimate Std. Error z value Pr(>|z|)
## (Intercept) 0.42917 0.11311 3.794 0.000148 ***
## scaleloghyper 0.16391 0.07510 2.182 0.029073 *
## modeldeepseek-chat 0.61391 0.11163 5.500 3.80e-08 ***
## modelgpt-4o 0.49506 0.11627 4.258 2.06e-05 ***
## scaleloghyper:modeldeepseek-chat -0.02575 0.11185 -0.230 0.817912
## scaleloghyper:modelgpt-4o -0.10721 0.11089 -0.967 0.333662
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Correlation of Fixed Effects:
## (Intr) scllgh mdldp- mdlg-4 scll:-
## scalelghypr 0.033
## mdldpsk-cht -0.440 -0.019
## modelgpt-4o -0.516 -0.036 0.434
## scllghypr:- -0.018 -0.672 0.049 0.013
## scllghyp:-4 -0.004 -0.677 0.014 0.030 0.454plot_model(goodness.glmer_scaleloghyper_model, type="pred", terms=c("scaleloghyper", "model"))## Data were 'prettified'. Consider using `terms="scaleloghyper [all]"` to
## get smooth plots.
ggsave(paste0(path, "./results/polysemes_scaleloghyper_model.png"), width=7, height=7)
data_monosemous_hyps_only$model_short <- factor(data_monosemous_hyps_only$model,
levels = c("claude-3-5-haiku-20241022", "deepseek-chat", "gpt-4o"),
labels = c("Claude", "Deepseek", "Gpt"))
# goodSentence ~ nws*scaleloghyper
# ---------------------------------
data$model_short <- factor(data$model,
levels = c("claude-3-5-haiku-20241022", "deepseek-chat", "gpt-4o"),
labels = c("Claude", "Deepseek", "ChatGPT"))
goodness.glmer_nws_scaleloghyper=glmer(as.factor(goodSentence) ~ numberWordSenses_fourBins*scaleloghyper +(1|sentenceLength), data=data_monosemous_hyps_only,family="binomial",control=glmerControl(optimizer="bobyqa"))
goodness.glmer_nws_scaleloghyper_add=glmer(as.factor(goodSentence) ~ numberWordSenses_fourBins+scaleloghyper +(1|sentenceLength), data=data_monosemous_hyps_only,family="binomial",control=glmerControl(optimizer="bobyqa"))
anova(goodness.glmer_nws_scaleloghyper, goodness.glmer_nws_scaleloghyper_add)## Data: data_monosemous_hyps_only
## Models:
## goodness.glmer_nws_scaleloghyper_add: as.factor(goodSentence) ~ numberWordSenses_fourBins + scaleloghyper + (1 | sentenceLength)
## goodness.glmer_nws_scaleloghyper: as.factor(goodSentence) ~ numberWordSenses_fourBins * scaleloghyper + (1 | sentenceLength)
## npar AIC BIC logLik -2*log(L)
## goodness.glmer_nws_scaleloghyper_add 4 2851.2 2874.2 -1421.6 2843.2
## goodness.glmer_nws_scaleloghyper 5 2852.2 2880.9 -1421.1 2842.2
## Chisq Df Pr(>Chisq)
## goodness.glmer_nws_scaleloghyper_add
## goodness.glmer_nws_scaleloghyper 1.0151 1 0.3137Anova(goodness.glmer_nws_scaleloghyper, type="III")## Analysis of Deviance Table (Type III Wald chisquare tests)
##
## Response: as.factor(goodSentence)
## Chisq Df Pr(>Chisq)
## (Intercept) 72.1439 1 < 2.2e-16 ***
## numberWordSenses_fourBins 1.9095 1 0.167017
## scaleloghyper 8.2359 1 0.004107 **
## numberWordSenses_fourBins:scaleloghyper 1.0222 1 0.311993
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1summary(goodness.glmer_nws_scaleloghyper)## Generalized linear mixed model fit by maximum likelihood (Laplace
## Approximation) [glmerMod]
## Family: binomial ( logit )
## Formula: as.factor(goodSentence) ~ numberWordSenses_fourBins * scaleloghyper +
## (1 | sentenceLength)
## Data: data_monosemous_hyps_only
## Control: glmerControl(optimizer = "bobyqa")
##
## AIC BIC logLik -2*log(L) df.resid
## 2852.2 2880.9 -1421.1 2842.2 2317
##
## Scaled residuals:
## Min 1Q Median 3Q Max
## -2.1096 -1.2470 0.5878 0.6886 0.9506
##
## Random effects:
## Groups Name Variance Std.Dev.
## sentenceLength (Intercept) 0.08392 0.2897
## Number of obs: 2322, groups: sentenceLength, 18
##
## Fixed effects:
## Estimate Std. Error z value Pr(>|z|)
## (Intercept) 0.84238 0.09918 8.494 < 2e-16
## numberWordSenses_fourBins -0.06692 0.04843 -1.382 0.16702
## scaleloghyper 0.14867 0.05180 2.870 0.00411
## numberWordSenses_fourBins:scaleloghyper -0.05171 0.05115 -1.011 0.31199
##
## (Intercept) ***
## numberWordSenses_fourBins
## scaleloghyper **
## numberWordSenses_fourBins:scaleloghyper
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Correlation of Fixed Effects:
## (Intr) nmWS_B scllgh
## nmbrWrdSn_B -0.288
## scalelghypr 0.070 -0.054
## nmbrWrdS_B: -0.045 0.018 -0.471report(goodness.glmer_nws_scaleloghyper)## We fitted a logistic mixed model (estimated using ML and BOBYQA optimizer) to
## predict goodSentence with numberWordSenses_fourBins and scaleloghyper (formula:
## as.factor(goodSentence) ~ numberWordSenses_fourBins * scaleloghyper). The model
## included sentenceLength as random effect (formula: ~1 | sentenceLength). The
## model's total explanatory power is weak (conditional R2 = 0.03) and the part
## related to the fixed effects alone (marginal R2) is of 6.18e-03. The model's
## intercept, corresponding to numberWordSenses_fourBins = 0 and scaleloghyper =
## 0, is at 0.84 (95% CI [0.65, 1.04], p < .001). Within this model:
##
## - The effect of numberWordSenses fourBins is statistically non-significant and
## negative (beta = -0.07, 95% CI [-0.16, 0.03], p = 0.167; Std. beta = -0.06, 95%
## CI [-0.15, 0.03])
## - The effect of scaleloghyper is statistically significant and positive (beta =
## 0.15, 95% CI [0.05, 0.25], p = 0.004; Std. beta = 0.12, 95% CI [0.03, 0.21])
## - The effect of numberWordSenses fourBins × scaleloghyper is statistically
## non-significant and negative (beta = -0.05, 95% CI [-0.15, 0.05], p = 0.312;
## Std. beta = -0.05, 95% CI [-0.14, 0.05])
##
## Standardized parameters were obtained by fitting the model on a standardized
## version of the dataset. 95% Confidence Intervals (CIs) and p-values were
## computed using a Wald z-distribution approximation.plot_model(goodness.glmer_nws_scaleloghyper, type="pred", terms=c("numberWordSenses_fourBins", "scaleloghyper"))
ggsave(paste0(path, "./results/polysemes_nws*scaleloghyper.png"), width= 7, height=7)
max(vif(goodness.glmer_nws_scaleloghyper))## [1] 1.289255# 1.1
goodness.glmer_poly_3way=glmer(as.factor(goodSentence) ~ scaleloghyper*numberWordSenses_fourBins*model +(1|sentenceLength), data=data_monosemous_hyps_only,family="binomial",control=glmerControl(optimizer="bobyqa"))
goodness.glmer_poly_2way=glmer(as.factor(goodSentence) ~ scaleloghyper + numberWordSenses_fourBins*model +(1|sentenceLength), data=data_monosemous_hyps_only,family="binomial",control=glmerControl(optimizer="bobyqa"))
anova(goodness.glmer_poly_3way, goodness.glmer_poly_2way)## Data: data_monosemous_hyps_only
## Models:
## goodness.glmer_poly_2way: as.factor(goodSentence) ~ scaleloghyper + numberWordSenses_fourBins * model + (1 | sentenceLength)
## goodness.glmer_poly_3way: as.factor(goodSentence) ~ scaleloghyper * numberWordSenses_fourBins * model + (1 | sentenceLength)
## npar AIC BIC logLik -2*log(L) Chisq Df
## goodness.glmer_poly_2way 8 2810.3 2856.3 -1397.1 2794.3
## goodness.glmer_poly_3way 13 2817.1 2891.8 -1395.5 2791.1 3.178 5
## Pr(>Chisq)
## goodness.glmer_poly_2way
## goodness.glmer_poly_3way 0.6726goodness.glmer_poly_2way_alt=glmer(as.factor(goodSentence) ~ scaleloghyper*model + numberWordSenses_fourBins +(1|sentenceLength), data=data_monosemous_hyps_only,family="binomial",control=glmerControl(optimizer="bobyqa"))
anova(goodness.glmer_poly_3way, goodness.glmer_poly_2way)## Data: data_monosemous_hyps_only
## Models:
## goodness.glmer_poly_2way: as.factor(goodSentence) ~ scaleloghyper + numberWordSenses_fourBins * model + (1 | sentenceLength)
## goodness.glmer_poly_3way: as.factor(goodSentence) ~ scaleloghyper * numberWordSenses_fourBins * model + (1 | sentenceLength)
## npar AIC BIC logLik -2*log(L) Chisq Df
## goodness.glmer_poly_2way 8 2810.3 2856.3 -1397.1 2794.3
## goodness.glmer_poly_3way 13 2817.1 2891.8 -1395.5 2791.1 3.178 5
## Pr(>Chisq)
## goodness.glmer_poly_2way
## goodness.glmer_poly_3way 0.6726anova(goodness.glmer_poly_2way_alt, goodness.glmer_poly_2way)## Data: data_monosemous_hyps_only
## Models:
## goodness.glmer_poly_2way_alt: as.factor(goodSentence) ~ scaleloghyper * model + numberWordSenses_fourBins + (1 | sentenceLength)
## goodness.glmer_poly_2way: as.factor(goodSentence) ~ scaleloghyper + numberWordSenses_fourBins * model + (1 | sentenceLength)
## npar AIC BIC logLik -2*log(L) Chisq Df
## goodness.glmer_poly_2way_alt 8 2823.0 2869.0 -1403.5 2807.0
## goodness.glmer_poly_2way 8 2810.3 2856.3 -1397.1 2794.3 12.684 0
## Pr(>Chisq)
## goodness.glmer_poly_2way_alt
## goodness.glmer_poly_2way# => 2way is the best fit
goodness.glmer_poly_1way=glmer(as.factor(goodSentence) ~ scaleloghyper + numberWordSenses_fourBins + model +(1|sentenceLength), data=data_monosemous_hyps_only,family="binomial",control=glmerControl(optimizer="bobyqa"))
anova(goodness.glmer_poly_2way, goodness.glmer_poly_1way)## Data: data_monosemous_hyps_only
## Models:
## goodness.glmer_poly_1way: as.factor(goodSentence) ~ scaleloghyper + numberWordSenses_fourBins + model + (1 | sentenceLength)
## goodness.glmer_poly_2way: as.factor(goodSentence) ~ scaleloghyper + numberWordSenses_fourBins * model + (1 | sentenceLength)
## npar AIC BIC logLik -2*log(L) Chisq Df
## goodness.glmer_poly_1way 6 2819.9 2854.4 -1404.0 2807.9
## goodness.glmer_poly_2way 8 2810.3 2856.3 -1397.1 2794.3 13.664 2
## Pr(>Chisq)
## goodness.glmer_poly_1way
## goodness.glmer_poly_2way 0.001079 **
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1Anova(goodness.glmer_poly_2way, type="III")## Analysis of Deviance Table (Type III Wald chisquare tests)
##
## Response: as.factor(goodSentence)
## Chisq Df Pr(>Chisq)
## (Intercept) 26.4341 1 2.727e-07 ***
## scaleloghyper 7.3985 1 0.0065279 **
## numberWordSenses_fourBins 11.4420 1 0.0007180 ***
## model 14.9213 2 0.0005753 ***
## numberWordSenses_fourBins:model 13.1602 2 0.0013877 **
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1summary(goodness.glmer_poly_2way)## Generalized linear mixed model fit by maximum likelihood (Laplace
## Approximation) [glmerMod]
## Family: binomial ( logit )
## Formula: as.factor(goodSentence) ~ scaleloghyper + numberWordSenses_fourBins *
## model + (1 | sentenceLength)
## Data: data_monosemous_hyps_only
## Control: glmerControl(optimizer = "bobyqa")
##
## AIC BIC logLik -2*log(L) df.resid
## 2810.3 2856.3 -1397.1 2794.3 2314
##
## Scaled residuals:
## Min 1Q Median 3Q Max
## -2.2949 -1.1776 0.5676 0.6712 1.4164
##
## Random effects:
## Groups Name Variance Std.Dev.
## sentenceLength (Intercept) 0.05836 0.2416
## Number of obs: 2322, groups: sentenceLength, 18
##
## Fixed effects:
## Estimate Std. Error z value
## (Intercept) 0.60168 0.11703 5.141
## scaleloghyper 0.12533 0.04608 2.720
## numberWordSenses_fourBins -0.27205 0.08043 -3.383
## modeldeepseek-chat 0.50860 0.13191 3.856
## modelgpt-4o 0.26022 0.13370 1.946
## numberWordSenses_fourBins:modeldeepseek-chat 0.18164 0.11491 1.581
## numberWordSenses_fourBins:modelgpt-4o 0.44419 0.12245 3.627
## Pr(>|z|)
## (Intercept) 2.73e-07 ***
## scaleloghyper 0.006528 **
## numberWordSenses_fourBins 0.000718 ***
## modeldeepseek-chat 0.000115 ***
## modelgpt-4o 0.051623 .
## numberWordSenses_fourBins:modeldeepseek-chat 0.113965
## numberWordSenses_fourBins:modelgpt-4o 0.000286 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Correlation of Fixed Effects:
## (Intr) scllgh nmWS_B mdldp- mdlg-4 nmWS_B:-
## scalelghypr 0.043
## nmbrWrdSn_B -0.414 -0.031
## mdldpsk-cht -0.515 0.011 0.349
## modelgpt-4o -0.584 -0.010 0.325 0.450
## nmbrWrS_B:- 0.290 0.005 -0.675 -0.531 -0.244
## nmbrWS_B:-4 0.303 0.006 -0.656 -0.231 -0.492 0.447report(goodness.glmer_poly_2way)## We fitted a logistic mixed model (estimated using ML and BOBYQA optimizer) to
## predict goodSentence with scaleloghyper, numberWordSenses_fourBins and model
## (formula: as.factor(goodSentence) ~ scaleloghyper + numberWordSenses_fourBins *
## model). The model included sentenceLength as random effect (formula: ~1 |
## sentenceLength). The model's total explanatory power is weak (conditional R2 =
## 0.05) and the part related to the fixed effects alone (marginal R2) is of 0.03.
## The model's intercept, corresponding to scaleloghyper = 0,
## numberWordSenses_fourBins = 0 and model = claude-3-5-haiku-20241022, is at 0.60
## (95% CI [0.37, 0.83], p < .001). Within this model:
##
## - The effect of scaleloghyper is statistically significant and positive (beta =
## 0.13, 95% CI [0.04, 0.22], p = 0.007; Std. beta = 0.13, 95% CI [0.04, 0.22])
## - The effect of numberWordSenses fourBins is statistically significant and
## negative (beta = -0.27, 95% CI [-0.43, -0.11], p < .001; Std. beta = -0.26, 95%
## CI [-0.41, -0.11])
## - The effect of model [deepseek-chat] is statistically significant and positive
## (beta = 0.51, 95% CI [0.25, 0.77], p < .001; Std. beta = 0.61, 95% CI [0.39,
## 0.83])
## - The effect of model [gpt-4o] is statistically non-significant and positive
## (beta = 0.26, 95% CI [-1.83e-03, 0.52], p = 0.052; Std. beta = 0.52, 95% CI
## [0.29, 0.74])
## - The effect of numberWordSenses fourBins × model [deepseek-chat] is
## statistically non-significant and positive (beta = 0.18, 95% CI [-0.04, 0.41],
## p = 0.114; Std. beta = 0.17, 95% CI [-0.04, 0.39])
## - The effect of numberWordSenses fourBins × model [gpt-4o] is statistically
## significant and positive (beta = 0.44, 95% CI [0.20, 0.68], p < .001; Std. beta
## = 0.42, 95% CI [0.19, 0.65])
##
## Standardized parameters were obtained by fitting the model on a standardized
## version of the dataset. 95% Confidence Intervals (CIs) and p-values were
## computed using a Wald z-distribution approximation.plot_model(goodness.glmer_poly_2way, type="pred", terms=c("model", "scaleloghyper", "numberWordSenses_fourBins"))
plot_model(goodness.glmer_poly_2way, type="pred", ncol = 4, terms=c("scaleloghyper [all]", "model", "numberWordSenses_fourBins"))
ggsave(paste0(path, "./results/polysemes_model_2way.png"), width= 7, height=7)
plot(allEffects(goodness.glmer_poly_2way))## Warning in Analyze.model(focal.predictors, mod, xlevels, default.levels, : the
## predictor scaleloghyper is a one-column matrix that was converted to a vector
## Warning in Analyze.model(focal.predictors, mod, xlevels, default.levels, : the
## predictor scaleloghyper is a one-column matrix that was converted to a vector
eff <- ggpredict(goodness.glmer_poly_2way, terms = c("numberWordSenses_fourBins", "model", "scaleloghyper"))
plot(eff)
emmeans(goodness.glmer_poly_3way, pairwise ~ model)## NOTE: Results may be misleading due to involvement in interactions## $emmeans
## model emmean SE df asymp.LCL asymp.UCL
## claude-3-5-haiku-20241022 0.450 0.107 Inf 0.240 0.661
## deepseek-chat 1.059 0.113 Inf 0.837 1.281
## gpt-4o 0.964 0.108 Inf 0.753 1.175
##
## Results are given on the logit (not the response) scale.
## Confidence level used: 0.95
##
## $contrasts
## contrast estimate SE df z.ratio
## (claude-3-5-haiku-20241022) - (deepseek-chat) -0.6088 0.112 Inf -5.418
## (claude-3-5-haiku-20241022) - (gpt-4o) -0.5140 0.117 Inf -4.389
## (deepseek-chat) - (gpt-4o) 0.0948 0.122 Inf 0.779
## p.value
## <0.0001
## <0.0001
## 0.7158
##
## Results are given on the log odds ratio (not the response) scale.
## P value adjustment: tukey method for comparing a family of 3 estimatesemm_model <- emmeans(goodness.glmer_poly_2way,
~ model,
type = "response")## NOTE: Results may be misleading due to involvement in interactionsemm_df <- as.data.frame(emm_model)
emm_df## model response SE df asymp.LCL asymp.UCL
## claude-3-5-haiku-20241022 0.6095984 0.02535466 Inf 0.5589285 0.6580090
## deepseek-chat 0.7423878 0.02145947 Inf 0.6981479 0.7821669
## gpt-4o 0.7232517 0.02134942 Inf 0.6795260 0.7630914
##
## Unknown transformation "as.factor": no transformation done
## Confidence level used: 0.95# Open a PNG device
png(paste0(path,"./results/polysemes_predictedProbabilityGoodSentence.png"), width = 1200, height = 800, res = 150)
ggplot(emm_df, aes(x = model,
y = response,
ymin = asymp.LCL,
ymax = asymp.UCL)) +
geom_pointrange(size = 0.8) +
geom_line(aes(group = 1), linetype = "dashed", color = "grey50") +
labs(
x = "Model",
y = "Predicted Probability of 'Good Sentence'",
title = "Estimated Marginal Means by Model"
) +
theme_minimal(base_size = 13) +
theme(
axis.text.x = element_text(angle = 25, hjust = 1),
plot.title = element_text(face = "bold")
)
# Close the device to save the file
dev.off()## quartz_off_screen
## 2plot_model(goodness.glmer_poly_3way,
type = "pred",
terms=c("model"),
transform_terms = function(term) {
dplyr::recode(term,
"claude-3-5-haiku-20241022" = "Claude",
"deepseek-chat" = "Deepseek",
"gpt-4o" = "ChatGPT"
)
})
# 3.10