{\rtf1\ansi\ansicpg1252\cocoartf2865
\cocoatextscaling0\cocoaplatform0{\fonttbl\f0\fswiss\fcharset0 Helvetica;}
{\colortbl;\red255\green255\blue255;}
{\*\expandedcolortbl;;}
\paperw11900\paperh16840\margl1440\margr1440\vieww11520\viewh8400\viewkind0
\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0

\f0\fs24 \cf0 ################################################################\
\
# Text Analysis and Visualization: Introduction to R\
# https://idev.letras.up.pt/cetapsdigitallab/node/83\
# Based on an ongoing by Luciano Moreira (2025)\
\
################################################################\
## First things first\
\
#create a project\
#choose the same template: Tools --> Global Options --> Appearance --> Crimson Editor\
\
getwd() # To know where the current directory is\
#setwd("/Users/lucianomoreira/Desktop/lisbon") # to set the work directory\
\
\
\
# Clean the environment (it's the windows that shows all the objects - top right of the screen)\
rm(list=ls()) \
\
\
# Packages to install (if needed)\
\
# install.packages("readxl")\
# install.packages("quanteda")\
# install.packages("quanteda.textstats")\
# install.packages("ggplot2")\
# install.packages("rainette")\
# install.packages("gtools")\
# install.packages("Xplortext") \
# install.packages("tidyr")\
# install.packages("dplyr")\
# install.packages("tidytext")\
# install.packages("wordcloud")\
# install.packages("reshape2")\
# install.packages("textdata")\
# install.packages("stringr")\
# install.packages("leaflet")\
# install.packages("ggtext")\
# install.packages("showtext")\
# install.packages("ggrepel")\
\
\
\
#call libraries\
import_libraries <- function() \{\
  library(readxl)\
  library(quanteda)\
  library(ggrepel)\
  library(ggtext)\
  library(showtext)\
  library(quanteda.textstats)\
  library(ggplot2)\
  library(rainette)\
  library(gtools)\
  library(Xplortext) \
  library(tidyr)\
  library(dplyr)\
  library(tidytext)\
  library(wordcloud)\
  library(reshape2)\
  library(textdata)\
  library(stringr)\
  library(leaflet)\
\}\
\
import_libraries()\
\
################################################################\
## Get and explore data\
\
# get data \
\
# library(readxl)\
lyman <- read_xlsx("lyman_edited_blue.xlsx")\
\
print(lyman)\
names(lyman) # identifies the name of each collumn\
length(lyman$id) # number of records\
\
#after exploring the dataset we make a subset to be easier to navigate\
\
lyman <- lyman[,c(1,2,4:8)]\
names(lyman) # identifies the name of each collumn\
\
# duplicate titles\
\
lyman[duplicated(lyman$title) == TRUE,] # it identifies a repeated title\
lyman[lyman$title == "Sea Change",] # Different authors though\
\
#duplicate authors\
\
length(unique(lyman$author)) ### cases of two authors.\
lyman[duplicated(lyman$author) == TRUE,] # it identifies a repeated title\
lyman$author[26]\
lyman$author[32]\
\
lyman[lyman$author == "Tidhar, Lavie",] # Different authors though\
lyman[lyman$author == "Congdon, Douglas E.",] # Different authors though\
\
\
################################################################\
## First visualizations\
\
\
# Computing production per year (all)\
year <- as.data.frame(table(lyman$year))\
print(year)\
plot(year)\
names(year)\
colnames(year)[1] <- "year" # change the name of the first collumn\
str(year) \
year$year <- as.numeric(as.character(year$year))\
min(year$year)\
max(year$year)\
c <- cor.test(as.numeric(year$year),year$Freq)\
print(c)\
\
# simple but beautiful plot\
#library(ggplot2)\
\
ggplot(year, aes(x=year, y=Freq, group=1)) +\
  geom_line(color= "red", group=1) +\
  labs(x = "Year", y = "Frequency") \
\
#note that we have missing years. In the final work, we would need to add information about missing years\
\
ggplot(year, aes(x=year, y=Freq)) + \
  geom_bar(stat = "identity", color = "red", fill= "red")\
\
\
# Now - try and do the same for gender\
\
\
################################################################\
## Text analysis\
# library(quanteda)\
\
print(lyman$annotation)\
\
#create a corpus\
l_corpus <- corpus(lyman$annotation) \
str(l_corpus)\
print(l_corpus)\
length(l_corpus)\
l_corpus[1] # gets the first text\
#create tokes\
l_tokens <- tokens(l_corpus)\
\
# Tokens in context - think of words and complete below\
\
collo <- kwic(l_tokens, \
              pattern = phrase(c("sea"), \
                               5)\
              )\
\
c_collo <- corpus(collo) \
print(c_collo)\
\
# clean the corpus\
\
dic <- c("s")\
\
\
t_collo <-  tokens_replace(tokens(tokens(c_collo,\
                                         remove_punct = TRUE,\
                                         remove_symbols = TRUE,\
                                         remove_separators = TRUE,\
                                         include_docvars = TRUE,\
                                         remove_numbers = TRUE) %>%\
                                    tokens_split(\
                                      separator = "\'92",\
                                      remove_separator = TRUE\
                                    )  %>%\
                                    tokens_tolower() %>%\
                                    tokens_remove(c(stopwords("english"), dic))\
                                  ),\
                           pattern = lexicon::hash_lemmas$token, # for example -- built becomes build\
                           replacement = lexicon::hash_lemmas$lemma\
                          ) \
dfm_collo <- dfm(t_collo) \
topfeatures(dfm_collo, 10) # ten or more occurrences\
# library(quanteda.textstats)\
l_freq <- textstat_frequency(dfm_collo, n = 20)\
# library(ggplot2)\
theme_set(theme_gray()) #https://ggplot2.tidyverse.org/reference/ggtheme.html\
\
ggplot(l_freq, aes(x = frequency, y = reorder(feature, frequency))) +\
  geom_point(color="dodgerblue4", size = 2) +\
  labs(x = "Frequency", y = "Feature") +\
  scale_x_continuous(breaks = c(2:10), minor_breaks = NULL)\
\
\
textstat_collocations(t_collo)\
\
\
\
# Dengrogram or a tree: HDA\
# library(rainette)\
\
\
com_trim <- dfm_trim(dfm_collo)\
comp_rai <-rainette(dfm_collo, k = 5)\
rainette_explor(comp_rai, com_trim, t_collo)\
\
rainette_plot(\
  comp_rai, com_trim, k = 3,\
  n_terms = 20,\
  free_scales = FALSE,\
  measure = "chi2",\
  show_negative = FALSE,\
  text_size = 12)\
\
\
## Correspondence analysis\
\
x <- as.data.frame(lyman)\
names(x)\
\
# preparing quantiles\
options(stringsAsFactors = TRUE)          # no automatic data transformation\
\
x$quantiles <- trimws(x$year, "both")\
x$quantiles <-  substr(x$quantiles, 1, 7)\
x$quantiles <- as.numeric(x$quantiles)\
# library(gtools)\
x$quantiles <- quantcut(x$quantiles, q = 4, na.rm = TRUE)\
names(x)\
\
#preparing \
x$annotation <- trimws(x$annotation, "both")\
\
class(x$annotation)\
\
\
# library(Xplortext)\
g <- TextData(x, \
              var.text = "annotation", \
              var.agg  = "quantiles",\
              context.quali = "gender",\
              stop.word.tm = TRUE,\
              Fmin = 5,\
              lminword = 4)\
\
ll <- LexCA(g, graph = FALSE)\
\
plot(ll, \
     selDoc = "ALL", #coord 10\
    selWord = "coord 30", # char 0.05\
     quali.sup = "coord 10",\
     col.doc = "gray",\
     col.word = "purple",\
    col.quali.sup = "darkgreen",\
     title = "Annotation on sea related literary works",\
     graph.type = "ggplot"\
)\
\
\
# Sentiment analysis\
\
names(x)\
\
toks <- tokens(x$annotation, \
               remove_punct = TRUE,\
               remove_symbols = TRUE,\
               remove_separators = TRUE,\
               include_docvars = TRUE) %>%\
  tokens_split(\
    separator = "\'92",\
    remove_separator = TRUE\
  )  %>%\
  tokens_tolower()  %>%\
  tokens_select(pattern = c(dic), \
                selection = "remove", \
                padding = FALSE) %>%\
  tokens_remove(stopwords("en")) \
\
tok_out <- as.list(toks)\
\
# create named array of equal lengths\
y <- sapply(tok_out, '[', seq(max(lengths(tok_out))))\
\
# library(tidyr)\
# library(dplyr)\
\
\
my_df_via_toks <- y %>% \
  as_tibble() %>% \
  pivot_longer(cols = everything(), names_to = "text", values_to = "tokens") %>% \
  filter(!is.na(tokens)) %>%  # remove NA values of each text\
  arrange(text)\
\
tidy_books <- my_df_via_toks\
names(tidy_books) <- c("text", "word") \
\
# library(tidytext)\
\
nrc_joy <- get_sentiments("nrc")\
\
\
tidy_books %>%\
  inner_join(nrc_joy, relationship = "many-to-many") %>%\
  count(word, sort = TRUE)\
\
bing_word_counts <- tidy_books %>%\
  inner_join(get_sentiments("bing")) %>%\
  count(word, sentiment, sort = TRUE) %>%\
  ungroup()\
\
bing_word_counts %>%\
  group_by(sentiment) %>%\
  slice_max(n, n = 10) %>% \
  ungroup() %>%\
  mutate(word = reorder(word, n)) %>%\
  ggplot(aes(n, word, fill = sentiment)) +\
  geom_col(show.legend = FALSE) +\
  facet_wrap(~sentiment, scales = "free_y") +\
  labs(x = "Contribution to sentiment",\
       y = NULL)\
\
#library(wordcloud)\
\
\
\
tidy_books %>%\
  count(word) %>%\
  with(wordcloud(word, n, max.words = 100))\
\
# library(reshape2)\
\
tt <- tidy_books %>%\
  left_join(get_sentiments("bing")) \
\
tt$sentiment <-   replace_na(tt$sentiment, "neutral")\
\
\
\
\
tt %>%\
  count(word, sentiment, sort = TRUE) %>%\
  acast(word ~ sentiment, value.var = "n", fill = 0.5) %>%\
  comparison.cloud(colors = c("purple","gray", "darkgreen"),\
                   max.words = 100)\
\
\
\
# graphs\
names(x)\
\
toks_dfm <- dfm(toks)\
\
library(quanteda.textplots)\
fcmat <- fcm(toks_dfm, \
             context = "document", \
             count = "boolean", \
             tri = FALSE)\
\
set.seed(100)\
feat <- names(topfeatures(fcmat, 30))\
fcm_select(fcmat, pattern = feat) %>%\
  textplot_network(min_freq = 1,\
                   omit_isolated = TRUE, \
                   vertex_color = "darkblue",\
                   vertex_labelcolor = "black",\
                   edge_color = "blue"\
  )\
\
\
# Last graphs\
\
# set template\
\
# library(ggtext)\
# library(showtext)\
# template from: https://r-graph-gallery.com/web-line-chart-with-labels-at-end-of-line.html\
font_add_google("Lato")\
showtext_auto()\
theme_set(theme_minimal(base_family = "Lato"))\
theme_update(\
  # Remove title for both x and y axes\
  # axis.title = element_blank(),\
  # Axes labels are grey\
  axis.text = element_text(color = "grey40"),\
  # The size of the axes labels are different for x and y.\
  axis.text.x = element_text(size = 10, margin = margin(t = 5)),\
  axis.text.y = element_text(size = 10, margin = margin(r = 5)),\
  # Also, the ticks have a very light grey color\
  axis.ticks = element_line(color = "grey91", size = .5),\
  # The length of the axis ticks is increased.\
  axis.ticks.length.x = unit(1.3, "lines"),\
  axis.ticks.length.y = unit(.7, "lines"),\
  # Remove the grid lines that come with ggplot2 plots by default\
  panel.grid = element_blank(),\
  # Customize margin values (top, right, bottom, left)\
  plot.margin = margin(20, 40, 20, 40),\
  # Use a light grey color for the background of both the plot and the panel\
  plot.background = element_rect(fill = "grey98", color = "grey98"),\
  panel.background = element_rect(fill = "grey98", color = "grey98"),\
  # Customize title appearence\
  plot.title = element_text(\
    color = "grey10", \
    size = 28, \
    face = "bold",\
    margin = margin(t = 15)\
  ),\
  # Customize subtitle appearence\
  plot.subtitle = element_markdown(\
    color = "grey30", \
    size = 16,\
    lineheight = 1.35,\
    margin = margin(t = 15, b = 40)\
  ),\
  # Title and caption are going to be aligned\
  plot.title.position = "plot",\
  plot.caption.position = "plot",\
  plot.caption = element_text(\
    color = "grey30", \
    size = 10,\
    lineheight = 1.2, \
    hjust = 0,\
    margin = margin(t = 40) # Large margin on the top of the caption.\
  ),\
  # Remove legend\
  legend.position = "none"\
)\
\
\
\
\
topfeatures(toks_dfm, 50)\
\
library(quanteda.textstats)\
tags_f  <- textstat_frequency(toks_dfm, groups = x$quantiles)\
\
\
tags_f$group <- factor(tags_f$group, \
                           levels = c("[1953,2015]","(2015,2018]",\
                                      "(2018,2020]", "(2020,2024]"\
                           ))\
don <- tags_f %>%\
  filter(tags_f$feature %in% c("climate", "sea", \
                        "future", "dystopia",\
                        "land", \
                        "water", \
                        "rise", "city", "flood", "society", \
                        "government", \
                        "woman", "ship",  "mermaid",\
                        "female"))\
names(don)\
\
don <- don %>%\
  group_by(feature, group) %>%\
  summarise(\
    Size_Sum = sum(frequency)\
  )\
\
\
plt <- don %>%\
  ggplot(aes(group, Size_Sum, group = feature)\
  ) + \
  geom_vline(\
    xintercept = seq(1, 5, by = 1),\
    color = "grey91", \
    size = .6\
  ) +\
  geom_line(\
    aes(color = feature),\
    size = .9\
  ) +\
  scale_x_discrete()\
\
\
xy_labs <- don %>%\
  group_by(feature) %>%\
  summarize(pos = which.max(group),\
            x = group[pos],\
            y = Size_Sum[pos])\
library(ggrepel)\
plt <- plt +\
  labs(x ="Years", y = "Frequency") +\
  geom_text_repel(data=xy_labs, \
                  aes(x=x, y=y, label = feature, color = feature),\
                  family = "Lato",\
                  fontface = "bold",\
                  size = 4,\
                  direction = "y",\
                  hjust = 0,\
                  segment.size = .7,\
                  segment.alpha = .5,\
                  segment.linetype = "dotted",\
                  box.padding = .4,\
                  segment.curvature = -0.1,\
                  segment.ncp = 3,\
                  segment.angle = 20,\
                  nudge_x = 1, inherit.aes=F, \
                  na.rm = FALSE)\
\
plt\
\
\
\
################################################################\
# A very simple map\
\
#library(leaflet)\
\
#assign latitude\
lyman$latitude <- ""\
names(lyman)\
lyman$latitude <- as.numeric(lyman$latitude)\
lyman[10,8] <- -37\
\
#assign longitude\
lyman$longitude <- ""\
lyman$longitude <- as.numeric(lyman$longitude)\
lyman[10,9] <- 144\
\
# marker_color <- "red"\
\
\
map <- leaflet() %>%\
  addProviderTiles(providers$CartoDB.PositronNoLabels) %>%\
  fitBounds(-7, 42, -13, 35) %>%\
  addCircleMarkers(\
    data = lyman,\
    lat = ~latitude,\
    lng = ~longitude,\
    # color = ~marker_color, \
    # fillColor = ~marker_color, \
    fillOpacity = 0.8,\
    radius = 4, \
    popup = box,\
    popupOptions = popupOptions(\
      minWidth = 100,\
      closeOnClick = FALSE\
    ),\
    group = "x"\
  )\
print(map)\
\
# explore these maps: \
# https://rstudio.up.pt/shiny/users/lucianomoreira/bluehumanities/\
\
}