################################################################

# Text Analysis and Visualization: Introduction to R
# https://idev.letras.up.pt/cetapsdigitallab/node/83
# Based on an ongoing by Luciano Moreira (2025)

################################################################
## First things first

#create a project
#choose the same template: Tools --> Global Options --> Appearance --> Crimson Editor

getwd() # To know where the current directory is
#setwd("/Users/lucianomoreira/Desktop/lisbon") # to set the work directory

# Clean the environment (it's the windows that shows all the objects - top right of the screen)
rm(list=ls()) 

# Packages to install (if needed)

# install.packages("readxl")
# install.packages("quanteda")
# install.packages("quanteda.textstats")
# install.packages("ggplot2")
# install.packages("rainette")
# install.packages("gtools")
# install.packages("Xplortext") 
# install.packages("tidyr")
# install.packages("dplyr")
# install.packages("tidytext")
# install.packages("wordcloud")
# install.packages("reshape2")
# install.packages("textdata")
# install.packages("stringr")
# install.packages("leaflet")
# install.packages("ggtext")
# install.packages("showtext")
# install.packages("ggrepel")

#call libraries
import_libraries <- function() {
  library(readxl)
  library(quanteda)
  library(ggrepel)
  library(ggtext)
  library(showtext)
  library(quanteda.textstats)
  library(ggplot2)
  library(rainette)
  library(gtools)
  library(Xplortext) 
  library(tidyr)
  library(dplyr)
  library(tidytext)
  library(wordcloud)
  library(reshape2)
  library(textdata)
  library(stringr)
  library(leaflet)
}

import_libraries()

################################################################
## Get and explore data

# get data 

# library(readxl)
lyman <- read_xlsx("lyman_edited_blue.xlsx")

print(lyman)
names(lyman) # identifies the name of each collumn
length(lyman$id) # number of records

#after exploring the dataset we make a subset to be easier to navigate

lyman <- lyman[,c(1,2,4:8)]
names(lyman) # identifies the name of each collumn

# duplicate titles

lyman[duplicated(lyman$title) == TRUE,] # it identifies a repeated title
lyman[lyman$title == "Sea Change",] # Different authors though

#duplicate authors

length(unique(lyman$author)) ### cases of two authors.
lyman[duplicated(lyman$author) == TRUE,] # it identifies a repeated title
lyman$author[26]
lyman$author[32]

lyman[lyman$author == "Tidhar, Lavie",] # Different authors though
lyman[lyman$author == "Congdon, Douglas E.",] # Different authors though

################################################################
## First visualizations

# Computing production per year (all)
year <- as.data.frame(table(lyman$year))
print(year)
plot(year)
names(year)
colnames(year)[1] <- "year" # change the name of the first collumn
str(year) 
year$year <- as.numeric(as.character(year$year))
min(year$year)
max(year$year)
c <- cor.test(as.numeric(year$year),year$Freq)
print(c)

# simple but beautiful plot
#library(ggplot2)

ggplot(year, aes(x=year, y=Freq, group=1)) +
  geom_line(color= "red", group=1) +
  labs(x = "Year", y = "Frequency") 

#note that we have missing years. In the final work, we would need to add information about missing years

ggplot(year, aes(x=year, y=Freq)) + 
  geom_bar(stat = "identity", color = "red", fill= "red")

# Now - try and do the same for gender

################################################################
## Text analysis
# library(quanteda)

print(lyman$annotation)

#create a corpus
l_corpus <- corpus(lyman$annotation) 
str(l_corpus)
print(l_corpus)
length(l_corpus)
l_corpus[1] # gets the first text
#create tokes
l_tokens <- tokens(l_corpus)

# Tokens in context - think of words and complete below

collo <- kwic(l_tokens, 
              pattern = phrase(c("sea"), 
                               5)
              )

c_collo <- corpus(collo) 
print(c_collo)

# clean the corpus

dic <- c("s")

t_collo <-  tokens_replace(tokens(tokens(c_collo,
                                         remove_punct = TRUE,
                                         remove_symbols = TRUE,
                                         remove_separators = TRUE,
                                         include_docvars = TRUE,
                                         remove_numbers = TRUE) %>%
                                    tokens_split(
                                      separator = "’",
                                      remove_separator = TRUE
                                    )  %>%
                                    tokens_tolower() %>%
                                    tokens_remove(c(stopwords("english"), dic))
                                  ),
                           pattern = lexicon::hash_lemmas$token, # for example -- built becomes build
                           replacement = lexicon::hash_lemmas$lemma
                          ) 
dfm_collo <- dfm(t_collo) 
topfeatures(dfm_collo, 10) # ten or more occurrences
# library(quanteda.textstats)
l_freq <- textstat_frequency(dfm_collo, n = 20)
# library(ggplot2)
theme_set(theme_gray()) #https://ggplot2.tidyverse.org/reference/ggtheme.html

ggplot(l_freq, aes(x = frequency, y = reorder(feature, frequency))) +
  geom_point(color="dodgerblue4", size = 2) +
  labs(x = "Frequency", y = "Feature") +
  scale_x_continuous(breaks = c(2:10), minor_breaks = NULL)

textstat_collocations(t_collo)

# Dengrogram or a tree: HDA
# library(rainette)

com_trim <- dfm_trim(dfm_collo)
comp_rai <-rainette(dfm_collo, k = 5)
rainette_explor(comp_rai, com_trim, t_collo)

rainette_plot(
  comp_rai, com_trim, k = 3,
  n_terms = 20,
  free_scales = FALSE,
  measure = "chi2",
  show_negative = FALSE,
  text_size = 12)

## Correspondence analysis

x <- as.data.frame(lyman)
names(x)

# preparing quantiles
options(stringsAsFactors = TRUE)          # no automatic data transformation

x$quantiles <- trimws(x$year, "both")
x$quantiles <-  substr(x$quantiles, 1, 7)
x$quantiles <- as.numeric(x$quantiles)
# library(gtools)
x$quantiles <- quantcut(x$quantiles, q = 4, na.rm = TRUE)
names(x)

#preparing 
x$annotation <- trimws(x$annotation, "both")

class(x$annotation)

# library(Xplortext)
g <- TextData(x, 
              var.text = "annotation", 
              var.agg  = "quantiles",
              context.quali = "gender",
              stop.word.tm = TRUE,
              Fmin = 5,
              lminword = 4)

ll <- LexCA(g, graph = FALSE)

plot(ll, 
     selDoc = "ALL", #coord 10
    selWord = "coord 30", # char 0.05
     quali.sup = "coord 10",
     col.doc = "gray",
     col.word = "purple",
    col.quali.sup = "darkgreen",
     title = "Annotation on sea related literary works",
     graph.type = "ggplot"
)

# Sentiment analysis

names(x)

toks <- tokens(x$annotation, 
               remove_punct = TRUE,
               remove_symbols = TRUE,
               remove_separators = TRUE,
               include_docvars = TRUE) %>%
  tokens_split(
    separator = "’",
    remove_separator = TRUE
  )  %>%
  tokens_tolower()  %>%
  tokens_select(pattern = c(dic), 
                selection = "remove", 
                padding = FALSE) %>%
  tokens_remove(stopwords("en")) 

tok_out <- as.list(toks)

# create named array of equal lengths
y <- sapply(tok_out, '[', seq(max(lengths(tok_out))))

# library(tidyr)
# library(dplyr)

my_df_via_toks <- y %>% 
  as_tibble() %>% 
  pivot_longer(cols = everything(), names_to = "text", values_to = "tokens") %>% 
  filter(!is.na(tokens)) %>%  # remove NA values of each text
  arrange(text)

tidy_books <- my_df_via_toks
names(tidy_books) <- c("text", "word") 

# library(tidytext)

nrc_joy <- get_sentiments("nrc")

tidy_books %>%
  inner_join(nrc_joy, relationship = "many-to-many") %>%
  count(word, sort = TRUE)

bing_word_counts <- tidy_books %>%
  inner_join(get_sentiments("bing")) %>%
  count(word, sentiment, sort = TRUE) %>%
  ungroup()

bing_word_counts %>%
  group_by(sentiment) %>%
  slice_max(n, n = 10) %>% 
  ungroup() %>%
  mutate(word = reorder(word, n)) %>%
  ggplot(aes(n, word, fill = sentiment)) +
  geom_col(show.legend = FALSE) +
  facet_wrap(~sentiment, scales = "free_y") +
  labs(x = "Contribution to sentiment",
       y = NULL)

#library(wordcloud)

tidy_books %>%
  count(word) %>%
  with(wordcloud(word, n, max.words = 100))

# library(reshape2)

tt <- tidy_books %>%
  left_join(get_sentiments("bing")) 

tt$sentiment <-   replace_na(tt$sentiment, "neutral")

tt %>%
  count(word, sentiment, sort = TRUE) %>%
  acast(word ~ sentiment, value.var = "n", fill = 0.5) %>%
  comparison.cloud(colors = c("purple","gray", "darkgreen"),
                   max.words = 100)

# graphs
names(x)

toks_dfm <- dfm(toks)

library(quanteda.textplots)
fcmat <- fcm(toks_dfm, 
             context = "document", 
             count = "boolean", 
             tri = FALSE)

set.seed(100)
feat <- names(topfeatures(fcmat, 30))
fcm_select(fcmat, pattern = feat) %>%
  textplot_network(min_freq = 1,
                   omit_isolated = TRUE, 
                   vertex_color = "darkblue",
                   vertex_labelcolor = "black",
                   edge_color = "blue"
  )

# Last graphs

# set template

# library(ggtext)
# library(showtext)
# template from: https://r-graph-gallery.com/web-line-chart-with-labels-at-end-of-line.html
font_add_google("Lato")
showtext_auto()
theme_set(theme_minimal(base_family = "Lato"))
theme_update(
  # Remove title for both x and y axes
  # axis.title = element_blank(),
  # Axes labels are grey
  axis.text = element_text(color = "grey40"),
  # The size of the axes labels are different for x and y.
  axis.text.x = element_text(size = 10, margin = margin(t = 5)),
  axis.text.y = element_text(size = 10, margin = margin(r = 5)),
  # Also, the ticks have a very light grey color
  axis.ticks = element_line(color = "grey91", size = .5),
  # The length of the axis ticks is increased.
  axis.ticks.length.x = unit(1.3, "lines"),
  axis.ticks.length.y = unit(.7, "lines"),
  # Remove the grid lines that come with ggplot2 plots by default
  panel.grid = element_blank(),
  # Customize margin values (top, right, bottom, left)
  plot.margin = margin(20, 40, 20, 40),
  # Use a light grey color for the background of both the plot and the panel
  plot.background = element_rect(fill = "grey98", color = "grey98"),
  panel.background = element_rect(fill = "grey98", color = "grey98"),
  # Customize title appearence
  plot.title = element_text(
    color = "grey10", 
    size = 28, 
    face = "bold",
    margin = margin(t = 15)
  ),
  # Customize subtitle appearence
  plot.subtitle = element_markdown(
    color = "grey30", 
    size = 16,
    lineheight = 1.35,
    margin = margin(t = 15, b = 40)
  ),
  # Title and caption are going to be aligned
  plot.title.position = "plot",
  plot.caption.position = "plot",
  plot.caption = element_text(
    color = "grey30", 
    size = 10,
    lineheight = 1.2, 
    hjust = 0,
    margin = margin(t = 40) # Large margin on the top of the caption.
  ),
  # Remove legend
  legend.position = "none"
)

topfeatures(toks_dfm, 50)

library(quanteda.textstats)
tags_f  <- textstat_frequency(toks_dfm, groups = x$quantiles)

tags_f$group <- factor(tags_f$group, 
                           levels = c("[1953,2015]","(2015,2018]",
                                      "(2018,2020]", "(2020,2024]"
                           ))
don <- tags_f %>%
  filter(tags_f$feature %in% c("climate", "sea", 
                        "future", "dystopia",
                        "land", 
                        "water", 
                        "rise", "city", "flood", "society", 
                        "government", 
                        "woman", "ship",  "mermaid",
                        "female"))
names(don)

don <- don %>%
  group_by(feature, group) %>%
  summarise(
    Size_Sum = sum(frequency)
  )

plt <- don %>%
  ggplot(aes(group, Size_Sum, group = feature)
  ) + 
  geom_vline(
    xintercept = seq(1, 5, by = 1),
    color = "grey91", 
    size = .6
  ) +
  geom_line(
    aes(color = feature),
    size = .9
  ) +
  scale_x_discrete()

xy_labs <- don %>%
  group_by(feature) %>%
  summarize(pos = which.max(group),
            x = group[pos],
            y = Size_Sum[pos])
library(ggrepel)
plt <- plt +
  labs(x ="Years", y = "Frequency") +
  geom_text_repel(data=xy_labs, 
                  aes(x=x, y=y, label = feature, color = feature),
                  family = "Lato",
                  fontface = "bold",
                  size = 4,
                  direction = "y",
                  hjust = 0,
                  segment.size = .7,
                  segment.alpha = .5,
                  segment.linetype = "dotted",
                  box.padding = .4,
                  segment.curvature = -0.1,
                  segment.ncp = 3,
                  segment.angle = 20,
                  nudge_x = 1, inherit.aes=F, 
                  na.rm = FALSE)

plt

################################################################
# A very simple map

#library(leaflet)

#assign latitude
lyman$latitude <- ""
names(lyman)
lyman$latitude <- as.numeric(lyman$latitude)
lyman[10,8] <- -37

#assign longitude
lyman$longitude <- ""
lyman$longitude <- as.numeric(lyman$longitude)
lyman[10,9] <- 144

# marker_color <- "red"

map <- leaflet() %>%
  addProviderTiles(providers$CartoDB.PositronNoLabels) %>%
  fitBounds(-7, 42, -13, 35) %>%
  addCircleMarkers(
    data = lyman,
    lat = ~latitude,
    lng = ~longitude,
    # color = ~marker_color, 
    # fillColor = ~marker_color, 
    fillOpacity = 0.8,
    radius = 4, 
    popup = box,
    popupOptions = popupOptions(
      minWidth = 100,
      closeOnClick = FALSE
    ),
    group = "x"
  )
print(map)

# explore these maps: 
# https://rstudio.up.pt/shiny/users/lucianomoreira/bluehumanities/