library(quanteda) library(tm) library(wordcloud) library(wordcloud2) library(dplyr) library(scales) # read in the corpus of rejections text = read.table("rejections.txt", sep = "\t", quote = "") # convert the text to the proper format - VectorSource interprets each element of text$V1 as a document myCorpus = Corpus(VectorSource(text$V1)) # applies function to make letters lowercase to all documents myCorpus = tm_map(myCorpus, content_transformer(tolower)) # remove URL's removeURL = function(x) gsub("http[[:alnum:]]*", "", x) myCorpus <- tm_map(myCorpus, content_transformer(removeURL)) # remove punctuation, numbers, and stopwords myCorpus = tm_map(myCorpus, removePunctuation) myCorpus = tm_map(myCorpus, removeNumbers) myCorpus = tm_map(myCorpus, function(x) removeWords(x,stopwords("english"))) #remove some unintersting words and words relating to the paper topics words_to_remove = c("said","from","what","told","over","more","other", "have","last","with","this","that","such","when", "been","says","will","also","where","why","would", "today","annals","neurology","actn","survivors","ich", "reflection","course","students","patients","stroke", "depression","cognitive","can","may","yes", "reflections","social","mri","cohort","sdoh", "icer","race","clinical","one","recovery","jaha", "time","see","csvd","hypertension","blood","pressure", "student","lobar","risk","like","dementia","section", "recurrence","clinically","program","women","died","macce", "postich","vessel","intracerebral","semesters", "burden","days","erich","cerebral","grades","performance", "mindset","impairment","antidepressant", "ischemic", "fee","htn","aha", "orcid","dpi","nih","sigcse", "hemorrhage", "disease","gender", '"', "age","racial","learning", "underrepresented","programming") myCorpus = tm_map(myCorpus, removeWords, words_to_remove) # compute term matrix & convert to matrix class, table summarizing the occurrence of each word in each class document_tm <- TermDocumentMatrix(myCorpus) document_tm_mat <- as.matrix(document_tm) # make matrix into dataframe and name columns to later combine with acceptances document_tm_matdf = as.data.frame(document_tm_mat) colnames(document_tm_matdf) = c(1:403) # create dataframe of words and frequencies, format wordcloud2 package reads words = sort(rowSums(document_tm_mat),decreasing=TRUE) df = data.frame(word = names(words),freq=words) # don't want to plot words that show up <6 times df5 = df[df$freq >= 6,] # get list of unique frequencies freqs <- unique(df5$freq)[order(unique(df5$freq))] # assign each unique frequency a color between 2 values cols <- seq_gradient_pal(low="#e6f5cb", high="#3262a8",)(seq(0,1,length.out=(length(freqs)))) # add color column to df5 containing the color for each word df5$color = NA for (i in 1:length(freqs)){ df5[df5$freq==freqs[i],]$color <- cols[i] } # plot the wordcloud with the color coming from the color column set.seed(1234) # for reproducibility wordcloud2(df5, color=df5$color, shuffle = FALSE, size = 0.75) # Now read in the acceptances corpus accept = read.table("../../data/PaperAcceptances.txt", sep = "\t", quote = "") # Same transformation as for rejections myCorpus2 = Corpus(VectorSource(accept$V1)) myCorpus2 = tm_map(myCorpus2, content_transformer(tolower)) removeURL = function(x) gsub("http[[:alnum:]]*", "", x) myCorpus2 <- tm_map(myCorpus2, content_transformer(removeURL)) # remove punctuation, numbers and stopwords myCorpus2 = tm_map(myCorpus2, removePunctuation) myCorpus2 = tm_map(myCorpus2, removeNumbers) myCorpus2 = tm_map(myCorpus2, function(x) removeWords(x,stopwords("english"))) # remove the same words as removed from rejections wordcloud myCorpus2 = tm_map(myCorpus2, removeWords, words_to_remove) # compute term matrix & convert to matrix class document_tm <- TermDocumentMatrix(myCorpus2) document_tm_mata <- as.matrix(document_tm) # save dataframe with column names different from rejection column names document_tm_matadf = as.data.frame(document_tm_mata) colnames(document_tm_matadf) = c(404:893) # format for wordcloud2 - dataframe with words and frequencies words = sort(rowSums(document_tm_mata),decreasing=TRUE) dfaccept = data.frame(word = names(words),freq=words) # keep only those occuring at least 6 times, get list of unique frequencies df5 = dfaccept[dfaccept$freq >= 6,] freqs <- unique(df5$freq)[order(unique(df5$freq))] # create color gradient with colors for each unique frequency that appears cols <- seq_gradient_pal(low="#e6f5cb", high="#3262a8",)(seq(0,1,length.out=(length(freqs)))) # assign frequencies to colors df5$color = NA for (i in 1:length(freqs)){ df5[df5$freq==freqs[i],]$color <- cols[i] } set.seed(1234) # for reproducibility # plot the wordcloud with the colors from the color column wordcloud2(df5, color=df5$color, shuffle = FALSE, size = 0.75) # Now create a comparison wordcloud # change frequencies from times each words appears to proportion of total words df$freq = df$freq/6465 # divide by total remaining words in rejections corpus dfaccept$freq = dfaccept$freq/7240 # divide by total remaining words in acceptances corpus # Rename frequency columns to be rejection in rejection dataframe and acceptance in acceptance dataframe colnames(df)[2] = "Rejection" colnames(dfaccept)[2] = "Acceptance" # join them together based on the word - now dfcompare contains all words appearing in either corpus, # with two other columns showing the proportion of that word in rejections and acceptances dfcompare = left_join(df, dfaccept) document_tm_matdf$Rejected = rowSums(document_tm_matdf) document_tm_matadf$Accepted = rowSums(document_tm_matadf) document_tm_matdf$word = rownames(document_tm_matdf) document_tm_matadf$word = rownames(document_tm_matadf) document_tm_matdf = document_tm_matdf[,which(colnames(document_tm_matdf) %in% c("Rejected","Accepted","word")) ] document_tm_matadf = document_tm_matadf[,which(colnames(document_tm_matadf) %in% c("Rejected","Accepted","word")) ] combined = dplyr::full_join(document_tm_matdf, document_tm_matadf, by = "word") rownames(combined) = combined$word combined = combined[,-which(colnames(combined) == "word")] combined[is.na(combined)] = 0 combined = as.matrix(combined) colnames(combined) = c("Rejected","Accepted") set.seed(1111) comparison.cloud(combined, random.order = FALSE, title.size = 2.5, max.words = 200, colors = c("#fc766a", "#5b84b1"))