-
Notifications
You must be signed in to change notification settings - Fork 0
/
load_repos.R
136 lines (112 loc) · 4.19 KB
/
load_repos.R
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
# Load project repo module.
library(here)
clone_repo <- function(repo_url){
dir.create(here("repos"), showWarnings = FALSE)
repo_name <- tail(unlist(strsplit(repo_url, "/")), n=1)
print(repo_name)
system(paste("git clone ", repo_url, " ", here("repos"), "/", repo_name, sep = ""))
return(repo_name)
}
clear_repos_dir <- function(){
unlink(here("repos"), recursive = TRUE)
}
get_file_info <- function(file_path, project_name){
file.info(here("repos", project_name, file_path))$size
}
get_project_name <- function(file_path){
tail(unlist(strsplit(head(unlist(strsplit(file_path, "[.]")), n=1), "/")), n=1)
}
get_file_type <- function(file_path){
extension <- tail(unlist(strsplit(file_path, "[.]")), n=1)
file_type <- switch(extension,
"py" = "Python",
"R" = "R",
"c" = "C",
"js" = "JavaScript",
"Other")
file_type
}
get_file_imports <- function(file_path, project_name){
file_type <- get_file_type(file_path)
if(file_type == "Other"){
return(character())
} else {
if(file_type == "Python"){
imports <- character()
con = file(here("repos", project_name, file_path), "r")
while ( TRUE ) {
line = readLines(con, n = 1)
if ( length(line) == 0 ) {
break
}
if(grepl("import", line))
{
if(grepl("from", line)){
import_statement <- head(unlist(strsplit(tail(unlist(strsplit(line, "from")), n=1), "import")), n=1)
} else {
if(grepl("as", line)){
import_statement <- head(unlist(strsplit(tail(unlist(strsplit(line, "import")), n=1), "as")), n=1)
} else {
import_statement <- head(unlist(strsplit(tail(unlist(strsplit(line, "import")), n=1), "\n")), n=1)
}
}
import_statement <- gsub(" ", "", import_statement)
imports <- c(imports, import_statement)
}
}
close(con)
return(imports)
}
}
}
make_graph <- function(imports, vertex){
edges <- matrix(nrow = 0, ncol = 2)
names(vertex) <- seq(length(vertex))
for(source_vertex in seq(length(imports))){
for(target_vertex in imports[source_vertex][[1]]){
target_id = which(target_vertex == vertex)
if(length(target_id) != 0){
edges <- rbind(edges, c(source_vertex, target_id))
} else {
edges <- rbind(edges, c(source_vertex, target_vertex))
}
}
}
colnames(edges) <- c('importer', 'imported')
edges
}
SaveDataAsCSV <- function(project_name){
project_files <- list.files(here("repos", project_name), recursive = TRUE)
project_names <- sapply(project_files, get_project_name)
file_sizes <- sapply(project_files, get_file_info, project_name = project_name)
file_types <- factor(sapply(project_files, get_file_type))
file_imports <- sapply(project_files, get_file_imports, project_name = project_name)
import_counts <- sapply(file_imports, length)
names(file_imports) <- sapply(names(file_imports), get_project_name)
graph <- make_graph(file_imports, project_names)
# Merge results
result <- data.frame(row.names = seq.int(length(project_files)),
project_files,
project_names,
file_sizes,
file_types,
import_counts)
#set your own working directory
WorkingDirectory <- paste(here(),"/",sep = "")
path <- paste(WorkingDirectory,project_name,sep = "")
print(path)
dir.create(path)
write.csv(graph,paste(path,"/GraphEdges.csv" ,sep = ""))
write.csv(result,paste(path,"/GraphData.csv", sep = ""))
}
MakeGraphData <- function(giturl){
clear_repos_dir()
name <- clone_repo(giturl)
SaveDataAsCSV(name)
return(name)
}
#MakeGraphData("https://github.com/kopok2/MachineLearningAlgorithms")
#MakeGraphData("https://github.com/skuam/PySDM")
#MakeGraphData("https://github.com/Kozea/Pyphen")
#MakeGraphData("https://github.com/yidao620c/python3-cookbook")
#MakeGraphData("https://github.com/OlafenwaMoses/ImageAI")