@@ -48,9 +48,9 @@ def scrape_downloads(projects):
48
48
res_dict [project ] = project_downloads
49
49
50
50
for project in projects :
51
- project_downloads = scrape_project_downloads (project )
51
+ project_downloads = scrape_project_downloads (project [ "pip" ] )
52
52
total += project_downloads
53
- res_dict [project ] = project_downloads
53
+ res_dict [project [ "repo" ] ] = project_downloads
54
54
55
55
print (f"Downloads: { res_dict } " )
56
56
return res_dict , total
@@ -61,6 +61,7 @@ def scrape_stars(projects):
61
61
total = 0
62
62
63
63
for project in projects :
64
+ project = project ["repo" ]
64
65
res = requests .get (REPOS_URLS .format (repo = project ))
65
66
res_json = res .json ()
66
67
project_stars = (
@@ -80,6 +81,7 @@ def scrape_colaborators(projects):
80
81
usernames = []
81
82
82
83
for project in projects :
84
+ project = project ["repo" ]
83
85
lastPage = False
84
86
page = 1
85
87
project_colaborators = 0
@@ -131,6 +133,7 @@ def scrape_repos_use(projects):
131
133
res_dict = {}
132
134
total = 0
133
135
for project in projects :
136
+ project = project ["repo" ]
134
137
try :
135
138
res = requests .get (REPOS_USE_URLS .format (repo = project ))
136
139
project_repos_use = retrieve_dependents_from_html (res .content )
@@ -162,6 +165,14 @@ def scrape_repos_use(projects):
162
165
return res_dict , total
163
166
164
167
168
+ def parse_project (project ):
169
+ repo = project ["github" ].split ("/" )[- 1 ]
170
+ return {
171
+ "repo" : repo ,
172
+ "pip" : project ["pip" ] if "pip" in project .keys () else repo ,
173
+ }
174
+
175
+
165
176
def scrape_stats ():
166
177
current_date = str (datetime .date .today ().strftime ("%Y-%m" ))
167
178
stats = {}
@@ -183,7 +194,7 @@ def scrape_stats():
183
194
)
184
195
with open (projects_yaml ) as fp :
185
196
projects = yaml .load (fp , SafeLoader )
186
- projects = list (map (lambda x : x [ "github" ]. split ( "/" )[ - 1 ]. rstrip ( "/" ), projects ))
197
+ projects = list (map (lambda x : parse_project ( x ), projects ))
187
198
188
199
for key in ["downloads" , "colaborators" , "repos_use" , "stars" ]:
189
200
scraped_val = None
0 commit comments