Dependabot Security Alerts Analyzer

Welcome to the documentation of Dependabot Security Alerts Analyzer!. Come here to visit the project.

Overview

Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magnaaliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

alternate text

This is blablabla

alternate text

Blablabla

Modules

DB Manager

Source code
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
DBNAME = 'mydb'

def record_point(point, bucket, org):
    """It writes a point into a InfluxDB Bucket.

        :param point: It represents the Point that will be added inside the bucket. For more information about 'Point' look 'DB Point' module.
        :type point: InfluxPoint

        :param write_api: It represents the API used to write the point on a bucket.
        :type write_api: WriteApi

        :param bucket: It contains the name of the bucket were the point will be written.
        :type bucket: str

        :param org: It contains the organization ID.
        :type org: str
        """

    host = 'localhost'
    port = 8086

    print(point)
    print("\n         ~~ INFO:Writing point into '" + DBNAME + "' ...")
    client = InfluxDBClient(host=host, port=port, username=USER, password=PASSWORD, database=DBNAME)
    json_body = [
        {
            "measurement": 'Record',
            "time": point.commited_date,
            "tags": {
                "repo_id": str(point.repo_id),
                "commit_hexsha":  str(point.commit_hexsha) ,
                "commit_author":  str(point.commit_author) ,
                "repo_name":  str(point.repo_name) ,
                "repo_owner": str(point.repo_owner) ,
                "email": str(point.email) ,
                "commit_message": str(point.commit_message) ,
                "has_config_file": str(point.has_config_file)
            },
            "fields": {
                "total_commit_dependencies": point.total_commit_dependencies,
                "new_critical_vulnerabilities": len(point.critical_severity["new_vulnerabilities"]),
                "fixed_critical_vulnerabilities": len(point.critical_severity["fixed_vulnerabilities"]),
                "removed_critical_vulnerabilities": len(point.critical_severity["removed_vulnerabilities"]),
                "revoked_fixed_critical_vulnerabilities": len(point.critical_severity["revoked_fixed_vulnerabilities"]),
                "revoked_removed_critical_vulnerabilities": len(point.critical_severity["revoked_removed_vulnerabilities"]),
                "kept_critical_vulnerabilities": len(point.critical_severity["kept_vulnerabilities"]),
                "new_high_vulnerabilities": len(point.high_severity["new_vulnerabilities"]),
                "fixed_high_vulnerabilities": len(point.high_severity["fixed_vulnerabilities"]),
                "removed_high_vulnerabilities": len(point.high_severity["removed_vulnerabilities"]),
                "revoked_fixed_high_vulnerabilities": len(point.high_severity["revoked_fixed_vulnerabilities"]),
                "revoked_removed_high_vulnerabilities": len(point.high_severity["revoked_removed_vulnerabilities"]),
                "kept_high_vulnerabilities": len(point.high_severity["kept_vulnerabilities"]),
                "new_moderate_vulnerabilities": len(point.moderate_severity["new_vulnerabilities"]),
                "fixed_moderate_vulnerabilities": len(point.moderate_severity["fixed_vulnerabilities"]),
                "removed_moderate_vulnerabilities": len(point.moderate_severity["removed_vulnerabilities"]),
                "revoked_fixed_moderate_vulnerabilities": len(point.moderate_severity["revoked_fixed_vulnerabilities"]),
                "revoked_removed_moderate_vulnerabilities": len(point.moderate_severity["revoked_removed_vulnerabilities"]),
                "kept_moderate_vulnerabilities": len(point.moderate_severity["kept_vulnerabilities"]),
                "new_low_vulnerabilities": len(point.low_severity["new_vulnerabilities"]),
                "fixed_low_vulnerabilities": len(point.low_severity["fixed_vulnerabilities"]),
                "removed_low_vulnerabilities": len(point.low_severity["removed_vulnerabilities"]),
                "revoked_fixed_low_vulnerabilities": len(point.low_severity["revoked_fixed_vulnerabilities"]),
                "revoked_removed_low_vulnerabilities": len(point.low_severity["revoked_removed_vulnerabilities"]),
                "kept_low_vulnerabilities": len(point.low_severity["kept_vulnerabilities"]),
                "total_new_vulnerabilities": len(point.summary["new_vulnerabilities"]),
                "total_fixed_vulnerabilities": len(point.summary["fixed_vulnerabilities"]),
                "total_removed_vulnerabilities": len(point.summary["removed_vulnerabilities"]),
                "total_revoked_fixed_vulnerabilities": len(point.summary["revoked_fixed_vulnerabilities"]),
                "total_revoked_removed_vulnerabilities": len(point.summary["revoked_removed_vulnerabilities"]),
                "total_kept_vulnerabilities": len(point.summary["kept_vulnerabilities"]),
                "total_commit_vulnerabilities": len(point.summary["total"])
            }
        }
    ]

    client.write_points(json_body, time_precision='s', protocol='json', database='mydb')
    #client_df = DataFrameClient(host=host, port=port, username=USER, password=PASSWORD, database=DBNAME)
    #results_df = client_df.query('SELECT * FROM Commit_22')

DB Point

class db_point.InfluxPoint(repo_id, commited_date, commit_hexsha, commit_author, repo_name, repo_owner, total_commit_dependencies, summary, critical_severity, high_severity, moderate_severity, low_severity, has_config_file, email, commit_message)

Bases: object

Conceptual class to represent each point present in a InfluxDB bucket.

Parameters:
  • repo_id (int) – Id of a repository.
  • commited_date (int) – Date when the commit was committed.
  • commit_hexsha (int) – It is the hexsha of a commit.
  • commit_author (int) – It contains the committed author name.
  • repo_name (str) – It represents the name of a repository.
  • repo_owner (str) – It represents the owner’s name of a repository.
  • total_commit_dependencies (int) – It contains the total number of dependencies present in a commit.
  • summary (Dictionary of lists.) – It contains a dictionary with the following keys: ‘new_vulnerabilities’, ‘fixed_vulnerabilities’, ‘removed_vulnerabilities’, ‘revoked_fixed_vulnerabilities’, ‘revoked_removed_vulnerabilities’, ‘kept_vulnerabilities’ and ‘total’. The values are the total number of vulnerabilities that correspond to those types.
  • critical_severity (Dictionary of lists.) – It contains a dictionary with the following keys: ‘new_vulnerabilities’, ‘fixed_vulnerabilities’, ‘removed_vulnerabilities’, ‘revoked_fixed_vulnerabilities’, ‘revoked_removed_vulnerabilities’, ‘kept_vulnerabilities’ and ‘total’. The values are critical severity vulnerabilities corresponding to those types.
  • high_severity (Dictionary of lists.) – It contains a dictionary with the following keys: ‘new_vulnerabilities’, ‘fixed_vulnerabilities’, ‘removed_vulnerabilities’, ‘revoked_fixed_vulnerabilities’, ‘revoked_removed_vulnerabilities’, ‘kept_vulnerabilities’ and ‘total’. The values are high severity vulnerabilities corresponding to those types.
  • moderate_severity (Dictionary of lists.) – It contains a dictionary with the following keys: ‘new_vulnerabilities’, ‘fixed_vulnerabilities’, ‘removed_vulnerabilities’, ‘revoked_fixed_vulnerabilities’, ‘revoked_removed_vulnerabilities’, ‘kept_vulnerabilities’ and ‘total’. The values are moderate severity vulnerabilities corresponding to those types.
  • low_severity (int) – It contains a dictionary with the following keys: ‘new_vulnerabilities’, ‘fixed_vulnerabilities’, ‘removed_vulnerabilities’, ‘revoked_fixed_vulnerabilities’, ‘revoked_removed_vulnerabilities’, ‘kept_vulnerabilities’ and ‘total’. The values are low severity vulnerabilities corresponding to those types.
  • has_config_file (bool) – It indicates if in the commit it is present the Dependabot configuration file.
  • email (str) – It contains the email of the commit author.
  • commit_message (str) – It contains the commit message.
Source code
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103


class InfluxPoint:
    """Conceptual class to represent each point present in a InfluxDB bucket.

        :param repo_id: Id of a repository.
        :type repo_id: int

        :param commited_date: Date when the commit was committed.
        :type commited_date: int

        :param commit_hexsha: It is the hexsha of a commit.
        :type commit_hexsha: int

        :param commit_author: It contains the committed author name.
        :type commit_author: int

        :param repo_name: It represents the name of a repository.
        :type repo_name: str

        :param repo_owner: It represents the owner's name of a repository.
        :type repo_owner: str

        :param total_commit_dependencies: It contains the total number of dependencies present in a commit.
        :type total_commit_dependencies: int

        :param summary: It contains a dictionary with the following keys: 'new_vulnerabilities', 'fixed_vulnerabilities', 'removed_vulnerabilities', 'revoked_fixed_vulnerabilities', 'revoked_removed_vulnerabilities', 'kept_vulnerabilities' and 'total'. The values are the total number of vulnerabilities that correspond to those types.
        :type summary: Dictionary of lists.

        :param critical_severity: It contains a dictionary with the following keys: 'new_vulnerabilities', 'fixed_vulnerabilities', 'removed_vulnerabilities', 'revoked_fixed_vulnerabilities', 'revoked_removed_vulnerabilities', 'kept_vulnerabilities' and 'total'. The values are critical severity vulnerabilities corresponding to those types.
        :type critical_severity: Dictionary of lists.

        :param high_severity: It contains a dictionary with the following keys: 'new_vulnerabilities', 'fixed_vulnerabilities', 'removed_vulnerabilities', 'revoked_fixed_vulnerabilities', 'revoked_removed_vulnerabilities', 'kept_vulnerabilities' and 'total'. The values are high severity vulnerabilities corresponding to those types.
        :type high_severity: Dictionary of lists.

        :param moderate_severity: It contains a dictionary with the following keys: 'new_vulnerabilities', 'fixed_vulnerabilities', 'removed_vulnerabilities', 'revoked_fixed_vulnerabilities', 'revoked_removed_vulnerabilities', 'kept_vulnerabilities' and 'total'. The values are moderate severity vulnerabilities corresponding to those types.
        :type moderate_severity: Dictionary of lists.

        :param low_severity: It contains a dictionary with the following keys: 'new_vulnerabilities', 'fixed_vulnerabilities', 'removed_vulnerabilities', 'revoked_fixed_vulnerabilities', 'revoked_removed_vulnerabilities', 'kept_vulnerabilities' and 'total'. The values are low severity vulnerabilities corresponding to those types.
        :type low_severity: int

        :param has_config_file: It indicates if in the commit it is present the Dependabot configuration file.
        :type has_config_file: bool

        :param email: It contains the email of the commit author.
        :type email: str

        :param commit_message: It contains the commit message.
        :type commit_message: str
        """

    def __init__(self
                     , repo_id
                     , commited_date
                     , commit_hexsha
                     , commit_author
                     , repo_name
                     , repo_owner
                     , total_commit_dependencies
                     , summary
                     , critical_severity
                     , high_severity
                     , moderate_severity
                     , low_severity
                     , has_config_file
                     , email
                     , commit_message):
        """Constructor method
        """
        self.repo_id = repo_id
        self.commited_date = commited_date
        self.commit_hexsha = commit_hexsha
        self.commit_author = commit_author
        self.repo_name = repo_name
        self.repo_owner = repo_owner
        self.total_commit_dependencies = total_commit_dependencies
        self.summary = summary
        self.critical_severity = critical_severity
        self.high_severity = high_severity
        self.moderate_severity = moderate_severity
        self.low_severity = low_severity
        self.has_config_file = has_config_file
        self.email = email
        self.commit_message = commit_message


    def __str__(self):
        """To String method
        """
        return "         - Commited date:'" + datetime.utcfromtimestamp(self.commited_date).strftime('%Y-%m-%d %H:%M:%S') + "'" + \
               "\n         - Repository id: '" + str(self.repo_id) + "'" + \
               "\n         - Commit author: '" + str(self.commit_author) + "'" + \
               "\n         - Repo name: '" + self.repo_name + "'" + \
               "\n         - Repo owner: '" + self.repo_owner + "'" + \
               "\n         - Total number of dependencies: " + str(self.total_commit_dependencies) 
              # "\n         - Dependabot configuration file: '" + str(self.has_config_file) + "'" + \
              # "\n         - Email address: '" + str(self.email) + "'" + \
              # "\n         - Commit message: '" + self.commit_message.title() + \
              # "\n         - Total number of vulnerabilities: " + str(self.summary) + \
              # "\n         - Critical severity: " + str(self.critical_severity) + \
              # "\n         - High severity: " + str(self.high_severity) + \
              #  "\n         - Moderate severity: " + str(self.moderate_severity) + \
              #  "\n         - Low severity: " + str(self.low_severity)

Dependencies Manager

dependencies_manager.calculate_summary()

It saves total number of different vulnerability ‘status’.

Parameters:vulnerability_id (int) – Vulnerability ID.
dependencies_manager.classify_vulnerabilities(vulnerabilities_list, vulnerability_type)

It appends each vulnerability to corresponding dictionary depending on severity level.

Parameters:
  • vulnerabilities_list ([]) – List of vulnerabilities to classify.
  • vulnerability_type (str) – Vulnerability type.
dependencies_manager.cleaner(version_to_clean)

It cleans a dependency version, removing spaces, ‘^’ and ‘~’. Semver does not read version strings that contain those unwanted characters.

Parameters:version_to_clean (str) – Dependency version to clean.
Returns:Dependency version cleaned.
Return type:str
dependencies_manager.clear_dictionaries()

It clears ‘summary’, ‘critical_vulnerabilities’, ‘high_vulnerabilities’, ‘moderate_vulnerabilities’ and ‘low_vulnerabilities’. To be ready to proccess next repository.

dependencies_manager.get_dependency(line)

It returns the dependency present in a line.

Parameters:line (str) – Line where the dependency is searched.
Returns:It returns the dependency present in a line.
Return type:Dependency
dependencies_manager.get_severity_level(vulnerability_id)

It return severity level of a vulnerability.

Parameters:vulnerability_id (int) – Vulnerability ID.
Returns:Vulnerability severity level.
Return type:str
dependencies_manager.is_vulnerable(dependency_version, vulnerable_range)

It analyzes if a dependency version is vulnerable.

Parameters:
  • dependency_version (str) – It represents the dependency version.
  • vulnerable_range (str) – It represents the vulnerable range.
Returns:

It returns ‘True’ if version is within range, if not it returns ‘False’.
Return type:

bool
dependencies_manager.set_dependencies_and_vulnerabilities(file_path, pattern_1, pattern_2, pattern_3, pattern_4, pattern_5, pattern_6)

It returns the total number of dependencies presented in a commit. If one of those dependencies contains a vulnerability it will be set into vulnerable dependencies list.

Parameters:
  • file_path (str) – Path of npm vulnerable library versions file.
  • pattern_1 (str) – Possible pattern of first level dependency.
  • pattern_2 (str) – Possible pattern of first level dependency.
  • pattern_3 (str) – Possible pattern of first level dependency.
  • pattern_4 (str) – Possible pattern of first level dependency.
  • pattern_5 (str) – Possible pattern of first level dependency.
  • pattern_6 (str) – Possible pattern of first level dependency.
Returns:

It returns the total number of dependencies presented in a commit.
Return type:

int
dependencies_manager.set_new_vulnerability(current_dependency)

If the current dependency is vulnerable, it will be classified as new, revoked or kept. At the end, the dependencies log is updated. It is added the current dependency.

param current_dependency:
 Dependency to classify.
type current_dependency:
 Dependency
dependencies_manager.set_npm_and_yarn_dependencies()

It saves last commit vulnerabilities and record the new dependencies and vulnerabilities.

Returns:It returns the total number of npm and yarn dependencies.
Return type:int
dependencies_manager.version_in_range(vulnerable_range, version)

It calculates if a version is within range.

Parameters:
  • vulnerable_range (str) – It represents the range.
  • version (str) – It represents the dependency version.
Returns:

It return ‘True’ if version is within range, if not it returns ‘False’.
Return type:

bool
dependencies_manager.vulnerabilities_per_severity_level()

It classifies each vulnerability depending on severity level.

Source code
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
new_vulnerabilities = []
last_commit_vulnerabilities = []
vulnerabilities_log = []
kept_vulnerabilities = []
total_commit_vulnerabilities = []
no_duplicated_vulnerabilities = []
severity_records = []
removed_vulnerabilities = []
fixed_vulnerabilities = []
revoked_fixed_vulnerabilities = []
revoked_removed_vulnerabilities = []
historial_fixes = []
total_dependencies = []

removed_vulnerabilities_historial = []

dependencies_set = []
previous_dependencies_affected = []

critical_vulnerabilities =	{
  "new_vulnerabilities": [],
  "fixed_vulnerabilities": [],
  "removed_vulnerabilities": [],
  "revoked_fixed_vulnerabilities": [],
  "revoked_removed_vulnerabilities": [],
  "kept_vulnerabilities": []
}

high_vulnerabilities =	{
  "new_vulnerabilities": [],
  "fixed_vulnerabilities": [],
  "removed_vulnerabilities": [],
  "revoked_fixed_vulnerabilities": [],
  "revoked_removed_vulnerabilities": [],
  "kept_vulnerabilities": []
}

moderate_vulnerabilities =	{
  "new_vulnerabilities": [],
  "fixed_vulnerabilities": [],
  "removed_vulnerabilities": [],
  "revoked_fixed_vulnerabilities": [],
  "revoked_removed_vulnerabilities": [],
  "kept_vulnerabilities": []
}

low_vulnerabilities =	{
  "new_vulnerabilities": [],
  "fixed_vulnerabilities": [],
  "removed_vulnerabilities": [],
  "revoked_fixed_vulnerabilities": [],
  "revoked_removed_vulnerabilities": [],
  "kept_vulnerabilities": []
}


summary = {
  "new_vulnerabilities": [],
  "fixed_vulnerabilities": [],
  "removed_vulnerabilities": [],
  "revoked_fixed_vulnerabilities": [],
  "revoked_removed_vulnerabilities": [],
  "kept_vulnerabilities": [],
  "total": []
}

vulnerable_versions_csv = pd.read_csv("../input/vulnerable_versions.csv", sep=';', dtype={"id": int})


def calculate_summary():
    """It saves total number of different vulnerability 'status'.

    :param vulnerability_id: Vulnerability ID.
    :type vulnerability_id: int

    """
    summary["new_vulnerabilities"] = new_vulnerabilities
    summary["fixed_vulnerabilities"] = fixed_vulnerabilities
    summary["removed_vulnerabilities"] = removed_vulnerabilities
    summary["revoked_fixed_vulnerabilities"] = revoked_fixed_vulnerabilities
    summary["revoked_removed_vulnerabilities"] = revoked_removed_vulnerabilities
    summary["kept_vulnerabilities"] = kept_vulnerabilities
    summary["total"] = total_commit_vulnerabilities


def get_severity_level(vulnerability_id):
    """It return severity level of a vulnerability.

    :param vulnerability_id: Vulnerability ID.
    :type vulnerability_id: int

    :return: Vulnerability severity level.
    :rtype: str
    """
    return vulnerable_versions_csv.loc[vulnerable_versions_csv['id'] == vulnerability_id, "severity"].to_numpy()


def classify_vulnerabilities(vulnerabilities_list, vulnerability_type):
    """It appends each vulnerability to corresponding dictionary depending on severity level.

    :param vulnerabilities_list: List of vulnerabilities to classify.
    :type vulnerabilities_list: []

    :param vulnerability_type: Vulnerability type.
    :type vulnerability_type: str
    """
    for vul in vulnerabilities_list:
        if get_severity_level(vul) == 1:
            low_vulnerabilities[vulnerability_type].append(vul)
        if get_severity_level(vul) == 2:
            moderate_vulnerabilities[vulnerability_type].append(vul)
        if get_severity_level(vul) == 3:
            high_vulnerabilities[vulnerability_type].append(vul)
        if get_severity_level(vul) == 4:
            critical_vulnerabilities[vulnerability_type].append(vul)


def vulnerabilities_per_severity_level():
    """It classifies each vulnerability depending on severity level.
    """
    classify_vulnerabilities(new_vulnerabilities, "new_vulnerabilities")
    classify_vulnerabilities(fixed_vulnerabilities, "fixed_vulnerabilities")
    classify_vulnerabilities(removed_vulnerabilities, "removed_vulnerabilities")
    classify_vulnerabilities(revoked_fixed_vulnerabilities, "revoked_fixed_vulnerabilities")
    classify_vulnerabilities(revoked_removed_vulnerabilities, "revoked_removed_vulnerabilities")
    classify_vulnerabilities(kept_vulnerabilities, "kept_vulnerabilities")


def clear_dictionaries():
    """It clears 'summary', 'critical_vulnerabilities', 'high_vulnerabilities', 'moderate_vulnerabilities' and 'low_vulnerabilities'. To be ready to proccess next repository.
    """
    for amount in summary:
        summary[amount].clear()
    for amount in critical_vulnerabilities:
        critical_vulnerabilities[amount].clear()
    for amount in high_vulnerabilities:
        high_vulnerabilities[amount].clear()
    for amount in moderate_vulnerabilities:
        moderate_vulnerabilities[amount].clear()
    for amount in low_vulnerabilities:
        low_vulnerabilities[amount].clear()


def cleaner(version_to_clean):
    """It cleans a dependency version, removing spaces, '^' and '~'.
    Semver does not read version strings that contain those unwanted characters.

    :param version_to_clean: Dependency version to clean.
    :type version_to_clean: str

    :return: Dependency version cleaned.
    :rtype: str
    """
    dependency_version_cleaned = version_to_clean.replace(" ", "")
    dependency_version_cleaned = dependency_version_cleaned.replace("^", "")
    dependency_version_cleaned = dependency_version_cleaned.replace("~", "")
    return dependency_version_cleaned


def version_in_range(vulnerable_range, version):
    """It calculates if a version is within range.

    :param vulnerable_range: It represents the range.
    :type vulnerable_range: str

    :param version: It represents the dependency version.
    :type version: str

    :return: It return 'True' if version is within range, if not it returns 'False'.
    :rtype: bool
    """
    range_cleaned = cleaner(vulnerable_range)

    if ">=" in vulnerable_range:
        range_cleaned = range_cleaned.replace(">=", "")
        result = semver.compare(range_cleaned, version)
        if result is -1 or 0:
            return True
    elif "<=" in vulnerable_range:
        range_cleaned = range_cleaned.replace("<=", "")
        result = semver.compare(range_cleaned, version)
        if result is 1 or 0:
            return True
    elif ">" in vulnerable_range:
        range_cleaned = range_cleaned.replace(">", "")
        result = semver.compare(range_cleaned, version)
        if result is -1:
            return True
    elif "<" in vulnerable_range:
        range_cleaned = range_cleaned.replace("<", "")
        result = semver.compare(range_cleaned, version)
        if result is 1:
            return True
    elif "=" in vulnerable_range:
        range_cleaned = range_cleaned.replace("=", "")

    result = semver.compare(range_cleaned, version)
    if result is 0:
        return True
    return False


def is_vulnerable(dependency_version, vulnerable_range):
    """It analyzes if a dependency version is vulnerable.

    :param dependency_version: It represents the dependency version.
    :type dependency_version: str

    :param vulnerable_range: It represents the vulnerable range.
    :type vulnerable_range: str

    :return: It returns 'True' if version is within range, if not it returns 'False'.
    :rtype: bool
    """
    if "," in vulnerable_range:
        return version_in_range(vulnerable_range.split(',')[0], dependency_version) and version_in_range(
            vulnerable_range.split(',')[1], dependency_version)
    else:
        return version_in_range(vulnerable_range, dependency_version)


def get_dependency(line):
    """It returns the dependency present in a line.

    :param line: Line where the dependency is searched.
    :type line: str

    :return: It returns the dependency present in a line.
    :rtype: Dependency
    """
    if line.startswith("@"):
        line = line[1:]
        return dependency.Dependency("@" + line.split("@")[0], line.split("@")[1].replace("extraneous", ""))
    else:
        return dependency.Dependency(line.split("@")[0], line.split("@")[1].replace("extraneous", ""))


def set_dependencies_and_vulnerabilities(file_path, pattern_1, pattern_2, pattern_3, pattern_4, pattern_5, pattern_6):
    """It returns the total number of dependencies presented in a commit. If one of those dependencies
    contains a vulnerability it will be set into vulnerable dependencies list.

    :param file_path: Path of npm vulnerable library versions file.
    :type file_path: str

    :param pattern_1: Possible pattern of first level dependency.
    :type pattern_1: str

    :param pattern_2: Possible pattern of first level dependency.
    :type pattern_2: str

    :param pattern_3: Possible pattern of first level dependency.
    :type pattern_3: str

    :param pattern_4: Possible pattern of first level dependency.
    :type pattern_4: str

    :param pattern_5: Possible pattern of first level dependency.
    :type pattern_5: str

    :param pattern_6: Possible pattern of first level dependency.
    :type pattern_6: str


    :return: It returns the total number of dependencies presented in a commit.
    :rtype: int
    """

    dependencies_set.clear()
    try:
        with open('../cloned_git/' + file_path) as file:
            if "npm" in file_path:
                file_content = file.readlines()[1:]
            else:
                file_content = file.readlines()
            file_content = [x.strip() for x in file_content]
            last = file_content[-2]
            for line in file_content:
                previous_len = len(dependencies_set)
                if line.startswith(pattern_1):
                    dependencies_set.append(get_dependency(line.split(pattern_1, 1)[1]))
                elif line.startswith(pattern_2):
                    dependencies_set.append(get_dependency(line.split(pattern_2, 1)[1]))
                elif line.startswith(pattern_3):
                    dependencies_set.append(get_dependency(line.split(pattern_3, 1)[1]))
                elif line.startswith(pattern_6):
                    dependencies_set.append(get_dependency(line.split(pattern_6, 1)[1]))
                elif line is last and line.startswith(pattern_4):
                    dependencies_set.append(get_dependency(line.split(pattern_4, 1)[1]))
                elif line is last and line.startswith(pattern_5):
                    dependencies_set.append(get_dependency(line.split(pattern_5, 1)[1]))
                if previous_len < len(dependencies_set):
                    set_new_vulnerability(dependencies_set[-1])
        file.close()
    except:
        pass
    total_dependencies = dependencies_set
    return len(dependencies_set)


def set_npm_and_yarn_dependencies():
    """It saves last commit vulnerabilities and record the new dependencies and vulnerabilities.

    :return: It returns the total number of npm and yarn dependencies.
    :rtype: int
    """
    vulnerable_dependencies = previous_dependencies_affected
    previous_commmit_vulnerabilities = last_commit_vulnerabilities
    last_commit_vulnerabilities.clear()
    previous_dependencies_affected.clear()

    for vul in total_commit_vulnerabilities:
        last_commit_vulnerabilities.append(vul)

    clear_dictionaries()

    total_commit_vulnerabilities.clear()
    kept_vulnerabilities.clear()
    new_vulnerabilities.clear()
    revoked_fixed_vulnerabilities.clear()
    revoked_removed_vulnerabilities.clear()
    removed_vulnerabilities.clear()


    npm = set_dependencies_and_vulnerabilities('npm_dependencies.txt', "└── UNMET DEPENDENCY ", "├── UNMET DEPENDENCY "
                                               , "├── ", "└── ", "└── UNMET DEPENDENCY ", "├─┬ UNMET DEPENDENCY ")

    yarn = set_dependencies_and_vulnerabilities('yarn_dependencies.txt', "└─  UNMET DEPENDENCY ", "├─ UNMET DEPENDENCY "
                                                , "├─ ", "└─  ", "└─  UNMET DEPENDENCY ", "├─┬ UNMET DEPENDENCY ")

    deleted_vulnerabilities = []
    for dependency in vulnerable_dependencies:
        if dependency.library_name not in dependencies_set:
            deleted_vulnerabilities.append(dependency.version)

    for vulnerability in previous_commmit_vulnerabilities:
        if vulnerability in deleted_vulnerabilities and vulnerability not in total_commit_vulnerabilities and vulnerability not in removed_vulnerabilities_historial:
            removed_vulnerabilities.append(vulnerability)
            removed_vulnerabilities_historial.append(vulnerability)
        elif vulnerability not in kept_vulnerabilities and vulnerability not in historial_fixes:
            fixed_vulnerabilities.append(vulnerability)
            historial_fixes.append(vulnerability)

    calculate_summary()
    vulnerabilities_per_severity_level()

    return npm + yarn


def set_new_vulnerability(current_dependency):
    """If the current dependency is vulnerable, it will be classified as new, revoked or kept.
    At the end, the dependencies log is updated. It is added the current dependency.

     :param current_dependency: Dependency to classify.
     :type current_dependency: Dependency

     """
    try:
        vulnerability_position = 0
        dependency_version = current_dependency.version
        dependency_version_clean = cleaner(dependency_version)

        if semver.VersionInfo.isvalid(dependency_version_clean):
            matched_vulnerable_version_ranges = vulnerable_versions_csv.loc[
                vulnerable_versions_csv['name'] == current_dependency.library_name, "vulnerableVersionRange"].to_numpy()

            matched_vulnerable_ids = vulnerable_versions_csv.loc[
                vulnerable_versions_csv['name'] == current_dependency.library_name, "id"].to_numpy()

            matched_severity_levels = vulnerable_versions_csv.loc[
                vulnerable_versions_csv['name'] == current_dependency.library_name, "severity"].to_numpy()

            for vulnerable_version_range in matched_vulnerable_version_ranges:
                vulnerability_id = matched_vulnerable_ids[vulnerability_position]
                if is_vulnerable(dependency_version_clean,
                                 vulnerable_version_range) and vulnerability_id not in no_duplicated_vulnerabilities:
                    previous_dependencies_affected.append(dependency.Dependency(current_dependency.library_name,vulnerability_id))
                    if vulnerability_id not in vulnerabilities_log:
                        if vulnerability_id not in last_commit_vulnerabilities:
                            new_vulnerabilities.append(vulnerability_id)
                    elif vulnerability_id in last_commit_vulnerabilities:
                        kept_vulnerabilities.append(vulnerability_id)
                    if vulnerability_id in vulnerabilities_log and vulnerability_id not in last_commit_vulnerabilities:
                        if vulnerability_id not in historial_fixes and vulnerability_id in removed_vulnerabilities_historial:
                            revoked_removed_vulnerabilities.append(vulnerability_id)
                            removed_vulnerabilities_historial.remove(vulnerability_id)
                        elif  vulnerability_id in historial_fixes and vulnerability_id not in removed_vulnerabilities_historial:
                            revoked_fixed_vulnerabilities.append(vulnerability_id)
                            historial_fixes.remove(vulnerability_id)
                    if vulnerability_id not in vulnerabilities_log:
                        vulnerabilities_log.append(vulnerability_id)
                    no_duplicated_vulnerabilities.append(vulnerability_id)
                    severity_records.append(str(matched_severity_levels[vulnerability_position]))

            for vul_id in new_vulnerabilities:
                if vul_id not in total_commit_vulnerabilities:
                    total_commit_vulnerabilities.append(vul_id)

            for vul_id in kept_vulnerabilities:
                if vul_id not in total_commit_vulnerabilities:
                    total_commit_vulnerabilities.append(vul_id)

            for vul_id in revoked_fixed_vulnerabilities:
                if vul_id not in total_commit_vulnerabilities:
                    total_commit_vulnerabilities.append(vul_id)

            for vul_id in revoked_removed_vulnerabilities:
                if vul_id not in total_commit_vulnerabilities:
                    total_commit_vulnerabilities.append(vul_id)

    except:
        error = sys.exc_info()[0]
        error_details = sys.exc_info()[1]
        print("Error: \n%s" % error)
        print("Error: \n%s" % error_details)

Dependency

class dependency.Dependency(library_name, version)

Bases: object

Conceptual class to represent each dependency.

Parameters:
  • library_name (str) – Name of a library.
  • version (str) – Version range of a library.
Source code
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
    """Conceptual class to represent each dependency.

        :param library_name: Name of a library.
        :type library_name: str

        :param version: Version range of a library.
        :type version: str
        """
    def __init__(self, library_name, version):
        """Constructor method
        """
        self.library_name = library_name
        self.version = version

Directories Manager

directories_manager.download_file(repo, commit_hexsha, file_name)

To Download file if it is presented in a repository commit.

Parameters:
  • repo (str) – Repository name that contains the commit.
  • commit_hexsha (str) – Commit where the file is searched.
  • file_name (str) – Name of the file to download.
Returns:

It returns ‘True’ if file exists in the commit and it can be downloaded, if not it returns ‘False’.
Return type:

Repo
directories_manager.first_level_dependencies_tree(repo, commit)

It downloads ‘package.json’, ‘package-lock.json’ and ‘npm-shrinkwrap.json’, ‘yarn.lock’ if it is possible. If’package.json’ was downloaded, it will generate the first level npm dependencies tree. If ‘yarn.lock’ was downloaded, it will create the first level yarn dependencies tree.

param repo:Repository that contains the commit that it is being analyzed.
type repo:Repo
param commit:Commit which dependencies first level tree is being built.
type commit:Commit
directories_manager.new_cloned_git_folder()

It deletes ‘cloned git’ folder, if possible. It generates a new ‘cloned git’ folder.

Parameters:file_directory (str) – Path where is the wanted to be delete file.
directories_manager.remove_file(file_directory)

To remove file.

Parameters:file_directory (str) – Path where is the wanted to be delete file.
Source code
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
def download_file(repo, commit_hexsha, file_name):
    """ To Download file if it is presented in a repository commit.

    :param repo: Repository name that contains the commit.
    :type repo: str

    :param commit_hexsha: Commit where the file is searched.
    :type commit_hexsha: str

    :param file_name: Name of the file to download.
    :type file_name: str

    :return: It returns 'True' if file exists in the commit and it can be downloaded, if not it returns 'False'.
    :rtype: Repo
    """
    try:
        repo.git.checkout(commit_hexsha, file_name)
        if path.exists("../cloned_git/" + file_name):
            return True
        return False
    except:
        # print("         INFO: It doesn't contain '" + file_name + "'")
        return False
        pass


def remove_file(file_directory):
    """ To remove file.

    :param file_directory: Path where is the wanted to be delete file.
    :type file_directory: str
    """
    try:
        os.remove(file_directory)
    except:
        pass


def new_cloned_git_folder():
    """ It deletes 'cloned git' folder, if possible.
    It generates a new 'cloned git' folder.

    :param file_directory: Path where is the wanted to be delete file.
    :type file_directory: str
    """
    try:
        shutil.rmtree("../cloned_git")
    except OSError as e:
        print("Error: %s - %s." % (e.filename, e.strerror))
    os.mkdir("../cloned_git")


def first_level_dependencies_tree(repo, commit):
    """ It downloads 'package.json', 'package-lock.json' and 'npm-shrinkwrap.json',
    'yarn.lock' if it is possible. If'package.json' was downloaded, it will generate the
    first level npm dependencies tree. If 'yarn.lock' was downloaded, it will create
    the first level yarn dependencies tree.

     :param repo: Repository that contains the commit that it is being analyzed.
     :type repo: Repo

     :param commit: Commit which dependencies first level tree is being built.
     :type commit: Commit
     """
    has_package_file = download_file(repo, commit.hexsha, 'package.json')
    download_file(repo, commit.hexsha, 'package-lock.json')
    download_file(repo, commit.hexsha, 'npm-shrinkwrap.json')
    has_yarn_dependencies = download_file(repo, commit.hexsha, 'yarn.lock')

    if has_package_file:
        scripts_manager.collect_npm_dependencies()

    if has_yarn_dependencies:
        scripts_manager.collect_yarn_dependencies()

    for file in ["../cloned_git/package.json", "../cloned_git/yarn.lock", "../cloned_git/package-lock.json",
                 "../cloned_git/package-lock.json"]:
        remove_file(file)

Git Manager

Source code
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48


def diff_in_dependencies(current_commit, previous_commit):
    """ It compares the current commit with the previous. To check if'package.json',
    'package-lock.json', 'npm-shrinkwrap.json' or 'yarn.lock' were changed.

    :param current_commit: Current commit.
    :type current_commit: Commit

    :param previous_commit: Previous commit.
    :type previous_commit: Commit

    :return: It returns 'True' if 'package.json', 'package-lock.json', 'npm-shrinkwrap.json' or 'yarn.lock' was modified in the current commit, if not it will return 'False'.
    :rtype: bool
    """
    diff_index = previous_commit.diff(current_commit)
    for diff in diff_index:
        if "package.json" in {diff.b_path} or "package.json" in {diff.a_path}:
            return True
        if "package-lock.json" in {diff.b_path} or "package-lock.json" in {diff.a_path}:
            return True
        if "npm-shrinkwrap.json" in {diff.b_path} or "npm-shrinkwrap.json" in {diff.a_path}:
            return True
        if "yarn.lock" in {diff.b_path} or "yarn.lock" in {diff.a_path}:
            return True
    return False


def clone_repo(name, owner, directory, id):
    """ To clone repository into '/cloned_git'

    :param name: Repository name.
    :type name: str

    :param owner: Repository owner.
    :type owner: str

    :param directory: Directory were the repository is cloned.
    :type directory: str

    :return: It returns an instance of the cloned repository.
    :rtype: Repo
    """
    print("\n~~ INFO:Cloning '" + owner + "/" + name + ".git' into '" + directory[:-3] + "cloned_git'")

    try:
        repo = Repo.clone_from(
        url="https://github.com/" + owner + "/" + name,

Helpers

Source code
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76

from src import dependencies_manager, directories_manager
from src.dependencies_manager import clear_dictionaries


def prepare_repo_environment():
    """ It clears 'total commit' and 'log' lists. It replaces the 'output' folder
    for a new one. After that the environment is ready for a new repository analysis.
        """
    clear_dictionaries()
    dependencies_manager.total_commit_vulnerabilities.clear()
    dependencies_manager.vulnerabilities_log.clear()
    dependencies_manager.historial_fixes.clear()
    dependencies_manager.removed_vulnerabilities_historial.clear()
    directories_manager.new_cloned_git_folder()


def prepare_commit_environment(repo, commit):
    """ It clears 'severity records' and 'no duplicated vulnerabilities' lists.
    After that it generates the 'first level dependencies tree' to extract the
    dependencies presented on the current commit.

    :param repo: Repository to prepare.
    :type repo: Repo

    :param commit: Current commit.
    :type commit: Commit
    """

    dependencies_manager.severity_records.clear()
    dependencies_manager.no_duplicated_vulnerabilities.clear()
    directories_manager.first_level_dependencies_tree(repo, commit)
    return False


def has_config_file(repo, commit):
    """ It sees if Dependabot's configuration is presented on a specific repository and commit.

    :param repo: Repository where the commit is.
    :type repo: Repo

    :param commit: Commit where it is searched the Dependabot's configuration file.
    :type commit: Commit

    :return It returns 'True' if Dependabot's configuration file is presented. If not it will return 'False'.
    :rtype
    """
    if directories_manager.download_file(repo, commit.hexsha, ".github/dependabot.yml"):
        return True
    return False


def print_current_time(state):
    """ It prints current time.
    :param state: It indicates if it has started o stopped recording.
    :type state: str
    """
    now = datetime.now()
    current_time = now.strftime("%Y-%m-%d %H:%M:%S")
    print(state +" recording - " + current_time)


def write_failed_repos_log(repo_id, repo_url):
    """ It records ID and url of repositories that were not downloaded properly.

    :param repo_id: Repository ID.
    :type repo_id: int

    :param repo_url: Repository url.
    :type repo_url: str
    """
    with open('../output/failed_repos.txt', 'a') as f1:
        failed_repo = str(repo_id) + "-" + repo_url
        f1.write(failed_repo + os.linesep)

    error = sys.exc_info()[0]

Scripts Manager

scripts_manager.collect_npm_dependencies()

It calls ‘npm_dependencies.sh’ to collect npm dependencies.

scripts_manager.collect_yarn_dependencies()

It calls ‘yarn_dependencies.sh’ to collect yarn dependencies.

Source code
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
def collect_npm_dependencies():
    """ It calls 'npm_dependencies.sh' to collect npm dependencies.
        """
    subprocess.call(['sh', '../scripts/npm_dependencies.sh'], stdout=open(os.devnull, 'w'), stderr=subprocess.STDOUT)


def collect_yarn_dependencies():
    """ It calls 'yarn_dependencies.sh' to collect yarn dependencies.
        """
    subprocess.call(['sh', '../scripts/yarn_dependencies.sh'], stdout=open(os.devnull, 'w'), stderr=subprocess.STDOUT)

Indices and tables