Skip to content

Commit

Permalink
chg: make all vuln IDs lowercase.
Browse files Browse the repository at this point in the history
  • Loading branch information
@Rafiot
Rafiot committed Dec 1, 2023
1 parent 79aebe4 commit 5037e83
Show file tree
Hide file tree
Showing 12 changed files with 127 additions and 74 deletions.
2 changes: 1 addition & 1 deletion vulnerabilitylookup/feeders/cisa_known_exploited.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ def update(self) -> bool:

index_cisa: Dict[str, float] = {}
for exploited_vuln in known_exploited_vulnerabilities['vulnerabilities']:
cve_id = exploited_vuln['cveID']
cve_id = exploited_vuln['cveID'].lower()
# do we already have a meta entry for this source?
if _meta_uuid := self.storage.hget(f'{cve_id}:meta', self.name):
meta_uuid = _meta_uuid.decode()
Expand Down
2 changes: 1 addition & 1 deletion vulnerabilitylookup/feeders/cvelistv5
Submodule cvelistv5 updated 249 files
58 changes: 33 additions & 25 deletions vulnerabilitylookup/feeders/cvelistv5.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,32 +57,40 @@ def update(self) -> bool:
else: # 'dateReserved' in vuln['cveMetadata']:
updated = fromisoformat_wrapper(vuln['cveMetadata']['dateReserved'])

cvelistv5ids[path.stem] = updated.timestamp()
p.set(path.stem, json.dumps(vuln))
# Check if we have a link with another known source we can link to
if ('containers' in vuln
and 'cna' in vuln['containers']
and 'source' in vuln['containers']['cna']
and 'advisory' in vuln['containers']['cna']['source']):
if vuln['containers']['cna']['source']['advisory'].startswith('GHSA'):
# got a github security advisory.
p.sadd(f'{path.stem}:link', vuln['containers']['cna']['source']['advisory'])
p.sadd(f"{vuln['containers']['cna']['source']['advisory']}:link", path.stem)
else:
self.logger.debug(f"[{path.stem}] Unknown advisory ID: {vuln['containers']['cna']['source']['advisory']}")

# Load affected products
vuln_id = path.stem.lower()

cvelistv5ids[vuln_id] = updated.timestamp()
p.set(vuln_id, json.dumps(vuln))
if ('containers' in vuln
and 'cna' in vuln['containers']
and 'affected' in vuln['containers']['cna']):
if ('vendor' in vuln['containers']['cna']['affected']
and 'product' in vuln['containers']['cna']['affected']):
vendor = vuln['containers']['cna']['affected']['vendor']
product = vuln['containers']['cna']['affected']['product']
p.sadd('vendors', vendor)
p.sadd(f'{vendor}:products', product)
p.sadd(f'{vendor}:vulnerabilities', path.stem)
p.sadd(f'{vendor}:{product}:vulnerabilities', path.stem)
and 'cna' in vuln['containers']):
# Check if we have a link with another known source we can link to
if ('source' in vuln['containers']['cna']
and 'advisory' in vuln['containers']['cna']['source']):
advisory = vuln['containers']['cna']['source']['advisory'].lower()
if advisory.startswith('ghsa'):
# got a github security advisory.
p.sadd(f'{vuln_id}:link', advisory)
p.sadd(f"{advisory}:link", vuln_id)
else:
self.logger.debug(f"[{vuln_id}] Unknown advisory ID: {advisory}")

if 'affected' in vuln['containers']['cna']:
for affected in vuln['containers']['cna']['affected']:
# Load affected products
if 'vendor' in affected and 'product' in affected:
vendor = affected['vendor'].strip().lower()
product = affected['product'].strip().lower()
if not vendor or not product:
# empty, or only spaces
continue
if 'n/a' in vendor or 'n/a' in product:
continue
p.sadd('vendors', vendor)
p.sadd(f'{vendor}:products', product)
p.sadd(f'{vendor}:vulnerabilities', vuln_id)
p.sadd(f'{vendor}:{product}:vulnerabilities', vuln_id)
else:
self.logger.info(f"[{vuln_id}] Missing vendor/product: {affected}")

if len(cvelistv5ids) > 1000:
# Avoid a massive execute on first import
Expand Down
2 changes: 1 addition & 1 deletion vulnerabilitylookup/feeders/github
Submodule github updated 211 files
10 changes: 6 additions & 4 deletions vulnerabilitylookup/feeders/github.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,12 +58,14 @@ def update(self) -> bool:
with path.open() as vuln_entry:
vuln = json.load(vuln_entry)
modified = fromisoformat_wrapper(vuln['modified'])
gsids[path.stem] = modified.timestamp()
vuln_id = path.stem.lower()
gsids[vuln_id] = modified.timestamp()
if 'aliases' in vuln and vuln.get('aliases'):
for alias in vuln.get('aliases'):
p.sadd(f'{path.stem}:link', alias)
p.sadd(f'{alias}:link', path.stem)
p.set(path.stem, json.dumps(vuln))
a = alias.lower()
p.sadd(f'{vuln_id}:link', a)
p.sadd(f'{a}:link', vuln_id)
p.set(vuln_id, json.dumps(vuln))
if len(gsids) > 1000:
# Avoid a massive execute on first import
p.zadd(f'index:{self.name}', gsids) # type: ignore
Expand Down
2 changes: 1 addition & 1 deletion vulnerabilitylookup/feeders/gsd
Submodule gsd updated 680 files
12 changes: 7 additions & 5 deletions vulnerabilitylookup/feeders/gsd.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,6 @@ def update(self) -> bool:
if not last_modified and 'OSV' in vuln:
if 'modified' in vuln['OSV']:
# Python < 3.11 cannot load times with a Z instead of +00:00

last_modified = fromisoformat_wrapper(vuln['OSV']['modified'])
elif 'withdrawn' in vuln['OSV']:
last_modified = fromisoformat_wrapper(vuln['OSV']['withdrawn'])
Expand All @@ -112,11 +111,14 @@ def update(self) -> bool:
commit = next(self.git.iter_commits(max_count=1, paths=path))
last_modified = commit.committed_datetime

gsids[path.stem] = last_modified.timestamp()
vuln_id = path.stem.lower()

gsids[vuln_id] = last_modified.timestamp()
if 'GSD' in vuln and 'alias' in vuln['GSD']:
p.sadd(f'{path.stem}:link', vuln['GSD']['alias'])
p.sadd(f'{vuln["GSD"]["alias"]}:link', path.stem)
p.set(path.stem, json.dumps(vuln))
a = vuln['GSD']['alias'].lower()
p.sadd(f'{vuln_id}:link', a)
p.sadd(f'{a}:link', vuln_id)
p.set(vuln_id, json.dumps(vuln))

if len(gsids) > 1000:
# Avoid a massive execute on first import
Expand Down
45 changes: 26 additions & 19 deletions vulnerabilitylookup/feeders/nvd.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,36 +60,43 @@ def update(self) -> bool:
p = self.storage.pipeline()
cves: Dict[str, float] = {}
for vuln in nvd_value['vulnerabilities']:
cve_id = vuln['cve']['id']
cve_id = vuln['cve']['id'].lower()
# do we already have a meta entry for this source?
if _meta_uuid := self.storage.hget(f'{cve_id}:meta', self.name):
meta_uuid = _meta_uuid.decode()
else:
meta_uuid = str(uuid4())
self.storage.hset(f'{cve_id}:meta', mapping={self.name: meta_uuid})
self.storage.set(f'{self.name}:{meta_uuid}', json.dumps(vuln))
cves[vuln['cve']['id']] = fromisoformat_wrapper(vuln['cve']['lastModified']).timestamp()
p.zadd(f'index:{self.name}', cves) # type: ignore

if not self.storage.exists(vuln['cve']['id']):
self.logger.warning(f"{vuln['cve']['id']} is missing.")
cves[cve_id] = fromisoformat_wrapper(vuln['cve']['lastModified']).timestamp()
if not self.storage.exists(cve_id):
self.logger.warning(f"{cve_id} is missing.")

if 'configurations' in vuln['cve']:
for configuration in vuln['cve']['configurations']:
if 'nodes' not in configuration:
continue
for node in configuration['nodes']:
if 'cpeMatch' not in node:
if 'configurations' in vuln['cve']:
for configuration in vuln['cve']['configurations']:
if 'nodes' not in configuration:
continue
for cpematch in node['cpeMatch']:
if 'criteria' in cpematch:
_, _, _, vendor, product, _ = cpematch['criteria'].split(':', 5)
p.sadd('vendors', vendor)
p.sadd(f'{vendor}:products', product)
p.sadd(f'{vendor}:vulnerabilities', vuln['cve']['id'])
p.sadd(f'{vendor}:{product}:vulnerabilities', vuln['cve']['id'])
for node in configuration['nodes']:
if 'cpeMatch' not in node:
continue
for cpematch in node['cpeMatch']:
if 'criteria' in cpematch:
_, _, _, vendor, product, _ = cpematch['criteria'].split(':', 5)
vendor = vendor.strip().lower()
product = product.strip().lower()
if not vendor or not product:
# empty, or only spaces
continue
if 'n/a' in vendor or 'n/a' in product:
continue
p.sadd('vendors', vendor)
p.sadd(f'{vendor}:products', product)
p.sadd(f'{vendor}:vulnerabilities', cve_id)
p.sadd(f'{vendor}:{product}:vulnerabilities', cve_id)

p.zadd(f'index:{self.name}', cves) # type: ignore
p.execute()

if results_per_page < max_results_per_page:
break
self.storage.hset('last_updates', mapping={self.name: last_update.isoformat()})
Expand Down
9 changes: 5 additions & 4 deletions vulnerabilitylookup/feeders/pysec.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,12 +79,13 @@ def update(self) -> bool:

commit = next(self.git.iter_commits(max_count=1, paths=path))
last_modified = commit.committed_datetime

pysecids[vuln['id']] = last_modified.timestamp()
vuln_id = vuln['id'].lower()
pysecids[vuln_id] = last_modified.timestamp()
if 'aliases' in vuln:
for alias in vuln['aliases']:
p.sadd(f"{vuln['id']}:link", alias)
p.sadd(f'{alias}:link', vuln['id'])
a = alias.lower()
p.sadd(f"{vuln_id}:link", a)
p.sadd(f'{a}:link', vuln_id)
p.set(path.stem, json.dumps(vuln, default=json_serial))

if len(pysecids) > 1000:
Expand Down
26 changes: 16 additions & 10 deletions website/web/templates/recent.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,10 @@ <h6>The vulnerabilities are sorted by update time (recent ot old)</h6>
</ul>
<div class="tab-content" id="vulnSourcesTabContent">
{% for source, vulns in recent.items() %}
<div class="tab-pane fade show {% if source == 'nvd' %} active {%endif%}"
<div class="tab-pane fade show {% if source == 'cvelistv5' %} active {%endif%}"
id="{{source}}-tab-pane" role="tabpanel" aria-labelledby="{{source}}-tab" tabindex="0">
<table class="table">
{% if source in ['nvd', 'gad'] %}
{% if source in ['cvelistv5', 'github'] %}
<thead>
<tr>
<th scope="col">Vulnerability ID</th>
Expand All @@ -45,14 +45,20 @@ <h6>The vulnerabilities are sorted by update time (recent ot old)</h6>
{%endif%}
<tbody>
{%for vuln in vulns %}
{% if source == "nvd"%}
<tr>
<th scope="row"><a href="https://nvd.nist.gov/vuln/detail/{{vuln['cve']['id']}}">
{{vuln['cve']['id']}}</a>
</th>
<td>{{vuln['cve']['descriptions'][0]['value']}}</td>
</tr>
{%elif source == "gad" %}
{%if source == "cvelistv5" %}
<tr>
<th scope="row">
<a href="https://nvd.nist.gov/vuln/detail/{{vuln['cveMetadata']['cveId']}}">{{vuln['cveMetadata']['cveId']}}</a>
</th>
<td>
{% if vuln['containers']['cna']['title'] %}
{{vuln['containers']['cna']['title']}}
{% else %}
{{vuln['containers']['cna']['descriptions'][0]['value']}}
{%endif%}
</td>
</tr>
{%elif source == "github" %}
<tr>
<th scope="row">
<a href="https://github.com/advisories/{{vuln['id']}}">{{vuln['id']}}</a>
Expand Down
12 changes: 9 additions & 3 deletions website/web/templates/search.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,9 @@
{% if source %}
{% if source == 'nvd' %}
{{vuln_templates.nvd_view(source=source, vulnerability_id=vulnerability_id, vulnerability_data=vulnerability_data)}}
{% elif source == 'gad' %}
{% elif source == 'cvelistv5' %}
{{vuln_templates.cvelistv5_view(source=source, vulnerability_id=vulnerability_id, vulnerability_data=vulnerability_data)}}
{% elif source == 'github' %}
{{vuln_templates.github_view(source=source, vulnerability_id=vulnerability_id, vulnerability_data=vulnerability_data)}}
{% elif source == 'gsd' %}
{{vuln_templates.gsd_view(source=source, vulnerability_id=vulnerability_id, vulnerability_data=vulnerability_data)}}
Expand All @@ -53,7 +55,9 @@ <h5>Vulnerabilites related to the one you searched</h5>
{% for vuln_id, vulnerability_data in vulnerabilities %}
{% if source == 'nvd' %}
{{vuln_templates.nvd_view(source=source, vulnerability_id=vuln_id, vulnerability_data=vulnerability_data)}}
{% elif source == 'gad' %}
{% elif source == 'cvelistv5' %}
{{vuln_templates.cvelistv5_view(source=source, vulnerability_id=vuln_id, vulnerability_data=vulnerability_data)}}
{% elif source == 'github' %}
{{vuln_templates.github_view(source=source, vulnerability_id=vuln_id, vulnerability_data=vulnerability_data)}}
{% elif source == 'gsd' %}
{{vuln_templates.gsd_view(source=source, vulnerability_id=vuln_id, vulnerability_data=vulnerability_data)}}
Expand Down Expand Up @@ -95,7 +99,9 @@ <h5>All the vulnerabilites related to {{vendor}} - {{product}}</h5>
{% for vuln_id, vulnerability_data in vulnerabilities %}
{% if source == 'nvd' %}
{{vuln_templates.nvd_view(source=source, vulnerability_id=vuln_id, vulnerability_data=vulnerability_data)}}
{% elif source == 'gad' %}
{% elif source == 'cvelistv5' %}
{{vuln_templates.cvelistv5_view(source=source, vulnerability_id=vuln_id, vulnerability_data=vulnerability_data)}}
{% elif source == 'github' %}
{{vuln_templates.github_view(source=source, vulnerability_id=vuln_id, vulnerability_data=vulnerability_data)}}
{% elif source == 'gsd' %}
{{vuln_templates.gsd_view(source=source, vulnerability_id=vuln_id, vulnerability_data=vulnerability_data)}}
Expand Down
21 changes: 21 additions & 0 deletions website/web/templates/vulnerability_templates.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,27 @@ <h6 class="card-subtitle mb-2 text-body-secondary">
</div>
{%- endmacro %}

{% macro cvelistv5_view(source, vulnerability_id, vulnerability_data) -%}
<div class="card">
<div class="card-body">
<h5 class="card-title">{{vulnerability_id}}</h5>
<h6 class="card-subtitle mb-2 text-body-secondary">
Vulnerability from <a href="https://github.com/CVEProject/cvelistV5">{{source}}</a>
</h6>
{% if vulnerability_data['containers']['cna']['title'] %}
<p class="card-text">{{vulnerability_data['containers']['cna']['title']}}</p>
{% else %}
<p class="card-text">{{vulnerability_data['containers']['cna']['descriptions'][0]['value']}}</p>
{%endif%}
{%if 'cisa_known_exploited' in vulnerability_data['meta']%}
{{ cisa_known_exploited_view(vulnerability_data['meta']['cisa_known_exploited']) }}
{%endif%}
<a href="https://nvd.nist.gov/vuln/detail/{{vulnerability_id}}" class="card-link">Show details on NVD website</a>
<pre>{{vulnerability_data|tojson(indent=2)}}</pre>
</div>
</div>
{%- endmacro %}

{% macro github_view(source, vulnerability_id, vulnerability_data) -%}
<div class="card">
<div class="card-body">
Expand Down

0 comments on commit 5037e83

Please sign in to comment.