Skip to content

Commit

Permalink
updated collection methods and brought cpe conversion in line with cv…
Browse files Browse the repository at this point in the history 
…esearch
  • Loading branch information
@P-T-I
P-T-I committed Sep 7, 2022
1 parent 7281997 commit 70c55a2
Show file tree
Hide file tree
Showing 7 changed files with 228 additions and 170 deletions.
2 changes: 1 addition & 1 deletion CveXplore/VERSION
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0.2.7.1
0.2.8
16 changes: 3 additions & 13 deletions CveXplore/cli_cmds/db_cmds/commands.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,15 +40,9 @@ def show_cmd(ctx, pretty):
config = Configuration()

if ctx.invoked_subcommand is None:
printer(
input_data=config.SOURCES,
pretty=pretty,
)
printer(input_data=config.SOURCES, pretty=pretty)
else:
printer(
input_data=config.SOURCES,
pretty=pretty,
)
printer(input_data=config.SOURCES, pretty=pretty)


@sources_cmd.group("set", invoke_without_command=True, help="Set sources")
Expand All @@ -58,11 +52,7 @@ def show_cmd(ctx, pretty):
help="Set the source key",
type=click.Choice(["capec", "cpe", "cwe", "via4", "cves"], case_sensitive=False),
)
@click.option(
"-v",
"--value",
help="Set the source key value",
)
@click.option("-v", "--value", help="Set the source key value")
@click.pass_context
def set_cmd(ctx, key, value):
config = Configuration()
Expand Down
33 changes: 7 additions & 26 deletions CveXplore/common/cpe_converters.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@
CPE Converters
==============
"""
from urllib.parse import unquote

from CveXplore.database.maintenance import cpe_conversion


def from2to3CPE(cpe, autofill=False):
Expand All @@ -19,19 +20,12 @@ def from2to3CPE(cpe, autofill=False):
cpe = cpe.strip()
if not cpe.startswith("cpe:2.3:"):
if not cpe.startswith("cpe:/"):
# can not do anything with this; returning original string
return cpe
cpe = cpe.replace("cpe:/", "cpe:2.3:")
cpe = cpe.replace("::", ":-:")
cpe = cpe.replace("~-", "~")
cpe = cpe.replace("~", ":-:")
cpe = cpe.replace("::", ":")
cpe = cpe.strip(":-")
cpe = unquote(cpe)
return False
cpe = cpe_conversion.cpe_uri_to_fs(cpe)
if autofill:
e = cpe.split(":")
for x in range(0, 13 - len(e)):
cpe += ":*"
cpe += ":-"
return cpe


Expand All @@ -47,21 +41,8 @@ def from3to2CPE(cpe):
cpe = cpe.strip()
if not cpe.startswith("cpe:/"):
if not cpe.startswith("cpe:2.3:"):
# can not do anything with this; returning original string
return cpe
cpe = cpe.replace("cpe:2.3:", "")
parts = cpe.split(":")
next = []
first = "cpe:/" + ":".join(parts[:5])
last = parts[5:]
if last:
for x in last:
next.append("~") if x == "-" else next.append(x)
if "~" in next:
pad(next, 6, "~")
cpe = "%s:%s" % (first, "".join(next))
cpe = cpe.replace(":-:", "::")
cpe = cpe.strip(":")
return False
cpe = cpe_conversion.cpe_fs_to_uri(cpe)
return cpe


Expand Down
6 changes: 3 additions & 3 deletions CveXplore/database/maintenance/DownloadHandler.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -338,12 +338,12 @@ def getTableNames(self):
return self.database.list_collection_names()

def setColInfo(self, collection, field, data):
self.database[collection].update(
self.database[collection].update_one(
{"db": collection}, {"$set": {field: data}}, upsert=True
)

def delColInfo(self, collection):
self.database["info"].remove({"db": collection})
self.database["info"].delete_one({"db": collection})

def getCPEVersionInformation(self, query):
return self.sanitize(self.database["cpe"].find_one(query))
Expand All @@ -362,7 +362,7 @@ def sanitize(self, x):
return x

def setColUpdate(self, collection, date):
self.database["info"].update(
self.database["info"].update_one(
{"db": collection}, {"$set": {"last-modified": date}}, upsert=True
)

Expand Down
92 changes: 21 additions & 71 deletions CveXplore/database/maintenance/Sources_process.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -878,25 +878,12 @@ def __init__(self):

self.indexes = {
"cpe": [
MongoUniqueIndex(
index=[("id", ASCENDING)],
name="id",
unique=True,
),
MongoAddIndex(
index=[("vendor", ASCENDING)], name="vendor"
),
MongoAddIndex(
index=[("product", ASCENDING)],
name="product",
),
MongoUniqueIndex(index=[("id", ASCENDING)], name="id", unique=True),
MongoAddIndex(index=[("vendor", ASCENDING)], name="vendor"),
MongoAddIndex(index=[("product", ASCENDING)], name="product"),
],
"cpeother": [
MongoUniqueIndex(
index=[("id", ASCENDING)],
name="id",
unique=True,
)
MongoUniqueIndex(index=[("id", ASCENDING)], name="id", unique=True)
],
"cves": [
MongoAddIndex(index=[("id", ASCENDING)], name="id"),
Expand All @@ -905,38 +892,18 @@ def __init__(self):
name="vulnerable_configuration",
),
MongoAddIndex(
index=[("vulnerable_product", ASCENDING)],
name="vulnerable_product",
),
MongoAddIndex(
index=[("Modified", ASCENDING)],
name="Modified",
),
MongoAddIndex(
index=[("Published", ASCENDING)],
name="Published",
),
MongoAddIndex(
index=[("last-modified", ASCENDING)],
name="last-modified",
),
MongoAddIndex(
index=[("cvss", ASCENDING)], name="cvss"
index=[("vulnerable_product", ASCENDING)], name="vulnerable_product"
),
MongoAddIndex(index=[("Modified", ASCENDING)], name="Modified"),
MongoAddIndex(index=[("Published", ASCENDING)], name="Published"),
MongoAddIndex(
index=[("cvss3", ASCENDING)], name="cvss3"
),
MongoAddIndex(
index=[("summary", TEXT)], name="summary"
),
MongoAddIndex(
index=[("vendors", ASCENDING)],
name="vendors",
),
MongoAddIndex(
index=[("products", ASCENDING)],
name="products",
index=[("last-modified", ASCENDING)], name="last-modified"
),
MongoAddIndex(index=[("cvss", ASCENDING)], name="cvss"),
MongoAddIndex(index=[("cvss3", ASCENDING)], name="cvss3"),
MongoAddIndex(index=[("summary", TEXT)], name="summary"),
MongoAddIndex(index=[("vendors", ASCENDING)], name="vendors"),
MongoAddIndex(index=[("products", ASCENDING)], name="products"),
MongoAddIndex(
index=[("vulnerable_product_stems", ASCENDING)],
name="vulnerable_product_stems",
Expand All @@ -946,19 +913,12 @@ def __init__(self):
name="vulnerable_configuration_stems",
),
],
"via4": [
MongoAddIndex(index=[("id", ASCENDING)], name="id")
],
"mgmt_whitelist": [
MongoAddIndex(index=[("id", ASCENDING)], name="id")
],
"mgmt_blacklist": [
MongoAddIndex(index=[("id", ASCENDING)], name="id")
],
"via4": [MongoAddIndex(index=[("id", ASCENDING)], name="id")],
"mgmt_whitelist": [MongoAddIndex(index=[("id", ASCENDING)], name="id")],
"mgmt_blacklist": [MongoAddIndex(index=[("id", ASCENDING)], name="id")],
"capec": [
MongoAddIndex(
index=[("related_weakness", ASCENDING)],
name="related_weakness",
index=[("related_weakness", ASCENDING)], name="related_weakness"
)
],
}
Expand All @@ -985,15 +945,10 @@ def create_indexes(self, collection=None):
for each in self.indexes[collection]:
if isinstance(each, MongoUniqueIndex):
self.setIndex(
collection,
each.index,
name=each.name,
unique=each.unique,
collection, each.index, name=each.name, unique=each.unique
)
elif isinstance(each, MongoAddIndex):
self.setIndex(
collection, each.index, name=each.name
)
self.setIndex(collection, each.index, name=each.name)
except KeyError:
# no specific index given, continue
self.logger.warning(
Expand All @@ -1011,15 +966,10 @@ def create_indexes(self, collection=None):
for each in self.indexes[collection]:
if isinstance(each, MongoUniqueIndex):
self.setIndex(
collection,
each.index,
name=each.name,
unique=each.unique,
collection, each.index, name=each.name, unique=each.unique
)
elif isinstance(each, MongoAddIndex):
self.setIndex(
collection, each.index, name=each.name
)
self.setIndex(collection, each.index, name=each.name)

def iter_indexes(self):
for each in self.get_via4_indexes():
Expand Down
58 changes: 2 additions & 56 deletions CveXplore/database/maintenance/Toolkit.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,59 +3,7 @@
import dateutil.parser
from dateutil import tz


# Note of warning: CPEs like cpe:/o:microsoft:windows_8:-:-:x64 are given to us by Mitre
# x64 will be parsed as Edition in this case, not Architecture
def toStringFormattedCPE(cpe, autofill=False):
cpe = cpe.strip()
if not cpe.startswith("cpe:2.3:"):
if not cpe.startswith("cpe:/"):
return False
cpe = cpe.replace("cpe:/", "cpe:2.3:")
cpe = cpe.replace("::", ":-:")
cpe = cpe.replace("~-", "~")
cpe = cpe.replace("~", ":-:")
cpe = cpe.replace("::", ":")
cpe = cpe.strip(":-")
cpe = unquote(cpe)
if autofill:
e = cpe.split(":")
for x in range(0, 13 - len(e)):
cpe += ":-"
return cpe


# Note of warning: Old CPE's can come in different formats, and are not uniform. Possibilities are:
# cpe:/a:7-zip:7-zip:4.65::~~~~x64~
# cpe:/a:7-zip:7-zip:4.65:-:~~~~x64~
# cpe:/a:7-zip:7-zip:4.65:-:~-~-~-~x64~
def toOldCPE(cpe):
cpe = cpe.strip()
if not cpe.startswith("cpe:/"):
if not cpe.startswith("cpe:2.3:"):
return False
cpe = cpe.replace("cpe:2.3:", "")
parts = cpe.split(":")
next = []
first = "cpe:/" + ":".join(parts[:5])
last = parts[5:]
if last:
for x in last:
next.append("~") if x == "-" else next.append(x)
if "~" in next:
pad(next, 6, "~")
cpe = "%s:%s" % (first, "".join(next))
cpe = cpe.replace(":-:", "::")
cpe = cpe.strip(":")
return cpe


def pad(seq, target_length, padding=None):
length = len(seq)
if length > target_length:
return seq
seq.extend([padding] * (target_length - length))
return seq
from CveXplore.database.maintenance import cpe_conversion


def currentTime(utc):
Expand Down Expand Up @@ -104,9 +52,7 @@ def tk_compile(regexes):
# Convert cpe2.2 url encoded to cpe2.3 char escaped
# cpe:2.3:o:cisco:ios:12.2%281%29 to cpe:2.3:o:cisco:ios:12.2\(1\)
def unquote(cpe):
return re.compile("%([0-9a-fA-F]{2})", re.M).sub(
lambda m: "\\" + chr(int(m.group(1), 16)), cpe
)
return cpe_conversion.unquote(cpe)


# Generates a human readable title from a CPE 2.3 string
Expand Down
Loading

0 comments on commit 70c55a2

Please sign in to comment.