diff --git a/Makefile b/Makefile index 60a33dc..2a9b032 100644 --- a/Makefile +++ b/Makefile @@ -19,11 +19,11 @@ dev: isort: @echo "-> Apply isort changes to ensure proper imports ordering" - ${VENV}/bin/isort --sl -l 100 src tests setup.py + ${VENV}/bin/isort --sl -l 100 --skip=src/fetchcode/vcs/pip setup.py src tests black: @echo "-> Apply black code formatter" - ${VENV}/bin/black -l 100 src tests setup.py + ${VENV}/bin/black -l 100 --exclude=src/fetchcode/vcs/pip src tests setup.py doc8: @echo "-> Run doc8 validation" @@ -33,11 +33,11 @@ valid: isort black check: @echo "-> Run pycodestyle (PEP8) validation" - @${ACTIVATE} pycodestyle --max-line-length=100 --exclude=.eggs,venv,lib,thirdparty,docs,migrations,settings.py,.cache . + @${ACTIVATE} pycodestyle --max-line-length=100 --exclude=.eggs,venv,lib,thirdparty,docs,migrations,settings.py,.cache,etc,src/fetchcode/vcs/pip,tests/data/ . @echo "-> Run isort imports ordering validation" - @${ACTIVATE} isort --sl --check-only -l 100 setup.py src tests . + @${ACTIVATE} isort --sl --check-only -l 100 --skip=src/fetchcode/vcs/pip setup.py src tests @echo "-> Run black validation" - @${ACTIVATE} black --check --check -l 100 src tests setup.py + @${ACTIVATE} black --check --check -l 100 --exclude=src/fetchcode/vcs/pip src tests setup.py clean: @echo "-> Clean the Python env" diff --git a/setup.cfg b/setup.cfg index f230c3d..60f0a03 100644 --- a/setup.cfg +++ b/setup.cfg @@ -84,3 +84,6 @@ docs = sphinx-rtd-dark-mode>=1.3.0 sphinx-copybutton +[pycodestyle] +max-line-length = 88 +ignore = E203,E701 diff --git a/src/fetchcode/__init__.py b/src/fetchcode/__init__.py index 115e4a7..5d05242 100644 --- a/src/fetchcode/__init__.py +++ b/src/fetchcode/__init__.py @@ -14,10 +14,10 @@ # CONDITIONS OF ANY KIND, either express or implied. See the License for the # specific language governing permissions and limitations under the License. -from ftplib import FTP -from mimetypes import MimeTypes import os import tempfile +from ftplib import FTP +from mimetypes import MimeTypes from urllib.parse import urlparse import requests @@ -40,19 +40,18 @@ def __init__(self, location, content_type, size, url): def fetch_http(url, location): """ - Return a `Response` object built from fetching the content at a HTTP/HTTPS based `url` URL string - saving the content in a file at `location` + Return a `Response` object built from fetching the content at a HTTP/HTTPS based + `url` URL string saving the content in a file at `location` """ r = requests.get(url) - with open(location, 'wb') as f: + with open(location, "wb") as f: f.write(r.content) - content_type = r.headers.get('content-type') - size = r.headers.get('content-length') + content_type = r.headers.get("content-type") + size = r.headers.get("content-length") size = int(size) if size else None - resp = Response(location=location, - content_type=content_type, size=size, url=url) + resp = Response(location=location, content_type=content_type, size=size, url=url) return resp @@ -80,19 +79,19 @@ def fetch_ftp(url, location): content_type = None ftp.cwd(dir) - file = 'RETR {}'.format(file) - with open(location, 'wb') as f: + file = "RETR {}".format(file) + with open(location, "wb") as f: ftp.retrbinary(file, f.write) ftp.close() - resp = Response(location=location, - content_type=content_type, size=size, url=url) + resp = Response(location=location, content_type=content_type, size=size, url=url) return resp def fetch(url): """ - Return a `Response` object built from fetching the content at the `url` URL string and store content at a temporary file. + Return a `Response` object built from fetching the content at the `url` URL string and + store content at a temporary file. """ temp = tempfile.NamedTemporaryFile(delete=False) @@ -101,9 +100,9 @@ def fetch(url): url_parts = urlparse(url) scheme = url_parts.scheme - fetchers = {'ftp': fetch_ftp, 'http': fetch_http, 'https': fetch_http} + fetchers = {"ftp": fetch_ftp, "http": fetch_http, "https": fetch_http} if scheme in fetchers: return fetchers.get(scheme)(url, location) - raise Exception('Not a supported/known scheme.') + raise Exception("Not a supported/known scheme.") diff --git a/src/fetchcode/package.py b/src/fetchcode/package.py index edba373..3755ae0 100644 --- a/src/fetchcode/package.py +++ b/src/fetchcode/package.py @@ -92,8 +92,7 @@ def get_cargo_data_from_purl(purl): ) versions = response.get("versions", []) for version in versions: - version_purl = PackageURL( - type=purl.type, name=name, version=version.get("num")) + version_purl = PackageURL(type=purl.type, name=name, version=version.get("num")) dl_path = version.get("dl_path") if dl_path: download_url = f"{base_url}/{dl_path}" @@ -145,9 +144,7 @@ def get_npm_data_from_purl(purl): tags = [] for num in versions: version = versions[num] - version_purl = PackageURL( - type=purl.type, name=name, version=version.get("version") - ) + version_purl = PackageURL(type=purl.type, name=name, version=version.get("version")) repository = version.get("repository") or {} bugs = response.get("bugs") or {} dist = version.get("dist") or {} @@ -309,12 +306,8 @@ def get_bitbucket_data_from_purl(purl): tags = tags_data.get("values") or {} for tag in tags: version = tag.get("name") or "" - version_purl = PackageURL( - type=purl.type, namespace=namespace, name=name, version=version - ) - download_url = ( - f"{base_path}/{namespace}/{name}/downloads/{name}-{version}.tar.gz" - ) + version_purl = PackageURL(type=purl.type, namespace=namespace, name=name, version=version) + download_url = f"{base_path}/{namespace}/{name}/downloads/{name}-{version}.tar.gz" code_view_url = f"{bitbucket_url}/{namespace}/{name}/src/{version}" yield Package( api_url=api_url, @@ -356,12 +349,9 @@ def get_gnu_data_from_purl(purl): purl = PackageURL.from_string(purl) source_archive_url = f"https://ftp.gnu.org/pub/gnu/{purl.name}/" version_regex_template = r"^({}-)(?P[\w.-]*)(.tar.gz)$" - version_regex = re.compile( - version_regex_template.format(re.escape(purl.name))) + version_regex = re.compile(version_regex_template.format(re.escape(purl.name))) - yield from extract_packages_from_listing( - purl, source_archive_url, version_regex, [] - ) + yield from extract_packages_from_listing(purl, source_archive_url, version_regex, []) @dataclasses.dataclass @@ -372,7 +362,9 @@ class DirectoryListedSource: is_nested: bool = dataclasses.field( default=False, metadata={ - "description": "Flag indicating whether the archives are nested within another directory" + "description": ( + "Flag indicating whether the archives are nested within another directory" + ) }, ) source_archive_regex: re.Pattern = dataclasses.field( @@ -410,9 +402,7 @@ def get_package_info(cls, package_url): # The udhcp is no longer maintained as a standalone project. # It has been fully integrated into busybox. class UdhcpDirectoryListedSource(DirectoryListedSource): - source_url = ( - "https://web.archive.org/web/20021209021312/http://udhcp.busybox.net/source/" - ) + source_url = "https://web.archive.org/web/20021209021312/http://udhcp.busybox.net/source/" @classmethod def get_package_info(cls, package_url): @@ -429,8 +419,7 @@ def get_package_info(cls, package_url): else: for version, data in UDHCP_RELEASES.items(): - purl = PackageURL( - type="generic", name="udhcp", version=version) + purl = PackageURL(type="generic", name="udhcp", version=version) yield Package( homepage_url=cls.source_url, download_url=data["url"], @@ -440,11 +429,11 @@ def get_package_info(cls, package_url): class IpkgDirectoryListedSource(DirectoryListedSource): - source_url = "https://web.archive.org/web/20090326020239/http://handhelds.org/download/packages/ipkg/" - is_nested = False - source_archive_regex = re.compile( - r"^(ipkg[-_])(?P[\w.-]*)(_arm.ipk|.tar.gz)$" + source_url = ( + "https://web.archive.org/web/20090326020239/http://handhelds.org/download/packages/ipkg/" ) + is_nested = False + source_archive_regex = re.compile(r"^(ipkg[-_])(?P[\w.-]*)(_arm.ipk|.tar.gz)$") ignored_files_and_dir = [] @classmethod @@ -484,8 +473,7 @@ class UtilLinuxDirectoryListedSource(DirectoryListedSource): class BusyBoxDirectoryListedSource(DirectoryListedSource): source_url = "https://www.busybox.net/downloads/" # Source archive ex: busybox-1.2.3.tar.bz2 - source_archive_regex = re.compile( - r"^(busybox-)(?P[\w.-]*)(.tar.bz2)$") + source_archive_regex = re.compile(r"^(busybox-)(?P[\w.-]*)(.tar.bz2)$") is_nested = False ignored_files_and_dir = [] @@ -493,8 +481,7 @@ class BusyBoxDirectoryListedSource(DirectoryListedSource): class UclibcDirectoryListedSource(DirectoryListedSource): source_url = "https://www.uclibc.org/downloads/" # Source archive ex: uClibc-1.2.3.tar.gz - source_archive_regex = re.compile( - r"^(uClibc-)(?P[\w.-]*)(.tar.gz)$") + source_archive_regex = re.compile(r"^(uClibc-)(?P[\w.-]*)(.tar.gz)$") is_nested = False ignored_files_and_dir = [] @@ -502,8 +489,7 @@ class UclibcDirectoryListedSource(DirectoryListedSource): class UclibcNGDirectoryListedSource(DirectoryListedSource): source_url = "https://downloads.uclibc-ng.org/releases/" # Source archive ex: uClibc-ng-1.2.3.tar.gz - source_archive_regex = re.compile( - r"^(uClibc-ng-)(?P[\w.-]*)(.tar.gz)$") + source_archive_regex = re.compile(r"^(uClibc-ng-)(?P[\w.-]*)(.tar.gz)$") is_nested = True ignored_files_and_dir = [] @@ -511,8 +497,7 @@ class UclibcNGDirectoryListedSource(DirectoryListedSource): class Bzip2DirectoryListedSource(DirectoryListedSource): source_url = "https://sourceware.org/pub/bzip2/" # Source archive ex: bzip2-1.2.3.tar.gz - source_archive_regex = re.compile( - r"^(bzip2-)(?P[\w.-]*)(.tar.gz)$") + source_archive_regex = re.compile(r"^(bzip2-)(?P[\w.-]*)(.tar.gz)$") is_nested = False ignored_files_and_dir = [] @@ -520,8 +505,7 @@ class Bzip2DirectoryListedSource(DirectoryListedSource): class OpenSSHDirectoryListedSource(DirectoryListedSource): source_url = "https://cdn.openbsd.org/pub/OpenBSD/OpenSSH/" # Source archive ex: openssh-1.2.3.tar.gz - source_archive_regex = re.compile( - r"^(openssh-)(?P[\w.-]*)(.tgz|.tar.gz)$") + source_archive_regex = re.compile(r"^(openssh-)(?P[\w.-]*)(.tgz|.tar.gz)$") is_nested = False ignored_files_and_dir = [] @@ -529,9 +513,7 @@ class OpenSSHDirectoryListedSource(DirectoryListedSource): class DnsmasqDirectoryListedSource(DirectoryListedSource): source_url = "https://thekelleys.org.uk/dnsmasq/" # Source archive ex: dnsmasq-1.2.3.tar.gz - source_archive_regex = re.compile( - r"^(dnsmasq-)(?P[\w.-]*)(.tar.xz|.tar.gz)$" - ) + source_archive_regex = re.compile(r"^(dnsmasq-)(?P[\w.-]*)(.tar.xz|.tar.gz)$") is_nested = False ignored_files_and_dir = [] @@ -539,8 +521,7 @@ class DnsmasqDirectoryListedSource(DirectoryListedSource): class EbtablesDirectoryListedSource(DirectoryListedSource): source_url = "https://www.netfilter.org/pub/ebtables/" # Source archive ex: ebtables-1.2.3.tar.gz - source_archive_regex = re.compile( - r"^(ebtables-)(?P[\w.-]*)(.tar.gz)$") + source_archive_regex = re.compile(r"^(ebtables-)(?P[\w.-]*)(.tar.gz)$") is_nested = False ignored_files_and_dir = [] @@ -548,8 +529,7 @@ class EbtablesDirectoryListedSource(DirectoryListedSource): class HostapdDirectoryListedSource(DirectoryListedSource): source_url = "https://w1.fi/releases/" # Source archive ex: hostapd-1.2.3.tar.gz - source_archive_regex = re.compile( - r"^(hostapd-)(?P[\w.-]*)(.tar.gz)$") + source_archive_regex = re.compile(r"^(hostapd-)(?P[\w.-]*)(.tar.gz)$") is_nested = False ignored_files_and_dir = [] @@ -567,8 +547,7 @@ class Iproute2DirectoryListedSource(DirectoryListedSource): class IptablesDirectoryListedSource(DirectoryListedSource): source_url = "https://www.netfilter.org/pub/iptables/" # Source archive ex: iptables-1.2.3.tar.bz2 - source_archive_regex = re.compile( - r"^(iptables-)(?P[\w.-]*)(.tar.bz2)$") + source_archive_regex = re.compile(r"^(iptables-)(?P[\w.-]*)(.tar.bz2)$") is_nested = False ignored_files_and_dir = [] @@ -576,8 +555,7 @@ class IptablesDirectoryListedSource(DirectoryListedSource): class LibnlDirectoryListedSource(DirectoryListedSource): source_url = "https://www.infradead.org/~tgr/libnl/files/" # Source archive ex: libnl-1.2.3.tar.gz - source_archive_regex = re.compile( - r"^(libnl-)(?P[\w.-]*)(.tar.gz)$") + source_archive_regex = re.compile(r"^(libnl-)(?P[\w.-]*)(.tar.gz)$") is_nested = False ignored_files_and_dir = [] @@ -585,8 +563,7 @@ class LibnlDirectoryListedSource(DirectoryListedSource): class LighttpdDirectoryListedSource(DirectoryListedSource): source_url = "https://download.lighttpd.net/lighttpd/releases-1.4.x/" # Source archive ex: lighttpd-1.2.3.tar.gz - source_archive_regex = re.compile( - r"^(lighttpd-)(?P[\w.-]*)(.tar.gz)$") + source_archive_regex = re.compile(r"^(lighttpd-)(?P[\w.-]*)(.tar.gz)$") is_nested = False ignored_files_and_dir = [] @@ -594,9 +571,7 @@ class LighttpdDirectoryListedSource(DirectoryListedSource): class NftablesDirectoryListedSource(DirectoryListedSource): source_url = "https://www.netfilter.org/pub/nftables/" # Source archive ex: nftables-1.2.3.tar.bz2 - source_archive_regex = re.compile( - r"^(nftables-)(?P[\w.-]*)(.tar.xz|.tar.bz2)$" - ) + source_archive_regex = re.compile(r"^(nftables-)(?P[\w.-]*)(.tar.xz|.tar.bz2)$") is_nested = False ignored_files_and_dir = [] @@ -604,9 +579,7 @@ class NftablesDirectoryListedSource(DirectoryListedSource): class WpaSupplicantDirectoryListedSource(DirectoryListedSource): source_url = "https://w1.fi/releases/" # Source archive ex: wpa_supplicant-1.2.3.tar.gz - source_archive_regex = re.compile( - r"^(wpa_supplicant-)(?P[\w.-]*)(.tar.gz)$" - ) + source_archive_regex = re.compile(r"^(wpa_supplicant-)(?P[\w.-]*)(.tar.gz)$") is_nested = False ignored_files_and_dir = [] @@ -614,8 +587,7 @@ class WpaSupplicantDirectoryListedSource(DirectoryListedSource): class SyslinuxDirectoryListedSource(DirectoryListedSource): source_url = "https://mirrors.edge.kernel.org/pub/linux/utils/boot/syslinux/" # Source archive ex: syslinux-1.2.3.tar.gz - source_archive_regex = re.compile( - r"^(syslinux-)(?P[\w.-]*)(.tar.gz)$") + source_archive_regex = re.compile(r"^(syslinux-)(?P[\w.-]*)(.tar.gz)$") is_nested = False ignored_files_and_dir = [] @@ -623,8 +595,7 @@ class SyslinuxDirectoryListedSource(DirectoryListedSource): class SyslinuxDirectoryListedSource(DirectoryListedSource): source_url = "https://mirrors.edge.kernel.org/pub/linux/utils/boot/syslinux/" # Source archive ex: syslinux-1.2.3.tar.gz - source_archive_regex = re.compile( - r"^(syslinux-)(?P[\w.-]*)(.tar.gz)$") + source_archive_regex = re.compile(r"^(syslinux-)(?P[\w.-]*)(.tar.gz)$") is_nested = False ignored_files_and_dir = [] @@ -632,9 +603,7 @@ class SyslinuxDirectoryListedSource(DirectoryListedSource): class ToyboxDirectoryListedSource(DirectoryListedSource): source_url = "http://www.landley.net/toybox/downloads/" # Source archive ex: toybox-1.2.3.tar.gz - source_archive_regex = re.compile( - r"^(toybox-)(?P[\w.-]*)(.tar.gz|.tar.bz2)$" - ) + source_archive_regex = re.compile(r"^(toybox-)(?P[\w.-]*)(.tar.gz|.tar.bz2)$") is_nested = False ignored_files_and_dir = [] @@ -642,9 +611,7 @@ class ToyboxDirectoryListedSource(DirectoryListedSource): class DropbearDirectoryListedSource(DirectoryListedSource): source_url = "https://matt.ucc.asn.au/dropbear/releases/" # Source archive ex: dropbear-1.2.3.tar.bz2 - source_archive_regex = re.compile( - r"^(dropbear-)(?P[\w.-]*)(.tar.bz2|_i386.deb)$" - ) + source_archive_regex = re.compile(r"^(dropbear-)(?P[\w.-]*)(.tar.bz2|_i386.deb)$") is_nested = False ignored_files_and_dir = [ "dropbear-0.44test1.tar.bz2", @@ -661,8 +628,7 @@ class DropbearDirectoryListedSource(DirectoryListedSource): class SambaDirectoryListedSource(DirectoryListedSource): source_url = "https://download.samba.org/pub/samba/stable/" # Source archive ex: samba-1.2.3.tar.gz - source_archive_regex = re.compile( - r"^(samba-)(?P[\w.-]*)(.tar.gz)$") + source_archive_regex = re.compile(r"^(samba-)(?P[\w.-]*)(.tar.gz)$") is_nested = False ignored_files_and_dir = [] @@ -670,8 +636,7 @@ class SambaDirectoryListedSource(DirectoryListedSource): class MtdUtilsDirectoryListedSource(DirectoryListedSource): source_url = "https://infraroot.at/pub/mtd/" # Source archive ex: mtd-utils-1.2.3.tar.bz2 - source_archive_regex = re.compile( - r"^(mtd-utils-)(?P[\w.-]*)(.tar.bz2)$") + source_archive_regex = re.compile(r"^(mtd-utils-)(?P[\w.-]*)(.tar.bz2)$") is_nested = False ignored_files_and_dir = [] @@ -679,8 +644,7 @@ class MtdUtilsDirectoryListedSource(DirectoryListedSource): class BareboxDirectoryListedSource(DirectoryListedSource): source_url = "https://www.barebox.org/download/" # Source archive ex: barebox-1.2.3.tar.bz2 - source_archive_regex = re.compile( - r"^(barebox-)(?P[\w.-]*)(.tar.bz2)$") + source_archive_regex = re.compile(r"^(barebox-)(?P[\w.-]*)(.tar.bz2)$") is_nested = False ignored_files_and_dir = [] @@ -688,8 +652,7 @@ class BareboxDirectoryListedSource(DirectoryListedSource): class LinuxDirectoryListedSource(DirectoryListedSource): source_url = "https://mirrors.edge.kernel.org/pub/linux/kernel/" # Source archive ex: linux-1.2.3.tar.gz - source_archive_regex = re.compile( - r"^(linux-)(?P[\w.-]*)(.tar.gz)$") + source_archive_regex = re.compile(r"^(linux-)(?P[\w.-]*)(.tar.gz)$") is_nested = True ignored_files_and_dir = [ "Historic/", @@ -707,12 +670,9 @@ class LinuxDirectoryListedSource(DirectoryListedSource): class E2fsprogsDirectoryListedSource(DirectoryListedSource): - source_url = ( - "https://mirrors.edge.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/" - ) + source_url = "https://mirrors.edge.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/" # Source archive ex: e2fsprogs-1.2.3.tar.gz - source_archive_regex = re.compile( - r"^(e2fsprogs-)(?P[\w.-]*)(.tar.gz)$") + source_archive_regex = re.compile(r"^(e2fsprogs-)(?P[\w.-]*)(.tar.gz)$") is_nested = True ignored_files_and_dir = ["testing/"] @@ -779,9 +739,7 @@ class E2fsprogsDirectoryListedSource(DirectoryListedSource): def get_htmllisting_data_from_purl(purl): """Generate `Package` object from the `purl` having directory listed source""" package_url = PackageURL.from_string(purl) - return DIR_LISTED_SOURCE_BY_PACKAGE_NAME[package_url.name].get_package_info( - package_url - ) + return DIR_LISTED_SOURCE_BY_PACKAGE_NAME[package_url.name].get_package_info(package_url) def get_packages_from_listing(purl, source_archive_url, regex, ignored_files_and_dir): @@ -823,9 +781,7 @@ def get_packages_from_listing(purl, source_archive_url, regex, ignored_files_and return packages -def extract_packages_from_listing( - purl, source_archive_url, regex, ignored_files_and_dir -): +def extract_packages_from_listing(purl, source_archive_url, regex, ignored_files_and_dir): """ Yield package data from a directory listing for the given source_archive_url. """ @@ -855,9 +811,7 @@ def extract_package_from_nested_listing(purl, source_url, regex, ignored_files_a directory_url = urljoin(source_url, directory.name) - for package in get_packages_from_listing( - purl, directory_url, regex, ignored_files_and_dir - ): + for package in get_packages_from_listing(purl, directory_url, regex, ignored_files_and_dir): # Don't yield all packages when a specific version is requested. if purl.version and package.version != purl.version: continue diff --git a/src/fetchcode/package_util.py b/src/fetchcode/package_util.py index 05c75ca..652276a 100644 --- a/src/fetchcode/package_util.py +++ b/src/fetchcode/package_util.py @@ -16,8 +16,8 @@ import dataclasses import json -from pathlib import Path import re +from pathlib import Path import attr @@ -33,9 +33,7 @@ def package_from_dict(package_data): Ignore unknown and unsupported fields. """ supported = {attr.name for attr in attr.fields(Package)} - cleaned_package_data = { - key: value for key, value in package_data.items() if key in supported - } + cleaned_package_data = {key: value for key, value in package_data.items() if key in supported} return Package(**cleaned_package_data) @@ -43,9 +41,7 @@ def package_from_dict(package_data): class GitHubSource: version_regex: re.Pattern = dataclasses.field( default=None, - metadata={ - "help_text": "Regular expression pattern to match and extract version from tag." - }, + metadata={"help_text": "Regular expression pattern to match and extract version from tag."}, ) ignored_tag_regex: re.Pattern = dataclasses.field( default=None, @@ -93,9 +89,7 @@ def get_github_packages(purl, version_regex, ignored_tag_regex, default_package) """ Yield package data from a directory listing for the given source_archive_url. """ - for package in _get_github_packages( - purl, version_regex, ignored_tag_regex, default_package - ): + for package in _get_github_packages(purl, version_regex, ignored_tag_regex, default_package): # Don't yield all packages when a specific version is requested. if purl.version and package.version != purl.version: continue @@ -110,9 +104,7 @@ def get_github_packages(purl, version_regex, ignored_tag_regex, default_package) def _get_github_packages(purl, version_regex, ignored_tag_regex, default_package): "Yield package for GitHub purl" - archive_download_url = ( - "https://github.com/{org}/{name}/archive/refs/tags/{tag_name}.tar.gz" - ) + archive_download_url = "https://github.com/{org}/{name}/archive/refs/tags/{tag_name}.tar.gz" package_dict = default_package.to_dict() for tag, date in utils.fetch_github_tags_gql(purl): @@ -137,9 +129,7 @@ def _get_github_packages(purl, version_regex, ignored_tag_regex, default_package if not version or not version[0].isdigit(): continue - download_url = archive_download_url.format( - org=purl.namespace, name=purl.name, tag_name=tag - ) + download_url = archive_download_url.format(org=purl.namespace, name=purl.name, tag_name=tag) date = date.strftime("%Y-%m-%dT%H:%M:%S") package_dict.update( @@ -169,8 +159,7 @@ class SquashfsToolsGitHubSource(GitHubSource): class PupnpGitHubSource(GitHubSource): - version_regex = re.compile( - r"\brelease-?(?P(?:\d+(\.\d+){1,2}))\b") + version_regex = re.compile(r"\brelease-?(?P(?:\d+(\.\d+){1,2}))\b") ignored_tag_regex = None @@ -185,8 +174,7 @@ class BpftoolGitHubSource(GitHubSource): class SqliteGitHubSource(GitHubSource): - version_regex = re.compile( - r"\bversion-?(?P(?:\d+(\.\d+){1,2}))\b") + version_regex = re.compile(r"\bversion-?(?P(?:\d+(\.\d+){1,2}))\b") ignored_tag_regex = None @@ -196,8 +184,7 @@ class LlvmGitHubSource(GitHubSource): class RpmGitHubSource(GitHubSource): - version_regex = re.compile( - r"rpm-(?P[^-]+(?:-(?!release).*)?|-release)") + version_regex = re.compile(r"rpm-(?P[^-]+(?:-(?!release).*)?|-release)") ignored_tag_regex = None @@ -276,9 +263,7 @@ class MiniupnpPackagesGitHubSource(GitHubSource): @classmethod def get_package_info(cls, gh_purl, package_name): - cls.version_regex = re.compile( - cls.version_regex_template.format(re.escape(package_name)) - ) + cls.version_regex = re.compile(cls.version_regex_template.format(re.escape(package_name))) packages = get_github_packages( gh_purl, @@ -298,13 +283,9 @@ def get_package_info(cls, gh_purl, package_name): # Archive source https://web.archive.org/web/20021209021312/http://udhcp.busybox.net/source/ -UDHCP_RELEASES = json.loads( - (DATA / "udhcp_releases.json").read_text(encoding="UTF-8") -) +UDHCP_RELEASES = json.loads((DATA / "udhcp_releases.json").read_text(encoding="UTF-8")) # Since there will be no new releases of ipkg, it's better to # store them in a dictionary rather than fetching them every time. -IPKG_RELEASES = json.loads( - (DATA / "ipkg_releases.json").read_text(encoding="UTF-8") -) +IPKG_RELEASES = json.loads((DATA / "ipkg_releases.json").read_text(encoding="UTF-8")) diff --git a/src/fetchcode/package_versions.py b/src/fetchcode/package_versions.py index e08fc4b..c0584eb 100644 --- a/src/fetchcode/package_versions.py +++ b/src/fetchcode/package_versions.py @@ -92,9 +92,7 @@ def get_launchpad_versions_from_purl(purl): for release in entries: source_package_version = release.get("source_package_version") - source_package_version = remove_debian_default_epoch( - version=source_package_version - ) + source_package_version = remove_debian_default_epoch(version=source_package_version) date_published = release.get("date_published") release_date = None if date_published and type(date_published) is str: @@ -135,9 +133,7 @@ def get_cargo_versions_from_purl(purl): """Fetch versions of Rust cargo packages from the crates.io API.""" purl = PackageURL.from_string(purl) url = f"https://crates.io/api/v1/crates/{purl.name}" - response = get_response( - url=url, content_type="json", headers={"User-Agent": "pm_bot"} - ) + response = get_response(url=url, content_type="json", headers={"User-Agent": "pm_bot"}) for version_info in response.get("versions"): yield PackageVersion( @@ -218,9 +214,7 @@ def get_maven_versions_from_purl(purl): return group_url = group_id.replace(".", "/") - endpoint = ( - f"https://repo1.maven.org/maven2/{group_url}/{artifact_id}/maven-metadata.xml" - ) + endpoint = f"https://repo1.maven.org/maven2/{group_url}/{artifact_id}/maven-metadata.xml" response = get_response(url=endpoint, content_type="binary") if response: xml_resp = ET.ElementTree(ET.fromstring(response.decode("utf-8"))) @@ -342,7 +336,8 @@ def trim_go_url_path(url_path: str) -> Optional[str]: >>> module = "github.com/xx/a" >>> assert trim_go_url_path("https://github.com/xx/a/b") == module """ - # some advisories contains this prefix in package name, e.g. https://github.com/advisories/GHSA-7h6j-2268-fhcm + # some advisories contains this prefix in package name, + # e.g. https://github.com/advisories/GHSA-7h6j-2268-fhcm go_url_prefix = "https://pkg.go.dev/" if url_path.startswith(go_url_prefix): url_path = url_path[len(go_url_prefix) :] @@ -488,7 +483,7 @@ def get_pypi_latest_date(downloads): { .... "upload_time_iso_8601": "2010-12-23T05:14:23.509436Z", - "url": "https://files.pythonhosted.org/packages/8f/1f/c20ca80fa5df025cc/Django-1.1.3.tar.gz", + "url": "https://files.pythonhosted.org/packages/8f/1f/c20ca80fa5df025cc/Django-1.1.3.tar.gz", # noqa }, { .... @@ -516,12 +511,13 @@ def get_response(url, content_type="json", headers=None): one of binary, text, yaml or json. """ try: - resp = requests.get(url=url, headers=headers) - except: - logger.error(traceback.format_exc()) - return - if not resp.status_code == 200: + resp = requests.get(url) + resp.raise_for_status() + except requests.HTTPError as http_err: logger.error(f"Error while fetching {url!r}: {resp.status_code!r}") + logger.error( + f"HTTP error occurred: {http_err} \n {traceback.format_exc()}", + ) return if content_type == "binary": diff --git a/src/fetchcode/packagedcode_models.py b/src/fetchcode/packagedcode_models.py index f76cae3..34bb85d 100644 --- a/src/fetchcode/packagedcode_models.py +++ b/src/fetchcode/packagedcode_models.py @@ -1,4 +1,4 @@ -# Copied from https://github.com/aboutcode-org/scancode-toolkit/blob/b4ea9c640f8ee4ed8851b5618c6d223bb1c02d47/src/packagedcode/models.py +# Copied from https://github.com/aboutcode-org/scancode-toolkit/blob/b4ea9c640f8ee4ed8851b5618c6d223bb1c02d47/src/packagedcode/models.py # noqa # # Copyright (c) 2017 nexB Inc. and others. All rights reserved. # http://nexb.com and https://github.com/aboutcode-org/scancode-toolkit/ @@ -27,15 +27,11 @@ from __future__ import print_function from __future__ import unicode_literals -from collections import OrderedDict import logging import sys +from collections import OrderedDict import attr -from packageurl import normalize_qualifiers -from packageurl import PackageURL - -from commoncode.datautils import choices from commoncode.datautils import Boolean from commoncode.datautils import Date from commoncode.datautils import Integer @@ -43,7 +39,9 @@ from commoncode.datautils import Mapping from commoncode.datautils import String from commoncode.datautils import TriBoolean - +from commoncode.datautils import choices +from packageurl import PackageURL +from packageurl import normalize_qualifiers """ Data models for package information and dependencies, abstracting the @@ -88,7 +86,7 @@ def logger_debug(*args): logger.setLevel(logging.DEBUG) def logger_debug(*args): - return logger.debug(' '.join(isinstance(a, str) and a or repr(a) for a in args)) + return logger.debug(" ".join(isinstance(a, str) and a or repr(a) for a in args)) class BaseModel(object): @@ -121,11 +119,11 @@ def fields(cls): return [a.name for a in attr.fields(cls)] -party_person = 'person' +party_person = "person" # often loosely defined -party_project = 'project' +party_project = "project" # more formally defined -party_org = 'organization' +party_org = "organization" PARTY_TYPES = ( None, party_person, @@ -142,27 +140,22 @@ class Party(BaseModel): type = String( validator=choices(PARTY_TYPES), - label='party type', - help='the type of this party: One of: ' - + ', '.join(p for p in PARTY_TYPES if p)) + label="party type", + help="the type of this party: One of: " + ", ".join(p for p in PARTY_TYPES if p), + ) role = String( - label='party role', - help='A role for this party. Something such as author, ' - 'maintainer, contributor, owner, packager, distributor, ' - 'vendor, developer, owner, etc.') + label="party role", + help="A role for this party. Something such as author, " + "maintainer, contributor, owner, packager, distributor, " + "vendor, developer, owner, etc.", + ) - name = String( - label='name', - help='Name of this party.') + name = String(label="name", help="Name of this party.") - email = String( - label='email', - help='Email for this party.') + email = String(label="email", help="Email for this party.") - url = String( - label='url', - help='URL to a primary web page for this party.') + url = String(label="url", help="URL to a primary web page for this party.") @attr.s() @@ -199,41 +192,38 @@ class BasePackage(BaseModel): type = String( repr=True, - label='package type', - help='Optional. A short code to identify what is the type of this ' - 'package. For instance gem for a Rubygem, docker for container, ' - 'pypi for Python Wheel or Egg, maven for a Maven Jar, ' - 'deb for a Debian package, etc.') + label="package type", + help="Optional. A short code to identify what is the type of this " + "package. For instance gem for a Rubygem, docker for container, " + "pypi for Python Wheel or Egg, maven for a Maven Jar, " + "deb for a Debian package, etc.", + ) namespace = String( - repr=True, - label='package namespace', - help='Optional namespace for this package.') + repr=True, label="package namespace", help="Optional namespace for this package." + ) - name = String( - repr=True, - label='package name', - help='Name of the package.') + name = String(repr=True, label="package name", help="Name of the package.") version = String( - repr=True, - label='package version', - help='Optional version of the package as a string.') + repr=True, label="package version", help="Optional version of the package as a string." + ) qualifiers = Mapping( default=None, value_type=str, converter=lambda v: normalize_qualifiers(v, encode=False), - label='package qualifiers', - help='Optional mapping of key=value pairs qualifiers for this package') + label="package qualifiers", + help="Optional mapping of key=value pairs qualifiers for this package", + ) subpath = String( - label='extra package subpath', - help='Optional extra subpath inside a package and relative to the root ' - 'of this package') + label="extra package subpath", + help="Optional extra subpath inside a package and relative to the root " "of this package", + ) def __attrs_post_init__(self, *args, **kwargs): - if not self.type and hasattr(self, 'default_type'): + if not self.type and hasattr(self, "default_type"): self.type = self.default_type @property @@ -244,8 +234,8 @@ def purl(self): if not self.name: return return PackageURL( - self.type, self.namespace, self.name, self.version, - self.qualifiers, self.subpath).to_string() + self.type, self.namespace, self.name, self.version, self.qualifiers, self.subpath + ).to_string() def repository_homepage_url(self, baseurl=default_web_baseurl): """ @@ -284,8 +274,7 @@ def set_purl(self, package_url): if not isinstance(package_url, PackageURL): package_url = PackageURL.from_string(package_url) - attribs = ['type', 'namespace', 'name', - 'version', 'qualifiers', 'subpath'] + attribs = ["type", "namespace", "name", "version", "qualifiers", "subpath"] for att in attribs: self_val = getattr(self, att) purl_val = getattr(package_url, att) @@ -297,14 +286,13 @@ def to_dict(self, **kwargs): Return an OrderedDict of primitive Python types. """ mapping = attr.asdict(self, dict_factory=OrderedDict) - if not kwargs.get('exclude_properties'): - mapping['purl'] = self.purl - mapping['repository_homepage_url'] = self.repository_homepage_url() - mapping['repository_download_url'] = self.repository_download_url() - mapping['api_data_url'] = self.api_data_url() + if not kwargs.get("exclude_properties"): + mapping["purl"] = self.purl + mapping["repository_homepage_url"] = self.repository_homepage_url() + mapping["repository_download_url"] = self.repository_download_url() + mapping["api_data_url"] = self.api_data_url() if self.qualifiers: - mapping['qualifiers'] = normalize_qualifiers( - self.qualifiers, encode=False) + mapping["qualifiers"] = normalize_qualifiers(self.qualifiers, encode=False) return mapping @classmethod @@ -314,6 +302,7 @@ def create(cls, ignore_unknown=True, **kwargs): Optionally `ignore_unknown` attributes provided in `kwargs`. """ from packagedcode import get_package_class + cls = get_package_class(kwargs, default=cls) return super(BasePackage, cls).create(ignore_unknown=ignore_unknown, **kwargs) @@ -326,37 +315,43 @@ class DependentPackage(BaseModel): purl = String( repr=True, - label='Dependent package URL', - help='A compact purl package URL. Typically when there is an unresolved requirement, there is no version. ' - 'If the dependency is resolved, the version should be added to the purl') + label="Dependent package URL", + help="A compact purl package URL. Typically when there is an unresolved requirement, there " + "is no version. If the dependency is resolved, the version should be added to the purl", + ) requirement = String( repr=True, - label='dependent package version requirement', - help='A string defining version(s)requirements. Package-type specific.') + label="dependent package version requirement", + help="A string defining version(s)requirements. Package-type specific.", + ) scope = String( repr=True, - label='dependency scope', - help='The scope of this dependency, such as runtime, install, etc. ' - 'This is package-type specific and is the original scope string.') + label="dependency scope", + help="The scope of this dependency, such as runtime, install, etc. " + "This is package-type specific and is the original scope string.", + ) is_runtime = Boolean( default=True, - label='is runtime flag', - help='True if this dependency is a runtime dependency.') + label="is runtime flag", + help="True if this dependency is a runtime dependency.", + ) is_optional = Boolean( default=False, - label='is optional flag', - help='True if this dependency is an optional dependency') + label="is optional flag", + help="True if this dependency is an optional dependency", + ) is_resolved = Boolean( default=False, - label='is resolved flag', - help='True if this dependency version requirement has ' - 'been resolved and this dependency url points to an ' - 'exact version.') + label="is resolved flag", + help="True if this dependency version requirement has " + "been resolved and this dependency url points to an " + "exact version.", + ) @attr.s() @@ -369,120 +364,109 @@ class Package(BasePackage): default_primary_language = None primary_language = String( - label='Primary programming language', - help='Primary programming language',) + label="Primary programming language", + help="Primary programming language", + ) description = String( - label='Description', - help='Description for this package. ' - 'By convention the first should be a summary when available.') + label="Description", + help="Description for this package. " + "By convention the first should be a summary when available.", + ) - release_date = Date( - label='release date', - help='Release date of the package') + release_date = Date(label="release date", help="Release date of the package") parties = List( item_type=Party, - label='parties', - help='A list of parties such as a person, project or organization.') + label="parties", + help="A list of parties such as a person, project or organization.", + ) - keywords = List( - item_type=str, - label='keywords', - help='A list of keywords.') + keywords = List(item_type=str, label="keywords", help="A list of keywords.") - homepage_url = String( - label='homepage URL', - help='URL to the homepage for this package.') + homepage_url = String(label="homepage URL", help="URL to the homepage for this package.") - download_url = String( - label='Download URL', - help='A direct download URL.') + download_url = String(label="Download URL", help="A direct download URL.") - api_url = String( - label='API URL', - help='URL of API for this package.') + api_url = String(label="API URL", help="URL of API for this package.") size = Integer( - default=None, - label='download size', - help='size of the package download in bytes') + default=None, label="download size", help="size of the package download in bytes" + ) - sha1 = String( - label='SHA1 checksum', - help='SHA1 checksum for this download in hexadecimal') + sha1 = String(label="SHA1 checksum", help="SHA1 checksum for this download in hexadecimal") - md5 = String( - label='MD5 checksum', - help='MD5 checksum for this download in hexadecimal') + md5 = String(label="MD5 checksum", help="MD5 checksum for this download in hexadecimal") sha256 = String( - label='SHA256 checksum', - help='SHA256 checksum for this download in hexadecimal') + label="SHA256 checksum", help="SHA256 checksum for this download in hexadecimal" + ) sha512 = String( - label='SHA512 checksum', - help='SHA512 checksum for this download in hexadecimal') + label="SHA512 checksum", help="SHA512 checksum for this download in hexadecimal" + ) bug_tracking_url = String( - label='bug tracking URL', - help='URL to the issue or bug tracker for this package') + label="bug tracking URL", help="URL to the issue or bug tracker for this package" + ) - code_view_url = String( - label='code view URL', - help='a URL where the code can be browsed online') + code_view_url = String(label="code view URL", help="a URL where the code can be browsed online") vcs_url = String( - help='a URL to the VCS repository in the SPDX form of: ' - 'https://github.com/nexb/scancode-toolkit.git@405aaa4b3 ' + help="a URL to the VCS repository in the SPDX form of: " + "https://github.com/nexb/scancode-toolkit.git@405aaa4b3 " 'See SPDX specification "Package Download Location" ' - 'at https://spdx.org/spdx-specification-21-web-version#h.49x2ik5 ') + "at https://spdx.org/spdx-specification-21-web-version#h.49x2ik5 " + ) copyright = String( - label='Copyright', - help='Copyright statements for this package. Typically one per line.') + label="Copyright", help="Copyright statements for this package. Typically one per line." + ) license_expression = String( - label='license expression', - help='The license expression for this package typically derived ' - 'from its declared license or .') + label="license expression", + help="The license expression for this package typically derived " + "from its declared license or .", + ) declared_license = String( - label='declared license', - help='The declared license mention, tag or text as found in a ' - 'package manifest.') + label="declared license", + help="The declared license mention, tag or text as found in a " "package manifest.", + ) - notice_text = String( - label='notice text', - help='A notice text for this package.') + notice_text = String(label="notice text", help="A notice text for this package.") root_path = String( - label='package root path', - help='The path to the root of the package documented in this manifest ' - 'if any, such as a Maven .pom or a npm package.json parent directory.') + label="package root path", + help="The path to the root of the package documented in this manifest " + "if any, such as a Maven .pom or a npm package.json parent directory.", + ) dependencies = List( item_type=DependentPackage, - label='dependencies', - help='A list of DependentPackage for this package. ') + label="dependencies", + help="A list of DependentPackage for this package. ", + ) contains_source_code = TriBoolean( - label='contains source code', - help='Flag set to True if this package contains its own source code, None ' - 'if this is unknown, False if not.') + label="contains source code", + help="Flag set to True if this package contains its own source code, None " + "if this is unknown, False if not.", + ) source_packages = List( item_type=String, - label='List of related source code packages', + label="List of related source code packages", help='A list of related source code Package URLs (aka. "purl") for ' - 'this package. For instance an SRPM is the "source package" for a ' - 'binary RPM.') + 'this package. For instance an SRPM is the "source package" for a ' + "binary RPM.", + ) def __attrs_post_init__(self, *args, **kwargs): - if not self.type and hasattr(self, 'default_type'): + if not self.type and hasattr(self, "default_type"): self.type = self.default_type - if not self.primary_language and hasattr(self, 'default_primary_language'): + if not self.primary_language and hasattr(self, "default_primary_language"): self.primary_language = self.default_primary_language @classmethod @@ -521,7 +505,9 @@ def get_package_resources(cls, package_root, codebase): """ if not Package.is_ignored_package_resource(package_root, codebase): yield package_root - for resource in package_root.walk(codebase, topdown=True, ignored=Package.is_ignored_package_resource): + for resource in package_root.walk( + codebase, topdown=True, ignored=Package.is_ignored_package_resource + ): yield resource @classmethod @@ -534,6 +520,7 @@ def ignore_resource(cls, resource, codebase): @staticmethod def is_ignored_package_resource(resource, codebase): from packagedcode import PACKAGE_TYPES + return any(pt.ignore_resource(resource, codebase) for pt in PACKAGE_TYPES) def compute_normalized_license(self): @@ -592,12 +579,13 @@ def compute_normalized_license(declared_license): return from packagedcode import licensing + try: return licensing.get_normalized_expression(declared_license) except Exception: # FIXME: add logging # we never fail just for this - return 'unknown' + return "unknown" # Package types @@ -607,79 +595,101 @@ def compute_normalized_license(declared_license): @attr.s() class DebianPackage(Package): - metafiles = ('*.control',) - extensions = ('.deb',) - filetypes = ('debian binary package',) - mimetypes = ('application/x-archive', - 'application/vnd.debian.binary-package',) - default_type = 'deb' + metafiles = ("*.control",) + extensions = (".deb",) + filetypes = ("debian binary package",) + mimetypes = ( + "application/x-archive", + "application/vnd.debian.binary-package", + ) + default_type = "deb" @attr.s() class JavaJar(Package): - metafiles = ('META-INF/MANIFEST.MF',) - extensions = ('.jar',) - filetypes = ('java archive ', 'zip archive',) - mimetypes = ('application/java-archive', 'application/zip',) - default_type = 'jar' - default_primary_language = 'Java' + metafiles = ("META-INF/MANIFEST.MF",) + extensions = (".jar",) + filetypes = ( + "java archive ", + "zip archive", + ) + mimetypes = ( + "application/java-archive", + "application/zip", + ) + default_type = "jar" + default_primary_language = "Java" @attr.s() class JavaWar(Package): - metafiles = ('WEB-INF/web.xml',) - extensions = ('.war',) - filetypes = ('java archive ', 'zip archive',) - mimetypes = ('application/java-archive', 'application/zip') - default_type = 'war' - default_primary_language = 'Java' + metafiles = ("WEB-INF/web.xml",) + extensions = (".war",) + filetypes = ( + "java archive ", + "zip archive", + ) + mimetypes = ("application/java-archive", "application/zip") + default_type = "war" + default_primary_language = "Java" @attr.s() class JavaEar(Package): - metafiles = ('META-INF/application.xml', 'META-INF/ejb-jar.xml') - extensions = ('.ear',) - filetypes = ('java archive ', 'zip archive',) - mimetypes = ('application/java-archive', 'application/zip') - default_type = 'ear' - default_primary_language = 'Java' + metafiles = ("META-INF/application.xml", "META-INF/ejb-jar.xml") + extensions = (".ear",) + filetypes = ( + "java archive ", + "zip archive", + ) + mimetypes = ("application/java-archive", "application/zip") + default_type = "ear" + default_primary_language = "Java" @attr.s() class Axis2Mar(Package): """Apache Axis2 module""" - metafiles = ('META-INF/module.xml',) - extensions = ('.mar',) - filetypes = ('java archive ', 'zip archive',) - mimetypes = ('application/java-archive', 'application/zip') - default_type = 'axis2' - default_primary_language = 'Java' + + metafiles = ("META-INF/module.xml",) + extensions = (".mar",) + filetypes = ( + "java archive ", + "zip archive", + ) + mimetypes = ("application/java-archive", "application/zip") + default_type = "axis2" + default_primary_language = "Java" @attr.s() class JBossSar(Package): - metafiles = ('META-INF/jboss-service.xml',) - extensions = ('.sar',) - filetypes = ('java archive ', 'zip archive',) - mimetypes = ('application/java-archive', 'application/zip') - default_type = 'jboss' - default_primary_language = 'Java' + metafiles = ("META-INF/jboss-service.xml",) + extensions = (".sar",) + filetypes = ( + "java archive ", + "zip archive", + ) + mimetypes = ("application/java-archive", "application/zip") + default_type = "jboss" + default_primary_language = "Java" @attr.s() class IvyJar(JavaJar): - metafiles = ('ivy.xml',) - default_type = 'ivy' - default_primary_language = 'Java' + metafiles = ("ivy.xml",) + default_type = "ivy" + default_primary_language = "Java" + # FIXME: move to bower.py @attr.s() class BowerPackage(Package): - metafiles = ('bower.json',) - default_type = 'bower' - default_primary_language = 'JavaScript' + metafiles = ("bower.json",) + default_type = "bower" + default_primary_language = "JavaScript" @classmethod def get_package_root(cls, manifest_resource, codebase): @@ -688,9 +698,9 @@ def get_package_root(cls, manifest_resource, codebase): @attr.s() class MeteorPackage(Package): - metafiles = ('package.js',) - default_type = 'meteor' - default_primary_language = 'JavaScript' + metafiles = ("package.js",) + default_type = "meteor" + default_primary_language = "JavaScript" @classmethod def get_package_root(cls, manifest_resource, codebase): @@ -700,27 +710,28 @@ def get_package_root(cls, manifest_resource, codebase): @attr.s() class CpanModule(Package): metafiles = ( - '*.pod', - '*.pm', - 'MANIFEST', - 'Makefile.PL', - 'META.yml', - 'META.json', - '*.meta', - 'dist.ini',) + "*.pod", + "*.pm", + "MANIFEST", + "Makefile.PL", + "META.yml", + "META.json", + "*.meta", + "dist.ini", + ) # TODO: refine me - extensions = ('.tar.gz',) - default_type = 'cpan' - default_primary_language = 'Perl' + extensions = (".tar.gz",) + default_type = "cpan" + default_primary_language = "Perl" # TODO: refine me: Go packages are a mess but something is emerging # TODO: move to and use godeps.py @attr.s() class Godep(Package): - metafiles = ('Godeps',) - default_type = 'golang' - default_primary_language = 'Go' + metafiles = ("Godeps",) + default_type = "golang" + default_primary_language = "Go" @classmethod def get_package_root(cls, manifest_resource, codebase): @@ -739,112 +750,126 @@ def get_package_root(cls, manifest_resource, codebase): @attr.s() class AndroidApp(Package): - filetypes = ('zip archive',) - mimetypes = ('application/zip',) - extensions = ('.apk',) - default_type = 'android' - default_primary_language = 'Java' + filetypes = ("zip archive",) + mimetypes = ("application/zip",) + extensions = (".apk",) + default_type = "android" + default_primary_language = "Java" # see http://tools.android.com/tech-docs/new-build-system/aar-formats @attr.s() class AndroidLibrary(Package): - filetypes = ('zip archive',) - mimetypes = ('application/zip',) + filetypes = ("zip archive",) + mimetypes = ("application/zip",) # note: Apache Axis also uses AAR extensions for plain Jars. # this could be decided based on internal structure - extensions = ('.aar',) - default_type = 'android-lib' - default_primary_language = 'Java' + extensions = (".aar",) + default_type = "android-lib" + default_primary_language = "Java" @attr.s() class MozillaExtension(Package): - filetypes = ('zip archive',) - mimetypes = ('application/zip',) - extensions = ('.xpi',) - default_type = 'mozilla' - default_primary_language = 'JavaScript' + filetypes = ("zip archive",) + mimetypes = ("application/zip",) + extensions = (".xpi",) + default_type = "mozilla" + default_primary_language = "JavaScript" @attr.s() class ChromeExtension(Package): - filetypes = ('data',) - mimetypes = ('application/octet-stream',) - extensions = ('.crx',) - default_type = 'chrome' - default_primary_language = 'JavaScript' + filetypes = ("data",) + mimetypes = ("application/octet-stream",) + extensions = (".crx",) + default_type = "chrome" + default_primary_language = "JavaScript" @attr.s() class IOSApp(Package): - filetypes = ('zip archive',) - mimetypes = ('application/zip',) - extensions = ('.ipa',) - default_type = 'ios' - default_primary_language = 'Objective-C' + filetypes = ("zip archive",) + mimetypes = ("application/zip",) + extensions = (".ipa",) + default_type = "ios" + default_primary_language = "Objective-C" @attr.s() class CabPackage(Package): - filetypes = ('microsoft cabinet',) - mimetypes = ('application/vnd.ms-cab-compressed',) - extensions = ('.cab',) - default_type = 'cab' + filetypes = ("microsoft cabinet",) + mimetypes = ("application/vnd.ms-cab-compressed",) + extensions = (".cab",) + default_type = "cab" @attr.s() class MsiInstallerPackage(Package): - filetypes = ('msi installer',) - mimetypes = ('application/x-msi',) - extensions = ('.msi',) - default_type = 'msi' + filetypes = ("msi installer",) + mimetypes = ("application/x-msi",) + extensions = (".msi",) + default_type = "msi" @attr.s() class InstallShieldPackage(Package): - filetypes = ('installshield',) - mimetypes = ('application/x-dosexec',) - extensions = ('.exe',) - default_type = 'installshield' + filetypes = ("installshield",) + mimetypes = ("application/x-dosexec",) + extensions = (".exe",) + default_type = "installshield" @attr.s() class NSISInstallerPackage(Package): - filetypes = ('nullsoft installer',) - mimetypes = ('application/x-dosexec',) - extensions = ('.exe',) - default_type = 'nsis' + filetypes = ("nullsoft installer",) + mimetypes = ("application/x-dosexec",) + extensions = (".exe",) + default_type = "nsis" @attr.s() class SharPackage(Package): - filetypes = ('posix shell script',) - mimetypes = ('text/x-shellscript',) - extensions = ('.sha', '.shar', '.bin',) - default_type = 'shar' + filetypes = ("posix shell script",) + mimetypes = ("text/x-shellscript",) + extensions = ( + ".sha", + ".shar", + ".bin", + ) + default_type = "shar" @attr.s() class AppleDmgPackage(Package): - filetypes = ('zlib compressed',) - mimetypes = ('application/zlib',) - extensions = ('.dmg', '.sparseimage',) - default_type = 'dmg' + filetypes = ("zlib compressed",) + mimetypes = ("application/zlib",) + extensions = ( + ".dmg", + ".sparseimage", + ) + default_type = "dmg" @attr.s() class IsoImagePackage(Package): - filetypes = ('iso 9660 cd-rom', 'high sierra cd-rom',) - mimetypes = ('application/x-iso9660-image',) - extensions = ('.iso', '.udf', '.img',) - default_type = 'iso' + filetypes = ( + "iso 9660 cd-rom", + "high sierra cd-rom", + ) + mimetypes = ("application/x-iso9660-image",) + extensions = ( + ".iso", + ".udf", + ".img", + ) + default_type = "iso" @attr.s() class SquashfsPackage(Package): - filetypes = ('squashfs',) - default_type = 'squashfs' + filetypes = ("squashfs",) + default_type = "squashfs" # TODO: Add VM images formats(VMDK, OVA, OVF, VDI, etc) and Docker/other containers diff --git a/src/fetchcode/utils.py b/src/fetchcode/utils.py index 3116cac..620b488 100644 --- a/src/fetchcode/utils.py +++ b/src/fetchcode/utils.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations under the License. import os + import requests from dateutil import parser as dateparser from dateutil.parser import ParserError @@ -141,8 +142,7 @@ def github_response(graphql_query): headers = {"Authorization": f"bearer {gh_token}"} endpoint = "https://api.github.com/graphql" - response = requests.post(endpoint, headers=headers, - json=graphql_query).json() + response = requests.post(endpoint, headers=headers, json=graphql_query).json() message = response.get("message") if message and message == "Bad credentials": diff --git a/src/fetchcode/vcs/__init__.py b/src/fetchcode/vcs/__init__.py index ef429ef..a4987cf 100644 --- a/src/fetchcode/vcs/__init__.py +++ b/src/fetchcode/vcs/__init__.py @@ -19,12 +19,12 @@ import tempfile from urllib.parse import urlparse +from fetchcode.vcs.pip._internal.utils import misc +from fetchcode.vcs.pip._internal.vcs import vcs from fetchcode.vcs.pip._internal.vcs.bazaar import Bazaar from fetchcode.vcs.pip._internal.vcs.git import Git from fetchcode.vcs.pip._internal.vcs.mercurial import Mercurial -from fetchcode.vcs.pip._internal.utils import misc from fetchcode.vcs.pip._internal.vcs.subversion import Subversion -from fetchcode.vcs.pip._internal.vcs import vcs class VCSResponse: diff --git a/src/fetchcode/vcs/git.py b/src/fetchcode/vcs/git.py index f602516..74d1984 100644 --- a/src/fetchcode/vcs/git.py +++ b/src/fetchcode/vcs/git.py @@ -18,10 +18,10 @@ import tempfile from urllib.parse import urlparse -from fetchcode.vcs.pip._internal.vcs.git import Git +from fetchcode.vcs import VCSResponse from fetchcode.vcs.pip._internal.utils import misc from fetchcode.vcs.pip._internal.vcs import vcs -from fetchcode.vcs import VCSResponse +from fetchcode.vcs.pip._internal.vcs.git import Git def fetch_via_git(url): diff --git a/tests/data/package/dirlisting/regenerate_mock_data.py b/tests/data/package/dirlisting/regenerate_mock_data.py index 067b830..6f56daa 100644 --- a/tests/data/package/dirlisting/regenerate_mock_data.py +++ b/tests/data/package/dirlisting/regenerate_mock_data.py @@ -16,6 +16,7 @@ from pathlib import Path + import requests data_location = Path(__file__).parent @@ -796,7 +797,6 @@ "filename": "generic/e2fsprogs/46.html", "url": "https://mirrors.edge.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/v1.47.0/", }, - ], }, ] diff --git a/tests/data/package/github/regenerate_mock_data.py b/tests/data/package/github/regenerate_mock_data.py index 9a2b33e..331ee9c 100644 --- a/tests/data/package/github/regenerate_mock_data.py +++ b/tests/data/package/github/regenerate_mock_data.py @@ -41,8 +41,7 @@ def fetch_github_mock_data(owner, name, subdir): while True: response = github_response(graphql_query) refs = response["data"]["repository"]["refs"] - mock_data_file = data_location / \ - f"{subdir}/github_mock_data_{file_count}.json" + mock_data_file = data_location / f"{subdir}/github_mock_data_{file_count}.json" with open(mock_data_file, "w") as file: json.dump(response, file, indent=2) diff --git a/tests/data/package_versions/regenerate_mock_data.py b/tests/data/package_versions/regenerate_mock_data.py index 9dbcaef..f007523 100644 --- a/tests/data/package_versions/regenerate_mock_data.py +++ b/tests/data/package_versions/regenerate_mock_data.py @@ -140,8 +140,7 @@ def fetch_golang_mock_data(): url = f"https://proxy.golang.org/github.com/blockloop/scan/@v/list" version_list = get_response(url=url, content_type="text") for version in version_list.split(): - file_name = data_location / \ - f"golang/versions/golang_mock_{version}_data.json" + file_name = data_location / f"golang/versions/golang_mock_{version}_data.json" response = get_response( url=f"https://proxy.golang.org/github.com/blockloop/scan/@v/{version}.info", content_type="json", @@ -169,8 +168,7 @@ def fetch_github_mock_data(): while True: response = github_response(graphql_query) refs = response["data"]["repository"]["refs"] - mock_data_file = data_location / \ - f"github/github_mock_data_{file_count}.json" + mock_data_file = data_location / f"github/github_mock_data_{file_count}.json" with open(mock_data_file, "w") as file: json.dump(response, file, indent=4) diff --git a/tests/test_fetch.py b/tests/test_fetch.py index 6366da3..1dcf746 100644 --- a/tests/test_fetch.py +++ b/tests/test_fetch.py @@ -21,45 +21,45 @@ from fetchcode import fetch -@mock.patch('fetchcode.requests.get') +@mock.patch("fetchcode.requests.get") def test_fetch_http_with_tempfile(mock_get): mock_get.return_value.headers = { - 'content-type': 'image/png', - 'content-length': '1000999', + "content-type": "image/png", + "content-length": "1000999", } - with mock.patch('fetchcode.open', mock.mock_open()) as mocked_file: - url = 'https://raw.githubusercontent.com/TG1999/converge/master/assets/Group%2022.png' + with mock.patch("fetchcode.open", mock.mock_open()) as mocked_file: + url = "https://raw.githubusercontent.com/TG1999/converge/master/assets/Group%2022.png" response = fetch(url=url) assert response is not None assert 1000999 == response.size assert url == response.url - assert 'image/png' == response.content_type + assert "image/png" == response.content_type -@mock.patch('fetchcode.FTP') +@mock.patch("fetchcode.FTP") def test_fetch_with_wrong_url(mock_get): with pytest.raises(Exception) as e_info: - url = 'ftp://speedtest/1KB.zip' + url = "ftp://speedtest/1KB.zip" response = fetch(url=url) - assert 'Not a valid URL' == e_info + assert "Not a valid URL" == e_info -@mock.patch('fetchcode.FTP', autospec=True) +@mock.patch("fetchcode.FTP", autospec=True) def test_fetch_ftp_with_tempfile(mock_ftp_constructor): mock_ftp = mock_ftp_constructor.return_value mock_ftp_constructor.return_value.size.return_value = 1024 - with mock.patch('fetchcode.open', mock.mock_open()) as mocked_file: - response = fetch('ftp://speedtest.tele2.net/1KB.zip') + with mock.patch("fetchcode.open", mock.mock_open()) as mocked_file: + response = fetch("ftp://speedtest.tele2.net/1KB.zip") assert 1024 == response.size - mock_ftp_constructor.assert_called_with('speedtest.tele2.net') - assert mock_ftp.login.called == True - mock_ftp.cwd.assert_called_with('/') + mock_ftp_constructor.assert_called_with("speedtest.tele2.net") + assert mock_ftp.login.called + mock_ftp.cwd.assert_called_with("/") assert mock_ftp.retrbinary.called def test_fetch_with_scheme_not_present(): with pytest.raises(Exception) as e_info: - url = 'abc://speedtest/1KB.zip' + url = "abc://speedtest/1KB.zip" response = fetch(url=url) - assert 'Not a supported/known scheme.' == e_info + assert "Not a supported/known scheme." == e_info diff --git a/tests/test_package.py b/tests/test_package.py index fa1f5ea..5ce7c1a 100644 --- a/tests/test_package.py +++ b/tests/test_package.py @@ -114,14 +114,11 @@ def check_result(self, filename, packages, regen=False): @mock.patch("fetchcode.utils.get_response") @mock.patch("fetchcode.utils.github_response") - def test_packages_github_source_avahi( - self, mock_github_response, mock_get_response - ): + def test_packages_github_source_avahi(self, mock_github_response, mock_get_response): test_data = [ "tests/data/package/github/avahi/github_mock_data_1.json", ] - mock_github_response.side_effect = [ - file_json(file) for file in test_data] + mock_github_response.side_effect = [file_json(file) for file in test_data] mock_get_response.return_value = file_json( "tests/data/package/github/avahi/github_mock_data_0.json" ) @@ -133,14 +130,11 @@ def test_packages_github_source_avahi( @mock.patch("fetchcode.utils.get_response") @mock.patch("fetchcode.utils.github_response") - def test_packages_github_source_avahi( - self, mock_github_response, mock_get_response - ): + def test_packages_github_source_avahi(self, mock_github_response, mock_get_response): test_data = [ "tests/data/package/github/avahi/github_mock_data_1.json", ] - mock_github_response.side_effect = [ - file_json(file) for file in test_data] + mock_github_response.side_effect = [file_json(file) for file in test_data] mock_get_response.return_value = file_json( "tests/data/package/github/avahi/github_mock_data_0.json" ) @@ -152,14 +146,11 @@ def test_packages_github_source_avahi( @mock.patch("fetchcode.utils.get_response") @mock.patch("fetchcode.utils.github_response") - def test_packages_github_source_bpftool( - self, mock_github_response, mock_get_response - ): + def test_packages_github_source_bpftool(self, mock_github_response, mock_get_response): test_data = [ "tests/data/package/github/bpftool/github_mock_data_1.json", ] - mock_github_response.side_effect = [ - file_json(file) for file in test_data] + mock_github_response.side_effect = [file_json(file) for file in test_data] mock_get_response.return_value = file_json( "tests/data/package/github/bpftool/github_mock_data_0.json" ) @@ -171,14 +162,11 @@ def test_packages_github_source_bpftool( @mock.patch("fetchcode.utils.get_response") @mock.patch("fetchcode.utils.github_response") - def test_packages_github_source_brotli( - self, mock_github_response, mock_get_response - ): + def test_packages_github_source_brotli(self, mock_github_response, mock_get_response): test_data = [ "tests/data/package/github/brotli/github_mock_data_1.json", ] - mock_github_response.side_effect = [ - file_json(file) for file in test_data] + mock_github_response.side_effect = [file_json(file) for file in test_data] mock_get_response.return_value = file_json( "tests/data/package/github/brotli/github_mock_data_0.json" ) @@ -190,14 +178,11 @@ def test_packages_github_source_brotli( @mock.patch("fetchcode.utils.get_response") @mock.patch("fetchcode.utils.github_response") - def test_packages_github_source_dosfstools( - self, mock_github_response, mock_get_response - ): + def test_packages_github_source_dosfstools(self, mock_github_response, mock_get_response): test_data = [ "tests/data/package/github/dosfstools/github_mock_data_1.json", ] - mock_github_response.side_effect = [ - file_json(file) for file in test_data] + mock_github_response.side_effect = [file_json(file) for file in test_data] mock_get_response.return_value = file_json( "tests/data/package/github/dosfstools/github_mock_data_0.json" ) @@ -209,14 +194,11 @@ def test_packages_github_source_dosfstools( @mock.patch("fetchcode.utils.get_response") @mock.patch("fetchcode.utils.github_response") - def test_packages_github_source_genext2fs( - self, mock_github_response, mock_get_response - ): + def test_packages_github_source_genext2fs(self, mock_github_response, mock_get_response): test_data = [ "tests/data/package/github/genext2fs/github_mock_data_1.json", ] - mock_github_response.side_effect = [ - file_json(file) for file in test_data] + mock_github_response.side_effect = [file_json(file) for file in test_data] mock_get_response.return_value = file_json( "tests/data/package/github/genext2fs/github_mock_data_0.json" ) @@ -228,14 +210,11 @@ def test_packages_github_source_genext2fs( @mock.patch("fetchcode.utils.get_response") @mock.patch("fetchcode.utils.github_response") - def test_packages_github_source_inotify_tools( - self, mock_github_response, mock_get_response - ): + def test_packages_github_source_inotify_tools(self, mock_github_response, mock_get_response): test_data = [ "tests/data/package/github/inotify-tools/github_mock_data_1.json", ] - mock_github_response.side_effect = [ - file_json(file) for file in test_data] + mock_github_response.side_effect = [file_json(file) for file in test_data] mock_get_response.return_value = file_json( "tests/data/package/github/inotify-tools/github_mock_data_0.json" ) @@ -247,16 +226,13 @@ def test_packages_github_source_inotify_tools( @mock.patch("fetchcode.utils.get_response") @mock.patch("fetchcode.utils.github_response") - def test_packages_github_source_llvm_project( - self, mock_github_response, mock_get_response - ): + def test_packages_github_source_llvm_project(self, mock_github_response, mock_get_response): test_data = [ "tests/data/package/github/llvm-project/github_mock_data_1.json", "tests/data/package/github/llvm-project/github_mock_data_2.json", "tests/data/package/github/llvm-project/github_mock_data_3.json", ] - mock_github_response.side_effect = [ - file_json(file) for file in test_data] + mock_github_response.side_effect = [file_json(file) for file in test_data] mock_get_response.return_value = file_json( "tests/data/package/github/llvm-project/github_mock_data_0.json" ) @@ -268,14 +244,11 @@ def test_packages_github_source_llvm_project( @mock.patch("fetchcode.utils.get_response") @mock.patch("fetchcode.utils.github_response") - def test_packages_github_source_miniupnpc( - self, mock_github_response, mock_get_response - ): + def test_packages_github_source_miniupnpc(self, mock_github_response, mock_get_response): test_data = [ "tests/data/package/github/miniupnp/github_mock_data_1.json", ] - mock_github_response.side_effect = [ - file_json(file) for file in test_data] + mock_github_response.side_effect = [file_json(file) for file in test_data] mock_get_response.return_value = file_json( "tests/data/package/github/miniupnp/github_mock_data_0.json" ) @@ -287,14 +260,11 @@ def test_packages_github_source_miniupnpc( @mock.patch("fetchcode.utils.get_response") @mock.patch("fetchcode.utils.github_response") - def test_packages_github_source_miniupnpd( - self, mock_github_response, mock_get_response - ): + def test_packages_github_source_miniupnpd(self, mock_github_response, mock_get_response): test_data = [ "tests/data/package/github/miniupnp/github_mock_data_1.json", ] - mock_github_response.side_effect = [ - file_json(file) for file in test_data] + mock_github_response.side_effect = [file_json(file) for file in test_data] mock_get_response.return_value = file_json( "tests/data/package/github/miniupnp/github_mock_data_0.json" ) @@ -306,14 +276,11 @@ def test_packages_github_source_miniupnpd( @mock.patch("fetchcode.utils.get_response") @mock.patch("fetchcode.utils.github_response") - def test_packages_github_source_minissdpd( - self, mock_github_response, mock_get_response - ): + def test_packages_github_source_minissdpd(self, mock_github_response, mock_get_response): test_data = [ "tests/data/package/github/miniupnp/github_mock_data_1.json", ] - mock_github_response.side_effect = [ - file_json(file) for file in test_data] + mock_github_response.side_effect = [file_json(file) for file in test_data] mock_get_response.return_value = file_json( "tests/data/package/github/miniupnp/github_mock_data_0.json" ) @@ -330,8 +297,7 @@ def test_packages_github_source_nix(self, mock_github_response, mock_get_respons "tests/data/package/github/nix/github_mock_data_1.json", "tests/data/package/github/nix/github_mock_data_2.json", ] - mock_github_response.side_effect = [ - file_json(file) for file in test_data] + mock_github_response.side_effect = [file_json(file) for file in test_data] mock_get_response.return_value = file_json( "tests/data/package/github/nix/github_mock_data_0.json" ) @@ -343,14 +309,11 @@ def test_packages_github_source_nix(self, mock_github_response, mock_get_respons @mock.patch("fetchcode.utils.get_response") @mock.patch("fetchcode.utils.github_response") - def test_packages_github_source_pupnp( - self, mock_github_response, mock_get_response - ): + def test_packages_github_source_pupnp(self, mock_github_response, mock_get_response): test_data = [ "tests/data/package/github/pupnp/github_mock_data_1.json", ] - mock_github_response.side_effect = [ - file_json(file) for file in test_data] + mock_github_response.side_effect = [file_json(file) for file in test_data] mock_get_response.return_value = file_json( "tests/data/package/github/pupnp/github_mock_data_0.json" ) @@ -362,9 +325,7 @@ def test_packages_github_source_pupnp( @mock.patch("fetchcode.utils.get_response") @mock.patch("fetchcode.utils.github_response") - def test_packages_github_source_cpython( - self, mock_github_response, mock_get_response - ): + def test_packages_github_source_cpython(self, mock_github_response, mock_get_response): test_data = [ "tests/data/package/github/cpython/github_mock_data_1.json", "tests/data/package/github/cpython/github_mock_data_2.json", @@ -373,8 +334,7 @@ def test_packages_github_source_cpython( "tests/data/package/github/cpython/github_mock_data_5.json", "tests/data/package/github/cpython/github_mock_data_6.json", ] - mock_github_response.side_effect = [ - file_json(file) for file in test_data] + mock_github_response.side_effect = [file_json(file) for file in test_data] mock_get_response.return_value = file_json( "tests/data/package/github/cpython/github_mock_data_0.json" ) @@ -391,8 +351,7 @@ def test_packages_github_source_rpm(self, mock_github_response, mock_get_respons "tests/data/package/github/rpm/github_mock_data_1.json", "tests/data/package/github/rpm/github_mock_data_2.json", ] - mock_github_response.side_effect = [ - file_json(file) for file in test_data] + mock_github_response.side_effect = [file_json(file) for file in test_data] mock_get_response.return_value = file_json( "tests/data/package/github/rpm/github_mock_data_0.json" ) @@ -404,14 +363,11 @@ def test_packages_github_source_rpm(self, mock_github_response, mock_get_respons @mock.patch("fetchcode.utils.get_response") @mock.patch("fetchcode.utils.github_response") - def test_packages_github_source_shadow( - self, mock_github_response, mock_get_response - ): + def test_packages_github_source_shadow(self, mock_github_response, mock_get_response): test_data = [ "tests/data/package/github/shadow/github_mock_data_1.json", ] - mock_github_response.side_effect = [ - file_json(file) for file in test_data] + mock_github_response.side_effect = [file_json(file) for file in test_data] mock_get_response.return_value = file_json( "tests/data/package/github/shadow/github_mock_data_0.json" ) @@ -423,15 +379,12 @@ def test_packages_github_source_shadow( @mock.patch("fetchcode.utils.get_response") @mock.patch("fetchcode.utils.github_response") - def test_packages_github_source_sqlite( - self, mock_github_response, mock_get_response - ): + def test_packages_github_source_sqlite(self, mock_github_response, mock_get_response): test_data = [ "tests/data/package/github/sqlite/github_mock_data_1.json", "tests/data/package/github/sqlite/github_mock_data_2.json", ] - mock_github_response.side_effect = [ - file_json(file) for file in test_data] + mock_github_response.side_effect = [file_json(file) for file in test_data] mock_get_response.return_value = file_json( "tests/data/package/github/sqlite/github_mock_data_0.json" ) @@ -443,14 +396,11 @@ def test_packages_github_source_sqlite( @mock.patch("fetchcode.utils.get_response") @mock.patch("fetchcode.utils.github_response") - def test_packages_github_source_squashfs_tools( - self, mock_github_response, mock_get_response - ): + def test_packages_github_source_squashfs_tools(self, mock_github_response, mock_get_response): test_data = [ "tests/data/package/github/squashfs-tools/github_mock_data_1.json", ] - mock_github_response.side_effect = [ - file_json(file) for file in test_data] + mock_github_response.side_effect = [file_json(file) for file in test_data] mock_get_response.return_value = file_json( "tests/data/package/github/squashfs-tools/github_mock_data_0.json" ) @@ -462,14 +412,11 @@ def test_packages_github_source_squashfs_tools( @mock.patch("fetchcode.utils.get_response") @mock.patch("fetchcode.utils.github_response") - def test_packages_github_source_wireless_tools( - self, mock_github_response, mock_get_response - ): + def test_packages_github_source_wireless_tools(self, mock_github_response, mock_get_response): test_data = [ "tests/data/package/github/wireless-tools/github_mock_data_1.json", ] - mock_github_response.side_effect = [ - file_json(file) for file in test_data] + mock_github_response.side_effect = [file_json(file) for file in test_data] mock_get_response.return_value = file_json( "tests/data/package/github/wireless-tools/github_mock_data_0.json" ) @@ -481,9 +428,7 @@ def test_packages_github_source_wireless_tools( @mock.patch("fetchcode.utils.get_response") @mock.patch("fetchcode.utils.github_response") - def test_packages_github_source_uboot( - self, mock_github_response, mock_get_response - ): + def test_packages_github_source_uboot(self, mock_github_response, mock_get_response): test_data = [ "tests/data/package/github/u-boot/github_mock_data_1.json", "tests/data/package/github/u-boot/github_mock_data_2.json", @@ -491,8 +436,7 @@ def test_packages_github_source_uboot( "tests/data/package/github/u-boot/github_mock_data_4.json", "tests/data/package/github/u-boot/github_mock_data_5.json", ] - mock_github_response.side_effect = [ - file_json(file) for file in test_data] + mock_github_response.side_effect = [file_json(file) for file in test_data] mock_get_response.return_value = file_json( "tests/data/package/github/u-boot/github_mock_data_0.json" ) @@ -504,14 +448,11 @@ def test_packages_github_source_uboot( @mock.patch("fetchcode.utils.get_response") @mock.patch("fetchcode.utils.github_response") - def test_packages_github_source_erofs_utils( - self, mock_github_response, mock_get_response - ): + def test_packages_github_source_erofs_utils(self, mock_github_response, mock_get_response): test_data = [ "tests/data/package/github/erofs-utils/github_mock_data_1.json", ] - mock_github_response.side_effect = [ - file_json(file) for file in test_data] + mock_github_response.side_effect = [file_json(file) for file in test_data] mock_get_response.return_value = file_json( "tests/data/package/github/erofs-utils/github_mock_data_0.json" ) @@ -523,17 +464,14 @@ def test_packages_github_source_erofs_utils( @mock.patch("fetchcode.utils.get_response") @mock.patch("fetchcode.utils.github_response") - def test_packages_github_source_openssl( - self, mock_github_response, mock_get_response - ): + def test_packages_github_source_openssl(self, mock_github_response, mock_get_response): test_data = [ "tests/data/package/github/openssl/github_mock_data_1.json", "tests/data/package/github/openssl/github_mock_data_2.json", "tests/data/package/github/openssl/github_mock_data_3.json", "tests/data/package/github/openssl/github_mock_data_4.json", ] - mock_github_response.side_effect = [ - file_json(file) for file in test_data] + mock_github_response.side_effect = [file_json(file) for file in test_data] mock_get_response.return_value = file_json( "tests/data/package/github/openssl/github_mock_data_0.json" ) @@ -739,9 +677,7 @@ def test_packages_wpa_supplicant(self, mock_get): for file in test_data ] - expected_file = ( - "tests/data/package/dirlisting/generic/wpa_supplicant-expected.json" - ) + expected_file = "tests/data/package/dirlisting/generic/wpa_supplicant-expected.json" result = info("pkg:generic/wpa_supplicant") self.check_result(expected_file, result) diff --git a/tests/test_package_versions.py b/tests/test_package_versions.py index 113cdbf..dc112c9 100644 --- a/tests/test_package_versions.py +++ b/tests/test_package_versions.py @@ -23,8 +23,7 @@ from fetchcode.package_versions import versions -FETCHCODE_REGEN_TEST_FIXTURES = os.getenv( - "FETCHCODE_REGEN_TEST_FIXTURES", False) +FETCHCODE_REGEN_TEST_FIXTURES = os.getenv("FETCHCODE_REGEN_TEST_FIXTURES", False) data_location = Path(__file__).parent / "data" / "package_versions" @@ -190,10 +189,7 @@ def test_get_golang_versions_from_purl(mock_get_response): for version in version_list.split(): side_effect.append( - get_json_data( - data_location / - f"golang/versions/golang_mock_{version}_data.json" - ) + get_json_data(data_location / f"golang/versions/golang_mock_{version}_data.json") ) purl = "pkg:golang/github.com/blockloop/scan" diff --git a/tests/test_skeleton_codestyle.py b/tests/test_skeleton_codestyle.py index b4ce8c1..446f095 100644 --- a/tests/test_skeleton_codestyle.py +++ b/tests/test_skeleton_codestyle.py @@ -7,9 +7,9 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +import configparser import subprocess import unittest -import configparser class BaseTests(unittest.TestCase): diff --git a/tests/test_vcs.py b/tests/test_vcs.py index 8249516..3bde7e8 100644 --- a/tests/test_vcs.py +++ b/tests/test_vcs.py @@ -29,44 +29,84 @@ def obtain(dest, url): @pytest.mark.parametrize( "url, vcs_type, domain", [ - pytest.param("git+http://github.com/jamesor/mongoose-versioner", - "git", "github.com", id="git_http"), - pytest.param("git+https://github.com/jamesor/mongoose-versioner", - "git", "github.com", id="git_https"), - pytest.param("git+ssh://github.com/jamesor/mongoose-versioner", - "git", "github.com", id="git_ssh"), - pytest.param("git+file://github.com/jamesor/mongoose-versioner", - "git", "github.com", id="git_file"), - pytest.param("git+git://github.com/jamesor/mongoose-versioner", - "git", "github.com", id="git_git"), - pytest.param("bzr+http://gitlab.com/jamesor/mongoose-versioner", - "bzr", "gitlab.com", id="bzr_http"), - pytest.param("bzr+https://gitlab.com/jamesor/mongoose-versioner", - "bzr", "gitlab.com", id="bzr_https"), - pytest.param("bzr+ssh://gitlab.com/jamesor/mongoose-versioner", - "bzr", "gitlab.com", id="bzr_ssh"), - pytest.param("bzr+ftp://gitlab.com/jamesor/mongoose-versioner", - "bzr", "gitlab.com", id="bzr_ftp"), - pytest.param("bzr+sftp://gitlab.com/jamesor/mongoose-versioner", - "bzr", "gitlab.com", id="bzr_sftp"), - pytest.param("bzr+lp://gitlab.com/jamesor/mongoose-versioner", - "bzr", "gitlab.com", id="bzr_lp"), - pytest.param("hg+file://bitbucket.com/jamesor/mongoose-versioner", - "hg", "bitbucket.com", id="hg_file"), - pytest.param("hg+http://bitbucket.com/jamesor/mongoose-versioner", - "hg", "bitbucket.com", id="hg_http"), - pytest.param("hg+https://bitbucket.com/jamesor/mongoose-versioner", - "hg", "bitbucket.com", id="hg_https"), - pytest.param("hg+ssh://bitbucket.com/jamesor/mongoose-versioner", - "hg", "bitbucket.com", id="hg_ssh"), - pytest.param("hg+static-http://bitbucket.com/jamesor/mongoose-versioner", - "hg", "bitbucket.com", id="hg_static_http"), - pytest.param("svn+http://bitbucket.com/jamesor/mongoose-versioner", - "svn", "bitbucket.com", id="svn_http"), - pytest.param("svn+https://bitbucket.com/jamesor/mongoose-versioner", - "svn", "bitbucket.com", id="svn_https"), - pytest.param("svn+svn://bitbucket.com/jamesor/mongoose-versioner", - "svn", "bitbucket.com", id="svn_svn") + pytest.param( + "git+http://github.com/jamesor/mongoose-versioner", "git", "github.com", id="git_http" + ), + pytest.param( + "git+https://github.com/jamesor/mongoose-versioner", "git", "github.com", id="git_https" + ), + pytest.param( + "git+ssh://github.com/jamesor/mongoose-versioner", "git", "github.com", id="git_ssh" + ), + pytest.param( + "git+file://github.com/jamesor/mongoose-versioner", "git", "github.com", id="git_file" + ), + pytest.param( + "git+git://github.com/jamesor/mongoose-versioner", "git", "github.com", id="git_git" + ), + pytest.param( + "bzr+http://gitlab.com/jamesor/mongoose-versioner", "bzr", "gitlab.com", id="bzr_http" + ), + pytest.param( + "bzr+https://gitlab.com/jamesor/mongoose-versioner", "bzr", "gitlab.com", id="bzr_https" + ), + pytest.param( + "bzr+ssh://gitlab.com/jamesor/mongoose-versioner", "bzr", "gitlab.com", id="bzr_ssh" + ), + pytest.param( + "bzr+ftp://gitlab.com/jamesor/mongoose-versioner", "bzr", "gitlab.com", id="bzr_ftp" + ), + pytest.param( + "bzr+sftp://gitlab.com/jamesor/mongoose-versioner", "bzr", "gitlab.com", id="bzr_sftp" + ), + pytest.param( + "bzr+lp://gitlab.com/jamesor/mongoose-versioner", "bzr", "gitlab.com", id="bzr_lp" + ), + pytest.param( + "hg+file://bitbucket.com/jamesor/mongoose-versioner", + "hg", + "bitbucket.com", + id="hg_file", + ), + pytest.param( + "hg+http://bitbucket.com/jamesor/mongoose-versioner", + "hg", + "bitbucket.com", + id="hg_http", + ), + pytest.param( + "hg+https://bitbucket.com/jamesor/mongoose-versioner", + "hg", + "bitbucket.com", + id="hg_https", + ), + pytest.param( + "hg+ssh://bitbucket.com/jamesor/mongoose-versioner", "hg", "bitbucket.com", id="hg_ssh" + ), + pytest.param( + "hg+static-http://bitbucket.com/jamesor/mongoose-versioner", + "hg", + "bitbucket.com", + id="hg_static_http", + ), + pytest.param( + "svn+http://bitbucket.com/jamesor/mongoose-versioner", + "svn", + "bitbucket.com", + id="svn_http", + ), + pytest.param( + "svn+https://bitbucket.com/jamesor/mongoose-versioner", + "svn", + "bitbucket.com", + id="svn_https", + ), + pytest.param( + "svn+svn://bitbucket.com/jamesor/mongoose-versioner", + "svn", + "bitbucket.com", + id="svn_svn", + ), ], ) @mock.patch("fetchcode.vcs.vcs.get_backend_for_scheme") diff --git a/tests/test_vcs_git.py b/tests/test_vcs_git.py index 08f3155..f2aa5de 100644 --- a/tests/test_vcs_git.py +++ b/tests/test_vcs_git.py @@ -28,16 +28,21 @@ def obtain(dest, url): @pytest.mark.parametrize( "url, vcs_type, domain", [ - pytest.param("git+http://github.com/jamesor/mongoose-versioner", - "git", "github.com", id="git_http"), - pytest.param("git+https://github.com/jamesor/mongoose-versioner", - "git", "github.com", id="git_https"), - pytest.param("git+ssh://github.com/jamesor/mongoose-versioner", - "git", "github.com", id="git_ssh"), - pytest.param("git+file://github.com/jamesor/mongoose-versioner", - "git", "github.com", id="git_file"), - pytest.param("git+git://github.com/jamesor/mongoose-versioner", - "git", "github.com", id="git_git") + pytest.param( + "git+http://github.com/jamesor/mongoose-versioner", "git", "github.com", id="git_http" + ), + pytest.param( + "git+https://github.com/jamesor/mongoose-versioner", "git", "github.com", id="git_https" + ), + pytest.param( + "git+ssh://github.com/jamesor/mongoose-versioner", "git", "github.com", id="git_ssh" + ), + pytest.param( + "git+file://github.com/jamesor/mongoose-versioner", "git", "github.com", id="git_file" + ), + pytest.param( + "git+git://github.com/jamesor/mongoose-versioner", "git", "github.com", id="git_git" + ), ], ) @mock.patch("fetchcode.vcs.git.vcs.get_backend")