Skip to content

Commit

Permalink
Provenance retrieval route (#16778)
Browse files Browse the repository at this point in the history
Co-authored-by: Mike Fiedler <[email protected]>
Co-authored-by: Dustin Ingram <[email protected]>
  • Loading branch information
3 people authored Oct 3, 2024
1 parent 2706bfb commit 50a58f3
Show file tree
Hide file tree
Showing 20 changed files with 523 additions and 442 deletions.
1 change: 0 additions & 1 deletion requirements/main.in
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,6 @@ requests
requests-aws4auth
redis>=2.8.0,<6.0.0
rfc3986
rfc8785
sentry-sdk
setuptools
sigstore~=3.3.0
Expand Down
9 changes: 9 additions & 0 deletions tests/common/db/packaging.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
JournalEntry,
ProhibitedProjectName,
Project,
Provenance,
Release,
Role,
RoleInvitation,
Expand Down Expand Up @@ -142,6 +143,14 @@ class Meta:
)


class ProvenanceFactory(WarehouseFactory):
class Meta:
model = Provenance

file = factory.SubFactory(FileFactory)
provenance = factory.Faker("json")


class FileEventFactory(WarehouseFactory):
class Meta:
model = File.Event
Expand Down
10 changes: 10 additions & 0 deletions tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,8 @@
from warehouse.accounts import services as account_services
from warehouse.accounts.interfaces import ITokenService, IUserService
from warehouse.admin.flags import AdminFlag, AdminFlagValue
from warehouse.attestations import services as attestations_services
from warehouse.attestations.interfaces import IIntegrityService
from warehouse.email import services as email_services
from warehouse.email.interfaces import IEmailSender
from warehouse.helpdesk import services as helpdesk_services
Expand Down Expand Up @@ -174,6 +176,7 @@ def pyramid_services(
project_service,
github_oidc_service,
activestate_oidc_service,
integrity_service,
macaroon_service,
helpdesk_service,
):
Expand All @@ -195,6 +198,7 @@ def pyramid_services(
services.register_service(
activestate_oidc_service, IOIDCPublisherService, None, name="activestate"
)
services.register_service(integrity_service, IIntegrityService, None)
services.register_service(macaroon_service, IMacaroonService, None, name="")
services.register_service(helpdesk_service, IHelpDeskService, None)

Expand Down Expand Up @@ -326,6 +330,7 @@ def get_app_config(database, nondefaults=None):
"docs.backend": "warehouse.packaging.services.LocalDocsStorage",
"sponsorlogos.backend": "warehouse.admin.services.LocalSponsorLogoStorage",
"billing.backend": "warehouse.subscriptions.services.MockStripeBillingService",
"integrity.backend": "warehouse.attestations.services.NullIntegrityService",
"billing.api_base": "http://stripe:12111",
"billing.api_version": "2020-08-27",
"mail.backend": "warehouse.email.services.SMTPEmailSender",
Expand Down Expand Up @@ -557,6 +562,11 @@ def dummy_attestation():
)


@pytest.fixture
def integrity_service(db_session):
return attestations_services.NullIntegrityService(db_session)


@pytest.fixture
def macaroon_service(db_session):
return macaroon_services.DatabaseMacaroonService(db_session)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
{
"version": 1,
"verification_material": {
"certificate": "MIIC6zCCAnGgAwIBAgIUFgmhIYx8gvBGePCTacG/4kbBdRwwCgYIKoZIzj0EAwMwNzEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MR4wHAYDVQQDExVzaWdzdG9yZS1pbnRlcm1lZGlhdGUwHhcNMjQwODI5MTcwOTM5WhcNMjQwODI5MTcxOTM5WjAAMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEtGrMPml4OtsRJ3Z6qRahs0kHCZxP4n9fvrJE957WVxgAGg4k6a1PbRJY9nT9wKpRrZmKV++AgA9ndhdruXXaAKOCAZAwggGMMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzAdBgNVHQ4EFgQUosNvhYEuTPfgyU/dZfu93lFGRNswHwYDVR0jBBgwFoAU39Ppz1YkEZb5qNjpKFWixi4YZD8wQAYDVR0RAQH/BDYwNIEyOTE5NDM2MTU4MjM2LWNvbXB1dGVAZGV2ZWxvcGVyLmdzZXJ2aWNlYWNjb3VudC5jb20wKQYKKwYBBAGDvzABAQQbaHR0cHM6Ly9hY2NvdW50cy5nb29nbGUuY29tMCsGCisGAQQBg78wAQgEHQwbaHR0cHM6Ly9hY2NvdW50cy5nb29nbGUuY29tMIGKBgorBgEEAdZ5AgQCBHwEegB4AHYA3T0wasbHETJjGR4cmWc3AqJKXrjePK3/h4pygC8p7o4AAAGRnx0/aQAABAMARzBFAiBogvcKHIIR9FcX1vQgDhGtAl0XQoMRiEB3OdUWO94P1gIhANdJlyISdtvVrHes25dWKTLepy+IzQmzfQU/S7cxWHmOMAoGCCqGSM49BAMDA2gAMGUCMGe2xTiuenbjdt1d2e4IaCiwRh2G4KAtyujRESSSUbpuGme/o9ouiApeONBv2CvvGAIxAOEkAGFO3aALE3IPNosxqaz9MbqJOdmYhB1Cz1D7xbFc/m243VxJWxaC/uOFEpyiYQ==",
"transparency_entries": [
{
"logIndex": "125970014",
"logId": {
"keyId": "wNI9atQGlz+VWfO6LRygH4QUfY/8W4RFwiT5i5WRgB0="
},
"kindVersion": {
"kind": "dsse",
"version": "0.0.1"
},
"integratedTime": "1724951379",
"inclusionPromise": {
"signedEntryTimestamp": "MEUCIQCHrKFTeXNY432S0bUSBS69S8d5JnNcDXa41q6OEvxEwgIgaZstc5Jpm0IgwFC7RDTXYEAKk+3aG/MkRkaPdJdyn8U="
},
"inclusionProof": {
"logIndex": "4065752",
"rootHash": "7jVDF3UNUZVEU85ffETQ3WKfXhOoMi4cgytJM250HTk=",
"treeSize": "4065754",
"hashes": [
"NwJgWJoxjearbnEIT9bnWXpzo0LGNrR1cpWId0g66rE=",
"kLjpW3Eh7pQJNOvyntghzF57tcfqk2IzX7cqiBDgGf8=",
"FW8y9LQ1i3q+MnbeGJipKGl4VfX1zRBOD7TmhbEw7uI=",
"mKcbGJDJ/+buNbXy9Eyv94nVoAyUauuIlN3cJg3qSBY=",
"5VytqqAHhfRkRWMrY43UXWCnRBb7JwElMlKpY5JueBc=",
"mZJnD39LTKdis2wUTz1OOMx3r7HwgJh9rnb2VwiPzts=",
"MXZOQFJFiOjREF0xwMOCXu29HwTchjTtl/BeFoI51wY=",
"g8zCkHnLwO3LojK7g5AnqE8ezSNRnCSz9nCL5GD3a8A=",
"RrZsD/RSxNoujlvq/MsCEvLSkKZfv0jmQM9Kp7qbJec=",
"QxmVWsbTp4cClxuAkuT51UH2EY7peHMVGKq7+b+cGwQ=",
"Q2LAtNzOUh+3PfwfMyNxYb06fTQmF3VeTT6Fr6Upvfc=",
"ftwAu6v62WFDoDmcZ1JKfrRPrvuiIw5v3BvRsgQj7N8="
],
"checkpoint": {
"envelope": "rekor.sigstore.dev - 1193050959916656506\n4065754\n7jVDF3UNUZVEU85ffETQ3WKfXhOoMi4cgytJM250HTk=\n\n— rekor.sigstore.dev wNI9ajBGAiEAhMomhZHOTNB5CVPO98CMXCv01ZlIF+C+CgzraAB01r8CIQCEuXbv6aqguUpB/ig5eXRIbarvxLXkg3nX48DzambktQ==\n"
}
},
"canonicalizedBody": "eyJhcGlWZXJzaW9uIjoiMC4wLjEiLCJraW5kIjoiZHNzZSIsInNwZWMiOnsiZW52ZWxvcGVIYXNoIjp7ImFsZ29yaXRobSI6InNoYTI1NiIsInZhbHVlIjoiOWRiNGJjMzE3MTgyZWI3NzljNDIyY2Q0NGI2ZDdlYTk5ZWM1M2Q3M2JiY2ZjZWVmZTIyNWVlYjQ3NTQyMjc4OCJ9LCJwYXlsb2FkSGFzaCI6eyJhbGdvcml0aG0iOiJzaGEyNTYiLCJ2YWx1ZSI6IjlkYjY0MjlhOTkzZGFiYTI4NzAwODk2ZTY2MzNjNzkxYWE0MDM3ODQ4NjJiYzY2MDBkM2E4NjYwMGQzYjA1NjMifSwic2lnbmF0dXJlcyI6W3sic2lnbmF0dXJlIjoiTUVVQ0lCaGlOL25NR0w3aHpZQk9QQjlUTGtuaEdTZEtuQ0Q0ekI3TDV5ZXc0QmJ3QWlFQXJzOHl6MCtCT2NnSEtzS0JzTXVOeVlhREdaRTBVV0JuMEdwNVpGMzUvU2M9IiwidmVyaWZpZXIiOiJMUzB0TFMxQ1JVZEpUaUJEUlZKVVNVWkpRMEZVUlMwdExTMHRDazFKU1VNMmVrTkRRVzVIWjBGM1NVSkJaMGxWUm1kdGFFbFplRGhuZGtKSFpWQkRWR0ZqUnk4MGEySkNaRkozZDBObldVbExiMXBKZW1vd1JVRjNUWGNLVG5wRlZrMUNUVWRCTVZWRlEyaE5UV015Ykc1ak0xSjJZMjFWZFZwSFZqSk5ValIzU0VGWlJGWlJVVVJGZUZaNllWZGtlbVJIT1hsYVV6RndZbTVTYkFwamJURnNXa2RzYUdSSFZYZElhR05PVFdwUmQwOUVTVFZOVkdOM1QxUk5OVmRvWTA1TmFsRjNUMFJKTlUxVVkzaFBWRTAxVjJwQlFVMUdhM2RGZDFsSUNrdHZXa2w2YWpCRFFWRlpTVXR2V2tsNmFqQkVRVkZqUkZGblFVVjBSM0pOVUcxc05FOTBjMUpLTTFvMmNWSmhhSE13YTBoRFduaFFORzQ1Wm5aeVNrVUtPVFUzVjFaNFowRkhaelJyTm1FeFVHSlNTbGs1YmxRNWQwdHdVbkphYlV0V0t5dEJaMEU1Ym1Sb1pISjFXRmhoUVV0UFEwRmFRWGRuWjBkTlRVRTBSd3BCTVZWa1JIZEZRaTkzVVVWQmQwbElaMFJCVkVKblRsWklVMVZGUkVSQlMwSm5aM0pDWjBWR1FsRmpSRUY2UVdSQ1owNVdTRkUwUlVablVWVnZjMDUyQ21oWlJYVlVVR1puZVZVdlpGcG1kVGt6YkVaSFVrNXpkMGgzV1VSV1VqQnFRa0puZDBadlFWVXpPVkJ3ZWpGWmEwVmFZalZ4VG1wd1MwWlhhWGhwTkZrS1drUTRkMUZCV1VSV1VqQlNRVkZJTDBKRVdYZE9TVVY1VDFSRk5VNUVUVEpOVkZVMFRXcE5Na3hYVG5aaVdFSXhaRWRXUVZwSFZqSmFWM2gyWTBkV2VRcE1iV1I2V2xoS01tRlhUbXhaVjA1cVlqTldkV1JETldwaU1qQjNTMUZaUzB0M1dVSkNRVWRFZG5wQlFrRlJVV0poU0ZJd1kwaE5Oa3g1T1doWk1rNTJDbVJYTlRCamVUVnVZakk1Ym1KSFZYVlpNamwwVFVOelIwTnBjMGRCVVZGQ1p6YzRkMEZSWjBWSVVYZGlZVWhTTUdOSVRUWk1lVGxvV1RKT2RtUlhOVEFLWTNrMWJtSXlPVzVpUjFWMVdUSTVkRTFKUjB0Q1oyOXlRbWRGUlVGa1dqVkJaMUZEUWtoM1JXVm5RalJCU0ZsQk0xUXdkMkZ6WWtoRlZFcHFSMUkwWXdwdFYyTXpRWEZLUzFoeWFtVlFTek12YURSd2VXZERPSEEzYnpSQlFVRkhVbTU0TUM5aFVVRkJRa0ZOUVZKNlFrWkJhVUp2WjNaalMwaEpTVkk1Um1OWUNqRjJVV2RFYUVkMFFXd3dXRkZ2VFZKcFJVSXpUMlJWVjA4NU5GQXhaMGxvUVU1a1NteDVTVk5rZEhaV2NraGxjekkxWkZkTFZFeGxjSGtyU1hwUmJYb0tabEZWTDFNM1kzaFhTRzFQVFVGdlIwTkRjVWRUVFRRNVFrRk5SRUV5WjBGTlIxVkRUVWRsTW5oVWFYVmxibUpxWkhReFpESmxORWxoUTJsM1VtZ3lSd28wUzBGMGVYVnFVa1ZUVTFOVlluQjFSMjFsTDI4NWIzVnBRWEJsVDA1Q2RqSkRkblpIUVVsNFFVOUZhMEZIUms4ellVRk1SVE5KVUU1dmMzaHhZWG81Q2sxaWNVcFBaRzFaYUVJeFEzb3hSRGQ0WWtaakwyMHlORE5XZUVwWGVHRkRMM1ZQUmtWd2VXbFpVVDA5Q2kwdExTMHRSVTVFSUVORlVsUkpSa2xEUVZSRkxTMHRMUzBLIn1dfX0="
}
]
},
"envelope": {
"statement": "eyJfdHlwZSI6Imh0dHBzOi8vaW4tdG90by5pby9TdGF0ZW1lbnQvdjEiLCJzdWJqZWN0IjpbeyJuYW1lIjoic2FtcGxlcHJvamVjdC0zLjAuMC50YXIuZ3oiLCJkaWdlc3QiOnsic2hhMjU2IjoiMTE3ZWQ4OGU1ZGIwNzNiYjkyOTY5YTc1NDU3NDVmZDk3N2VlODViNzAxOTcwNmRkMjU2YTY0MDU4ZjcwOTYzZCJ9fV0sInByZWRpY2F0ZVR5cGUiOiJodHRwczovL2RvY3MucHlwaS5vcmcvYXR0ZXN0YXRpb25zL3B1Ymxpc2gvdjEiLCJwcmVkaWNhdGUiOm51bGx9",
"signature": "MEUCIBhiN/nMGL7hzYBOPB9TLknhGSdKnCD4zB7L5yew4BbwAiEArs8yz0+BOcgHKsKBsMuNyYaDGZE0UWBn0Gp5ZF35/Sc="
}
}
58 changes: 58 additions & 0 deletions tests/functional/api/test_integrity.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import json

from http import HTTPStatus
from pathlib import Path

from ...common.db.packaging import (
FileFactory,
ProjectFactory,
ProvenanceFactory,
ReleaseFactory,
)

_HERE = Path(__file__).parent
_ASSETS = _HERE.parent / "_fixtures"
assert _ASSETS.is_dir()


def test_provenance_available(webtest):
with open(
_ASSETS / "sampleproject-3.0.0.tar.gz.publish.attestation",
) as f:
attestation_contents = f.read()
attestation_json = json.loads(attestation_contents)

project = ProjectFactory.create()
release = ReleaseFactory.create(project=project)
file_ = FileFactory.create(release=release, packagetype="sdist")
ProvenanceFactory.create(
file=file_,
provenance={"attestation_bundles": [{"attestations": [attestation_json]}]},
)

response = webtest.get(
f"/integrity/{project.name}/{release.version}/{file_.filename}/provenance",
status=HTTPStatus.OK,
)
assert response.json
assert "attestation_bundles" in response.json
attestation_bundles = response.json["attestation_bundles"]
assert len(attestation_bundles) == 1
attestation_bundle = attestation_bundles[0]
assert "attestations" in attestation_bundle
attestations = attestation_bundle["attestations"]
assert len(attestations) == 1
attestation = attestations[0]
assert attestation == attestation_json
101 changes: 80 additions & 21 deletions tests/functional/forklift/test_legacy.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,19 +11,27 @@
# limitations under the License.

import base64
import json

from http import HTTPStatus
from pathlib import Path

import pymacaroons
import pytest

from webob.multidict import MultiDict

from tests.common.db.oidc import GitHubPublisherFactory
from tests.common.db.packaging import ProjectFactory, RoleFactory
from warehouse.macaroons import caveats

from ...common.db.accounts import UserFactory
from ...common.db.macaroons import MacaroonFactory

_HERE = Path(__file__).parent
_ASSETS = _HERE.parent / "_fixtures"
assert _ASSETS.is_dir()


def test_incorrect_post_redirect(webtest):
"""
Expand Down Expand Up @@ -68,13 +76,7 @@ def test_remove_doc_upload(webtest):
],
)
def test_file_upload(webtest, upload_url, additional_data):
user = UserFactory.create(
with_verified_primary_email=True,
password=( # 'password'
"$argon2id$v=19$m=1024,t=6,p=6$EiLE2Nsbo9S6N+acs/beGw$ccyZDCZstr1/+Y/1s3BVZ"
"HOJaqfBroT0JCieHug281c"
),
)
user = UserFactory.create(with_verified_primary_email=True, clear_pwd="password")

# Construct the macaroon
dm = MacaroonFactory.create(
Expand Down Expand Up @@ -135,13 +137,7 @@ def test_file_upload(webtest, upload_url, additional_data):


def test_duplicate_file_upload_error(webtest):
user = UserFactory.create(
with_verified_primary_email=True,
password=( # 'password'
"$argon2id$v=19$m=1024,t=6,p=6$EiLE2Nsbo9S6N+acs/beGw$ccyZDCZstr1/+Y/1s3BVZ"
"HOJaqfBroT0JCieHug281c"
),
)
user = UserFactory.create(with_verified_primary_email=True, clear_pwd="password")

# Construct the macaroon
dm = MacaroonFactory.create(
Expand Down Expand Up @@ -215,13 +211,7 @@ def test_duplicate_file_upload_error(webtest):


def test_invalid_classifier_upload_error(webtest):
user = UserFactory.create(
with_verified_primary_email=True,
password=( # 'password'
"$argon2id$v=19$m=1024,t=6,p=6$EiLE2Nsbo9S6N+acs/beGw$ccyZDCZstr1/+Y/1s3BVZ"
"HOJaqfBroT0JCieHug281c"
),
)
user = UserFactory.create(with_verified_primary_email=True, clear_pwd="password")

# Construct the macaroon
dm = MacaroonFactory.create(
Expand Down Expand Up @@ -270,3 +260,72 @@ def test_invalid_classifier_upload_error(webtest):
status=HTTPStatus.BAD_REQUEST,
)
assert "'This :: Is :: Invalid' is not a valid classifier" in resp.body.decode()


def test_provenance_upload(webtest):
user = UserFactory.create(with_verified_primary_email=True, clear_pwd="password")
project = ProjectFactory.create(name="sampleproject")
RoleFactory.create(user=user, project=project, role_name="Owner")
publisher = GitHubPublisherFactory.create(projects=[project])

# Construct the macaroon. This needs to be based on a Trusted Publisher, which is
# required to upload attestations
dm = MacaroonFactory.create(
oidc_publisher_id=publisher.id,
caveats=[
caveats.OIDCPublisher(oidc_publisher_id=str(publisher.id)),
caveats.ProjectID(project_ids=[str(p.id) for p in publisher.projects]),
],
additional={"oidc": {"ref": "someref", "sha": "somesha"}},
)

m = pymacaroons.Macaroon(
location="localhost",
identifier=str(dm.id),
key=dm.key,
version=pymacaroons.MACAROON_V2,
)
for caveat in dm.caveats:
m.add_first_party_caveat(caveats.serialize(caveat))
serialized_macaroon = f"pypi-{m.serialize()}"

with open(_ASSETS / "sampleproject-3.0.0.tar.gz", "rb") as f:
content = f.read()

with open(
_ASSETS / "sampleproject-3.0.0.tar.gz.publish.attestation",
) as f:
attestation_contents = f.read()

webtest.set_authorization(("Basic", ("__token__", serialized_macaroon)))
webtest.post(
"/legacy/?:action=file_upload",
params={
"name": "sampleproject",
"sha256_digest": (
"117ed88e5db073bb92969a7545745fd977ee85b7019706dd256a64058f70963d"
),
"filetype": "sdist",
"metadata_version": "2.1",
"version": "3.0.0",
"attestations": f"[{attestation_contents}]",
},
upload_files=[("content", "sampleproject-3.0.0.tar.gz", content)],
status=HTTPStatus.OK,
)

assert len(project.releases) == 1
release = project.releases[0]
assert release.files.count() == 1
file_ = project.releases[0].files[0]
assert file_.provenance is not None
provenance = file_.provenance.provenance
assert "attestation_bundles" in provenance
attestation_bundles = provenance["attestation_bundles"]
assert len(attestation_bundles) == 1
bundle = provenance["attestation_bundles"][0]
assert "attestations" in bundle
attestations = bundle["attestations"]
assert len(attestations) == 1
attestation = attestations[0]
assert attestation == json.loads(attestation_contents)
54 changes: 54 additions & 0 deletions tests/unit/api/test_integrity.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import pretend
import pytest

from warehouse.api import integrity


def test_select_content_type(db_request):
db_request.accept = "application/json"

assert (
integrity._select_content_type(db_request)
== integrity.MIME_PYPI_INTEGRITY_V1_JSON
)


# Backstop; can be removed/changed once this view supports HTML.
@pytest.mark.parametrize(
"content_type",
[integrity.MIME_TEXT_HTML, integrity.MIME_PYPI_INTEGRITY_V1_HTML],
)
def test_provenance_for_file_bad_accept(db_request, content_type):
db_request.accept = content_type
response = integrity.provenance_for_file(pretend.stub(), db_request)
assert response.status_code == 406
assert response.json == {"message": "Request not acceptable"}


def test_provenance_for_file_not_enabled(db_request, monkeypatch):
monkeypatch.setattr(db_request, "flags", pretend.stub(enabled=lambda *a: True))

response = integrity.provenance_for_file(pretend.stub(), db_request)
assert response.status_code == 403
assert response.json == {"message": "Attestations temporarily disabled"}


def test_provenance_for_file_not_present(db_request, monkeypatch):
monkeypatch.setattr(db_request, "flags", pretend.stub(enabled=lambda *a: False))
file = pretend.stub(provenance=None, filename="fake-1.2.3.tar.gz")

response = integrity.provenance_for_file(file, db_request)
assert response.status_code == 404
assert response.json == {"message": "No provenance available for fake-1.2.3.tar.gz"}
Loading

0 comments on commit 50a58f3

Please sign in to comment.