diff --git a/.github/workflows/precommit.yml b/.github/workflows/precommit.yml new file mode 100644 index 0000000..c2f7e71 --- /dev/null +++ b/.github/workflows/precommit.yml @@ -0,0 +1,14 @@ +name: pre-commit + +on: + pull_request: + push: + branches: [main] + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 + - uses: pre-commit/action@v3.0.0 diff --git a/.gitignore b/.gitignore index e69de29..da299c3 100644 --- a/.gitignore +++ b/.gitignore @@ -0,0 +1,165 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +.ruff_cache + +# vscode +.vscode/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..a6dbc84 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,34 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-added-large-files + - id: pretty-format-json + exclude: archive|examples + - id: check-json + - id: check-docstring-first + - id: trailing-whitespace + exclude: archive|examples + - id: end-of-file-fixer + exclude: archive|examples +- repo: https://github.com/psf/black + rev: 23.3.0 + hooks: + - id: black + pass_filenames: true + exclude: _vendor|vendored|examples|archive +- repo: https://github.com/charliermarsh/ruff-pre-commit + rev: v0.0.265 + hooks: + - id: ruff +- repo: https://github.com/python-jsonschema/check-jsonschema + rev: 0.23.0 + hooks: + - id: check-github-workflows +- repo: https://github.com/econchick/interrogate + rev: 1.5.0 + hooks: + - id: interrogate + types: [python] + # exclude: ^(docs/conf.py|setup.py|tests/functional/sample) + args: [--config=pyproject.toml] diff --git a/channeldata-1.schema.json b/archive/channeldata-1.schema.json similarity index 100% rename from channeldata-1.schema.json rename to archive/channeldata-1.schema.json diff --git a/common-1.schema.json b/archive/common-1.schema.json similarity index 100% rename from common-1.schema.json rename to archive/common-1.schema.json diff --git a/examples/repodata.json b/archive/examples/repodata.json similarity index 100% rename from examples/repodata.json rename to archive/examples/repodata.json diff --git a/examples/testfile.json b/archive/examples/testfile.json similarity index 100% rename from examples/testfile.json rename to archive/examples/testfile.json diff --git a/generate-index.py b/archive/generate-index.py similarity index 95% rename from generate-index.py rename to archive/generate-index.py index 68bb982..a30dd1c 100644 --- a/generate-index.py +++ b/archive/generate-index.py @@ -93,10 +93,10 @@ """ -files=sorted(p for p in os.listdir(".") if p.endswith(".schema.json")) +files = sorted(p for p in os.listdir(".") if p.endswith(".schema.json")) rendered = Template(_TEMPLATE).render(files=files) print(rendered, file=sys.stderr) with open(join(dirname(__file__), "index.html"), "w") as fh: - fh.write(rendered) \ No newline at end of file + fh.write(rendered) diff --git a/info-security-1.schema.json b/archive/info-security-1.schema.json similarity index 100% rename from info-security-1.schema.json rename to archive/info-security-1.schema.json diff --git a/notes.txt b/archive/notes.txt similarity index 100% rename from notes.txt rename to archive/notes.txt diff --git a/repodata-1.schema.json b/archive/repodata-1.schema.json similarity index 100% rename from repodata-1.schema.json rename to archive/repodata-1.schema.json diff --git a/repodata-record-1.schema.json b/archive/repodata-record-1.schema.json similarity index 100% rename from repodata-record-1.schema.json rename to archive/repodata-record-1.schema.json diff --git a/conda_models/__init__.py b/conda_models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/conda_models/_base.py b/conda_models/_base.py new file mode 100644 index 0000000..1571cde --- /dev/null +++ b/conda_models/_base.py @@ -0,0 +1,27 @@ +from functools import lru_cache +from typing import Optional, Type + +from pydantic import BaseModel, Extra, create_model + + +class ExtrasForbiddenModel(BaseModel): + class Config: + extra = Extra.forbid + + +@lru_cache(maxsize=None) +def make_optional(baseclass: Type[BaseModel]) -> Type[BaseModel]: + """Extracts the fields and validators from the baseclass and make fields optional""" + fields = baseclass.__fields__ + validators = {"__validators__": baseclass.__validators__} + optional_fields = {key: (Optional[item.type_], None) for key, item in fields.items()} + return create_model( + f"Optional{baseclass.__name__}", + **optional_fields, + __validators__=validators, + ) + + +def export_to_json(model, path): + with open(path, "w") as f: + f.write(model.json(indent=2)) diff --git a/conda_models/channeldata.py b/conda_models/channeldata.py new file mode 100644 index 0000000..c3e61f4 --- /dev/null +++ b/conda_models/channeldata.py @@ -0,0 +1,67 @@ +from typing import Iterable, Optional + +from pydantic import AnyUrl, Field + +from ._base import ExtrasForbiddenModel +from .run_exports import RunExports +from .types import PackageNameStr, VersionStr + + +class ChannelDataPackage(ExtrasForbiddenModel): + has_activate_scripts: bool = Field(..., alias="activate.d") + "Whether the package contains activation scripts" + has_deactivate_scripts: bool = Field(..., alias="deactivate.d") + "Whether the package contains deactivation scripts" + binary_prefix: bool + "Whether the package contains binary files that hardcode the installation prefix" + description: Optional[str] = None + "The description of the package" + dev_url: Optional[Iterable[AnyUrl]] = None + "The URL to the development page of the package" + doc_url: Optional[Iterable[AnyUrl]] = None + "The URL to the documentation page of the package" + home: Optional[Iterable[AnyUrl]] = None + "The URL to the homepage of the package" + source_url: Optional[Iterable[AnyUrl]] = None + "The URL to the latest source code of the package" + license: Optional[str] = None + "The license of the package" + has_post_link_scripts: bool + "Whether the package contains post-link scripts" + has_pre_link_scripts: bool + "Whether the package contains pre-link scripts" + has_pre_unlink_scripts: bool + "Whether the package contains pre-unlink scripts" + run_exports: dict[VersionStr, RunExports] + "The run exports of the package, per package version (not build!)" + subdirs: Iterable[str] + "Which subdirs the package is available in" + summary: Optional[str] = None + "The summary of the package. Shorter than description." + text_prefix: bool + "Whether the package contains text files that hardcode the installation prefix" + timestamp: Optional[int] = None + "Last update time for artifacts of this package" + version: Optional[VersionStr] = None + "The latest version of the package" + + +class ChannelData(ExtrasForbiddenModel): + """ + Data structures that are present in a `channeldata.json` file. Some channels on anaconda.org + contain a `channeldata.json` file which describes the subdirs the channel contains, the + packages stored in the channel and additional data about them like their latest version. + + The `ChannelData` struct represents the data found within the `channeldata.json` file. The + `ChannelDataPackage` contains information about a package. + + Note that certain aspects of `ChannelData` are broken (e.g. run_exports is only available for a + single package variant) and therefore the `ChannelData` struct is not really used much more. + """ + + channeldata_version: int + "The version of the channeldata format" + packages: dict[PackageNameStr, ChannelDataPackage] + "The packages available in the channel, grouped by name" + subdirs: Iterable[str] + "The subdirs available in the channel" diff --git a/conda_models/conda_build_config.py b/conda_models/conda_build_config.py new file mode 100644 index 0000000..7f3bcc5 --- /dev/null +++ b/conda_models/conda_build_config.py @@ -0,0 +1,3 @@ +""" +WIP +""" diff --git a/conda_models/conda_lock.py b/conda_models/conda_lock.py new file mode 100644 index 0000000..7f3bcc5 --- /dev/null +++ b/conda_models/conda_lock.py @@ -0,0 +1,3 @@ +""" +WIP +""" diff --git a/conda_models/condarc.py b/conda_models/condarc.py new file mode 100644 index 0000000..8ae6d09 --- /dev/null +++ b/conda_models/condarc.py @@ -0,0 +1,619 @@ +""" +WIP +""" +from typing import Dict, Iterable, Literal, Optional, Union + +from pydantic import AnyUrl, Field, FilePath + +from ._base import ExtrasForbiddenModel +from .types import ChannelNameOrUrl, NonEmptyStr, PackageNameStr, MatchSpecStr + + +class Condarc(ExtrasForbiddenModel): + """ + This is the configuration file for conda. It can be used to configure + several aspects of its behavior. + """ + + #################################################### + # Channel Configuration # + #################################################### + + channels: Iterable[ + Union[ + ChannelNameOrUrl, + Dict[ChannelNameOrUrl, Dict[NonEmptyStr, str]], # Check channel options schema + ] + ] = Field( + default=("defaults",), + alias="channel", + description=""" + The list of conda channels to include for relevant operations. + """, + ) + + channel_alias: AnyUrl = "https://conda.anaconda.org" + """ + The prepended url location to associate with channel names. + """ + + default_channels: Iterable[ChannelNameOrUrl] = ( + "https://repo.anaconda.com/pkgs/main", + "https://repo.anaconda.com/pkgs/r", + ) + """ + The list of channel names and/or urls used for the 'defaults' + multichannel. + """ + + override_channels_enabled: bool = True + """ + Permit use of the --overide-channels command-line flag. + """ + + use_local: bool = False + """ + Add the 'local' channel (as configured with conda-build) to the channel list. + """ + + allowlist_channels: Iterable[ChannelNameOrUrl] = () + """ + The exclusive list of channels allowed to be used on the system. Use + of any other channels will result in an error. If conda-build channels + are to be allowed, along with the --use-local command line flag, be + sure to include the 'local' channel in the list. If the list is empty + or left undefined, no channel exclusions will be enforced. + """ + + custom_channels: Dict[NonEmptyStr, ChannelNameOrUrl] = {"pkgs/pro": "https://repo.anaconda.com"}, + """ + A map of key-value pairs where the key is a channel name and the value + is a channel location. Channels defined here override the default + 'channel_alias' value. The channel name (key) is not included in the + channel location (value). For example, to override the location of + the 'conda-forge' channel where the url to repodata is + https://anaconda-repo.dev/packages/conda-forge/linux-64/repodata.json, + add an entry 'conda-forge: https://anaconda-repo.dev/packages'. + """ + + custom_multichannels: Dict[NonEmptyStr, Iterable[ChannelNameOrUrl]] = Field( + default_factory=dict, + description=""" + A multichannel is a metachannel composed of multiple channels. The two + reserved multichannels are 'defaults' and 'local'. The 'defaults' + multichannel is customized using the 'default_channels' parameter. The + 'local' multichannel is a list of file:// channel locations where + conda-build stashes successfully-built packages. Other multichannels + can be defined with custom_multichannels, where the key is the + multichannel name and the value is a list of channel names and/or + channel urls. + """, + ) + + migrated_channel_aliases: Iterable[ChannelNameOrUrl] = () + """ + A list of previously-used channel_alias values. Useful when switching + between different Anaconda Repository instances. + """ + + migrated_custom_channels: Dict[NonEmptyStr, ChannelNameOrUrl] = Field( + default_factory=dict, + description=""" + A map of key-value pairs where the key is a channel name and the value + is the previous location of the channel. + """, + ) + + add_anaconda_token: bool = Field( + default=True, + alias="add_binstar_token", + description=""" + In conjunction with the anaconda command-line client (installed with + `conda install anaconda-client`), and following logging into an + Anaconda Server API site using `anaconda login`, automatically apply a + matching private token to enable access to private packages and + channels. + """, + ) + + allow_non_channel_urls: bool = False + """ + Warn, but do not fail, when conda detects a channel url is not a valid + channel. + """ + + restore_free_channel: bool = False + """ + Add the "free" channel back into defaults, behind "main" in priority. + The "free" channel was removed from the collection of default channels + in conda 4.7.0. + """ + + repodata_fns: Iterable[NonEmptyStr] = ("current_repodata.json", "repodata.json") + """ + Specify filenames for repodata fetching. The default is + ('current_repodata.json', 'repodata.json'), which tries a subset of + the full index containing only the latest version for each package, + then falls back to repodata.json. You may want to specify something + else to use an alternate index that has been reduced somehow. + """ + + use_only_tar_bz2: bool = False + """ + A boolean indicating that only .tar.bz2 conda packages should be + downloaded. This is forced to True if conda-build is installed and + older than 3.18.3, because older versions of conda break when conda + feeds it the new file format. + """ + + repodata_threads: int = 0 + """ + Threads to use when downloading and reading repodata. When not set, + defaults to None, which uses the default ThreadPoolExecutor behavior. + """ + + #################################################### + # Basic Conda Configuration # + #################################################### + + envs_dirs: Iterable[FilePath] = Field( + default=(), + alias="envs_path", + description=""" + The list of directories to search for named environments. When + creating a new named environment, the environment will be placed in + the first writable location. + + **env_var_string_delimiter** -> ':' + """, + ) + + pkgs_dirs: Iterable[FilePath] = () + """ + The list of directories where locally-available packages are linked + from at install time. Packages not locally available are downloaded + and extracted into the first writable directory. + """ + + default_threads: int = 0 + """ + Threads to use by default for parallel operations. Default is None, + which allows operations to choose themselves. For more specific + control, see the other *_threads parameters: + + - repodata_threads: for fetching/loading repodata + - verify_threads: for verifying package contents in transactions + - execute_threads: for carrying out the unlinking and linking steps + """ + + #################################################### + # Network Configuration # + #################################################### + + client_ssl_cert: Optional[FilePath] = Field( + default=None, + alias="client_cert", + description=""" + **aliases** -> client_cert + + A path to a single file containing a private key and certificate (e.g. + .pem file). Alternately, use client_ssl_cert_key in conjunction with + client_ssl_cert for individual files. + """, + ) + + client_ssl_cert_key: Optional[str] = Field( + default=None, + alias="client_cert_key", + description=""" + **aliases** -> client_cert_key + + Used in conjunction with client_ssl_cert for a matching key file. + """, + ) + + local_repodata_ttl: Union[int, bool] = 1 + """ + For a value of False or 0, always fetch remote repodata (HTTP 304 + responses respected). For a value of True or 1, respect the HTTP + Cache-Control max-age header. Any other positive integer values is the + number of seconds to locally cache repodata before checking the remote + server for an update. + """ + + offline: bool = False + """ + Restrict conda to cached download content and file:// based urls. + """ + + proxy_servers: Dict[NonEmptyStr, AnyUrl] = Field( + default_factory=dict, + description=""" + A mapping to enable proxy settings. Keys can be either (1) a + scheme://hostname form, which will match any request to the given + scheme and exact hostname, or (2) just a scheme, which will match + requests to that scheme. Values are are the actual proxy server, and + are of the form 'scheme://[user:password@]host[:port]'. The optional + 'user:password' inclusion enables HTTP Basic Auth with your proxy. + """, + ) + + remote_connect_timeout_secs: float = 9.15 + """ + The number seconds conda will wait for your client to establish a + connection to a remote url resource. + """ + + remote_max_retries: int = 3 + """ + The maximum number of retries each HTTP connection should attempt. + """ + + remote_backoff_factor: int = 1 + """ + The factor determines the time HTTP connection should wait for + attempt. + """ + + remote_read_timeout_secs: float = 60.0 + """ + Once conda has connected to a remote resource and sent an HTTP + request, the read timeout is the number of seconds conda will wait for + the server to send a response. + """ + + ssl_verify: bool = Field( + default=True, + alias="verify_ssl", + description=""" + **aliases** -> verify_ssl + + Conda verifies SSL certificates for HTTPS requests, just like a web + browser. By default, SSL verification is enabled, and conda operations + will fail if a required url's certificate cannot be verified. Setting + ssl_verify to False disables certification verification. The value for + ssl_verify can also be (1) a path to a CA bundle file, or (2) a path + to a directory containing certificates of trusted CA. + """, + ) + + #################################################### + # Solver Configuration # + #################################################### + + aggressive_update_packages: Iterable[PackageNameStr] = ("ca-certificates", "certifi", "openssl"), + """ + A list of packages that, if installed, are always updated to the + latest possible version. + """ + + auto_update_conda: bool = Field( + default=True, + alias="self_update", + description=""" + **aliases** -> self_update + + Automatically update conda when a newer or higher priority version is + detected. + """, + ) + + channel_priority: Union[Literal["flexible", "strict", "disabled"], bool] = "flexible" + """ + Accepts values of 'strict', 'flexible', and 'disabled'. The default + value is 'flexible'. With strict channel priority, packages in lower + priority channels are not considered if a package with the same name + appears in a higher priority channel. With flexible channel priority, + the solver may reach into lower priority channels to fulfill + dependencies, rather than raising an unsatisfiable error. With channel + priority disabled, package version takes precedence, and the + configured priority of channels is used only to break ties. In + previous versions of conda, this parameter was configured as either + True or False. True is now an alias to 'flexible'. + """ + + create_default_packages: Iterable[MatchSpecStr] = () + """ + Packages that are by default added to newly created environments. + """ + + disallowed_packages: Iterable[MatchSpecStr] = Field( + default=(), + alias="disallow", + description=""" + **aliases** -> disallow + + Package specifications to disallow installing. The default is to allow + all packages. + **env_var_string_delimiter** -> '&' + """, + ) + + force_reinstall: bool = False + """ + Ensure that any user-requested package for the current operation is + uninstalled and reinstalled, even if that package already exists in + the environment. + """ + + pinned_packages: Iterable[MatchSpecStr] = () + """ + A list of package specs to pin for every environment resolution. This + parameter is in BETA, and its behavior may change in a future release. + + **env_var_string_delimiter** -> '&' + """ + + pip_interop_enabled: bool = False + """ + Allow the conda solver to interact with non-conda-installed Python + packages. + """ + + track_features: Iterable[NonEmptyStr] = () + """ + DEPRECATED. + + A list of features that are tracked by default. An entry here is + similar to adding an entry to the create_default_packages list. + """ + + solver: NonEmptyStr = "classic" + """ + A string to choose between the different solver logics implemented in + conda. A solver logic takes care of turning your requested packages + into a list of specs to add and/or remove from a given environment, + based on their dependencies and specified constraints. + """ + + #################################################### + # Package Linking and Install-time Configuration # + #################################################### + + allow_softlinks: bool = False + """ + When allow_softlinks is True, conda uses hard-links when possible, and + soft-links (symlinks) when hard-links are not possible, such as when + installing on a different filesystem than the one that the package + cache is on. When allow_softlinks is False, conda still uses hard- + links when possible, but when it is not possible, conda copies files. + Individual packages can override this setting, specifying that certain + files should never be soft-linked (see the no_link option in the build + recipe documentation). + """ + + always_copy: bool = Field( + default=False, + alias="copy", + description=""" + **aliases** -> copy + + Register a preference that files be copied into a prefix during + install rather than hard-linked. + """, + ) + + always_softlink: bool = Field( + default=False, + alias="softlink", + description=""" + **aliases** -> softlink + + Register a preference that files be soft-linked (symlinked) into a + prefix during install rather than hard-linked. The link source is the + 'pkgs_dir' package cache from where the package is being linked. + WARNING: Using this option can result in corruption of long-lived + conda environments. Package caches are *caches*, which means there is + some churn and invalidation. With this option, the contents of + environments can be switched out (or erased) via operations on other + environments. + """, + ) + + path_conflict: Literal["clobber", "warn", "prevent"] = "clobber" + """ + The method by which conda handle's conflicting/overlapping paths + during a create, install, or update operation. The value must be one + of 'clobber', 'warn', or 'prevent'. The '--clobber' command-line flag + or clobber configuration parameter overrides path_conflict set to + 'prevent'. + """ + + rollback_enabled: bool = True + """ + Should any error occur during an unlink/link transaction, revert any + disk mutations made to that point in the transaction. + """ + + safety_checks: Literal["warn", "enabled", "disabled"] = "warn" + """ + Enforce available safety guarantees during package installation. The + value must be one of 'enabled', 'warn', or 'disabled'. + """ + + extra_safety_checks: bool = False + """ + Perform additional validation on package contents. Currently, runs sha256 + verification on every file within each package during installation. + """ + + signing_metadata_url_base: Optional[str] = None + """ + Base URL for obtaining trust metadata updates (i.e., the `*.root.json` + and `key_mgr.json` files) used to verify metadata and (eventually) + package signatures. + """ + + shortcuts: bool = True + """ + Allow packages to create OS-specific shortcuts (e.g. in the Windows + Start Menu) at install time. + """ + + non_admin_enabled: bool = True + """ + Allows completion of conda's create, install, update, and remove + operations, for non-privileged (non-root or non-administrator) users. + """ + + separate_format_cache: bool = False + """ + Treat .tar.bz2 files as different from .conda packages when filenames + are otherwise similar. This defaults to False, so that your package + cache doesn't churn when rolling out the new package format. If you'd + rather not assume that a .tar.bz2 and .conda from the same place + represent the same content, set this to True. + """ + + verify_threads: int = 1 + """ + Threads to use when performing the transaction verification step. + """ + + execute_threads: int = 1 + """ + Threads to use when performing the unlink/link transaction. + This step is pretty strongly I/O limited, and you may not see much + benefit here. + """ + + #################################################### + # Conda-build Configuration # + #################################################### + + bld_path: FilePath = "" + """ + The location where conda-build will put built packages. Same as + 'croot', but 'croot' takes precedence when both are defined. Also used + in construction of the 'local' multichannel. + """ + + croot: FilePath = "" + """ + The location where conda-build will put built packages. Same as + 'bld_path', but 'croot' takes precedence when both are defined. Also + used in construction of the 'local' multichannel. + """ + + anaconda_upload: bool = Field( + default=False, + alias="binstar_upload", + description=""" + **aliases** -> binstar_upload + + Automatically upload packages built with conda build to anaconda.org. + """, + ) + + conda_build: dict = Field( + default_factory=dict, + alias="conda-build", + description=""" + **aliases** -> conda-build + + General configuration parameters for conda-build. + """, + ) + + #################################################### + # Output, Prompt, and Flow Control Configuration # + #################################################### + + always_yes: bool = Field( + default=False, + alias="yes", + description=""" + **aliases** -> yes + + Automatically choose the 'yes' option whenever asked to proceed with a + conda operation, such as when running `conda install`. + """, + ) + + auto_activate_base: bool = True + """ + Automatically activate the base environment during shell + initialization. + """ + + auto_stack: int = 0 + """ + Implicitly use --stack when using activate if current level of nesting + (as indicated by CONDA_SHLVL environment variable) is less than or + equal to specified value. 0 or false disables automatic stacking, 1 or + true enables it for one level. + """ + + changeps1: bool = True + """ + When using activate, change the command prompt ($PS1) to include the + activated environment. + """ + + env_prompt: str = "({default_env})" + """ + Template for prompt modification based on the active environment. + Currently supported template variables are '{prefix}', '{name}', and + '{default_env}'. '{prefix}' is the absolute path to the active + environment. '{name}' is the basename of the active environment + prefix. '{default_env}' holds the value of '{name}' if the active + environment is a conda named environment ('-n' flag), or otherwise + holds the value of '{prefix}'. Templating uses python's str.format() + method. + """ + + json__: bool = Field( + default=False, + alias="json", + description=""" + Ensure all output written to stdout is structured json. + """, + ) + + notify_outdated_conda: bool = True + """ + Notify if a newer version of conda is detected during a create, + install, update, or remove operation. + """ + + quiet: bool = False + """ + Disable progress bar display and other output. + """ + + report_errors: bool = False + """ + Opt in, or opt out, of automatic error reporting to core maintainers. + Error reports are anonymous, with only the error stack trace and + information given by `conda info` being sent. + """ + + show_channel_urls: bool = False + """ + Show channel URLs when displaying what is going to be downloaded. + """ + + verbosity: int = Field( + default=0, + alias="verbose", + description=""" + **aliases** -> verbose + + Sets output log level. 0 is warn. 1 is info. 2 is debug. 3 is trace. + """, + ) + + unsatisfiable_hints: bool = True + """ + A boolean to determine if conda should find conflicting packages in + the case of a failed install. + """ + + unsatisfiable_hints_check_depth: int = 2 + """ + An integer that specifies how many levels deep to search for + unsatisfiable dependencies. If this number is 1 it will complete the + unsatisfiable hints fastest (but perhaps not the most complete). The + higher this number, the longer the generation of the unsat hint will + take. Defaults to 3. + """ diff --git a/conda_models/constructor.py b/conda_models/constructor.py new file mode 100644 index 0000000..7f3bcc5 --- /dev/null +++ b/conda_models/constructor.py @@ -0,0 +1,3 @@ +""" +WIP +""" diff --git a/conda_models/environment_file.py b/conda_models/environment_file.py new file mode 100644 index 0000000..143b4d8 --- /dev/null +++ b/conda_models/environment_file.py @@ -0,0 +1,27 @@ +""" +WIP +""" +from typing import Dict, Iterable, Literal, Optional, Union + +from pydantic import DirectoryPath + +from ._base import ExtrasForbiddenModel +from .types import MatchSpecStr, NonEmptyStr, SubdirStr + + +class EnvironmentYaml(ExtrasForbiddenModel): + name: Optional[str] = None + "Name for the environment" + channels: Optional[Iterable[NonEmptyStr]] = None + "Channels to search for packages" + dependencies: Union[ + NonEmptyStr, + Iterable[Union[MatchSpecStr, Dict[Literal["pip"], Iterable[NonEmptyStr]]]], + ] = () + "Packages to install into the environment, as a series of match specifications." + variables: Optional[Dict[NonEmptyStr, str]] = None + "Shell variables to define for the environment." + prefix: Optional[DirectoryPath] = None + "Path where the environment should be created." + platforms: Optional[Iterable[SubdirStr]] = None + "Platforms to search for packages." diff --git a/conda_models/exceptions.py b/conda_models/exceptions.py new file mode 100644 index 0000000..be81eeb --- /dev/null +++ b/conda_models/exceptions.py @@ -0,0 +1,3 @@ +""" +Validation errors that can be raised by the models. +""" diff --git a/conda_models/history.py b/conda_models/history.py new file mode 100644 index 0000000..0d1ff5a --- /dev/null +++ b/conda_models/history.py @@ -0,0 +1,60 @@ +""" +WIP +""" +from typing import Iterable, Union + +from pydantic import Field + +from ._base import ExtrasForbiddenModel +from .match_spec import MatchSpec +from .package_record import PackageRecord +from .types import MatchSpecStr + + +class HistoryRecord(ExtrasForbiddenModel): + """ + A single record in the conda history. + """ + + timestamp: int + "Date when the action was recorded" + command: str + "The command that was run" + specs: Iterable[Union[MatchSpec, MatchSpecStr]] + "The specs that the user asked for explicitly" + added: Iterable[PackageRecord] = () + "The packages that were added to the environment as a result of the transaction" + removed: Iterable[PackageRecord] = () + "The packages that were removed from the environment as a result of the transaction" + info: dict[str, str] = Field(default_factory=dict) + "Arbitrary metadata to be stored with the history record, like the conda client version" + + @classmethod + def from_str(cls, value: str): + raise NotImplementedError("TODO") + + +class History(ExtrasForbiddenModel): + """ + Placed in conda-meta/history, holds a series of actions formatted like: + + ``` + ==> 2019-01-29 21:16:11 <== + # cmd: /abs/path/to/bin/conda install -c conda-forge numpy + # conda version: 4.6.1 + -conda-forge::certifi-2018.11.29-py36_1000 + -defaults::blas-1.0-mkl + -defaults::chardet-3.0.4-py36h0f667ec_1 + -defaults::numpy-1.12.1-py36he24570b_1 + +conda-forge::blas-1.1-openblas + +conda-forge::wheel-0.32.3-py37_0 + +defaults::python-3.7.2-h0371630_0 + # update specs: ['numpy'] + ``` + """ + + records: Iterable[HistoryRecord] + + @classmethod + def from_str(cls, value: str): + raise NotImplementedError("TODO") diff --git a/conda_models/match_spec.py b/conda_models/match_spec.py new file mode 100644 index 0000000..6dbfa80 --- /dev/null +++ b/conda_models/match_spec.py @@ -0,0 +1,107 @@ +""" +A [`MatchSpec`] is, fundamentally, a query language for conda packages. Any of the fields that +comprise a [`crate::PackageRecord`] can be used to compose a [`MatchSpec`]. + +[`MatchSpec`] can be composed with keyword arguments, where keys are any of the attributes of +[`crate::PackageRecord`]. Values for keyword arguments are the exact values the attribute should +match against. Many fields can also be matched against non-exact values -- by including wildcard +`*` and `>`/`<` ranges--where supported. Any non-specified field is the equivalent of a full +wildcard match. + +MatchSpecs can also be composed using a single positional argument, with optional keyword +arguments. Keyword arguments also override any conflicting information provided in the positional +argument. Conda has historically had several string representations for equivalent MatchSpecs. + +A series of rules are now followed for creating the canonical string representation of a MatchSpec +instance. The canonical string representation can generically be represented by + +(channel(/subdir):(namespace):)name(version(build))[key1=value1,key2=value2] + +where `()` indicate optional fields. + +The rules for constructing a canonical string representation are: + +1. `name` (i.e. "package name") is required, but its value can be '*'. Its position is always + outside the key-value brackets. +2. If `version` is an exact version, it goes outside the key-value brackets and is prepended by + `==`. If `version` is a "fuzzy" value (e.g. `1.11.*`), it goes outside the key-value brackets + with the `.*` left off and is prepended by `=`. Otherwise `version` is included inside key-value + brackets. +3. If `version` is an exact version, and `build` is an exact value, `build` goes outside key-value + brackets prepended by a `=`. Otherwise, `build` goes inside key-value brackets. `build_string` + is an alias for `build`. +4. The `namespace` position is being held for a future feature. It is currently ignored. +5. If `channel` is included and is an exact value, a `::` separator is used between `channel` and + `name`. `channel` can either be a canonical channel name or a channel url. In the canonical + string representation, the canonical channel name will always be used. +6. If `channel` is an exact value and `subdir` is an exact value, `subdir` is appended to `channel` + with a `/` separator. Otherwise, `subdir` is included in the key-value brackets. +7. Key-value brackets can be delimited by comma, space, or comma+space. Value can optionally be + wrapped in single or double quotes, but must be wrapped if `value` contains a comma, space, or + equal sign. The canonical format uses comma delimiters and single quotes. +8. When constructing a `MatchSpec` instance from a string, any key-value pair given inside the + key-value brackets overrides any matching parameter given outside the brackets. + +When `MatchSpec` attribute values are simple strings, the are interpreted using the following +conventions: + - If the string begins with `^` and ends with `$`, it is converted to a regex. + - If the string contains an asterisk (`*`), it is transformed from a glob to a regex. + - Otherwise, an exact match to the string is sought. + +To fully-specify a package with a full, exact spec, the following fields must be given as exact +values: + + - channel + - subdir + - name + - version + - build + +In the future, the namespace field might be added to this list. + +Alternatively, an exact spec is given by +`*[sha256=01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b]`. +""" +from typing import Literal, Optional, Union + +from ._base import ExtrasForbiddenModel +from .types import ( + BuildSpecStr, + MD5Str, + PackageFileNameStr, + PackageNameStr, + SHA256Str, + SubdirStr, + VersionSpecStr, +) + + +class MatchSpec(ExtrasForbiddenModel): + """ + TODO: In theory, any PackageRecord (scalar) keys should be admitted here. + """ + + name: Optional[Union[PackageNameStr, Literal["*"]]] = None + "The name of the package" + version: Optional[VersionSpecStr] = None + "The version spec of the package (e.g. `1.2.3`, `>=1.2.3`, `1.2.*`)" + build: Optional[BuildSpecStr] = None + "The build string of the package (e.g. `py37_0`, `py37h6de7cb9_0`, `py*`)" + build_number: Optional[int] = None + "The build number of the package" + file_name: Optional[PackageFileNameStr] = None + "Match the specific filename of the package" + channel: Optional[str] = None + "The channel of the package" + subdir: Optional[SubdirStr] = None + "The subdir of the channel" + namespace: Optional[str] = None + "The namespace of the package (currently not used)" + md5: Optional[MD5Str] = None + "The MD5 hash of the package archive" + sha256: Optional[SHA256Str] = None + "The SHA256 hash of the package archive" + + @classmethod + def from_str(cls, value: str): + raise NotImplementedError diff --git a/conda_models/menuinst.py b/conda_models/menuinst.py new file mode 100644 index 0000000..7f3bcc5 --- /dev/null +++ b/conda_models/menuinst.py @@ -0,0 +1,3 @@ +""" +WIP +""" diff --git a/conda_models/meta_yaml.py b/conda_models/meta_yaml.py new file mode 100644 index 0000000..6125b39 --- /dev/null +++ b/conda_models/meta_yaml.py @@ -0,0 +1,414 @@ +""" +WIP +""" +from typing import Iterable, Literal, Union + +from pydantic import AnyUrl, Field, PositiveInt, constr + +from ._base import ExtrasForbiddenModel +from .types import ( + BuildStr, + EntryPointStr, + MD5Str, + NameVersionBuildMatchSpecStr, + NoarchStr, + NonEmptyStr, + PackageNameStr, + SHA1Str, + SHA256Str, + VersionStr, +) + + +class _RunExports(ExtrasForbiddenModel): + weak: Iterable[NameVersionBuildMatchSpecStr] = None + """ + Dependencies to be exported to runtime requirements when package is added as a host + dependency. + """ + strong: Iterable[NameVersionBuildMatchSpecStr] = None + """ + Dependencies to be exported to runtime requirements when package is added as a build + dependency. + """ + weak_constrains: Iterable[NameVersionBuildMatchSpecStr] = None + """ + Dependencies to be exported to runtime constrains when package is added as a host + dependency. + """ + strong_constrains: Iterable[NameVersionBuildMatchSpecStr] = None + """ + Dependencies to be exported to runtime constrains when package is added as a build + dependency. + """ + + +class _Package(ExtrasForbiddenModel): + name: PackageNameStr + version: VersionStr + + +class _Source(ExtrasForbiddenModel): + fn: str = None + url: Union[AnyUrl, Iterable[AnyUrl]] = None + md5: MD5Str = None + sha1: SHA1Str = None + sha256: SHA256Str = None + path: NonEmptyStr = None + git_url: NonEmptyStr = None + "URL or (relative) path to git repository" + git_tag: NonEmptyStr = None + git_branch: NonEmptyStr = None + git_rev: NonEmptyStr = None + git_depth: int = -1 + hg_url: NonEmptyStr = None + hg_tag: NonEmptyStr = None + svn_url: NonEmptyStr = None + svn_rev: NonEmptyStr = None + svn_ignore_externals: bool = False + svn_username: NonEmptyStr = None + svn_password: NonEmptyStr = None + folder: NonEmptyStr = None + patches: Iterable[NonEmptyStr] = None + no_hoist: bool = None # ??? + "UNDOCUMENTED" + path_via_symlink: str = None # ??? + "UNDOCUMENTED" + + +class _Build(ExtrasForbiddenModel): + number: PositiveInt = 0 + "Identifier for subsequent builds of the same version." + string: BuildStr = None + """ + Third field in the final package filename. + Usually automatically generated from the build number and package contents. + """ + entry_points: Iterable[EntryPointStr] = None + """ + List of Python entry points to be generated when the package is installed. + """ + osx_is_app: bool = False + "Make entry points use python.app instead of Python in macOS" + track_features: Iterable[NonEmptyStr] = None + """ + Adding track_features to one or more of the package variants will cause conda to de-prioritize + it or "weigh it down". The lowest priority package is the one that would cause the most + track_features to be activated in the environment. The default package among many variants is + the one that would cause the least track_features to be activated. + + No two packages in a given subdir should ever have the same track_feature. + """ + preserve_egg_dir: bool = False + "Needed for some packages that use features specific to setuptools." + no_link: Iterable[NonEmptyStr] = None + "A list of globs for files that should always be copied and never soft linked or hard linked." + binary_relocation: bool = True + """ + Whether binary files should be made relocatable using install_name_tool on macOS or patchelf on + Linux. The default is True. It also accepts False, which indicates no relocation for any files, + or a list of files, which indicates relocation only for listed files. + """ + script: Union[NonEmptyStr, Iterable[NonEmptyStr]] = None + """ + Used instead of build.sh or bld.bat. For short build scripts, this can be more convenient. You + may need to use selectors to use different scripts for different platforms. + """ + noarch: NoarchStr = None + "Make this package noarch (architecture independent)." + noarch_python: bool = False + "DEPRECATED. Use 'noarch: python' instead." + has_prefix_files: None + """ + Text files (files containing no NULL bytes) may contain the build prefix and need it replaced + with the install prefix at installation time. Conda will automatically register such files. + Binary files that contain the build prefix are generally handled differently (see + 'binary_has_prefix_files') but there may be cases where such a binary file needs to be treated + as an ordinary text file, in which case they need to be identified. + """ + binary_has_prefix_files: Iterable[NonEmptyStr] = None + """ + By default, conda-build tries to detect prefixes in all files. You may also elect to specify + files with binary prefixes individually. This allows you to specify the type of file as binary, + when it may be incorrectly detected as text for some reason. Binary files are those containing + NULL bytes + """ + ignore_prefix_files: Union[bool, Iterable[NonEmptyStr]] = None + """ + Used to exclude some or all of the files in the build recipe from the list of files that have + the build prefix replaced with the install prefix. Use 'True' to ignore all files, or a list of + paths to specify individual filenames. This setting is independent of RPATH replacement. Use + the 'detect_binary_files_with_prefix' setting to control that behavior. + """ + detect_binary_files_with_prefix: bool = True + """ + Binary files may contain the build prefix and need it replaced with the install prefix at + installation time. Conda can automatically identify and register such files. + """ + skip_compile_pyc: Iterable[NonEmptyStr] = None + """ + List of globs that will not be compiled to bytecode. + """ + rpaths: Iterable[NonEmptyStr] = ("lib/",) + """ + Set which RPATHs are used when making executables relocatable on Linux. This is a Linux feature + that is ignored on other systems. The default is lib/. + """ + script_env: Iterable[constr(min_length=3, regex=r"[A-z0-9_]+(=.+)?")] = None + """ + Allow these environment variables to be seen by the build process. You can also (re)define + their values with the `NAME=VAR` syntax. + """ + always_include_files: Iterable[NonEmptyStr] = None + """ + Force files to always be included, even if they are already in the environment from the build + dependencies. This may be needed, for example, to create a recipe for conda itself. + """ + skip: bool = False + """ + Specifies whether conda-build should skip the build of this recipe. Particularly useful for + defining recipes that are platform specific, thanks to selectors. + """ + pin_depends: Literal["record", "strict"] = None + """ + EXPERIMENTAL. Enforce pinning behaviour on the output recipe or built package. + + With a value of record, conda-build will record all requirements exactly as they would be + installed in a file called info/requires. These pins will not show up in the output of conda + render and they will not affect the actual run dependencies of the output package. It is only + adding in this new file. + + With a value of strict, conda-build applies the pins to the actual metadata. This does affect + the output of conda render and also affects the end result of the build. The package + dependencies will be strictly pinned down to the build string level. This will supersede any + dynamic or compatible pinning that conda-build may otherwise be doing. + """ + include_recipe: bool = True + """ + The full conda-build recipe and rendered meta.yaml file is included in the Package metadata by + default. You can disable it here. + """ + run_exports: Union[Iterable[PackageNameStr], _RunExports] = None + """ + List of packages that will be injected as a runtime dependency in other recipes. + Use the 'strong' key to indicate which dependencies will be injected when this package is used + as a build dependency. Use 'weak' for dependencies that will be injected when this package is + used as a host dependency. If you do not specify a category and just write a list of packages, + 'weak' is assumed. + """ + ignore_run_exports: Iterable[PackageNameStr] = None + "Ignore these injected run exports, regardless the origin." + ignore_run_exports_from: Iterable[PackageNameStr] = None + "Ignore the injected run exports coming from these packages." + force_use_keys: Iterable[PackageNameStr] = None + "Ensure these packages are considered for the build hash" + force_ignore_keys: Iterable[PackageNameStr] = None + "Ensure these packages are NOT considered for the build hash" + merge_build_host: bool = False + "Whether to merge the build and host dependencies into a single environment." + missing_dso_whitelist: Iterable[NonEmptyStr] = None + """ + List of globs for dynamic shared object (DSO) files that should be ignored when examining + linkage information. + """ + overlinking_ignore_patterns: Iterable[NonEmptyStr] = None + """ + Used to ignore patterns of files for the overlinking and overdepending checks. This is + sometimes useful to speed up builds that have many files (large repackage jobs) or builds where + you know only a small fraction of the files should be checked. + + Glob patterns are allowed here, but mind your quoting, especially with leading wildcards. + + Use this sparingly, as the overlinking checks generally do prevent you from making mistakes. + """ + runpath_whitelist: Iterable[NonEmptyStr] = None + """ + List of globs for paths which are allowed to appear as runpaths in the package's shared + libraries. All other runpaths will cause a warning message to be printed during the build. + """ + activate_in_script: bool = True + "UNDOCUMENTED. Whether the environments should be activated before the build script runs." + # These are mentioned in conda_build.metadata.FIELDS but not documented? + disable_pip: bool = False + "UNDOCUMENTED." + error_overdepending: None + "UNDOCUMENTED." + error_overlinking: None + "UNDOCUMENTED." + features: Iterable[NonEmptyStr] = None + "DEPRECATED." + msvc_compiler: str + "UNDOCUMENTED." + noarch_python_build_age: int = None + "UNDOCUMENTED." + no_move_top_level_workdir_loops: bool = None + "UNDOCUMENTED." + postlink: str + "UNDOCUMENTED." + preferred_env_executable_paths: list + "UNDOCUMENTED." + preferred_env: str + "UNDOCUMENTED." + prelink: str + "UNDOCUMENTED." + preunlink: str + "UNDOCUMENTED." + provides_features: dict + "UNDOCUMENTED." + requires_features: dict + "UNDOCUMENTED." + rpaths_patcher: None + "UNDOCUMENTED." + + +class _Requirements(ExtrasForbiddenModel): + """ + Specifies the build and runtime requirements. Dependencies of these requirements are included + automatically. + """ + + build: Iterable[NameVersionBuildMatchSpecStr] = None + """ + Tools required to build the package. These packages are run on the build system and include + things such as revision control systems (Git, SVN) make tools (GNU make, Autotool, CMake) and + compilers (real cross, pseudo-cross, or native when not cross-compiling), and any source + pre-processors. + + Packages which provide "sysroot" files, like the CDT packages (see below) also belong in the + build section. + """ + host: Iterable[NameVersionBuildMatchSpecStr] = None + """ + Packages that need to be specific to the target platform when the target platform is not + necessarily the same as the native build platform. + + For example, in order for a recipe to be "cross-capable", shared libraries requirements must be + listed in the host section, rather than the build section, so that the shared libraries that + get linked are ones for the target platform, rather than the native build platform. + + You should also include the base interpreter for packages that need one. In other words, a + Python package would list 'python' here and an R package would list 'mro-base' or 'r-base'. + """ + run: Iterable[NameVersionBuildMatchSpecStr] = None + """ + Packages required to run the package. These are the dependencies that are installed + automatically whenever the package is installed. + """ + run_constrained: Iterable[NameVersionBuildMatchSpecStr] = None + """ + Packages that are optional at runtime but must obey the supplied additional constraint if they + are installed. + """ + conflicts: Iterable[NameVersionBuildMatchSpecStr] = None + "UNDOCUMENTED." + + +class _Test(ExtrasForbiddenModel): + files: Iterable[NonEmptyStr] = None + """ + Test files that are copied from the recipe into the temporary test directory and are needed + during testing. If providing a path, forward slashes must be used. Allows globs. + """ + source_files: Iterable[NonEmptyStr] = None + """ + Test files that are copied from the source work directory into the temporary test directory and + are needed during testing. Allows globs. + """ + requires: Iterable[NameVersionBuildMatchSpecStr] = None + """ + In addition to the runtime requirements, you can specify requirements needed during testing. + The runtime requirements that you specified in the "run" section described above are + automatically included during testing. + """ + commands: Iterable[NonEmptyStr] = None + """ + Shell commands that are run as part of the test. + """ + imports: Iterable[constr(regex=r"[A-z0-9\._]+")] = None + """ + Python modules that will be imported as part of the test checks. + """ + downstreams: Iterable[PackageNameStr] = None + """ + Run the bundled test suite of the listed packages ensuring the package being built is used as + a dependency. + """ + + +class _App(ExtrasForbiddenModel): + """ + If the app section is present, the package is an app, meaning that it appears in Anaconda + Navigator. + """ + + entry: NonEmptyStr = None + "The command that is called to launch the app in Navigator." + icon: NonEmptyStr = None + "The icon file contained in the recipe." + summary: NonEmptyStr = None + "Summary of the package used in Navigator." + own_environment: bool = False + "Whether to install the app through Navigator into its own environment." + type: NonEmptyStr = None + "UNDOCUMENTED." + cli_opts: NonEmptyStr = None + "UNDOCUMENTED." + + +class _About(ExtrasForbiddenModel): + home: NonEmptyStr = None + dev_url: AnyUrl = None + doc_url: AnyUrl = None + doc_source_url: AnyUrl = None + license_url: AnyUrl = None + license: NonEmptyStr = None + summary: NonEmptyStr = None + description: NonEmptyStr = None + license_family: NonEmptyStr = None + identifiers: Iterable[NonEmptyStr] = None + tags: Iterable[NonEmptyStr] = None + keywords: Iterable[NonEmptyStr] = None + license_file: Union[NonEmptyStr, Iterable[NonEmptyStr]] = None + prelink_message: NonEmptyStr = None + readme: Union[NonEmptyStr, Iterable[NonEmptyStr]] = None + + +class _OutputTest(ExtrasForbiddenModel): + script: NonEmptyStr = None + + +_Extra = dict + + +class _Output(ExtrasForbiddenModel): + name: PackageNameStr + version: VersionStr + number: PositiveInt + entry_points: Iterable[EntryPointStr] = None + script: NonEmptyStr = None + "Script that will be run to prepare or distribute the files being packaged." + script_interpreter: NonEmptyStr = None + "Interpreter to use to run the script." + files: Iterable[NonEmptyStr] = None + "List of files to include in the package, run after 'script', if any." + build: dict # Is this really allowed / used ??? + requirements: Union[Iterable[NameVersionBuildMatchSpecStr], _Requirements] = None + run_exports: Union[Iterable[NameVersionBuildMatchSpecStr], _RunExports] = None + test: _OutputTest = None + type_: Literal["conda", "conda_v2", "wheel"] = Field("conda", alias="type") + about: _About = None + extra: _Extra = None + target: NonEmptyStr = None + "UNDOCUMENTED." + + +class MetaYaml(ExtrasForbiddenModel): + package: _Package + source: Union[_Source, Iterable[_Source]] + build: _Build + requirements: _Requirements + test: _Test + outputs: Iterable[_Output] = None + about: _About = None + app: _App = None + extra: _Extra = None diff --git a/conda_models/package_info.py b/conda_models/package_info.py new file mode 100644 index 0000000..7f3bcc5 --- /dev/null +++ b/conda_models/package_info.py @@ -0,0 +1,3 @@ +""" +WIP +""" diff --git a/conda_models/package_record.py b/conda_models/package_record.py new file mode 100644 index 0000000..31243ae --- /dev/null +++ b/conda_models/package_record.py @@ -0,0 +1,88 @@ +""" +Definitions for the repodata.json files served in conda channels. +""" +from typing import Iterable, Optional, Union + +from pydantic import PositiveInt + +from ._base import ExtrasForbiddenModel +from .types import ( + BuildNumber, + BuildStr, + MD5Str, + NameVersionBuildMatchSpecStr, + NoarchStr, + NonEmptyStr, + PackageNameStr, + SHA256Str, + SubdirStr, + VersionStr, +) + + +class RepodataRecord(ExtrasForbiddenModel): + """ + A single record in the conda repodata. + + A single record refers to a single binary distribution of a package on a conda channel. + """ + + arch: Optional[str] = None + "Optionally the architecture the package supports" + build: BuildStr + "The build string of the package." + build_number: BuildNumber + "The build number of the package." + constrains: Iterable[NameVersionBuildMatchSpecStr] + """ + Additional constraints on packages. `constrains` are different from `depends` in that packages + specified in `depends` must be installed next to this package, whereas packages specified in + `constrains` are not required to be installed, but if they are installed they must follow these + constraints. + """ + depends: Iterable[NameVersionBuildMatchSpecStr] + "Specification of packages this package depends on." + features: Optional[NonEmptyStr] = None + """ + Features are a deprecated way to specify different feature sets for the conda solver. This is + not supported anymore and should not be used. Instead, `mutex` packages should be used to + specify mutually exclusive features. + """ + legacy_bz2_md5: Optional[NonEmptyStr] = None + "A deprecated md5 hash" + legacy_bz2_size: Optional[PositiveInt] = None + "A deprecated package archive size" + license: Optional[NonEmptyStr] = None + "The specific license of the package" + license_family: Optional[NonEmptyStr] = None + "The specific license of the package" + md5: Optional[MD5Str] = None + "The md5 hash of the package archive" + name: PackageNameStr + "The name of the package" + noarch: NoarchStr + "Whether the package is architecture independent, and in which way." + platform: Optional[str] = None + "The platform the package supports" + sha256: Optional[SHA256Str] = None + "The sha256 hash of the package archive" + size: Optional[PositiveInt] = None + "The size of the package archive, in bytes" + subdir: SubdirStr + "The subdirectory of the channel this package is in" + timestamp: Optional[PositiveInt] = None + "The date this entry was created" + track_features: Optional[Union[NonEmptyStr, Iterable[NonEmptyStr]]] = None + """ + Nowadays only used to downweight package variants (ie. give a variant less priority). To that + effect, the number of track features is counted (number of commas) and the package is + downweighted by the number of track_features. + """ + version: VersionStr + "The version of the package" + preferred_env: Optional[str] = None + "Unused" + date: Optional[str] = None + "Unused" + package_type: Optional[str] = None + "Unused" diff --git a/conda_models/prefix_record.py b/conda_models/prefix_record.py new file mode 100644 index 0000000..7f3bcc5 --- /dev/null +++ b/conda_models/prefix_record.py @@ -0,0 +1,3 @@ +""" +WIP +""" diff --git a/conda_models/repodata.py b/conda_models/repodata.py new file mode 100644 index 0000000..7147b6d --- /dev/null +++ b/conda_models/repodata.py @@ -0,0 +1,61 @@ +""" +Definitions for the repodata.json files served in conda channels. +""" +from pydantic import AnyUrl, Field + +from ._base import ExtrasForbiddenModel, make_optional +from .package_record import PackageRecord +from .types import ( + CondaPackageFileNameStr, + NonEmptyStr, + PackageFileNameStr, + Subdir, + TarBz2PackageFileNameStr, +) + + +class RepodataRecord(PackageRecord): + """ + A single record in the conda repodata. + + A single record refers to a single binary distribution of a package on a conda channel. + """ + + filename: str = Field(..., alias="fn") + "The filename of the package archive" + url: AnyUrl + "The canonical URL from where to get this package" + channel: NonEmptyStr + """ + String representation of the channel where the package comes from. + It can be a URL (preferred) or a channel name. + """ + revoked: bool = None + "DEPRECATED." + + +OptionalRepodataRecord = make_optional(RepodataRecord) + + +class ChannelInfo(ExtrasForbiddenModel): + """ """ + + subdir: Subdir + + +class Repodata(ExtrasForbiddenModel): + """ """ + + info: ChannelInfo + "Information about the repodata" + packages: dict[TarBz2PackageFileNameStr, OptionalRepodataRecord] + "The .tar.bz2 packages in the repodata" + packages_conda: dict[CondaPackageFileNameStr, OptionalRepodataRecord] = Field( + ..., + alias="packages.conda", + ) + "The .conda packages in the repodata" + removed: set[PackageFileNameStr] + "The packages that have been removed from the repodata" + version: int = Field(..., alias="repodata_version") + "The version of the repodata" diff --git a/conda_models/repodata_patch_instructions.py b/conda_models/repodata_patch_instructions.py new file mode 100644 index 0000000..edc4e68 --- /dev/null +++ b/conda_models/repodata_patch_instructions.py @@ -0,0 +1,24 @@ +""" +WIP +""" +from pydantic import Field + +from ._base import ExtrasForbiddenModel +from .repodata import OptionalRepodataRecord +from .types import CondaPackageFileNameStr, PackageFileNameStr, TarBz2PackageFileNameStr + + +class RepodataPatchInstructions(ExtrasForbiddenModel): + packages: dict[TarBz2PackageFileNameStr, OptionalRepodataRecord] + "The .tar.bz2 packages in the repodata that will be patched" + packages_conda: dict[CondaPackageFileNameStr, OptionalRepodataRecord] = Field( + ..., + alias="packages.conda", + ) + "The .conda packages in the repodata that will be patched" + remove: set[PackageFileNameStr] + "The packages (conda or tar.bz2) that should be removed from the index" + revoke: set[str] + "DEPRECATED." + patch_instructions_version: int + "Version of the patch instructions schema" diff --git a/conda_models/repodata_state.py b/conda_models/repodata_state.py new file mode 100644 index 0000000..7f3bcc5 --- /dev/null +++ b/conda_models/repodata_state.py @@ -0,0 +1,3 @@ +""" +WIP +""" diff --git a/conda_models/requirements_file.py b/conda_models/requirements_file.py new file mode 100644 index 0000000..ee4998e --- /dev/null +++ b/conda_models/requirements_file.py @@ -0,0 +1,34 @@ +""" +WIP +""" +from typing import Iterable, Optional + +from ._base import ExtrasForbiddenModel +from .types import MatchSpecStr, SubdirStr + + +class RequirementsTxt(ExtrasForbiddenModel): + """ + A requirements.txt file as produced by conda list --export or conda list --explicit + + ``` + # This file may be used to create an environment using: + # $ conda create --name --file + # platform: osx-64 + @EXPLICIT + https://repo.anaconda.com/pkgs/free/osx-64/mkl-11.3.3-0.tar.bz2 + https://repo.anaconda.com/pkgs/free/osx-64/numpy-1.11.1-py35_0.tar.bz2 + https://repo.anaconda.com/pkgs/free/osx-64/openssl-1.0.2h-1.tar.bz2 + https://repo.anaconda.com/pkgs/free/osx-64/pip-8.1.2-py35_0.tar.bz2 + https://repo.anaconda.com/pkgs/free/osx-64/python-3.5.2-0.tar.bz2 + https://repo.anaconda.com/pkgs/free/osx-64/readline-6.2-2.tar.bz2 + ``` + """ + + platform: Optional[SubdirStr] = None + explicit: bool = False + records: Iterable[MatchSpecStr] = () + + @classmethod + def from_str(cls, value: str): + raise NotImplementedError("TODO") diff --git a/conda_models/types.py b/conda_models/types.py new file mode 100644 index 0000000..34b54bd --- /dev/null +++ b/conda_models/types.py @@ -0,0 +1,107 @@ +""" +conda-specific constrains for scalar types. +""" +from enum import Enum, auto +from typing import Union + +from pydantic import AnyUrl, constr + + +class NoarchStr(str, Enum): + Python = "python" + """ + A noarch python package is a python package without any precompiled python files (`.pyc` or + `__pycache__`). Normally these files are bundled with the package. However, these files are + tied to a specific version of Python and must therefor be generated for every target + platform and architecture. This complicates the build process. + + For noarch python packages these files are generated when installing the package by invoking + the compilation process through the python binary that is installed in the same environment. + + This introductory blog post highlights some of specific of noarch python packages: + + + Or read the docs for more information: + + """ + Generic = "generic" + """ + Noarch generic packages allow users to distribute docs, datasets, and source code in conda + packages. + """ + + +class NoarchType(str, Enum): + GenericV1 = auto() + "An old-format generic noarch package" + GenericV2 = auto() + "A new-format generic noarch package" + Python = auto() + "A noarch Python package" + + +class SubdirStr(str, Enum): + Linux64 = "linux-64" + Linux32 = "linux-32" + LinuxArmV6l = "linux-armv6l" + LinuxArmV7l = "linux-armv7l" + LinuxPPC64le = "linux-ppc64le" + LinuxS390x = "linux-s390x" + OSX64 = "osx-64" + OSX32 = "osx-32" + OSXArm64 = "osx-arm64" + Win64 = "win-64" + Win32 = "win-32" + WinArm64 = "win-arm64" + ZosZ = "zos-z" + Noarch = "noarch" + + +class PackageType(str, Enum): + pass + + +class Platform(str, Enum): + pass + + +NonEmptyStr = constr(min_length=1) + +package_name_regex = r"[0-9a-zA-Z\._-]+" +version_regex = r"([0-9]!)?[0-9a-z\._]+" +version_spec_regex = r"[0-9a-z<>=!\.\*]+" +build_string_regex = r"[0-9a-zA-Z\._]+" +build_string_spec_regex = r"[0-9a-zA-Z\._\*]+" + +MD5Str = constr(min_length=32, max_length=32, regex=r"[a-fA-F0-9]{32}") +SHA1Str = constr(min_length=40, max_length=40, regex=r"[a-fA-F0-9]{40}") +SHA256Str = constr(min_length=64, max_length=64, regex=r"[a-fA-F0-9]{64}") +MatchSpecStr = NonEmptyStr # TODO: implement regex??? +BuildStr = constr(min_length=1, regex=build_string_regex) +BuildSpecStr = constr(min_length=1, regex=build_string_spec_regex) +PackageNameStr = constr(min_length=1, regex=package_name_regex) +PackageFileNameStr = constr( + min_length=1, + regex=rf"({package_name_regex})-({version_regex})-({build_string_regex})\.(conda|tar\.bz2)", +) +TarBz2PackageFileNameStr = constr( + min_length=1, + regex=rf"({package_name_regex})-({version_regex})-({build_string_regex})\.tar\.bz2", +) +CondaPackageFileNameStr = constr( + min_length=1, + regex=rf"({package_name_regex})-({version_regex})-({build_string_regex})\.conda", +) +NameVersionBuildMatchSpecStr = constr( + min_length=1, + regex=rf"({package_name_regex})\s+(" + rf"({version_spec_regex})" + rf"|({version_spec_regex})?\s+({build_string_spec_regex})" + rf")?", +) +VersionStr = constr(min_length=1, regex=version_regex) +VersionSpecStr = constr(min_length=1, regex=version_spec_regex) + +EntryPointStr = constr(min_length=5, regex=r"\S+\s*=\s*[A-z0-9_\.]:[A-z0-9_]") + +ChannelNameOrUrl = Union[NonEmptyStr, AnyUrl] diff --git a/conda_models/version.py b/conda_models/version.py new file mode 100644 index 0000000..7f3bcc5 --- /dev/null +++ b/conda_models/version.py @@ -0,0 +1,3 @@ +""" +WIP +""" diff --git a/dev/README.md b/dev/README.md new file mode 100644 index 0000000..df27248 --- /dev/null +++ b/dev/README.md @@ -0,0 +1 @@ + diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..df27248 --- /dev/null +++ b/docs/README.md @@ -0,0 +1 @@ + diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..df27248 --- /dev/null +++ b/examples/README.md @@ -0,0 +1 @@ + diff --git a/index.html b/index.html index 6f53702..c27926d 100644 --- a/index.html +++ b/index.html @@ -67,7 +67,7 @@

Conda Schemas

- + diff --git a/models/README.md b/models/README.md new file mode 100644 index 0000000..df27248 --- /dev/null +++ b/models/README.md @@ -0,0 +1 @@ + diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..3713901 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,66 @@ +[tool.ruff] +line-length = 99 +select = [ + "E", "F", "W", #flake8 + "UP", # pyupgrade + "I", # isort + "YTT", #flake8-2020 + "TCH", # flake8-type-checing + "BLE", # flake8-blind-exception + "B", # flake8-bugbear + "A", # flake8-builtins + "C4", # flake8-comprehensions + "ISC", # flake8-implicit-str-concat + "G", # flake8-logging-format + "PIE", # flake8-pie + "COM", # flake8-commas + "SIM", # flake8-simplify + "INP", # flake8-no-pep420 + "PYI", # flake8-pyi + "Q", # flake8-quotes + "RSE", # flake8-raise + "RET", # flake8-return + "TID", # flake8-tidy-imports # replace absolutify import + "TRY", # tryceratops + "ICN", # flake8-import-conventions + "RUF", # ruff specyfic rules +] +ignore = [ + "A003", # shadowed builtin in class +] +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".mypy_cache", + ".pants.d", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + "__pypackages__", + "_build", + "generate-index.py", +] + +target-version = "py38" +fix = true + +[tool.ruff.flake8-quotes] +docstring-quotes = "double" + +[tool.ruff.isort] +known-first-party=['conda_models'] +combine-as-imports = true + +[tool.black] +target-version = ['py38', 'py39', 'py310'] +line-length = 99 + +[tool.interrogate] +# TODO: https://github.com/econchick/interrogate/issues/129 +ignore-module = true +fail-under = 100 +exclude = ["archive", "docs", "tests", "conda_models/_version.py"] +verbose = 2 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..7f3bcc5 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,3 @@ +""" +WIP +"""
File