forked from spietras/rules_conda
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Sebastian Pietras
committed
Aug 26, 2020
1 parent
0d5208a
commit f315956
Showing
20 changed files
with
617 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,148 @@ | ||
########################################## COMMON ########################################## | ||
|
||
config/*/.ijwb/ | ||
|
||
########################################## BAZEL ########################################## | ||
|
||
# Bazel links | ||
/example/bazel-* | ||
|
||
########################################## PYTHON ########################################## | ||
|
||
# Byte-compiled / optimized / DLL files | ||
__pycache__/ | ||
*.py[cod] | ||
*$py.class | ||
|
||
# C extensions | ||
*.so | ||
|
||
# Distribution / packaging | ||
.Python | ||
build/ | ||
develop-eggs/ | ||
dist/ | ||
downloads/ | ||
eggs/ | ||
.eggs/ | ||
lib64/ | ||
parts/ | ||
sdist/ | ||
var/ | ||
wheels/ | ||
share/python-wheels/ | ||
*.egg-info/ | ||
.installed.cfg | ||
*.egg | ||
MANIFEST | ||
|
||
# PyInstaller | ||
# Usually these files are written by a python script from a template | ||
# before PyInstaller builds the exe, so as to inject date/other infos into it. | ||
*.manifest | ||
*.spec | ||
|
||
# Installer logs | ||
pip-log.txt | ||
pip-delete-this-directory.txt | ||
|
||
# Unit test / coverage reports | ||
htmlcov/ | ||
.tox/ | ||
.nox/ | ||
.coverage | ||
.coverage.* | ||
.cache | ||
nosetests.xml | ||
coverage.xml | ||
*.cover | ||
*.py,cover | ||
.hypothesis/ | ||
.pytest_cache/ | ||
cover/ | ||
|
||
# Translations | ||
*.mo | ||
*.pot | ||
|
||
# Django stuff: | ||
*.log | ||
local_settings.py | ||
db.sqlite3 | ||
db.sqlite3-journal | ||
|
||
# Flask stuff: | ||
instance/ | ||
.webassets-cache | ||
|
||
# Scrapy stuff: | ||
.scrapy | ||
|
||
# Sphinx documentation | ||
docs/_build/ | ||
|
||
# PyBuilder | ||
.pybuilder/ | ||
target/ | ||
|
||
# Jupyter Notebook | ||
.ipynb_checkpoints | ||
|
||
# IPython | ||
profile_default/ | ||
ipython_config.py | ||
|
||
# pyenv | ||
# For a library or package, you might want to ignore these files since the code is | ||
# intended to run in multiple environments; otherwise, check them in: | ||
# .python-version | ||
|
||
# pipenv | ||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. | ||
# However, in case of collaboration, if having platform-specific dependencies or dependencies | ||
# having no cross-platform support, pipenv may install dependencies that don't work, or not | ||
# install all needed dependencies. | ||
#Pipfile.lock | ||
|
||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow | ||
__pypackages__/ | ||
|
||
# Celery stuff | ||
celerybeat-schedule | ||
celerybeat.pid | ||
|
||
# SageMath parsed files | ||
*.sage.py | ||
|
||
# Environments | ||
.env | ||
.venv | ||
env/ | ||
venv/ | ||
ENV/ | ||
env.bak/ | ||
venv.bak/ | ||
|
||
# Spyder project settings | ||
.spyderproject | ||
.spyproject | ||
|
||
# Rope project settings | ||
.ropeproject | ||
|
||
# mkdocs documentation | ||
/site | ||
|
||
# mypy | ||
.mypy_cache/ | ||
.dmypy.json | ||
dmypy.json | ||
|
||
# Pyre type checker | ||
.pyre/ | ||
|
||
# pytype static type analyzer | ||
.pytype/ | ||
|
||
# Cython debug symbols | ||
cython_debug/ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
workspace(name = "rules_conda") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,117 @@ | ||
# TODO: other architectures | ||
ARCH = "x86_64" | ||
|
||
# CONDA CONFIGURATION | ||
CONDA_MAJOR = "3" | ||
CONDA_MINOR = "py38_4.8.3" | ||
CONDA_SHA = { | ||
"Windows": "1f4ff67f051c815b6008f144fdc4c3092af2805301d248b56281c36c1f4333e5", | ||
"MacOSX": "9b9a353fadab6aa82ac0337c367c23ef842f97868dcbb2ff25ec3aa463afc871", | ||
"Linux": "879457af6a0bf5b34b48c12de31d4df0ee2f06a8e68768e5758c3293b2daf688" | ||
} | ||
CONDA_INSTALLER_NAME_TEMPLATE = "Miniconda{major}-{minor}-{os}-{arch}.{ext}" | ||
CONDA_BASE_URL = "https://repo.anaconda.com/miniconda/" | ||
CONDA_INSTALLER_FLAGS = { | ||
"Windows": ["/InstallationType=JustMe", "/AddToPath=0", "/RegisterPython=0", "/S", "/D={}"], | ||
"MacOSX": ["-b", "-f", "-p", "{}"], | ||
"Linux": ["-b", "-f", "-p", "{}"] | ||
} | ||
|
||
OS_EXT_MAP = { | ||
"Windows": "exe", | ||
"MacOSX": "sh", | ||
"Linux": "sh" | ||
} | ||
|
||
INSTALLER_DIR = "installer" | ||
|
||
CONDA_BUILD_FILE_TEMPLATE = """# This file was automatically generated by rules_conda | ||
exports_files(['{conda}']) | ||
""" | ||
|
||
|
||
def _get_installer_flags(repository_ctx, dir): | ||
os = _get_os(repository_ctx) | ||
flags = CONDA_INSTALLER_FLAGS[os] | ||
# insert directory | ||
return flags[:-1] + [flags[-1].format(dir)] | ||
|
||
|
||
def _get_os(repository_ctx): | ||
os_family = repository_ctx.os.name.lower() | ||
if "windows" in os_family: | ||
return "Windows" | ||
if "mac" in os_family: | ||
return "MacOSX" | ||
if "linux" in os_family or "unix" in os_family: | ||
return "Linux" | ||
fail("Unsupported OS: {}".format(os_family)) | ||
|
||
|
||
# download conda installer | ||
def _download_conda(repository_ctx): | ||
repository_ctx.report_progress("Downloading conda installer") | ||
os = _get_os(repository_ctx) | ||
# TODO: check architecture also, for now downloading everything as 64 bit | ||
ext = OS_EXT_MAP[os] | ||
url = CONDA_BASE_URL + CONDA_INSTALLER_NAME_TEMPLATE.format(major=CONDA_MAJOR, minor=CONDA_MINOR, os=os, arch=ARCH, ext=ext) | ||
output = "{}/install.{}".format(INSTALLER_DIR, ext) | ||
# download from url to output | ||
repository_ctx.download( | ||
url = url, | ||
output = output, | ||
sha256 = CONDA_SHA[os], | ||
executable = True | ||
) | ||
return output | ||
|
||
|
||
# install conda locally | ||
def _install_conda(repository_ctx, installer): | ||
repository_ctx.report_progress("Installing conda") | ||
os = _get_os(repository_ctx) | ||
installer_flags = _get_installer_flags(repository_ctx, repository_ctx.attr.conda_dir) | ||
args = [installer] + installer_flags | ||
# execute installer with flags adjusted to OS | ||
result = repository_ctx.execute(args, quiet=repository_ctx.attr.quiet) | ||
if result.return_code: | ||
fail("Failure installing conda.\n{}\n{}".format(result.stdout, result.stderr)) | ||
return "condabin/conda" | ||
|
||
|
||
# use conda to update itself | ||
def _update_conda(repository_ctx, executable): | ||
args = [executable, "update", "conda"] | ||
# update conda itself | ||
result = repository_ctx.execute(args, quiet=repository_ctx.attr.quiet, working_directory=repository_ctx.attr.conda_dir) | ||
if result.return_code: | ||
fail("Failure updating conda.\n{}\n{}".format(result.stdout, result.stderr)) | ||
|
||
|
||
# create BUILD file with exposed conda binary | ||
def _create_conda_build_file(repository_ctx, executable): | ||
conda = "{}/{}".format(repository_ctx.attr.conda_dir, executable) | ||
repository_ctx.file( | ||
"BUILD", | ||
content = CONDA_BUILD_FILE_TEMPLATE.format(conda=conda) | ||
) | ||
|
||
|
||
def _load_conda_impl(repository_ctx): | ||
installer = _download_conda(repository_ctx) | ||
executable = _install_conda(repository_ctx, installer) | ||
_update_conda(repository_ctx, executable) | ||
_create_conda_build_file(repository_ctx, executable) | ||
|
||
|
||
load_conda_rule = repository_rule( | ||
_load_conda_impl, | ||
attrs = { | ||
"conda_dir": attr.string(mandatory=True), | ||
"quiet": attr.bool( | ||
default = True, | ||
doc = "False if conda output should be shown" | ||
) | ||
} | ||
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,48 @@ | ||
load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") | ||
load(":conda.bzl", "load_conda_rule") | ||
load(":env.bzl", "conda_create_rule") | ||
load(":toolchain.bzl", "toolchain_rule") | ||
|
||
CONDA_REPO_NAME = "conda" | ||
CONDA_DIR = "conda" | ||
DEFAULT_ENV_NAME = "my_env" | ||
DEFAULT_TOOLCHAIN_REPO_NAME = "conda_tools" | ||
DEFAULT_TOOLCHAIN_NAME = "python_toolchain" | ||
|
||
# download and install conda | ||
def load_conda(**kwargs): | ||
|
||
maybe( | ||
load_conda_rule, | ||
CONDA_REPO_NAME, | ||
conda_dir = CONDA_DIR, | ||
**kwargs | ||
) | ||
|
||
# create conda environment | ||
def conda_create(name = DEFAULT_ENV_NAME, **kwargs): | ||
|
||
conda = "@{}//:{}/condabin/conda".format(CONDA_REPO_NAME, CONDA_DIR) | ||
|
||
maybe( | ||
conda_create_rule, | ||
name, | ||
conda = conda, | ||
**kwargs | ||
) | ||
|
||
# register python toolchain from environments | ||
def register_toolchain(py3_env, py2_env=None, name=DEFAULT_TOOLCHAIN_REPO_NAME, toolchain_name=DEFAULT_TOOLCHAIN_NAME, **kwargs): | ||
py2_runtime = "@{}//:python_runtime".format(py2_env) if py2_env else None | ||
py3_runtime = "@{}//:python_runtime".format(py3_env) | ||
|
||
maybe( | ||
toolchain_rule, | ||
name, | ||
py2_runtime = py2_runtime, | ||
py3_runtime = py3_runtime, | ||
toolchain_name = toolchain_name, | ||
**kwargs | ||
) | ||
|
||
native.register_toolchains("@{}//:{}".format(name, toolchain_name)) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,66 @@ | ||
BUILD_FILE_CONTENT = """# This file was automatically generated by rules_conda | ||
package(default_visibility = ["//visibility:public"]) | ||
load("@bazel_tools//tools/python:toolchain.bzl", "py_runtime_pair") | ||
py_runtime( | ||
name = "python_runtime", | ||
files = glob(["{env_path}/**/*"], exclude_directories = 0), | ||
interpreter = "{env_path}/bin/python", | ||
python_version = "PY{py_major}" | ||
) | ||
""" | ||
|
||
|
||
# create new local conda environment from file | ||
def _create_environment(repository_ctx, executable, env_name): | ||
repository_ctx.report_progress("Creating conda environment") | ||
|
||
# path to env file as string | ||
env_file = str(repository_ctx.path(repository_ctx.attr.environment)) | ||
|
||
args = [executable, "env", "create", "-f", env_file, "-p", "./{}".format(env_name)] | ||
|
||
result = repository_ctx.execute(args, quiet=repository_ctx.attr.quiet) | ||
if result.return_code: | ||
fail("Failure creating environment.\n{}\n{}".format(result.stdout, result.stderr)) | ||
|
||
|
||
# check if python2 or python3 has been installed | ||
def _get_py_major(repository_ctx, env_name): | ||
if repository_ctx.path("{}/bin/python3".format(env_name)).exists: | ||
return 3 | ||
return 2 | ||
|
||
|
||
# create BUILD file with py_runtime | ||
def _create_env_build_file(repository_ctx, env_name): | ||
py_major = _get_py_major(repository_ctx, env_name) | ||
repository_ctx.file( | ||
"BUILD", | ||
content = BUILD_FILE_CONTENT.format(env_path=env_name, py_major=py_major) | ||
) | ||
|
||
|
||
def _conda_create_impl(repository_ctx): | ||
executable = str(repository_ctx.path(repository_ctx.attr.conda)) | ||
env_name = repository_ctx.name | ||
_create_environment(repository_ctx, executable, env_name) | ||
_create_env_build_file(repository_ctx, env_name) | ||
|
||
|
||
conda_create_rule = repository_rule( | ||
_conda_create_impl, | ||
attrs = { | ||
"conda": attr.label(mandatory=True), | ||
"environment": attr.label( | ||
mandatory = True, | ||
allow_single_file = True, | ||
doc = "The label of the environment.yml file.", | ||
), | ||
"quiet": attr.bool( | ||
default = True, | ||
doc = "False if conda output should be shown" | ||
) | ||
} | ||
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
USE_BAZEL_VERSION=3.4.1 |
Oops, something went wrong.