diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000000..0fe2019442 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,21 @@ +# What does this PR do? + + + + + +Fixes # (issue) + + +## Before submitting +- [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). +- [ ] Did you make sure to update the documentation with your changes? +- [ ] Did you write any new necessary tests? diff --git a/.github/workflows/check_code_quality.yml b/.github/workflows/check_code_quality.yml new file mode 100644 index 0000000000..49f04e00a2 --- /dev/null +++ b/.github/workflows/check_code_quality.yml @@ -0,0 +1,50 @@ +name: check_code_quality + +on: + push: + branches: [ main ] + paths: + - "optimum/**.py" + - "tests/**.py" + - "examples/**.py" + + pull_request: + branches: [ main ] + paths: + - "optimum/**.py" + - "tests/**.py" + - "examples/**.py" + +jobs: + build: + strategy: + fail-fast: false + matrix: + python-version: [3.8] + os: [ubuntu-20.04] + + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Create and start a virtual environment + run: | + python -m venv venv + source venv/bin/activate + - name: Install dependencies + run: | + source venv/bin/activate + pip install --upgrade pip + pip install isort + pip install black + - name: Check style with black + run: | + source venv/bin/activate + black --check . + - name: Check style with isort + run: | + source venv/bin/activate + isort --check . diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000000..d786fdfc6d --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,16 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +include README.md +include LICENSE diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000..254b9d2b36 --- /dev/null +++ b/Makefile @@ -0,0 +1,41 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +.PHONY: style test + +# Run code quality checks +style_check: + black --check . + isort --check . + +style: + black . + isort . + +# Run tests for the library +test: + python -m pytest tests + +# Utilities to release to PyPi +build_dist_install_tools: + pip install build + pip install twine + +build_dist: + rm -fr build + rm -fr dist + python -m build + +pypi_upload: build_dist + python -m twine upload dist/* diff --git a/examples/question-answering/README.md b/examples/question-answering/README.md new file mode 100644 index 0000000000..6f258e717b --- /dev/null +++ b/examples/question-answering/README.md @@ -0,0 +1,17 @@ + + +# SQuAD diff --git a/examples/text-classification/README.md b/examples/text-classification/README.md new file mode 100644 index 0000000000..56e1be632f --- /dev/null +++ b/examples/text-classification/README.md @@ -0,0 +1,17 @@ + + +# MRPC diff --git a/optimum/habana/__init__.py b/optimum/habana/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/optimum/habana/version.py b/optimum/habana/version.py new file mode 100644 index 0000000000..6b1d65ceaa --- /dev/null +++ b/optimum/habana/version.py @@ -0,0 +1,15 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__version__ = "0.1.0" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000..b2a989ef21 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,17 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[tool.black] +line-length = 119 +target-version = ['py35'] diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000000..3f739e2fe3 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,54 @@ +[isort] +default_section = FIRSTPARTY +ensure_newline_before_comments = True +force_grid_wrap = 0 +include_trailing_comma = True +known_first_party = transformers +known_third_party = + absl + conllu + datasets + elasticsearch + fairseq + faiss-cpu + fastprogress + fire + fugashi + git + h5py + matplotlib + nltk + numpy + packaging + pandas + PIL + psutil + pytest + pytorch_lightning + rouge_score + sacrebleu + seqeval + sklearn + streamlit + tensorboardX + tensorflow + tensorflow_datasets + timeout_decorator + torch + torchaudio + torchtext + torchvision + torch_xla + tqdm + +line_length = 119 +lines_after_imports = 2 +multi_line_output = 3 +use_parentheses = True + +[flake8] +ignore = E203, E501, E741, W503, W605 +max-line-length = 119 + +[tool:pytest] +doctest_optionflags=NUMBER NORMALIZE_WHITESPACE ELLIPSIS diff --git a/setup.py b/setup.py new file mode 100644 index 0000000000..b51750ae46 --- /dev/null +++ b/setup.py @@ -0,0 +1,57 @@ +import re + +from setuptools import find_namespace_packages, setup + + +# Ensure we match the version set in optimum/version.py +try: + filepath = "optimum/habana/version.py" + with open(filepath) as version_file: + (__version__,) = re.findall('__version__ = "(.*)"', version_file.read()) +except Exception as error: + assert False, "Error: Could not open '%s' due %s\n" % (filepath, error) + + +install_requires = [ + "optimum", + "datasets", + "tokenizers", + "torch", + "sentencepiece", + "scipy", + "pillow", + "pytest", + "pytest-pythonpath", + "parameterized", + "GitPython", +] + +setup( + name="optimum-habana", + version=__version__, + description="Optimum Library is an extension of the Hugging Face Transformers library, providing a framework to " + "integrate third-party libraries from Hardware Partners and interface with their specific " + "functionality.", + long_description=open("README.md", "r", encoding="utf-8").read(), + long_description_content_type="text/markdown", + classifiers=[ + "Development Status :: 3 - Alpha", + "License :: OSI Approved :: Apache Software License", + "Intended Audience :: Developers", + "Intended Audience :: Education", + "Intended Audience :: Science/Research", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + ], + keywords="transformers, quantization, fine-tuning, gaudi, hpu", + url="https://huggingface.co/hardware", + author="HuggingFace Inc. Special Ops Team", + author_email="hardware@huggingface.co", + license="Apache", + packages=find_namespace_packages(include=["optimum*"]), + install_requires=install_requires, + include_package_data=True, + zip_safe=False, +) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2