From a9a17ce626b50eeb64c27f329b5f2f944576b32e Mon Sep 17 00:00:00 2001 From: Eli Uriegas Date: Mon, 25 Apr 2022 13:27:25 -0700 Subject: [PATCH] third_party: Fix build_bundled script This script was hanging up on a couple of licenses not being parsed correctly / not accounted for in the original pass through Should resolve issues with tests not running 100% correctly Signed-off-by: Eli Uriegas Pull Request resolved: https://github.com/pytorch/pytorch/pull/76336 Signed-off-by: Eli Uriegas Approved by: https://github.com/ezyang --- .gitattributes | 1 + third_party/LICENSES_BUNDLED.txt | 189 ++++++++++++++++++++++++------- third_party/build_bundled.py | 54 ++++++--- 3 files changed, 184 insertions(+), 60 deletions(-) diff --git a/.gitattributes b/.gitattributes index d87495166e5c2..8bccf04bbb7dc 100644 --- a/.gitattributes +++ b/.gitattributes @@ -3,3 +3,4 @@ .github/workflows/generated-*.yml linguist-generated=true .github/generated-* linguist-generated=true .github/scripts/gql_mocks.json linguist-generated=true +third_party/LICENSES_BUNDLED.txt linguist-generated=true diff --git a/third_party/LICENSES_BUNDLED.txt b/third_party/LICENSES_BUNDLED.txt index c1c9a1783964f..9b61374c0aa76 100644 --- a/third_party/LICENSES_BUNDLED.txt +++ b/third_party/LICENSES_BUNDLED.txt @@ -6,11 +6,21 @@ License: MIT Files: third_party/FP16 For details, see third_party/FP16/LICENSE +Name: FP16-source +License: MIT +Files: third_party/XNNPACK/build/FP16-source + For details, see third_party/XNNPACK/build/FP16-source/LICENSE + Name: FXdiv License: MIT Files: third_party/FXdiv For details, see third_party/FXdiv/LICENSE +Name: FXdiv-source +License: MIT +Files: third_party/XNNPACK/build/FXdiv-source + For details, see third_party/XNNPACK/build/FXdiv-source/LICENSE + Name: NNPACK License: BSD-2-Clause Files: third_party/NNPACK @@ -29,22 +39,36 @@ Files: third_party/XNNPACK Name: benchmark License: Apache-2.0 Files: third_party/benchmark, - third_party/protobuf/third_party/benchmark, + third_party/onnx/third_party/benchmark, third_party/onnx-tensorrt/third_party/onnx/third_party/benchmark, - third_party/onnx/third_party/benchmark + third_party/protobuf/third_party/benchmark For details, see third_party/benchmark/LICENSE, - third_party/protobuf/third_party/benchmark/LICENSE, + third_party/onnx/third_party/benchmark/LICENSE, third_party/onnx-tensorrt/third_party/onnx/third_party/benchmark/LICENSE, - third_party/onnx/third_party/benchmark/LICENSE + third_party/protobuf/third_party/benchmark/LICENSE + +Name: breakpad +License: BSD-3-Clause +Files: third_party/breakpad + For details, see third_party/breakpad/LICENSE Name: clog License: BSD-2-Clause -Files: third_party/cpuinfo/deps/clog, - third_party/fbgemm/third_party/cpuinfo/deps/clog, - third_party/QNNPACK/deps/clog - For details, see third_party/cpuinfo/deps/clog/LICENSE, - third_party/fbgemm/third_party/cpuinfo/deps/clog/LICENSE, - third_party/QNNPACK/deps/clog/LICENSE +Files: third_party/QNNPACK/deps/clog, + third_party/XNNPACK/build/clog-source/deps/clog, + third_party/XNNPACK/build/cpuinfo-source/deps/clog, + third_party/cpuinfo/deps/clog, + third_party/fbgemm/third_party/cpuinfo/deps/clog + For details, see third_party/QNNPACK/deps/clog/LICENSE, + third_party/XNNPACK/build/clog-source/deps/clog/LICENSE, + third_party/XNNPACK/build/cpuinfo-source/deps/clog/LICENSE, + third_party/cpuinfo/deps/clog/LICENSE, + third_party/fbgemm/third_party/cpuinfo/deps/clog/LICENSE + +Name: clog-source +License: BSD-2-Clause +Files: third_party/XNNPACK/build/clog-source + For details, see third_party/XNNPACK/build/clog-source/LICENSE Name: cpuinfo License: BSD-2-Clause @@ -53,6 +77,21 @@ Files: third_party/cpuinfo, For details, see third_party/cpuinfo/LICENSE, third_party/fbgemm/third_party/cpuinfo/LICENSE +Name: cpuinfo-source +License: BSD-2-Clause +Files: third_party/XNNPACK/build/cpuinfo-source + For details, see third_party/XNNPACK/build/cpuinfo-source/LICENSE + +Name: cudnn_frontend +License: MIT +Files: third_party/cudnn_frontend + For details, see third_party/cudnn_frontend/LICENSE.txt + +Name: dart +License: Apache-2.0 +Files: third_party/flatbuffers/dart + For details, see third_party/flatbuffers/dart/LICENSE + Name: eigen License: BSD-3-Clause Files: third_party/eigen @@ -68,12 +107,17 @@ License: BSD-3-Clause Files: third_party/fbgemm For details, see third_party/fbgemm/LICENSE +Name: flatbuffers +License: Apache-2.0 +Files: third_party/flatbuffers + For details, see third_party/flatbuffers/LICENSE.txt + Name: fmt License: MIT with exception -Files: third_party/kineto/libkineto/third_party/fmt, - third_party/fmt - For details, see third_party/kineto/libkineto/third_party/fmt/LICENSE.rst, - third_party/fmt/LICENSE.rst +Files: third_party/fmt, + third_party/kineto/libkineto/third_party/fmt + For details, see third_party/fmt/LICENSE.rst, + third_party/kineto/libkineto/third_party/fmt/LICENSE.rst Name: foxi License: MIT @@ -87,14 +131,18 @@ Files: third_party/gemmlowp/gemmlowp Name: generator License: Apache-2.0 -Files: third_party/kineto/libkineto/third_party/googletest/googlemock/scripts/generator, - third_party/googletest/googlemock/scripts/generator, +Files: third_party/XNNPACK/build/googletest-source/googlemock/scripts/generator, + third_party/benchmark/build/third_party/googletest/src/googlemock/scripts/generator, third_party/fbgemm/third_party/googletest/googlemock/scripts/generator, + third_party/googletest/googlemock/scripts/generator, + third_party/kineto/libkineto/third_party/googletest/googlemock/scripts/generator, third_party/protobuf/third_party/googletest/googlemock/scripts/generator, third_party/tensorpipe/third_party/googletest/googlemock/scripts/generator - For details, see third_party/kineto/libkineto/third_party/googletest/googlemock/scripts/generator/LICENSE, - third_party/googletest/googlemock/scripts/generator/LICENSE, + For details, see third_party/XNNPACK/build/googletest-source/googlemock/scripts/generator/LICENSE, + third_party/benchmark/build/third_party/googletest/src/googlemock/scripts/generator/LICENSE, third_party/fbgemm/third_party/googletest/googlemock/scripts/generator/LICENSE, + third_party/googletest/googlemock/scripts/generator/LICENSE, + third_party/kineto/libkineto/third_party/googletest/googlemock/scripts/generator/LICENSE, third_party/protobuf/third_party/googletest/googlemock/scripts/generator/LICENSE, third_party/tensorpipe/third_party/googletest/googlemock/scripts/generator/LICENSE @@ -103,46 +151,58 @@ License: BSD-3-Clause Files: third_party/gloo For details, see third_party/gloo/LICENSE +Name: googlebenchmark-source +License: Apache-2.0 +Files: third_party/XNNPACK/build/googlebenchmark-source + For details, see third_party/XNNPACK/build/googlebenchmark-source/LICENSE + Name: googlemock License: BSD-3-Clause -Files: third_party/kineto/libkineto/third_party/googletest/googlemock, - third_party/googletest/googlemock, +Files: third_party/XNNPACK/build/googletest-source/googlemock, third_party/fbgemm/third_party/googletest/googlemock, + third_party/kineto/libkineto/third_party/googletest/googlemock, third_party/protobuf/third_party/googletest/googlemock, third_party/tensorpipe/third_party/googletest/googlemock - For details, see third_party/kineto/libkineto/third_party/googletest/googlemock/LICENSE, - third_party/googletest/googlemock/LICENSE, + For details, see third_party/XNNPACK/build/googletest-source/googlemock/LICENSE, third_party/fbgemm/third_party/googletest/googlemock/LICENSE, + third_party/kineto/libkineto/third_party/googletest/googlemock/LICENSE, third_party/protobuf/third_party/googletest/googlemock/LICENSE, third_party/tensorpipe/third_party/googletest/googlemock/LICENSE Name: googletest License: BSD-3-Clause -Files: third_party/kineto/libkineto/third_party/googletest, - third_party/kineto/libkineto/third_party/googletest/googletest, - third_party/googletest, - third_party/googletest/googletest, +Files: third_party/XNNPACK/build/googletest-source/googletest, third_party/fbgemm/third_party/googletest, third_party/fbgemm/third_party/googletest/googletest, + third_party/googletest, + third_party/kineto/libkineto/third_party/googletest, + third_party/kineto/libkineto/third_party/googletest/googletest, third_party/protobuf/third_party/googletest, third_party/protobuf/third_party/googletest/googletest, third_party/tensorpipe/third_party/googletest, third_party/tensorpipe/third_party/googletest/googletest - For details, see third_party/kineto/libkineto/third_party/googletest/LICENSE, - third_party/kineto/libkineto/third_party/googletest/googletest/LICENSE, - third_party/googletest/LICENSE, - third_party/googletest/googletest/LICENSE, + For details, see third_party/XNNPACK/build/googletest-source/googletest/LICENSE, third_party/fbgemm/third_party/googletest/LICENSE, third_party/fbgemm/third_party/googletest/googletest/LICENSE, + third_party/googletest/LICENSE, + third_party/kineto/libkineto/third_party/googletest/LICENSE, + third_party/kineto/libkineto/third_party/googletest/googletest/LICENSE, third_party/protobuf/third_party/googletest/LICENSE, third_party/protobuf/third_party/googletest/googletest/LICENSE, third_party/tensorpipe/third_party/googletest/LICENSE, third_party/tensorpipe/third_party/googletest/googletest/LICENSE +Name: googletest-source +License: BSD-3-Clause +Files: third_party/XNNPACK/build/googletest-source + For details, see third_party/XNNPACK/build/googletest-source/LICENSE + Name: gtest License: BSD-3-Clause -Files: third_party/ideep/mkl-dnn/tests/gtests/gtest - For details, see third_party/ideep/mkl-dnn/tests/gtests/gtest/LICENSE +Files: third_party/ideep/mkl-dnn/tests/gtest, + third_party/ideep/mkl-dnn/third_party/oneDNN/tests/gtests/gtest + For details, see third_party/ideep/mkl-dnn/tests/gtest/LICENSE, + third_party/ideep/mkl-dnn/third_party/oneDNN/tests/gtests/gtest/LICENSE Name: ideep License: MIT @@ -154,11 +214,21 @@ License: BSD-3-Clause Files: third_party/ios-cmake For details, see third_party/ios-cmake/LICENSE +Name: json +License: MIT +Files: third_party/cudnn_frontend/include/contrib/nlohmann/json + For details, see third_party/cudnn_frontend/include/contrib/nlohmann/json/LICENSE.txt + Name: kineto License: BSD-3-Clause Files: third_party/kineto For details, see third_party/kineto/LICENSE +Name: libdisasm +License: Clarified Artistic License +Files: third_party/breakpad/src/third_party/libdisasm + For details, see third_party/breakpad/src/third_party/libdisasm/LICENSE + Name: libnop License: Apache-2.0 Files: third_party/tensorpipe/third_party/libnop @@ -169,6 +239,11 @@ License: MIT Files: third_party/tensorpipe/third_party/libuv For details, see third_party/tensorpipe/third_party/libuv/LICENSE +Name: lss +License: BSD-3-Clause +Files: third_party/breakpad/src/third_party/lss + For details, see third_party/breakpad/src/third_party/lss/LICENSE + Name: miniz-2.0.8 License: MIT Files: third_party/miniz-2.0.8 @@ -189,12 +264,20 @@ License: BSD-Source-Code Files: third_party/neon2sse For details, see third_party/neon2sse/LICENSE +Name: oneDNN +License: Apache-2.0 +Files: third_party/ideep/mkl-dnn/third_party/oneDNN + For details, see third_party/ideep/mkl-dnn/third_party/oneDNN/LICENSE + +Name: onnx +License: Apache-2.0 +Files: third_party/onnx + For details, see third_party/onnx/LICENSE + Name: onnx License: MIT -Files: third_party/onnx-tensorrt/third_party/onnx, - third_party/onnx - For details, see third_party/onnx-tensorrt/third_party/onnx/LICENSE, - third_party/onnx/LICENSE +Files: third_party/onnx-tensorrt/third_party/onnx + For details, see third_party/onnx-tensorrt/third_party/onnx/LICENSE Name: onnx-tensorrt License: MIT @@ -208,23 +291,30 @@ Files: third_party/protobuf Name: psimd License: MIT -Files: third_party/psimd - For details, see third_party/psimd/LICENSE +Files: third_party/XNNPACK/deps/psimd, + third_party/psimd + For details, see third_party/XNNPACK/deps/psimd/LICENSE, + third_party/psimd/LICENSE Name: pthreadpool License: BSD-2-Clause Files: third_party/pthreadpool For details, see third_party/pthreadpool/LICENSE +Name: pthreadpool-source +License: BSD-2-Clause +Files: third_party/XNNPACK/build/pthreadpool-source + For details, see third_party/XNNPACK/build/pthreadpool-source/LICENSE + Name: pybind11 License: BSD-3-Clause -Files: third_party/pybind11, +Files: third_party/onnx/third_party/pybind11, third_party/onnx-tensorrt/third_party/onnx/third_party/pybind11, - third_party/onnx/third_party/pybind11, + third_party/pybind11, third_party/tensorpipe/third_party/pybind11 - For details, see third_party/pybind11/LICENSE, + For details, see third_party/onnx/third_party/pybind11/LICENSE, third_party/onnx-tensorrt/third_party/onnx/third_party/pybind11/LICENSE, - third_party/onnx/third_party/pybind11/LICENSE, + third_party/pybind11/LICENSE, third_party/tensorpipe/third_party/pybind11/LICENSE Name: python-peachpy @@ -242,6 +332,21 @@ License: BSL-1.0 Files: third_party/sleef For details, see third_party/sleef/LICENSE.txt +Name: src +License: BSD-3-Clause +Files: third_party/benchmark/build/third_party/googletest/src + For details, see third_party/benchmark/build/third_party/googletest/src/LICENSE + +Name: swift +License: Apache-2.0 +Files: third_party/flatbuffers/swift + For details, see third_party/flatbuffers/swift/LICENSE + +Name: tb_plugin +License: BSD-3-Clause +Files: third_party/kineto/tb_plugin + For details, see third_party/kineto/tb_plugin/LICENSE + Name: tbb License: Apache-2.0 Files: third_party/tbb diff --git a/third_party/build_bundled.py b/third_party/build_bundled.py index 0e1da44565edc..c05e1c3642fe1 100644 --- a/third_party/build_bundled.py +++ b/third_party/build_bundled.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +import argparse import os @@ -52,7 +53,7 @@ def create_bundled(d, outstream): outstream.write(f"Files: {files}\n") outstream.write(' For details, see ') outstream.write(license_file) - outstream.write('\n\n') + outstream.write('\n\n') def identify_license(f, exception=''): @@ -89,6 +90,8 @@ def squeeze(t): elif 'BoostSoftwareLicense-Version1.0' in txt: # Hmm, do we need to check the text? return 'BSL-1.0' + elif squeeze("Clarified Artistic License") in txt: + return 'Clarified Artistic License' elif all([squeeze(m) in txt.lower() for m in bsd3_txt]): return 'BSD-3-Clause' elif all([squeeze(m) in txt.lower() for m in bsd3_v1_txt]): @@ -97,30 +100,30 @@ def squeeze(t): return 'BSD-2-Clause' elif all([squeeze(m) in txt.lower() for m in bsd3_src_txt]): return 'BSD-Source-Code' - elif all([squeeze(m) in txt.lower() for m in mit_txt]): + elif any([squeeze(m) in txt.lower() for m in mit_txt]): return 'MIT' else: raise ValueError('unknown license') -mit_txt = ['permission is hereby granted, free of charge, to any person ' - 'obtaining a copy of this software and associated documentation ' - 'files (the "software"), to deal in the software without ' - 'restriction, including without limitation the rights to use, copy, ' - 'modify, merge, publish, distribute, sublicense, and/or sell copies ' - 'of the software, and to permit persons to whom the software is ' +mit_txt = ['permission is hereby granted, free of charge, to any person ', + 'obtaining a copy of this software and associated documentation ', + 'files (the "software"), to deal in the software without ', + 'restriction, including without limitation the rights to use, copy, ', + 'modify, merge, publish, distribute, sublicense, and/or sell copies ', + 'of the software, and to permit persons to whom the software is ', 'furnished to do so, subject to the following conditions:', - 'the above copyright notice and this permission notice shall be ' + 'the above copyright notice and this permission notice shall be ', 'included in all copies or substantial portions of the software.', - 'the software is provided "as is", without warranty of any kind, ' - 'express or implied, including but not limited to the warranties of ' - 'merchantability, fitness for a particular purpose and ' - 'noninfringement. in no event shall the authors or copyright holders ' - 'be liable for any claim, damages or other liability, whether in an ' - 'action of contract, tort or otherwise, arising from, out of or in ' - 'connection with the software or the use or other dealings in the ' - 'software.' + 'the software is provided "as is", without warranty of any kind, ', + 'express or implied, including but not limited to the warranties of ', + 'merchantability, fitness for a particular purpose and ', + 'noninfringement. in no event shall the authors or copyright holders ', + 'be liable for any claim, damages or other liability, whether in an ', + 'action of contract, tort or otherwise, arising from, out of or in ', + 'connection with the software or the use or other dealings in the ', + 'software.', ] bsd3_txt = ['redistribution and use in source and binary forms, with or without ' @@ -154,6 +157,21 @@ def squeeze(t): if __name__ == '__main__': third_party = os.path.join(mydir) - fname = os.path.join(third_party, 'LICENSES_BUNDLED.txt') + parser = argparse.ArgumentParser( + description="Generate bundled licenses file", + ) + parser.add_argument( + "--out-file", + type=str, + default=os.environ.get( + "PYTORCH_THIRD_PARTY_BUNDLED_LICENSE_FILE", + str(os.path.join(third_party, 'LICENSES_BUNDLED.txt')) + ), + help="location to output new bundled licenses file", + ) + + args = parser.parse_args() + fname = args.out_file + print(f"+ Writing bundled licenses to {args.out_file}") with open(fname, 'w') as fid: create_bundled(third_party, fid)