From 20f55375dc3ec969c24d39e9d97830d84be345a8 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Tue, 5 Jul 2022 16:51:38 +0200 Subject: gnu: python-pytorch: Update to 1.11.0. * gnu/packages/machine-learning.scm (python-pytorch): Update to 1.11.0. [source](snippet): Remove now unnecessary "caffe2/onnx/backend.cc" substitution. [arguments]: Add 'remove-caffe2-onnx-scripts' phase. * gnu/packages/patches/python-pytorch-runpath.patch, gnu/packages/patches/python-pytorch-system-libraries.patch: Update. --- gnu/packages/machine-learning.scm | 31 ++++++++++++++------- gnu/packages/patches/python-pytorch-runpath.patch | 25 ++++++++++------- .../patches/python-pytorch-system-libraries.patch | 32 +++++++++++----------- 3 files changed, 52 insertions(+), 36 deletions(-) (limited to 'gnu') diff --git a/gnu/packages/machine-learning.scm b/gnu/packages/machine-learning.scm index 4015d77726..9e8792e9cf 100644 --- a/gnu/packages/machine-learning.scm +++ b/gnu/packages/machine-learning.scm @@ -2869,7 +2869,7 @@ TensorFlow.js, PyTorch, and MediaPipe.") (define-public python-pytorch (package (name "python-pytorch") - (version "1.10.2") + (version "1.11.0") (source (origin (method git-fetch) (uri (git-reference @@ -2879,7 +2879,7 @@ TensorFlow.js, PyTorch, and MediaPipe.") (file-name (git-file-name name version)) (sha256 (base32 - "15fi3nr7fx2zc9j2xf0jq627zfmnvs8hijyifg9769arm8kfijs1")) + "1zbk7y74r0ycsfa7x59jnhwhs1gj5rs3n89p15y0212iszgbljq8")) (patches (search-patches "python-pytorch-system-libraries.patch" "python-pytorch-runpath.patch")) (modules '((guix build utils))) @@ -2899,12 +2899,7 @@ TensorFlow.js, PyTorch, and MediaPipe.") "gloo" "googletest" "ios-cmake" "NNPACK" "onnx" "protobuf" "pthreadpool" "pybind11" "python-enum" "python-peachpy" - "python-six" "tbb" "XNNPACK" "zstd")) - - ;; Adjust references to the onnx-optimizer headers. - (substitute* "caffe2/onnx/backend.cc" - (("onnx/optimizer/") - "onnxoptimizer/")))))) + "python-six" "tbb" "XNNPACK" "zstd")))))) (build-system python-build-system) (arguments '(#:phases (modify-phases %standard-phases @@ -2942,7 +2937,23 @@ TensorFlow.js, PyTorch, and MediaPipe.") (let ((python-site (site-packages inputs outputs))) (for-each delete-file (find-files python-site - "(^test_cpp_rpc|_test)$")))))) + "(^test_cpp_rpc|_test)$"))))) + (add-after 'install 'remove-caffe2-onnx-scripts + (lambda* (#:key outputs #:allow-other-keys) + (let* ((out (assoc-ref outputs "out")) + (bin (string-append out "/bin"))) + ;; Remove 'convert-caffe2-to-onnx' and + ;; 'convert-onnx-to-caffe2': they seem to be + ;; deprecated and they cause a failure of the + ;; 'sanity-check' phase: + ;; + ;; ImportError: cannot import name 'metanet_pb2' from partially initialized module 'caffe2.proto' (most likely due to a circular import) + (for-each delete-file + (find-files bin "^convert-.*caffe2")) + + (substitute* (find-files out "^entry_points\\.txt$") + (("^convert-.*" all) + (string-append "# " all "\n"))))))) ;; XXX: Tests attempt to download data such as ;; . @@ -2977,7 +2988,7 @@ TensorFlow.js, PyTorch, and MediaPipe.") python-future python-six python-requests - onnx ;propagated for its Python modules + onnx ;propagated for its Python modules onnx-optimizer cpuinfo)) (home-page "https://pytorch.org/") diff --git a/gnu/packages/patches/python-pytorch-runpath.patch b/gnu/packages/patches/python-pytorch-runpath.patch index 6f270ef9b1..7f95b88a2b 100644 --- a/gnu/packages/patches/python-pytorch-runpath.patch +++ b/gnu/packages/patches/python-pytorch-runpath.patch @@ -2,6 +2,21 @@ Libraries (such as 'libtorch_cpu.so') and executables (such as 'torch_shm_manage get installed, quite surprisingly, to 'lib/python3.8/site-packages/{bin,lib}'. Make sure RUNPATH matches that. +diff --git a/caffe2/CMakeLists.txt b/caffe2/CMakeLists.txt +index 5b5622f0..30d27e57 100644 +--- a/caffe2/CMakeLists.txt ++++ b/caffe2/CMakeLists.txt +@@ -1909,7 +1909,7 @@ if(BUILD_PYTHON) + if(${BUILDING_WITH_TORCH_LIBS}) + # site-packages/caffe2/python/caffe2_pybind11_state + # site-packages/torch/lib +- set(caffe2_pybind11_rpath "${_rpath_portable_origin}/../../torch/lib") ++ set(caffe2_pybind11_rpath $ORIGIN/../../torch/lib) + endif(${BUILDING_WITH_TORCH_LIBS}) + + # Must also include `CMAKE_SHARED_LINKER_FLAGS` in linker flags for +diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake +index ee9cf410..f190e69b 100644 --- a/cmake/Dependencies.cmake +++ b/cmake/Dependencies.cmake @@ -4,7 +4,7 @@ if(APPLE) @@ -13,13 +28,3 @@ Make sure RUNPATH matches that. endif(APPLE) # Use separate rpaths during build and install phases set(CMAKE_SKIP_BUILD_RPATH FALSE) - ---- a/caffe2/CMakeLists.txt -+++ b/caffe2/CMakeLists.txt -@@ -1797,5 +1797,5 @@ if(BUILD_PYTHON) - if(${BUILDING_WITH_TORCH_LIBS}) - # site-packages/caffe2/python/caffe2_pybind11_state - # site-packages/torch/lib -- set(caffe2_pybind11_rpath "${_rpath_portable_origin}/../../torch/lib") -+ set(caffe2_pybind11_rpath $ORIGIN/../../torch/lib) - endif(${BUILDING_WITH_TORCH_LIBS}) diff --git a/gnu/packages/patches/python-pytorch-system-libraries.patch b/gnu/packages/patches/python-pytorch-system-libraries.patch index ae872cfd84..fd849fd9e2 100644 --- a/gnu/packages/patches/python-pytorch-system-libraries.patch +++ b/gnu/packages/patches/python-pytorch-system-libraries.patch @@ -2,10 +2,10 @@ Use our own googletest rather than the bundled one. Get NNPACK to use our own PeachPy rather than the bundled one. diff --git a/caffe2/CMakeLists.txt b/caffe2/CMakeLists.txt -index 26210cb5..18b6df27 100644 +index d57d7ebb..5b5622f0 100644 --- a/caffe2/CMakeLists.txt +++ b/caffe2/CMakeLists.txt -@@ -1723,7 +1723,7 @@ if(BUILD_TEST) +@@ -1736,7 +1736,7 @@ if(BUILD_TEST) if(NOT MSVC) add_executable(${test_name}_${CPU_CAPABILITY} "${test_src}" ../aten/src/ATen/native/quantized/affine_quantizer_base.cpp) # TODO: Get rid of c10 dependency (which is only needed for the implementation of AT_ERROR) @@ -14,7 +14,7 @@ index 26210cb5..18b6df27 100644 if(USE_FBGEMM) target_link_libraries(${test_name}_${CPU_CAPABILITY} fbgemm) endif() -@@ -1746,7 +1746,7 @@ if(BUILD_TEST) +@@ -1759,7 +1759,7 @@ if(BUILD_TEST) foreach(test_src ${Caffe2_CPU_TEST_SRCS}) get_filename_component(test_name ${test_src} NAME_WE) add_executable(${test_name} "${test_src}") @@ -23,16 +23,16 @@ index 26210cb5..18b6df27 100644 if(USE_OPENMP) # -fopenmp is a compile time flag and as result not guaranteed # to link executable against OpenMP runtime library -@@ -1769,7 +1769,7 @@ if(BUILD_TEST) +@@ -1785,7 +1785,7 @@ if(BUILD_TEST) foreach(test_src ${Caffe2_GPU_TEST_SRCS}) get_filename_component(test_name ${test_src} NAME_WE) - cuda_add_executable(${test_name} "${test_src}") + add_executable(${test_name} "${test_src}") - target_link_libraries(${test_name} torch_library gtest_main) + target_link_libraries(${test_name} torch_library gtest_main gtest) target_include_directories(${test_name} PRIVATE $) target_include_directories(${test_name} PRIVATE ${Caffe2_CPU_INCLUDE}) add_test(NAME ${test_name} COMMAND $) -@@ -1787,7 +1787,7 @@ if(BUILD_TEST) +@@ -1803,7 +1803,7 @@ if(BUILD_TEST) foreach(test_src ${Caffe2_VULKAN_TEST_SRCS}) get_filename_component(test_name ${test_src} NAME_WE) add_executable(${test_name} "${test_src}") @@ -41,7 +41,7 @@ index 26210cb5..18b6df27 100644 target_include_directories(${test_name} PRIVATE $) target_include_directories(${test_name} PRIVATE ${Caffe2_CPU_INCLUDE}) add_test(NAME ${test_name} COMMAND $) -@@ -1805,7 +1805,7 @@ if(BUILD_TEST) +@@ -1821,7 +1821,7 @@ if(BUILD_TEST) foreach(test_src ${Caffe2_HIP_TEST_SRCS}) get_filename_component(test_name ${test_src} NAME_WE) add_executable(${test_name} "${test_src}") @@ -51,10 +51,10 @@ index 26210cb5..18b6df27 100644 target_include_directories(${test_name} PRIVATE ${Caffe2_CPU_INCLUDE} ${Caffe2_HIP_INCLUDE}) target_compile_options(${test_name} PRIVATE ${HIP_CXX_FLAGS}) diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake -index ca560288..c0696e53 100644 +index 557ab649..ee9cf410 100644 --- a/cmake/Dependencies.cmake +++ b/cmake/Dependencies.cmake -@@ -684,11 +684,6 @@ if(BUILD_TEST OR BUILD_MOBILE_BENCHMARK OR BUILD_MOBILE_TEST) +@@ -732,11 +732,6 @@ if(BUILD_TEST OR BUILD_MOBILE_BENCHMARK OR BUILD_MOBILE_TEST) # this shouldn't be necessary anymore. get_property(INC_DIR_temp DIRECTORY PROPERTY INCLUDE_DIRECTORIES) set_property(DIRECTORY PROPERTY INCLUDE_DIRECTORIES "") @@ -66,7 +66,7 @@ index ca560288..c0696e53 100644 # We will not need to test benchmark lib itself. set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "Disable benchmark testing as we don't need it.") -@@ -1537,7 +1532,7 @@ if(CAFFE2_CMAKE_BUILDING_WITH_MAIN_REPO AND NOT INTERN_DISABLE_ONNX) +@@ -1543,7 +1538,7 @@ if(CAFFE2_CMAKE_BUILDING_WITH_MAIN_REPO AND NOT INTERN_DISABLE_ONNX) endif() set_property(TARGET onnx_proto PROPERTY IMPORTED_LOCATION ${ONNX_PROTO_LIBRARY}) message("-- Found onnx: ${ONNX_LIBRARY} ${ONNX_PROTO_LIBRARY}") @@ -98,10 +98,10 @@ index a41343cb..6075bdd0 100644 +set(NNPACK_FOUND TRUE) +set(USE_NNPACK ON) diff --git a/test/cpp/c10d/CMakeLists.txt b/test/cpp/c10d/CMakeLists.txt -index 2e48773a..a70506ce 100644 +index bf91460c..ef56948f 100644 --- a/test/cpp/c10d/CMakeLists.txt +++ b/test/cpp/c10d/CMakeLists.txt -@@ -17,14 +17,14 @@ function(c10d_add_test test_src) +@@ -16,14 +16,14 @@ function(c10d_add_test test_src) add_test(NAME ${test_name} COMMAND $) endfunction() @@ -119,7 +119,7 @@ index 2e48773a..a70506ce 100644 if(INSTALL_TEST) install(TARGETS HashStoreTest DESTINATION bin) endif() -@@ -32,11 +32,11 @@ endif() +@@ -31,11 +31,11 @@ endif() if(USE_CUDA) if(USE_GLOO AND USE_C10D_GLOO) @@ -133,7 +133,7 @@ index 2e48773a..a70506ce 100644 endif() if(USE_NCCL AND USE_C10D_NCCL) # NCCL is a private dependency of libtorch, but the tests include some -@@ -57,7 +57,7 @@ if(USE_CUDA) +@@ -56,7 +56,7 @@ if(USE_CUDA) endif() else() if(USE_GLOO AND USE_C10D_GLOO) @@ -143,10 +143,10 @@ index 2e48773a..a70506ce 100644 endif() diff --git a/test/cpp/tensorexpr/CMakeLists.txt b/test/cpp/tensorexpr/CMakeLists.txt -index 213e99bd..ecaae840 100644 +index 8fc5a0a1..643202f6 100644 --- a/test/cpp/tensorexpr/CMakeLists.txt +++ b/test/cpp/tensorexpr/CMakeLists.txt -@@ -46,7 +46,7 @@ target_include_directories(tutorial_tensorexpr PRIVATE ${ATen_CPU_INCLUDE}) +@@ -53,7 +53,7 @@ target_include_directories(tutorial_tensorexpr PRIVATE ${ATen_CPU_INCLUDE}) # pthreadpool header. For some build environment we need add the dependency # explicitly. if(USE_PTHREADPOOL) -- cgit 1.4.1