--- a/setup.py
+++ b/setup.py
@@ -256,6 +256,7 @@
 import setuptools.command.sdist
 from setuptools import Extension, find_packages, setup
 from setuptools.dist import Distribution
+sys.path[:0] = os.getcwd()
 from tools.build_pytorch_libs import build_pytorch
 from tools.generate_torch_version import get_torch_version
 from tools.setup_helpers.cmake import CMake
@@ -483,14 +483,6 @@
     check_submodules()
     check_pydep("yaml", "pyyaml")
     build_python = not BUILD_LIBTORCH_WHL
-    build_pytorch(
-        version=version,
-        cmake_python_library=cmake_python_library,
-        build_python=build_python,
-        rerun_cmake=RERUN_CMAKE,
-        cmake_only=CMAKE_ONLY,
-        cmake=cmake,
-    )
 
     if CMAKE_ONLY:
         report(