use bzip2 for exported libraries to save space
Opened this issue · 3 comments
fwiw, I tried this once, then gave it up because I could only get it to work partway. (Below is a diff off the #259 src.) I could definitely make the bz2 export tarballs. I don't remember if I couldn't get the subproject/externalproject to unpack them as naturally as tgz or if conda-build didn't want to fetch them as source. The exports were definitely smaller (by hundreds of MB when you're getting to the ~2GB size) and feasible for export-only.
(base) psilocaluser@bash:psinet:/psi/gits/libint2-Jan2023: (new-cmake-2023-take2-b) cat diff_export_compressor_work
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 0f13ae1..ad3b0f6 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -89,6 +89,7 @@ option_with_default(LIBINT2_PREFIX_PYTHON_INSTALL "For LIBINT2_ENABLE_PYTHON=ON,
option_with_print(BUILD_SHARED_LIBS "Build Libint library as shared, not static" OFF)
option_with_print(LIBINT2_BUILD_SHARED_AND_STATIC_LIBS "Build both shared and static Libint libraries in one shot. Uses -fPIC." OFF)
+option_with_default(EXPORT_COMPRESSOR "??? gzip or bzip2" gzip)
# <<< Which Integrals Classes, Which Derivative Levels >>>
@@ -310,6 +311,16 @@ booleanize01(LIBINT_GENERATE_FMA)
booleanize01(LIBINT_ENABLE_GENERIC_CODE)
booleanize01(SUPPORT_T1G12)
+if (EXPORT_COMPRESSOR STREQUAL "gzip")
+ set(EXPORT_COMPRESSOR_CMD "cfz")
+ set(EXPORT_COMPRESSOR_EXT "tgz")
+elseif (EXPORT_COMPRESSOR STREQUAL "bzip2")
+ set(EXPORT_COMPRESSOR_CMD "jcf")
+ set(EXPORT_COMPRESSOR_EXT "tbz2")
+else()
+ message(FATAL_ERROR "No valid compressor; invoke CMake with -DEXPORT_COMPRESSOR=gzip|bzip2")
+endif()
+
################################# Main Project #################################
set(EXPORT_STAGE_DIR ${PROJECT_BINARY_DIR}/libint-${LIBINT_EXT_VERSION})
set(STAGED_INSTALL_PREFIX "${PROJECT_BINARY_DIR}/library-install-stage")
diff --git a/src/lib/libint/CMakeLists.txt b/src/lib/libint/CMakeLists.txt
index 1e1b5d8..42ead8c 100644
--- a/src/lib/libint/CMakeLists.txt
+++ b/src/lib/libint/CMakeLists.txt
@@ -36,17 +36,17 @@ add_custom_target(libint-library-populate DEPENDS ${EXPORT_STAGE_DIR}/CMakeLists
# <<< Export The Library Source >>>
-add_custom_command(OUTPUT "${EXPORT_STAGE_DIR}.tgz"
- COMMAND ${CMAKE_COMMAND} -E tar "cfz" "${EXPORT_STAGE_DIR}.tgz" "${EXPORT_STAGE_DIR}"
+add_custom_command(OUTPUT "${EXPORT_STAGE_DIR}.${EXPORT_COMPRESSOR_EXT}"
+ COMMAND ${CMAKE_COMMAND} -E tar ${EXPORT_COMPRESSOR_CMD} "${EXPORT_STAGE_DIR}.${EXPORT_COMPRESSOR_EXT}" "${EXPORT_STAGE_DIR}"
WORKING_DIRECTORY "${EXPORT_STAGE_DIR}/.."
DEPENDS libint-library-generate libint-library-populate
COMMENT "Exporting tarball of Libint2 library source")
if (LIBINT_BUILD_LIBRARY_AS_SUBPROJECT)
# rerun cmake if building as subproject to unpack and consume the library as subproject
- add_custom_target(libint-library-export DEPENDS "${EXPORT_STAGE_DIR}.tgz"
+ add_custom_target(libint-library-export DEPENDS "${EXPORT_STAGE_DIR}.${EXPORT_COMPRESSOR_EXT}"
COMMAND ${CMAKE_COMMAND} -S "${CMAKE_SOURCE_DIR}" -B "${CMAKE_BINARY_DIR}")
else()
- add_custom_target(libint-library-export DEPENDS "${EXPORT_STAGE_DIR}.tgz")
+ add_custom_target(libint-library-export DEPENDS "${EXPORT_STAGE_DIR}.${EXPORT_COMPRESSOR_EXT}")
endif()
add_custom_target(export DEPENDS libint-library-export)
@loriab thanks for the info. I just tried consuming a .tbz2 export as a subproject and seemed to work fine (I am sure ExternalProject_add has had support for bzip2 since forever). It would be useful to know if indeed your use cases are dealbreakers.
If it works cmake-side, conda-build shouldn't be a dealbreaker. The one note I found suggests I got the build to the point where a bzip2 gave up for other too-big-AM reasons. At worst, I can unpack and repack the export tarball.