2023-03-21 15:29:41 +00:00
cmake_minimum_required ( VERSION 3.12 ) # Don't bump this version for no reason
2023-03-21 00:37:16 +00:00
project ( "llama.cpp" C CXX )
2023-03-13 17:12:33 +00:00
2023-03-21 15:29:41 +00:00
set ( CMAKE_EXPORT_COMPILE_COMMANDS ON )
2023-03-13 19:22:15 +00:00
if ( NOT XCODE AND NOT MSVC AND NOT CMAKE_BUILD_TYPE )
set ( CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE )
set_property ( CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo" )
endif ( )
2023-03-21 15:29:41 +00:00
set ( CMAKE_RUNTIME_OUTPUT_DIRECTORY ${ CMAKE_BINARY_DIR } /bin )
if ( CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR )
set ( LLAMA_STANDALONE ON )
# configure project version
# TODO
else ( )
set ( LLAMA_STANDALONE OFF )
endif ( )
if ( EMSCRIPTEN )
set ( BUILD_SHARED_LIBS_DEFAULT OFF )
option ( LLAMA_WASM_SINGLE_FILE "llama: embed WASM inside the generated llama.js" ON )
else ( )
if ( MINGW )
set ( BUILD_SHARED_LIBS_DEFAULT OFF )
else ( )
set ( BUILD_SHARED_LIBS_DEFAULT ON )
endif ( )
endif ( )
2023-03-21 00:37:16 +00:00
#
# Option list
#
2023-03-13 17:12:33 +00:00
2023-03-21 00:37:16 +00:00
# general
2023-05-25 21:07:29 +00:00
option ( LLAMA_STATIC "llama: static link libraries" OFF )
option ( LLAMA_NATIVE "llama: enable -march=native flag" OFF )
option ( LLAMA_LTO "llama: enable link time optimization" OFF )
2023-03-13 17:12:33 +00:00
2023-03-21 00:37:16 +00:00
# debug
2023-05-25 21:07:29 +00:00
option ( LLAMA_ALL_WARNINGS "llama: enable all compiler warnings" ON )
option ( LLAMA_ALL_WARNINGS_3RD_PARTY "llama: enable all compiler warnings in 3rd party libs" OFF )
option ( LLAMA_GPROF "llama: enable gprof" OFF )
2023-03-21 00:37:16 +00:00
# sanitizers
2023-05-25 21:07:29 +00:00
option ( LLAMA_SANITIZE_THREAD "llama: enable thread sanitizer" OFF )
option ( LLAMA_SANITIZE_ADDRESS "llama: enable address sanitizer" OFF )
option ( LLAMA_SANITIZE_UNDEFINED "llama: enable undefined sanitizer" OFF )
2023-03-21 00:37:16 +00:00
# instruction set specific
2023-05-25 21:07:29 +00:00
option ( LLAMA_AVX "llama: enable AVX" ON )
option ( LLAMA_AVX2 "llama: enable AVX2" ON )
option ( LLAMA_AVX512 "llama: enable AVX512" OFF )
option ( LLAMA_AVX512_VBMI "llama: enable AVX512-VBMI" OFF )
option ( LLAMA_AVX512_VNNI "llama: enable AVX512-VNNI" OFF )
option ( LLAMA_FMA "llama: enable FMA" ON )
2023-04-13 12:48:21 +00:00
# in MSVC F16C is implied with AVX2/AVX512
if ( NOT MSVC )
2023-05-25 21:07:29 +00:00
option ( LLAMA_F16C "llama: enable F16C" ON )
2023-04-13 12:48:21 +00:00
endif ( )
2023-03-21 00:37:16 +00:00
# 3rd party libs
2023-06-04 20:34:30 +00:00
option ( LLAMA_ACCELERATE "llama: enable Accelerate framework" ON )
option ( LLAMA_BLAS "llama: use BLAS" OFF )
2023-05-27 14:24:06 +00:00
set ( LLAMA_BLAS_VENDOR "Generic" CACHE STRING "llama: BLAS library vendor" )
2023-06-04 20:34:30 +00:00
option ( LLAMA_CUBLAS "llama: use cuBLAS" OFF )
set ( LLAMA_CUDA_DMMV_X "32" CACHE STRING "llama: x stride for dmmv CUDA kernels" )
set ( LLAMA_CUDA_DMMV_Y "1" CACHE STRING "llama: y block size for dmmv CUDA kernels" )
option ( LLAMA_CLBLAST "llama: use CLBlast" OFF )
option ( LLAMA_METAL "llama: use Metal" OFF )
2023-05-25 21:07:29 +00:00
option ( LLAMA_BUILD_TESTS "llama: build tests" ${ LLAMA_STANDALONE } )
option ( LLAMA_BUILD_EXAMPLES "llama: build examples" ${ LLAMA_STANDALONE } )
option ( LLAMA_BUILD_SERVER "llama: build server example" OFF )
2023-03-21 15:29:41 +00:00
2023-05-01 16:23:47 +00:00
#
# Build info header
#
# Generate initial build-info.h
include ( ${ CMAKE_CURRENT_SOURCE_DIR } /scripts/build-info.cmake )
if ( EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git" )
2023-05-03 00:43:43 +00:00
set ( GIT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/.git" )
# Is git submodule
if ( NOT IS_DIRECTORY "${GIT_DIR}" )
file ( READ ${ GIT_DIR } REAL_GIT_DIR_LINK )
string ( REGEX REPLACE "gitdir: (.*)\n$" "\\1" REAL_GIT_DIR ${ REAL_GIT_DIR_LINK } )
set ( GIT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/${REAL_GIT_DIR}" )
endif ( )
2023-05-01 16:23:47 +00:00
# Add a custom target for build-info.h
add_custom_target ( BUILD_INFO ALL DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/build-info.h" )
# Add a custom command to rebuild build-info.h when .git/index changes
add_custom_command (
O U T P U T " $ { C M A K E _ C U R R E N T _ S O U R C E _ D I R } / b u i l d - i n f o . h "
C O M M E N T " G e n e r a t i n g b u i l d d e t a i l s f r o m G i t "
C O M M A N D $ { C M A K E _ C O M M A N D } - P " $ { C M A K E _ C U R R E N T _ S O U R C E _ D I R } / s c r i p t s / b u i l d - i n f o . c m a k e "
W O R K I N G _ D I R E C T O R Y $ { C M A K E _ C U R R E N T _ S O U R C E _ D I R }
2023-05-03 00:43:43 +00:00
D E P E N D S " $ { G I T _ D I R } / i n d e x "
2023-05-01 16:23:47 +00:00
V E R B A T I M
)
else ( )
message ( WARNING "Git repository not found; to enable automatic generation of build info, make sure Git is installed and the project is a Git repository." )
endif ( )
2023-03-21 00:37:16 +00:00
#
# Compile flags
#
2023-03-31 19:19:16 +00:00
set ( CMAKE_CXX_STANDARD 11 )
2023-03-21 00:37:16 +00:00
set ( CMAKE_CXX_STANDARD_REQUIRED true )
2023-03-31 19:19:16 +00:00
set ( CMAKE_C_STANDARD 11 )
2023-03-21 00:37:16 +00:00
set ( CMAKE_C_STANDARD_REQUIRED true )
set ( THREADS_PREFER_PTHREAD_FLAG ON )
find_package ( Threads REQUIRED )
2023-03-13 17:12:33 +00:00
if ( NOT MSVC )
if ( LLAMA_SANITIZE_THREAD )
2023-03-21 00:37:16 +00:00
add_compile_options ( -fsanitize=thread )
2023-03-25 21:38:11 +00:00
link_libraries ( -fsanitize=thread )
2023-03-13 17:12:33 +00:00
endif ( )
if ( LLAMA_SANITIZE_ADDRESS )
2023-03-21 00:37:16 +00:00
add_compile_options ( -fsanitize=address -fno-omit-frame-pointer )
2023-03-25 21:38:11 +00:00
link_libraries ( -fsanitize=address )
2023-03-13 17:12:33 +00:00
endif ( )
if ( LLAMA_SANITIZE_UNDEFINED )
2023-03-21 00:37:16 +00:00
add_compile_options ( -fsanitize=undefined )
2023-03-25 21:38:11 +00:00
link_libraries ( -fsanitize=undefined )
2023-03-13 17:12:33 +00:00
endif ( )
endif ( )
2023-03-21 00:37:16 +00:00
if ( APPLE AND LLAMA_ACCELERATE )
2023-03-13 17:12:33 +00:00
find_library ( ACCELERATE_FRAMEWORK Accelerate )
if ( ACCELERATE_FRAMEWORK )
message ( STATUS "Accelerate framework found" )
2023-03-21 00:37:16 +00:00
add_compile_definitions ( GGML_USE_ACCELERATE )
set ( LLAMA_EXTRA_LIBS ${ LLAMA_EXTRA_LIBS } ${ ACCELERATE_FRAMEWORK } )
2023-03-13 17:12:33 +00:00
else ( )
message ( WARNING "Accelerate framework not found" )
endif ( )
endif ( )
2023-04-20 01:14:14 +00:00
2023-05-20 14:58:31 +00:00
if ( LLAMA_BLAS )
2023-03-21 00:37:16 +00:00
if ( LLAMA_STATIC )
set ( BLA_STATIC ON )
endif ( )
2023-05-20 14:58:31 +00:00
if ( $ ( CMAKE_VERSION ) VERSION_GREATER_EQUAL 3.22 )
set ( BLA_SIZEOF_INTEGER 8 )
endif ( )
set ( BLA_VENDOR ${ LLAMA_BLAS_VENDOR } )
2023-03-21 00:37:16 +00:00
find_package ( BLAS )
if ( BLAS_FOUND )
2023-05-20 14:58:31 +00:00
message ( STATUS "BLAS found, Libraries: ${BLAS_LIBRARIES}" )
2023-03-21 00:37:16 +00:00
2023-05-20 14:58:31 +00:00
add_compile_options ( ${ BLAS_LINKER_FLAGS } )
2023-03-21 00:37:16 +00:00
add_compile_definitions ( GGML_USE_OPENBLAS )
2023-05-20 14:58:31 +00:00
set ( LLAMA_EXTRA_LIBS ${ LLAMA_EXTRA_LIBS } ${ BLAS_LIBRARIES } )
message ( "${BLAS_LIBRARIES} ${BLAS_INCLUDE_DIRS}" )
include_directories ( ${ BLAS_INCLUDE_DIRS } )
2023-03-21 00:37:16 +00:00
else ( )
2023-05-20 14:58:31 +00:00
message ( WARNING "BLAS not found, please refer to "
" h t t p s : / / c m a k e . o r g / c m a k e / h e l p / l a t e s t / m o d u l e / F i n d B L A S . h t m l #blas-lapack-vendors"
" t o s e t c o r r e c t L L A M A _ B L A S _ V E N D O R " )
2023-03-21 00:37:16 +00:00
endif ( )
endif ( )
2023-03-13 17:12:33 +00:00
2023-04-19 09:22:45 +00:00
if ( LLAMA_CUBLAS )
cmake_minimum_required ( VERSION 3.17 )
find_package ( CUDAToolkit )
if ( CUDAToolkit_FOUND )
message ( STATUS "cuBLAS found" )
2023-04-20 01:14:14 +00:00
enable_language ( CUDA )
2023-06-04 20:34:30 +00:00
set ( GGML_SOURCES_CUDA ggml-cuda.cu ggml-cuda.h )
2023-04-20 01:14:14 +00:00
2023-04-19 09:22:45 +00:00
add_compile_definitions ( GGML_USE_CUBLAS )
2023-05-25 21:07:29 +00:00
add_compile_definitions ( GGML_CUDA_DMMV_X= ${ LLAMA_CUDA_DMMV_X } )
add_compile_definitions ( GGML_CUDA_DMMV_Y= ${ LLAMA_CUDA_DMMV_Y } )
2023-04-19 09:22:45 +00:00
if ( LLAMA_STATIC )
set ( LLAMA_EXTRA_LIBS ${ LLAMA_EXTRA_LIBS } CUDA::cudart_static CUDA::cublas_static CUDA::cublasLt_static )
else ( )
set ( LLAMA_EXTRA_LIBS ${ LLAMA_EXTRA_LIBS } CUDA::cudart CUDA::cublas CUDA::cublasLt )
endif ( )
else ( )
message ( WARNING "cuBLAS not found" )
endif ( )
endif ( )
2023-06-04 20:34:30 +00:00
if ( LLAMA_METAL )
find_library ( FOUNDATION_LIBRARY Foundation REQUIRED )
find_library ( METAL_FRAMEWORK Metal REQUIRED )
find_library ( METALKIT_FRAMEWORK MetalKit REQUIRED )
find_library ( METALPERFORMANCE_FRAMEWORK MetalPerformanceShaders REQUIRED )
set ( GGML_SOURCES_METAL ggml-metal.m ggml-metal.h )
add_compile_definitions ( GGML_USE_METAL )
add_compile_definitions ( GGML_METAL_NDEBUG )
# get full path to the file
#add_compile_definitions(GGML_METAL_DIR_KERNELS="${CMAKE_CURRENT_SOURCE_DIR}/")
# copy ggml-metal.metal to bin directory
configure_file ( ggml-metal.metal bin/ggml-metal.metal COPYONLY )
set ( LLAMA_EXTRA_LIBS ${ LLAMA_EXTRA_LIBS }
$ { F O U N D A T I O N _ L I B R A R Y }
$ { M E T A L _ F R A M E W O R K }
$ { M E T A L K I T _ F R A M E W O R K }
$ { M E T A L P E R F O R M A N C E _ F R A M E W O R K }
)
endif ( )
ggml : add CLBlast support (#1164)
* Allow use of OpenCL GPU-based BLAS using ClBlast instead of OpenBLAS for context processing
* Improve ClBlast implementation, avoid recreating buffers, remove redundant transfers
* Finish merge of ClBlast support
* Move CLBlast implementation to separate file
Add buffer reuse code (adapted from slaren's cuda implementation)
* Add q4_2 and q4_3 CLBlast support, improve code
* Double CLBlast speed by disabling OpenBLAS thread workaround
Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
Co-authored-by: slaren <2141330+slaren@users.noreply.github.com>
* Fix device selection env variable names
* Fix cast in opencl kernels
* Add CLBlast to CMakeLists.txt
* Replace buffer pool with static buffers a, b, qb, c
Fix compile warnings
* Fix typos, use GGML_TYPE defines, improve code
* Improve btype dequant kernel selection code, add error if type is unsupported
* Improve code quality
* Move internal stuff out of header
* Use internal enums instead of CLBlast enums
* Remove leftover C++ includes and defines
* Make event use easier to read
Co-authored-by: Henri Vasserman <henv@hot.ee>
* Use c compiler for opencl files
* Simplify code, fix include
* First check error, then release event
* Make globals static, fix indentation
* Rename dequant kernels file to conform with other file names
* Fix import cl file name
---------
Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
Co-authored-by: slaren <2141330+slaren@users.noreply.github.com>
Co-authored-by: Henri Vasserman <henv@hot.ee>
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
2023-04-28 14:57:16 +00:00
if ( LLAMA_CLBLAST )
find_package ( CLBlast )
if ( CLBlast_FOUND )
message ( STATUS "CLBlast found" )
2023-06-04 20:34:30 +00:00
set ( GGML_SOURCES_OPENCL ggml-opencl.cpp ggml-opencl.h )
ggml : add CLBlast support (#1164)
* Allow use of OpenCL GPU-based BLAS using ClBlast instead of OpenBLAS for context processing
* Improve ClBlast implementation, avoid recreating buffers, remove redundant transfers
* Finish merge of ClBlast support
* Move CLBlast implementation to separate file
Add buffer reuse code (adapted from slaren's cuda implementation)
* Add q4_2 and q4_3 CLBlast support, improve code
* Double CLBlast speed by disabling OpenBLAS thread workaround
Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
Co-authored-by: slaren <2141330+slaren@users.noreply.github.com>
* Fix device selection env variable names
* Fix cast in opencl kernels
* Add CLBlast to CMakeLists.txt
* Replace buffer pool with static buffers a, b, qb, c
Fix compile warnings
* Fix typos, use GGML_TYPE defines, improve code
* Improve btype dequant kernel selection code, add error if type is unsupported
* Improve code quality
* Move internal stuff out of header
* Use internal enums instead of CLBlast enums
* Remove leftover C++ includes and defines
* Make event use easier to read
Co-authored-by: Henri Vasserman <henv@hot.ee>
* Use c compiler for opencl files
* Simplify code, fix include
* First check error, then release event
* Make globals static, fix indentation
* Rename dequant kernels file to conform with other file names
* Fix import cl file name
---------
Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
Co-authored-by: slaren <2141330+slaren@users.noreply.github.com>
Co-authored-by: Henri Vasserman <henv@hot.ee>
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
2023-04-28 14:57:16 +00:00
add_compile_definitions ( GGML_USE_CLBLAST )
set ( LLAMA_EXTRA_LIBS ${ LLAMA_EXTRA_LIBS } clblast )
else ( )
message ( WARNING "CLBlast not found" )
endif ( )
endif ( )
2023-03-13 17:12:33 +00:00
if ( LLAMA_ALL_WARNINGS )
if ( NOT MSVC )
2023-03-21 00:37:16 +00:00
set ( c_flags
- W a l l
- W e x t r a
- W p e d a n t i c
- W c a s t - q u a l
2023-03-28 16:48:20 +00:00
- W d o u b l e - p r o m o t i o n
- W s h a d o w
2023-03-21 00:37:16 +00:00
- W s t r i c t - p r o t o t y p e s
- W p o i n t e r - a r i t h
)
set ( cxx_flags
- W a l l
- W e x t r a
- W p e d a n t i c
- W c a s t - q u a l
2023-03-28 16:51:55 +00:00
- W n o - u n u s e d - f u n c t i o n
Rewrite loading code to try to satisfy everyone:
- Support all three formats (ggml, ggmf, ggjt). (However, I didn't
include the hack needed to support GPT4All files without conversion.
Those can still be used after converting them with convert.py from my
other PR.)
- Support both mmap and read (mmap is used by default, but can be
disabled with `--no-mmap`, and is automatically disabled for pre-ggjt
files or on platforms where mmap is not supported).
- Support multi-file models like before, but automatically determine the
number of parts rather than requiring `--n_parts`.
- Improve validation and error checking.
- Stop using the per-file type field (f16) entirely in favor of just
relying on the per-tensor type/size fields. This has no immediate
benefit, but makes it easier to experiment with different formats, and
should make it easier to support the new GPTQ-for-LLaMa models in the
future (I have some work in progress on that front).
- Support VirtualLock on Windows (using the same `--mlock` option as on
Unix).
- Indicate loading progress when using mmap + mlock. (Which led me
to the interesting observation that on my Linux machine, with a
warm file cache, mlock actually takes some time, whereas mmap
without mlock starts almost instantly...)
- To help implement this, move mlock support from ggml to the
loading code.
- madvise/PrefetchVirtualMemory support (based on #740)
- Switch from ifstream to the `fopen` family of functions to avoid
unnecessary copying and, when mmap is enabled, allow reusing the same
file descriptor for both metadata reads and mmap (whereas the existing
implementation opens the file a second time to mmap).
- Quantization now produces a single-file output even with multi-file
inputs (not really a feature as much as 'it was easier this way').
Implementation notes:
I tried to factor the code into more discrete pieces than before.
Regarding code style: I tried to follow the code style, but I'm naughty
and used a few advanced C++ features repeatedly:
- Destructors to make it easier to ensure everything gets cleaned up.
- Exceptions. I don't even usually use exceptions when writing C++, and
I can remove them if desired... but here they make the loading code
much more succinct while still properly handling a variety of errors,
ranging from API calls failing to integer overflow and allocation
failure. The exceptions are converted to error codes at the
API boundary.)
Co-authored-by: Pavol Rusnak <pavol@rusnak.io> (for the bit I copied from #740)
2023-04-08 19:24:37 +00:00
- W n o - m u l t i c h a r
2023-03-21 00:37:16 +00:00
)
2023-03-13 17:12:33 +00:00
else ( )
# todo : msvc
endif ( )
2023-03-21 00:37:16 +00:00
add_compile_options (
" $ < $ < C O M P I L E _ L A N G U A G E : C > : $ { c _ f l a g s } > "
" $ < $ < C O M P I L E _ L A N G U A G E : C X X > : $ { c x x _ f l a g s } > "
)
2023-03-13 17:12:33 +00:00
endif ( )
Rewrite loading code to try to satisfy everyone:
- Support all three formats (ggml, ggmf, ggjt). (However, I didn't
include the hack needed to support GPT4All files without conversion.
Those can still be used after converting them with convert.py from my
other PR.)
- Support both mmap and read (mmap is used by default, but can be
disabled with `--no-mmap`, and is automatically disabled for pre-ggjt
files or on platforms where mmap is not supported).
- Support multi-file models like before, but automatically determine the
number of parts rather than requiring `--n_parts`.
- Improve validation and error checking.
- Stop using the per-file type field (f16) entirely in favor of just
relying on the per-tensor type/size fields. This has no immediate
benefit, but makes it easier to experiment with different formats, and
should make it easier to support the new GPTQ-for-LLaMa models in the
future (I have some work in progress on that front).
- Support VirtualLock on Windows (using the same `--mlock` option as on
Unix).
- Indicate loading progress when using mmap + mlock. (Which led me
to the interesting observation that on my Linux machine, with a
warm file cache, mlock actually takes some time, whereas mmap
without mlock starts almost instantly...)
- To help implement this, move mlock support from ggml to the
loading code.
- madvise/PrefetchVirtualMemory support (based on #740)
- Switch from ifstream to the `fopen` family of functions to avoid
unnecessary copying and, when mmap is enabled, allow reusing the same
file descriptor for both metadata reads and mmap (whereas the existing
implementation opens the file a second time to mmap).
- Quantization now produces a single-file output even with multi-file
inputs (not really a feature as much as 'it was easier this way').
Implementation notes:
I tried to factor the code into more discrete pieces than before.
Regarding code style: I tried to follow the code style, but I'm naughty
and used a few advanced C++ features repeatedly:
- Destructors to make it easier to ensure everything gets cleaned up.
- Exceptions. I don't even usually use exceptions when writing C++, and
I can remove them if desired... but here they make the loading code
much more succinct while still properly handling a variety of errors,
ranging from API calls failing to integer overflow and allocation
failure. The exceptions are converted to error codes at the
API boundary.)
Co-authored-by: Pavol Rusnak <pavol@rusnak.io> (for the bit I copied from #740)
2023-04-08 19:24:37 +00:00
if ( MSVC )
add_compile_definitions ( _CRT_SECURE_NO_WARNINGS )
2023-04-22 08:18:20 +00:00
if ( BUILD_SHARED_LIBS )
set ( CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON )
endif ( )
Rewrite loading code to try to satisfy everyone:
- Support all three formats (ggml, ggmf, ggjt). (However, I didn't
include the hack needed to support GPT4All files without conversion.
Those can still be used after converting them with convert.py from my
other PR.)
- Support both mmap and read (mmap is used by default, but can be
disabled with `--no-mmap`, and is automatically disabled for pre-ggjt
files or on platforms where mmap is not supported).
- Support multi-file models like before, but automatically determine the
number of parts rather than requiring `--n_parts`.
- Improve validation and error checking.
- Stop using the per-file type field (f16) entirely in favor of just
relying on the per-tensor type/size fields. This has no immediate
benefit, but makes it easier to experiment with different formats, and
should make it easier to support the new GPTQ-for-LLaMa models in the
future (I have some work in progress on that front).
- Support VirtualLock on Windows (using the same `--mlock` option as on
Unix).
- Indicate loading progress when using mmap + mlock. (Which led me
to the interesting observation that on my Linux machine, with a
warm file cache, mlock actually takes some time, whereas mmap
without mlock starts almost instantly...)
- To help implement this, move mlock support from ggml to the
loading code.
- madvise/PrefetchVirtualMemory support (based on #740)
- Switch from ifstream to the `fopen` family of functions to avoid
unnecessary copying and, when mmap is enabled, allow reusing the same
file descriptor for both metadata reads and mmap (whereas the existing
implementation opens the file a second time to mmap).
- Quantization now produces a single-file output even with multi-file
inputs (not really a feature as much as 'it was easier this way').
Implementation notes:
I tried to factor the code into more discrete pieces than before.
Regarding code style: I tried to follow the code style, but I'm naughty
and used a few advanced C++ features repeatedly:
- Destructors to make it easier to ensure everything gets cleaned up.
- Exceptions. I don't even usually use exceptions when writing C++, and
I can remove them if desired... but here they make the loading code
much more succinct while still properly handling a variety of errors,
ranging from API calls failing to integer overflow and allocation
failure. The exceptions are converted to error codes at the
API boundary.)
Co-authored-by: Pavol Rusnak <pavol@rusnak.io> (for the bit I copied from #740)
2023-04-08 19:24:37 +00:00
endif ( )
2023-03-21 00:37:16 +00:00
if ( LLAMA_LTO )
include ( CheckIPOSupported )
check_ipo_supported ( RESULT result OUTPUT output )
if ( result )
set ( CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE )
else ( )
message ( WARNING "IPO is not supported: ${output}" )
endif ( )
endif ( )
# Architecture specific
# TODO: probably these flags need to be tweaked on some architectures
# feel free to update the Makefile for your architecture and send a pull request or issue
2023-03-13 17:12:33 +00:00
message ( STATUS "CMAKE_SYSTEM_PROCESSOR: ${CMAKE_SYSTEM_PROCESSOR}" )
2023-03-21 00:37:16 +00:00
if ( NOT MSVC )
if ( LLAMA_STATIC )
add_link_options ( -static )
if ( MINGW )
add_link_options ( -static-libgcc -static-libstdc++ )
endif ( )
endif ( )
if ( LLAMA_GPROF )
add_compile_options ( -pg )
endif ( )
if ( LLAMA_NATIVE )
add_compile_options ( -march=native )
endif ( )
endif ( )
2023-03-13 17:12:33 +00:00
if ( ${ CMAKE_SYSTEM_PROCESSOR } MATCHES "arm" OR ${ CMAKE_SYSTEM_PROCESSOR } MATCHES "aarch64" )
message ( STATUS "ARM detected" )
2023-03-21 00:37:16 +00:00
if ( MSVC )
# TODO: arm msvc?
else ( )
if ( ${ CMAKE_SYSTEM_PROCESSOR } MATCHES "aarch64" )
2023-04-30 18:48:38 +00:00
# Apple M1, M2, etc.
# Raspberry Pi 3, 4, Zero 2 (64-bit)
2023-03-21 00:37:16 +00:00
add_compile_options ( -mcpu=native )
endif ( )
2023-04-30 18:48:38 +00:00
if ( ${ CMAKE_SYSTEM_PROCESSOR } MATCHES "armv6" )
# Raspberry Pi 1, Zero
add_compile_options ( -mfpu=neon-fp-armv8 -mfp16-format=ieee -mno-unaligned-access )
endif ( )
if ( ${ CMAKE_SYSTEM_PROCESSOR } MATCHES "armv7" )
# Raspberry Pi 2
add_compile_options ( -mfpu=neon-fp-armv8 -mfp16-format=ieee -mno-unaligned-access -funsafe-math-optimizations )
endif ( )
if ( ${ CMAKE_SYSTEM_PROCESSOR } MATCHES "armv8" )
# Raspberry Pi 3, 4, Zero 2 (32-bit)
add_compile_options ( -mfp16-format=ieee -mno-unaligned-access )
endif ( )
2023-03-21 00:37:16 +00:00
endif ( )
elseif ( ${ CMAKE_SYSTEM_PROCESSOR } MATCHES "^(x86_64|i686|AMD64)$" )
2023-03-13 17:12:33 +00:00
message ( STATUS "x86 detected" )
if ( MSVC )
2023-03-25 21:38:11 +00:00
if ( LLAMA_AVX512 )
2023-04-20 01:14:14 +00:00
add_compile_options ( $< $<COMPILE_LANGUAGE:C > :/arch:AVX512> )
add_compile_options ( $< $<COMPILE_LANGUAGE:CXX > :/arch:AVX512> )
2023-04-17 13:10:57 +00:00
# MSVC has no compile-time flags enabling specific
# AVX512 extensions, neither it defines the
# macros corresponding to the extensions.
# Do it manually.
if ( LLAMA_AVX512_VBMI )
2023-04-20 01:14:14 +00:00
add_compile_definitions ( $< $<COMPILE_LANGUAGE:C > :__AVX512VBMI__> )
add_compile_definitions ( $< $<COMPILE_LANGUAGE:CXX > :__AVX512VBMI__> )
2023-04-17 13:10:57 +00:00
endif ( )
if ( LLAMA_AVX512_VNNI )
2023-04-20 01:14:14 +00:00
add_compile_definitions ( $< $<COMPILE_LANGUAGE:C > :__AVX512VNNI__> )
add_compile_definitions ( $< $<COMPILE_LANGUAGE:CXX > :__AVX512VNNI__> )
2023-04-17 13:10:57 +00:00
endif ( )
2023-03-25 21:38:11 +00:00
elseif ( LLAMA_AVX2 )
2023-04-20 01:14:14 +00:00
add_compile_options ( $< $<COMPILE_LANGUAGE:C > :/arch:AVX2> )
add_compile_options ( $< $<COMPILE_LANGUAGE:CXX > :/arch:AVX2> )
2023-03-21 00:37:16 +00:00
elseif ( LLAMA_AVX )
2023-04-20 01:14:14 +00:00
add_compile_options ( $< $<COMPILE_LANGUAGE:C > :/arch:AVX> )
add_compile_options ( $< $<COMPILE_LANGUAGE:CXX > :/arch:AVX> )
2023-03-21 00:37:16 +00:00
endif ( )
2023-03-13 17:12:33 +00:00
else ( )
2023-04-13 12:48:21 +00:00
if ( LLAMA_F16C )
add_compile_options ( -mf16c )
endif ( )
2023-03-21 00:37:16 +00:00
if ( LLAMA_FMA )
add_compile_options ( -mfma )
2023-03-13 17:12:33 +00:00
endif ( )
2023-03-21 00:37:16 +00:00
if ( LLAMA_AVX )
add_compile_options ( -mavx )
2023-03-13 17:12:33 +00:00
endif ( )
2023-03-21 00:37:16 +00:00
if ( LLAMA_AVX2 )
add_compile_options ( -mavx2 )
2023-03-13 17:12:33 +00:00
endif ( )
2023-03-25 21:38:11 +00:00
if ( LLAMA_AVX512 )
add_compile_options ( -mavx512f )
2023-04-17 13:10:57 +00:00
add_compile_options ( -mavx512bw )
endif ( )
if ( LLAMA_AVX512_VBMI )
add_compile_options ( -mavx512vbmi )
endif ( )
if ( LLAMA_AVX512_VNNI )
add_compile_options ( -mavx512vnni )
2023-03-25 21:38:11 +00:00
endif ( )
2023-03-13 17:12:33 +00:00
endif ( )
2023-05-02 16:42:16 +00:00
elseif ( ${ CMAKE_SYSTEM_PROCESSOR } MATCHES "ppc64" )
message ( STATUS "PowerPC detected" )
add_compile_options ( -mcpu=native -mtune=native )
#TODO: Add targets for Power8/Power9 (Altivec/VSX) and Power10(MMA) and query for big endian systems (ppc64/le/be)
2023-03-21 00:37:16 +00:00
else ( )
message ( STATUS "Unknown architecture" )
2023-03-13 17:12:33 +00:00
endif ( )
2023-03-21 00:37:16 +00:00
#
2023-03-22 05:32:36 +00:00
# Build libraries
2023-03-21 00:37:16 +00:00
#
2023-03-21 15:29:41 +00:00
add_library ( ggml OBJECT
g g m l . c
2023-04-20 01:14:14 +00:00
g g m l . h
ggml : add SOTA 2,3,4,5,6 bit k-quantizations (#1684)
* Starting to add k-quantization to ggml
I think it is better to have quantization separate from
ggml. For now just adding the k-quants there, but it would be
better to also factor out the existing ggml quantizations.
* Adding Q3_K and Q8_K (de)-quantization
* Q3_K now working on CUDA and AVX2/scalar
CUDA is not ideal - ~50% slower than Q4_0 for
single token prediction, about the same in batch
mode (perplexity). CPU single token is ~55 ms
(on Ryzen 7950X).
* Some improvement for Q3_K on CUDA
It is now ~22.5 ms/token on my GPU, so ~30% slower than Q4_0.
* Some more CUDA optimizations for Q3_K
Single token is now 20.5 ms/token (~20% slower than Q4_0).
Perplexity is on par with Q4_0.
* Adding Q4_K - scalar, AVX2, CUDA
Performance is the same or perhaps very slightly better than Q4_0 on the CPU.
On the GPU, single token prediction is ~10% better than Q4_0,
batch mode (perplexity is about the same).
* Adding Q6_K - scalar, AVX2, CUDA
Performance is ~40% lower compared to Q4_K on the CPU.
This is to be expected, considering that we are memory bound
on the CPU and the 6-bit model is ~44% larger than the 4-bit.
On the GPU, single token prediction is ~6% lower than Q4_0,
batch mode (perplexity) is even closer (but still slower).
* Adding Q5_K - scalar, AVX2, CUDA
Performance is ~20% lower compared to Q4_K on the CPU.
This is to be expected, considering that we are memory bound
on the CPU and the 5-bit model is ~22% larger than the 4-bit.
On the GPU, single token prediction is about the same as Q4_0
for both, single token and batch prediction.
* Per convention, all QX_K quantizations use Q5_K for output.weight
* Adding quantization mixes
* Quantization mixes: didn't quite get what I wanted in the last commit
* Q4_K dot product for ARM_NEON
* Q6_K dot product for ARM_NEON
* Q5_K dot product for ARM_NEON
* Adding Q3_K dot for ARM_NEON
It is 22% slower than Q4_K, despite the smaller model size.
On x86_64, where we are memory bound, the Q3_K model is
quite a bit faster than Q4_K.
* A very slightly faster ARM_NEON Q3_K dot
* Adding Q2_K - just CUDA for now
Token prediction is pretty good - about 15.5 ms on a RTX 4080.
Perplexity is about the same as Q4_K.
* Adding scalar and AVX2 Q2_K dot
* Adding ARM_NEON Q2_K dot
About the same performance as Q4_K.
* A slightly faster ARM_NEON Q2_K dot
Single token prediction is now ~36 ms on M2 Max.
The code is much simpler too.
* Fixed bug in Q2_K CUDA dot product kernel
Stranegly enough, for the few prompts I tried with the 7B model
the responses looked perfectly reasonable. Only realized something
is not quite right when I tried the larger models and started getting
nonse back.
In any case, Q2_K single token evaluation time on an RTX 4080 in a Ryzen7950X
box iusing CUDA and model fully loaded on the GPU are
~15.5 ms for 7B, ~25.4 ms for 13B, and ~55.8 ms for 30B.
The max number of layers that fit in VRAM for The 65B is 32.
With that, we get ~330 ms per token, which is not that much faster
than just running on the CPU (~470 ms per token).
* Don't print zeros/NaNs when no count histogram has been collected
* A 10% faster CUDA vector dot kernel for Q3_K
Q3_K is now running at ~18.5 ms / token on CUDA,
so the gap to Q4_0 is only 10%.
It seems memory acccess pattern is more important for
performance than the amount of computation the kernel
does.
* A slightly daster Q4_K AVX2 dot product
For perplexity, where we are less memory bound, time per
pass drops by ~5%. Barely measurable difference for single
token prediction.
* A slightly faster ARM_NEON A4_K dot product
* Minor
* Fix quantization error test
We cannot possibly be expecting rmse < 0.002 for 2- and 3-bit
quantization variants.
* Fix docker build
I have been sloppy with vector reinterpret casts on ARM_NEON.
It seems clang is very forgiving in that regard.
* Added forgotten ggml.o dependence on k_quants.h to the Makefile
* Had unintentionally committed the Makefile with -Ofast enabled
* ggml : rename k_quants -> ggml-quants-k, use lowercase in code
---------
Co-authored-by: Iwan Kawrakow <iwan.kawrakow@gmail.com>
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
2023-06-05 19:56:18 +00:00
g g m l - q u a n t s - k . h
g g m l - q u a n t s - k . c
2023-06-04 20:34:30 +00:00
$ { G G M L _ S O U R C E S _ C U D A }
$ { G G M L _ S O U R C E S _ O P E N C L }
$ { G G M L _ S O U R C E S _ M E T A L }
)
2023-03-21 15:29:41 +00:00
2023-03-13 17:12:33 +00:00
target_include_directories ( ggml PUBLIC . )
2023-03-21 15:29:41 +00:00
target_compile_features ( ggml PUBLIC c_std_11 ) # don't bump
2023-04-21 18:27:06 +00:00
target_link_libraries ( ggml PUBLIC Threads::Threads ${ LLAMA_EXTRA_LIBS } )
2023-04-22 13:31:56 +00:00
2023-03-23 20:16:48 +00:00
if ( BUILD_SHARED_LIBS )
set_target_properties ( ggml PROPERTIES POSITION_INDEPENDENT_CODE ON )
endif ( )
2023-03-22 05:32:36 +00:00
2023-03-22 16:37:10 +00:00
add_library ( llama
2023-03-22 05:32:36 +00:00
l l a m a . c p p
Rewrite loading code to try to satisfy everyone:
- Support all three formats (ggml, ggmf, ggjt). (However, I didn't
include the hack needed to support GPT4All files without conversion.
Those can still be used after converting them with convert.py from my
other PR.)
- Support both mmap and read (mmap is used by default, but can be
disabled with `--no-mmap`, and is automatically disabled for pre-ggjt
files or on platforms where mmap is not supported).
- Support multi-file models like before, but automatically determine the
number of parts rather than requiring `--n_parts`.
- Improve validation and error checking.
- Stop using the per-file type field (f16) entirely in favor of just
relying on the per-tensor type/size fields. This has no immediate
benefit, but makes it easier to experiment with different formats, and
should make it easier to support the new GPTQ-for-LLaMa models in the
future (I have some work in progress on that front).
- Support VirtualLock on Windows (using the same `--mlock` option as on
Unix).
- Indicate loading progress when using mmap + mlock. (Which led me
to the interesting observation that on my Linux machine, with a
warm file cache, mlock actually takes some time, whereas mmap
without mlock starts almost instantly...)
- To help implement this, move mlock support from ggml to the
loading code.
- madvise/PrefetchVirtualMemory support (based on #740)
- Switch from ifstream to the `fopen` family of functions to avoid
unnecessary copying and, when mmap is enabled, allow reusing the same
file descriptor for both metadata reads and mmap (whereas the existing
implementation opens the file a second time to mmap).
- Quantization now produces a single-file output even with multi-file
inputs (not really a feature as much as 'it was easier this way').
Implementation notes:
I tried to factor the code into more discrete pieces than before.
Regarding code style: I tried to follow the code style, but I'm naughty
and used a few advanced C++ features repeatedly:
- Destructors to make it easier to ensure everything gets cleaned up.
- Exceptions. I don't even usually use exceptions when writing C++, and
I can remove them if desired... but here they make the loading code
much more succinct while still properly handling a variety of errors,
ranging from API calls failing to integer overflow and allocation
failure. The exceptions are converted to error codes at the
API boundary.)
Co-authored-by: Pavol Rusnak <pavol@rusnak.io> (for the bit I copied from #740)
2023-04-08 19:24:37 +00:00
l l a m a . h
2023-06-04 20:34:30 +00:00
l l a m a - u t i l . h
)
2023-03-22 05:32:36 +00:00
target_include_directories ( llama PUBLIC . )
target_compile_features ( llama PUBLIC cxx_std_11 ) # don't bump
2023-06-04 20:34:30 +00:00
target_link_libraries ( llama PRIVATE
g g m l
$ { L L A M A _ E X T R A _ L I B S }
)
2023-04-22 13:31:56 +00:00
2023-03-23 20:16:48 +00:00
if ( BUILD_SHARED_LIBS )
set_target_properties ( llama PROPERTIES POSITION_INDEPENDENT_CODE ON )
target_compile_definitions ( llama PRIVATE LLAMA_SHARED LLAMA_BUILD )
endif ( )
2023-03-21 00:37:16 +00:00
2023-06-04 20:34:30 +00:00
if ( GGML_SOURCES_CUDA )
2023-04-20 01:14:14 +00:00
message ( STATUS "GGML CUDA sources found, configuring CUDA architecture" )
2023-06-04 20:34:30 +00:00
set_property ( TARGET ggml PROPERTY CUDA_ARCHITECTURES OFF )
set_property ( TARGET ggml PROPERTY CUDA_SELECT_NVCC_ARCH_FLAGS "Auto" )
2023-04-20 01:14:14 +00:00
set_property ( TARGET llama PROPERTY CUDA_ARCHITECTURES OFF )
endif ( )
2023-03-21 15:29:41 +00:00
#
# programs, examples and tests
#
if ( LLAMA_BUILD_TESTS AND NOT CMAKE_JS_VERSION )
2023-03-30 17:56:59 +00:00
include ( CTest )
2023-03-21 15:29:41 +00:00
add_subdirectory ( tests )
endif ( )
2023-03-25 18:26:40 +00:00
if ( LLAMA_BUILD_EXAMPLES )
add_subdirectory ( examples )
2023-04-18 19:00:14 +00:00
add_subdirectory ( pocs )
2023-03-25 18:26:40 +00:00
endif ( )
