From 7a198bad4ebdfcfde882d42a2f9e8a2256fe9edf Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Mon, 17 Aug 2026 15:12:41 +0200 Subject: [PATCH] vendor: move hash to vendor --- examples/gguf-hash/CMakeLists.txt | 10 ++-- .../gguf-hash/deps/rotate-bits/package.json | 13 ----- examples/gguf-hash/deps/sha1/package.json | 9 ---- examples/gguf-hash/deps/sha256/package.json | 15 ------ examples/gguf-hash/deps/xxhash/clib.json | 12 ----- scripts/sync_vendor.py | 51 +++++++++++++++++++ vendor/rotate-bits/LICENSE.md | 21 ++++++++ .../deps => vendor}/rotate-bits/rotate-bits.h | 0 .../gguf-hash/deps => vendor}/sha1/sha1.c | 0 .../gguf-hash/deps => vendor}/sha1/sha1.h | 0 vendor/sha256/LICENSE | 1 + .../gguf-hash/deps => vendor}/sha256/sha256.c | 0 .../gguf-hash/deps => vendor}/sha256/sha256.h | 0 vendor/xxhash/LICENSE | 26 ++++++++++ .../gguf-hash/deps => vendor}/xxhash/xxhash.c | 0 .../gguf-hash/deps => vendor}/xxhash/xxhash.h | 0 16 files changed, 104 insertions(+), 54 deletions(-) delete mode 100644 examples/gguf-hash/deps/rotate-bits/package.json delete mode 100644 examples/gguf-hash/deps/sha1/package.json delete mode 100644 examples/gguf-hash/deps/sha256/package.json delete mode 100644 examples/gguf-hash/deps/xxhash/clib.json create mode 100644 vendor/rotate-bits/LICENSE.md rename {examples/gguf-hash/deps => vendor}/rotate-bits/rotate-bits.h (100%) rename {examples/gguf-hash/deps => vendor}/sha1/sha1.c (100%) rename {examples/gguf-hash/deps => vendor}/sha1/sha1.h (100%) create mode 100644 vendor/sha256/LICENSE rename {examples/gguf-hash/deps => vendor}/sha256/sha256.c (100%) rename {examples/gguf-hash/deps => vendor}/sha256/sha256.h (100%) create mode 100644 vendor/xxhash/LICENSE rename {examples/gguf-hash/deps => vendor}/xxhash/xxhash.c (100%) rename {examples/gguf-hash/deps => vendor}/xxhash/xxhash.h (100%) diff --git a/examples/gguf-hash/CMakeLists.txt b/examples/gguf-hash/CMakeLists.txt index 15c5c68c6f..f16bb3c17c 100644 --- a/examples/gguf-hash/CMakeLists.txt +++ b/examples/gguf-hash/CMakeLists.txt @@ -2,20 +2,20 @@ set(TARGET llama-gguf-hash) add_executable(${TARGET} gguf-hash.cpp) install(TARGETS ${TARGET} RUNTIME) -# clibs dependencies -include_directories(deps/) +set(VENDOR_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../vendor) +include_directories(${VENDOR_DIR}) -add_library(xxhash OBJECT deps/xxhash/xxhash.c deps/xxhash/xxhash.h) +add_library(xxhash OBJECT ${VENDOR_DIR}/xxhash/xxhash.c ${VENDOR_DIR}/xxhash/xxhash.h) target_link_libraries(${TARGET} PRIVATE xxhash) -add_library(sha1 OBJECT deps/sha1/sha1.c deps/sha1/sha1.h) +add_library(sha1 OBJECT ${VENDOR_DIR}/sha1/sha1.c ${VENDOR_DIR}/sha1/sha1.h) target_link_libraries(${TARGET} PRIVATE sha1) if (NOT MSVC) # disable warnings in 3rd party code target_compile_options(sha1 PRIVATE -w) endif() -add_library(sha256 OBJECT deps/sha256/sha256.c deps/sha256/sha256.h) +add_library(sha256 OBJECT ${VENDOR_DIR}/sha256/sha256.c ${VENDOR_DIR}/sha256/sha256.h) target_link_libraries(${TARGET} PRIVATE sha256) target_link_libraries(${TARGET} PRIVATE ggml ${CMAKE_THREAD_LIBS_INIT}) diff --git a/examples/gguf-hash/deps/rotate-bits/package.json b/examples/gguf-hash/deps/rotate-bits/package.json deleted file mode 100644 index 74c0bef68d..0000000000 --- a/examples/gguf-hash/deps/rotate-bits/package.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "name": "rotate-bits", - "version": "0.1.1", - "repo": "jb55/rotate-bits.h", - "description": "rotate bits", - "keywords": ["rotl", "rotr"], - "src": ["rotate-bits.h"], - "license": "Public Domain", - "development": { - "thlorenz/tap.c": "*" - } -} - diff --git a/examples/gguf-hash/deps/sha1/package.json b/examples/gguf-hash/deps/sha1/package.json deleted file mode 100644 index 6a5843dd1e..0000000000 --- a/examples/gguf-hash/deps/sha1/package.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "name": "sha1", - "version": "0.0.1", - "repo": "clibs/sha1", - "description": "sha1 hash algorithm", - "keywords": ["sha1", "hash"], - "license": "public domain", - "src": ["sha1.c", "sha1.h"] -} diff --git a/examples/gguf-hash/deps/sha256/package.json b/examples/gguf-hash/deps/sha256/package.json deleted file mode 100644 index b92a041273..0000000000 --- a/examples/gguf-hash/deps/sha256/package.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "name": "sha256", - "version": "0.0.2", - "repo": "jb55/sha256.c", - "description": "sha256 in c", - "keywords": ["sha256", "sha2"], - "src": ["sha256.c", "sha256.h"], - "dependencies": { - "jb55/rotate-bits.h": "0.1.1" - }, - "development": { - "thlorenz/tap.c": "*" - } -} - diff --git a/examples/gguf-hash/deps/xxhash/clib.json b/examples/gguf-hash/deps/xxhash/clib.json deleted file mode 100644 index 242343c5d9..0000000000 --- a/examples/gguf-hash/deps/xxhash/clib.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "name": "xxhash", - "version": "0.8.2", - "repo": "Cyan4973/xxhash", - "description": "Extremely fast non-cryptographic hash algorithm", - "keywords": ["xxhash", "hashing"], - "license": "BSD-2-Clause", - "src": [ - "xxhash.c", - "xxhash.h" - ] -} diff --git a/scripts/sync_vendor.py b/scripts/sync_vendor.py index 4ecf50f908..3db57d428a 100755 --- a/scripts/sync_vendor.py +++ b/scripts/sync_vendor.py @@ -7,6 +7,12 @@ import subprocess HTTPLIB_VERSION = "refs/tags/v0.53.1" +# used by examples/gguf-hash, these repos have no release tag, so we pin a commit +XXHASH_COMMIT = "9f465f1ea932d6ad9a26cd77496311ffa544cd68" +SHA1_COMMIT = "e1e2536fcf6a8f9703be8c85d58724b408552287" +SHA256_COMMIT = "5e637272c13f200872d55ff579f7e2ab6c3f252f" +ROTATE_BITS_COMMIT = "27e784942f67db44abf2115c6638e735b579acd1" + vendor = { "https://github.com/nlohmann/json/releases/latest/download/json.hpp": "vendor/nlohmann/json.hpp", "https://github.com/nlohmann/json/releases/latest/download/json_fwd.hpp": "vendor/nlohmann/json_fwd.hpp", @@ -22,12 +28,57 @@ vendor = { f"https://raw.githubusercontent.com/yhirose/cpp-httplib/{HTTPLIB_VERSION}/LICENSE": "vendor/cpp-httplib/LICENSE", "https://raw.githubusercontent.com/sheredom/subprocess.h/9ce0d701b6fb10f8f8c4445edd31e7c60a1237e3/subprocess.h": "vendor/sheredom/subprocess.h", + + f"https://raw.githubusercontent.com/Cyan4973/xxHash/{XXHASH_COMMIT}/xxhash.c": "vendor/xxhash/xxhash.c", + f"https://raw.githubusercontent.com/Cyan4973/xxHash/{XXHASH_COMMIT}/xxhash.h": "vendor/xxhash/xxhash.h", + f"https://raw.githubusercontent.com/Cyan4973/xxHash/{XXHASH_COMMIT}/LICENSE": "vendor/xxhash/LICENSE", + + # clibs/sha1 ships no license file, the source header says public domain + f"https://raw.githubusercontent.com/clibs/sha1/{SHA1_COMMIT}/sha1.c": "vendor/sha1/sha1.c", + f"https://raw.githubusercontent.com/clibs/sha1/{SHA1_COMMIT}/sha1.h": "vendor/sha1/sha1.h", + + f"https://raw.githubusercontent.com/jb55/sha256.c/{SHA256_COMMIT}/sha256.c": "vendor/sha256/sha256.c", + f"https://raw.githubusercontent.com/jb55/sha256.c/{SHA256_COMMIT}/sha256.h": "vendor/sha256/sha256.h", + f"https://raw.githubusercontent.com/jb55/sha256.c/{SHA256_COMMIT}/LICENSE": "vendor/sha256/LICENSE", + + f"https://raw.githubusercontent.com/jb55/rotate-bits.h/{ROTATE_BITS_COMMIT}/rotate-bits.h": "vendor/rotate-bits/rotate-bits.h", + f"https://raw.githubusercontent.com/jb55/rotate-bits.h/{ROTATE_BITS_COMMIT}/LICENSE.md": "vendor/rotate-bits/LICENSE.md", +} + +# local changes kept on top of the upstream sources +patches = { + "vendor/xxhash/xxhash.h": [( + '#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 */\n', + '/* Windows SDK under 10.0.22000 is missing stdalign.h so we add a check\n' + ' before allowing the windows compiler to use the C11 form.\n' + ' Reference: https://github.com/Cyan4973/xxHash/issues/955 */\n' + '#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) \\\n' + ' && (defined(_MSC_VER) && (_MSC_VER >= 1000) || !defined(_MSC_VER)) /* >= C11 */\n' + )], + + # silence a maybe-uninitialized warning + "vendor/sha256/sha256.c": [( + " uint32_t W[16];\n", + " uint32_t W[16] = {0};\n" + )], } for url, filename in vendor.items(): print(f"downloading {url} to {filename}") # noqa: NP100 urllib.request.urlretrieve(url, filename) +for filename, replacements in patches.items(): + print(f"patching {filename}") # noqa: NP100 + with open(filename, "r", encoding="utf-8", newline="") as f: + content = f.read() + for old, new in replacements: + if content.count(old) != 1: + print(f"Error: cannot apply patch on {filename}, upstream code has changed") # noqa: NP100 + sys.exit(1) + content = content.replace(old, new) + with open(filename, "w", encoding="utf-8", newline="") as f: + f.write(content) + print("Splitting httplib.h...") # noqa: NP100 try: subprocess.check_call([ diff --git a/vendor/rotate-bits/LICENSE.md b/vendor/rotate-bits/LICENSE.md new file mode 100644 index 0000000000..4a07c57f26 --- /dev/null +++ b/vendor/rotate-bits/LICENSE.md @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 William Casarin + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/examples/gguf-hash/deps/rotate-bits/rotate-bits.h b/vendor/rotate-bits/rotate-bits.h similarity index 100% rename from examples/gguf-hash/deps/rotate-bits/rotate-bits.h rename to vendor/rotate-bits/rotate-bits.h diff --git a/examples/gguf-hash/deps/sha1/sha1.c b/vendor/sha1/sha1.c similarity index 100% rename from examples/gguf-hash/deps/sha1/sha1.c rename to vendor/sha1/sha1.c diff --git a/examples/gguf-hash/deps/sha1/sha1.h b/vendor/sha1/sha1.h similarity index 100% rename from examples/gguf-hash/deps/sha1/sha1.h rename to vendor/sha1/sha1.h diff --git a/vendor/sha256/LICENSE b/vendor/sha256/LICENSE new file mode 100644 index 0000000000..950b19483a --- /dev/null +++ b/vendor/sha256/LICENSE @@ -0,0 +1 @@ +2010-06-11 : Igor Pavlov : Public domain diff --git a/examples/gguf-hash/deps/sha256/sha256.c b/vendor/sha256/sha256.c similarity index 100% rename from examples/gguf-hash/deps/sha256/sha256.c rename to vendor/sha256/sha256.c diff --git a/examples/gguf-hash/deps/sha256/sha256.h b/vendor/sha256/sha256.h similarity index 100% rename from examples/gguf-hash/deps/sha256/sha256.h rename to vendor/sha256/sha256.h diff --git a/vendor/xxhash/LICENSE b/vendor/xxhash/LICENSE new file mode 100644 index 0000000000..e4c5da7234 --- /dev/null +++ b/vendor/xxhash/LICENSE @@ -0,0 +1,26 @@ +xxHash Library +Copyright (c) 2012-2021 Yann Collet +All rights reserved. + +BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/examples/gguf-hash/deps/xxhash/xxhash.c b/vendor/xxhash/xxhash.c similarity index 100% rename from examples/gguf-hash/deps/xxhash/xxhash.c rename to vendor/xxhash/xxhash.c diff --git a/examples/gguf-hash/deps/xxhash/xxhash.h b/vendor/xxhash/xxhash.h similarity index 100% rename from examples/gguf-hash/deps/xxhash/xxhash.h rename to vendor/xxhash/xxhash.h