diff --git a/.github/workflows/kickmix.yml b/.github/workflows/kickmix.yml index 58458c7a01..d9e966cc3b 100644 --- a/.github/workflows/kickmix.yml +++ b/.github/workflows/kickmix.yml @@ -24,6 +24,77 @@ on: - main permissions: {} jobs: + build_dist: + runs-on: ${{ matrix.os_dist.os }} + permissions: + contents: read + strategy: + fail-fast: false + matrix: + os_dist: [ + {os: ubuntu-24.04, dist: cp310-manylinux_x86_64}, + {os: ubuntu-24.04, dist: cp311-manylinux_x86_64}, + {os: ubuntu-24.04, dist: cp312-manylinux_x86_64}, + {os: ubuntu-24.04, dist: cp313-manylinux_x86_64}, + {os: ubuntu-24.04, dist: cp314-manylinux_x86_64}, + {os: macos-15-intel, dist: cp310-macosx_x86_64}, + {os: macos-15-intel, dist: cp311-macosx_x86_64}, + {os: macos-15-intel, dist: cp312-macosx_x86_64}, + {os: macos-15-intel, dist: cp313-macosx_x86_64}, + {os: macos-15-intel, dist: cp314-macosx_x86_64}, + {os: macos-15, dist: cp310-macosx_arm64}, + {os: macos-15, dist: cp311-macosx_arm64}, + {os: macos-15, dist: cp312-macosx_arm64}, + {os: macos-15, dist: cp313-macosx_arm64}, + {os: macos-15, dist: cp314-macosx_arm64}, + {os: windows-2025, dist: cp310-win_amd64}, + {os: windows-2025, dist: cp311-win_amd64}, + {os: windows-2025, dist: cp312-win_amd64}, + {os: windows-2025, dist: cp313-win_amd64}, + {os: windows-2025, dist: cp314-win_amd64}, + ] + env: + CIBW_BUILD: "${{ matrix.os_dist.dist }}" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + - run: python -m pip install pybind11==2.12.0 cibuildwheel~=3.3.1 setuptools wheel + - run: python -m cibuildwheel kickmix --print-build-identifiers + - run: python -m cibuildwheel kickmix --output-dir dist + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: "dist-kickmix-${{ matrix.os_dist.os }}-${{ matrix.os_dist.dist }}" + path: dist/* + build_sdist: + runs-on: ubuntu-24.04 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + - run: python -m pip install setuptools pybind11==2.12.0 wheel + - run: cd kickmix && python setup.py sdist + - run: pip install kickmix/dist/*.tar.gz + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: "dist-kickmix-sdist" + path: kickmix/dist/*.tar.gz + merge_upload_artifacts: + needs: ["build_dist", "build_sdist"] + runs-on: ubuntu-24.04 + permissions: + contents: read + actions: read + steps: + - name: Merge Artifacts + uses: actions/upload-artifact/merge@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: dist-kickmix + pattern: dist-kickmix-* code_is_autoformatted: runs-on: ubuntu-26.04 steps: diff --git a/.gitignore b/.gitignore index 25a0a0a111..2ca71012c9 100644 --- a/.gitignore +++ b/.gitignore @@ -34,8 +34,12 @@ parts/ sdist/ var/ wheels/ +wheelhouse/ +python_build_kickmix/ pip-wheel-metadata/ share/python-wheels/ +wheelhouse/ +python_build_kickmix/ *.egg-info/ .installed.cfg *.egg diff --git a/docs/kickmix/dev.md b/docs/kickmix/dev.md index 5b63715c14..053bb2bb7c 100644 --- a/docs/kickmix/dev.md +++ b/docs/kickmix/dev.md @@ -39,6 +39,19 @@ pip install bazel-bin/kickmix/kickmix-0.0.dev0-py3-none-any.whl PYTHONPATH=kickmix pytest kickmix/src ``` +## Build Portable Python Wheel (with cibuildwheel) + +```bash +sudo apt install podman +pip install cibuildwheel + +# See https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip for --only values +STORAGE_OPTS="overlay.ignore_chown_errors=true" \ +CIBW_CONTAINER_ENGINE=podman \ +cibuildwheel kickmix --only cp313-manylinux_x86_64 +# output is in wheelhouse/ +``` + ## Auto-format code (using clang-format-21) The project includes a `.clang-format` file that clang format will use automatically. diff --git a/kickmix/MANIFEST.in b/kickmix/MANIFEST.in new file mode 100644 index 0000000000..3375315df8 --- /dev/null +++ b/kickmix/MANIFEST.in @@ -0,0 +1,2 @@ +recursive-include src *.h *.inl *.cc +global-exclude *.test.cc *.test.h *.perf.cc *.perf.h *_test.py diff --git a/kickmix/pyproject.toml b/kickmix/pyproject.toml new file mode 100644 index 0000000000..b756d5fde0 --- /dev/null +++ b/kickmix/pyproject.toml @@ -0,0 +1,24 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[build-system] +requires = ["setuptools", "wheel", "pybind11==2.12.0"] +build-backend = "setuptools.build_meta" + +[tool.cibuildwheel] +test-requires = ["pytest"] +test-command = "pytest {package}/src" + +[tool.pytest.ini_options] +pythonpath = ["."] diff --git a/kickmix/setup.py b/kickmix/setup.py new file mode 100644 index 0000000000..bf3154e46d --- /dev/null +++ b/kickmix/setup.py @@ -0,0 +1,68 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import glob +import platform + +import pybind11 +from pybind11.setup_helpers import ParallelCompile +from setuptools import Extension, setup + +ParallelCompile().install() + +ALL_SOURCE_FILES = glob.glob("src/**/*.cc", recursive=True) +TEST_FILES = glob.glob("src/**/*.test.cc", recursive=True) +PERF_FILES = glob.glob("src/**/*.perf.cc", recursive=True) +MAIN_FILES = glob.glob("src/**/main.cc", recursive=True) +HEADER_FILES = glob.glob("src/**/*.h", recursive=True) + glob.glob("src/**/*.inl", recursive=True) +RELEVANT_SOURCE_FILES = sorted(set(ALL_SOURCE_FILES) - set(TEST_FILES + PERF_FILES + MAIN_FILES)) + +__version__ = '0.1.dev0' + +if platform.system().startswith('Win'): + common_compile_args = ['/std:c++20', '/O2', f'/DVERSION_INFO={__version__}'] +else: + common_compile_args = [ + '-std=c++20', + '-fno-strict-aliasing', + '-fvisibility=hidden', + '-O3', + '-g0', + f'-DVERSION_INFO={__version__}', + ] + if any(arch in platform.machine().lower() for arch in ['x86', 'amd64', 'i386', 'i686']): + common_compile_args.append('-mno-avx') + +kickmix_extension = Extension( + 'kickmix', + sources=RELEVANT_SOURCE_FILES, + depends=HEADER_FILES, + include_dirs=[pybind11.get_include(), 'src'], + language='c++', + extra_compile_args=common_compile_args, +) + +setup( + name='kickmix', + version=__version__, + author='Qualtran Developers', + url='https://github.com/quantumlib/Qualtran', + license='Apache-2.0', + description='Fast construction and simulation of quantum circuits with classical operations, phasing, and X basis demolition measurement.', + ext_modules=[kickmix_extension], + python_requires='>=3.10.0', + install_requires=['numpy'], + # Needed on Windows/macOS to avoid the default `build` colliding with Bazel's `BUILD`. + options={'build': {'build_base': 'python_build_kickmix'}}, +) diff --git a/kickmix/src/kickmix/build/circuit_builder.cc b/kickmix/src/kickmix/build/circuit_builder.cc index 73d1582e46..cb165727a0 100644 --- a/kickmix/src/kickmix/build/circuit_builder.cc +++ b/kickmix/src/kickmix/build/circuit_builder.cc @@ -1,6 +1,8 @@ #include "kickmix/build/circuit_builder.h" +#include #include +#include using namespace kickmix; diff --git a/kickmix/src/kickmix/build/circuit_builder.h b/kickmix/src/kickmix/build/circuit_builder.h index 3b11b60a81..fd05956a28 100644 --- a/kickmix/src/kickmix/build/circuit_builder.h +++ b/kickmix/src/kickmix/build/circuit_builder.h @@ -1,8 +1,13 @@ #ifndef KICKMIX_CIRCUIT_BUILDER_H #define KICKMIX_CIRCUIT_BUILDER_H +#include #include #include +#include +#include +#include +#include #include "circuit_gen_ctx.h" #include "circuit_pattern.h" diff --git a/kickmix/src/kickmix/circuit/circuit.cc b/kickmix/src/kickmix/circuit/circuit.cc index 8592659a3d..fa9ce4093c 100644 --- a/kickmix/src/kickmix/circuit/circuit.cc +++ b/kickmix/src/kickmix/circuit/circuit.cc @@ -30,51 +30,51 @@ void Circuit::clear() { num_info_tags = 0; num_info_tag_chars = 0; if (op_types != nullptr) { - free(op_types); + free_aligned(op_types); op_types = nullptr; } if (qqq0 != nullptr) { - free(qqq0); + free_aligned(qqq0); qqq0 = nullptr; } if (qqq1 != nullptr) { - free(qqq1); + free_aligned(qqq1); qqq1 = nullptr; } if (qqq2 != nullptr) { - free(qqq2); + free_aligned(qqq2); qqq2 = nullptr; } if (qq0 != nullptr) { - free(qq0); + free_aligned(qq0); qq0 = nullptr; } if (qq1 != nullptr) { - free(qq1); + free_aligned(qq1); qq1 = nullptr; } if (q0 != nullptr) { - free(q0); + free_aligned(q0); q0 = nullptr; } if (bc != nullptr) { - free(bc); + free_aligned(bc); bc = nullptr; } if (b0 != nullptr) { - free(b0); + free_aligned(b0); b0 = nullptr; } if (angles != nullptr) { - free(angles); + free_aligned(angles); angles = nullptr; } if (info_tag_end_offsets != nullptr) { - free(info_tag_end_offsets); + free_aligned(info_tag_end_offsets); info_tag_end_offsets = nullptr; } if (info_tag_chars != nullptr) { - free(info_tag_chars); + free_aligned(info_tag_chars); info_tag_chars = nullptr; } } @@ -135,7 +135,7 @@ Circuit Circuit::from_kmx_or_kmb_file(FILE *file) { // Read rest of line. std::string line; while (true) { - if (i == EOF || i == '\n') { + if (i == EOF || i == '\n' || i == '\r') { break; } line.push_back((char)i); @@ -275,24 +275,6 @@ Circuit::Circuit(Circuit &&other) noexcept { other.num_info_tag_chars = 0; } -template -static T *aligned_alloc_32_else_throw(size_t count) { - if (count == 0) { - return nullptr; - } - if (count > (SIZE_MAX - 31) / sizeof(T)) { - throw std::invalid_argument("Too much memory to allocate."); - } - size_t bytes = count * sizeof(T); - bytes += size_t{31}; - bytes &= ~size_t{31}; - T *result = (T *)std::aligned_alloc(32, bytes); - if (result == nullptr) { - throw std::invalid_argument("Failed to allocate."); - } - return result; -} - Circuit::Circuit(const Circuit &other) { register_data = other.register_data; num_ops = other.num_ops; @@ -307,18 +289,18 @@ Circuit::Circuit(const Circuit &other) { num_info_tags = other.num_info_tags; num_info_tag_chars = other.num_info_tag_chars; - op_types = aligned_alloc_32_else_throw(num_ops); - qqq0 = aligned_alloc_32_else_throw(num_qqq_ops); - qqq1 = aligned_alloc_32_else_throw(num_qqq_ops); - qqq2 = aligned_alloc_32_else_throw(num_qqq_ops); - qq0 = aligned_alloc_32_else_throw(num_qq_ops); - qq1 = aligned_alloc_32_else_throw(num_qq_ops); - q0 = aligned_alloc_32_else_throw(num_q_ops); - bc = aligned_alloc_32_else_throw(num_c_ops); - b0 = aligned_alloc_32_else_throw(num_b_ops); - angles = aligned_alloc_32_else_throw(num_angle_ops); - info_tag_end_offsets = aligned_alloc_32_else_throw(num_info_tags); - info_tag_chars = aligned_alloc_32_else_throw(num_info_tag_chars); + op_types = aligned_alloc_else_throw(num_ops); + qqq0 = aligned_alloc_else_throw(num_qqq_ops); + qqq1 = aligned_alloc_else_throw(num_qqq_ops); + qqq2 = aligned_alloc_else_throw(num_qqq_ops); + qq0 = aligned_alloc_else_throw(num_qq_ops); + qq1 = aligned_alloc_else_throw(num_qq_ops); + q0 = aligned_alloc_else_throw(num_q_ops); + bc = aligned_alloc_else_throw(num_c_ops); + b0 = aligned_alloc_else_throw(num_b_ops); + angles = aligned_alloc_else_throw(num_angle_ops); + info_tag_end_offsets = aligned_alloc_else_throw(num_info_tags); + info_tag_chars = aligned_alloc_else_throw(num_info_tag_chars); memcpy(op_types, other.op_types, num_ops); memcpy(qqq0, other.qqq0, num_qqq_ops * sizeof(uint32_t)); @@ -353,18 +335,18 @@ Circuit &Circuit::operator=(const Circuit &other) { num_info_tags = other.num_info_tags; num_info_tag_chars = other.num_info_tag_chars; - op_types = aligned_alloc_32_else_throw(num_ops); - qqq0 = aligned_alloc_32_else_throw(num_qqq_ops); - qqq1 = aligned_alloc_32_else_throw(num_qqq_ops); - qqq2 = aligned_alloc_32_else_throw(num_qqq_ops); - qq0 = aligned_alloc_32_else_throw(num_qq_ops); - qq1 = aligned_alloc_32_else_throw(num_qq_ops); - q0 = aligned_alloc_32_else_throw(num_q_ops); - bc = aligned_alloc_32_else_throw(num_c_ops); - b0 = aligned_alloc_32_else_throw(num_b_ops); - angles = aligned_alloc_32_else_throw(num_angle_ops); - info_tag_end_offsets = aligned_alloc_32_else_throw(num_info_tags); - info_tag_chars = aligned_alloc_32_else_throw(num_info_tag_chars); + op_types = aligned_alloc_else_throw(num_ops); + qqq0 = aligned_alloc_else_throw(num_qqq_ops); + qqq1 = aligned_alloc_else_throw(num_qqq_ops); + qqq2 = aligned_alloc_else_throw(num_qqq_ops); + qq0 = aligned_alloc_else_throw(num_qq_ops); + qq1 = aligned_alloc_else_throw(num_qq_ops); + q0 = aligned_alloc_else_throw(num_q_ops); + bc = aligned_alloc_else_throw(num_c_ops); + b0 = aligned_alloc_else_throw(num_b_ops); + angles = aligned_alloc_else_throw(num_angle_ops); + info_tag_end_offsets = aligned_alloc_else_throw(num_info_tags); + info_tag_chars = aligned_alloc_else_throw(num_info_tag_chars); memcpy(op_types, other.op_types, num_ops); memcpy(qqq0, other.qqq0, num_qqq_ops * sizeof(uint32_t)); @@ -1314,7 +1296,7 @@ inline static uint32_t read_u32_be(FILE *file) { uint32_t result{}; fread_else_throw(&result, sizeof(uint32_t), file); if constexpr (std::endian::native == std::endian::big) { - result = __builtin_bswap32(result); + result = bswap32(result); } return result; } @@ -1323,7 +1305,7 @@ inline static uint64_t read_u64_be(FILE *file) { uint64_t result{}; fread_else_throw(&result, sizeof(uint64_t), file); if constexpr (std::endian::native == std::endian::big) { - result = __builtin_bswap64(result); + result = bswap64(result); } return result; } @@ -1349,7 +1331,7 @@ template static void alloc_and_read_T_blob_packet(FILE *file, KmbPacketType type, T **data_out, size_t count) { read_u64_be_expect(file, (uint64_t)type, "Payload packets in wrong order."); read_u64_be_expect(file, count * sizeof(T), "Size mismatch: actual payload size vs size specified in header."); - *data_out = aligned_alloc_32_else_throw(count); + *data_out = aligned_alloc_else_throw(count); fread_else_throw(*data_out, count * sizeof(T), file); } @@ -1417,7 +1399,7 @@ Circuit Circuit::from_kmb_file(FILE *file, bool skip_magic) { read_u64_be_expect(file, (uint64_t)KmbPacketType::OPERATION_TYPES, "Expected operation type data."); read_u64_be_expect( file, result.num_ops, "Operation type payload size was inconsistent with amount specified in header."); - result.op_types = aligned_alloc_32_else_throw(result.num_ops); + result.op_types = aligned_alloc_else_throw(result.num_ops); fread_else_throw((void *)result.op_types, result.num_ops, file); // Operation target data. @@ -1445,9 +1427,9 @@ Circuit Circuit::from_kmb_file(FILE *file, bool skip_magic) { (result.num_info_tags + 1) * sizeof(uint64_t) + result.num_info_tag_chars, "Size mismatch: actual payload size vs size specified in header."); read_u64_be_expect(file, result.num_info_tags, "Size mismatch: actual payload size vs size specified in header."); - result.info_tag_end_offsets = aligned_alloc_32_else_throw(result.num_info_tags); + result.info_tag_end_offsets = aligned_alloc_else_throw(result.num_info_tags); fread_else_throw(result.info_tag_end_offsets, result.num_info_tags * sizeof(uint64_t), file); - result.info_tag_chars = aligned_alloc_32_else_throw(result.num_info_tag_chars); + result.info_tag_chars = aligned_alloc_else_throw(result.num_info_tag_chars); fread_else_throw(result.info_tag_chars, result.num_info_tag_chars, file); if (fgetc(file) != EOF) { diff --git a/kickmix/src/kickmix/circuit/circuit_diagram.cc b/kickmix/src/kickmix/circuit/circuit_diagram.cc index 0f239440e0..9bc1d00dec 100644 --- a/kickmix/src/kickmix/circuit/circuit_diagram.cc +++ b/kickmix/src/kickmix/circuit/circuit_diagram.cc @@ -1,3 +1,4 @@ +#include #include #include #include diff --git a/kickmix/src/kickmix/circuit/circuit_diagram_html.cc b/kickmix/src/kickmix/circuit/circuit_diagram_html.cc index 8c37971111..cf4bd18a60 100644 --- a/kickmix/src/kickmix/circuit/circuit_diagram_html.cc +++ b/kickmix/src/kickmix/circuit/circuit_diagram_html.cc @@ -1,6 +1,9 @@ +#include +#include #include #include #include +#include #include #include #include diff --git a/kickmix/src/kickmix/circuit/circuit_util.cc b/kickmix/src/kickmix/circuit/circuit_util.cc index d213b227a9..256afad753 100644 --- a/kickmix/src/kickmix/circuit/circuit_util.cc +++ b/kickmix/src/kickmix/circuit/circuit_util.cc @@ -1,5 +1,7 @@ #include "kickmix/circuit/circuit_util.h" +#include + using namespace kickmix; size_t kickmix::compute_reaction_depth(size_t num_qubits, size_t num_bits, std::span operations) { diff --git a/kickmix/src/kickmix/circuit/mutable_circuit.cc b/kickmix/src/kickmix/circuit/mutable_circuit.cc index 9c2fb38a69..5eb19d23b3 100644 --- a/kickmix/src/kickmix/circuit/mutable_circuit.cc +++ b/kickmix/src/kickmix/circuit/mutable_circuit.cc @@ -1,9 +1,11 @@ #include "mutable_circuit.h" +#include #include #include #include "kickmix/id/register_id.h" +#include "kickmix/mem/util.h" using namespace kickmix; @@ -20,7 +22,7 @@ static constexpr uint8_t name_hash(std::string_view name) { // 1. Very efficient to evaluate. // 2. Result in a small range (currently < 64). // 3. Different value for every defined operation name. - uint8_t result = name.size(); + uint8_t result = static_cast(name.size()); if (name.size() > 0) { result += name.back() * 3; result ^= name[0] * 24; @@ -330,7 +332,7 @@ static void parse_debug_print_line(MutableCircuit &builder, const char *c, const } void MutableCircuit::append_from_kmx_line(std::string_view kmx_line) { - const char *end = kmx_line.end(); + const char *end = kmx_line.data() + kmx_line.size(); const char *c = kmx_line.data(); std::string_view op_name = read_word(c, end); @@ -466,7 +468,7 @@ void MutableCircuit::append_from_kmx_file(FILE *file) { } void MutableCircuit::append_from_kmx_text(std::string_view kmx_text) { - const char *end = kmx_text.end(); + const char *end = kmx_text.data() + kmx_text.size(); const char *c = kmx_text.data(); while (c < end) { // Scan for end of line (or beginning of comment). @@ -489,24 +491,6 @@ void MutableCircuit::append_from_kmx_text(std::string_view kmx_text) { } } -template -static T *aligned_alloc_32_else_throw(size_t count) { - if (count == 0) { - return nullptr; - } - if (count > (SIZE_MAX - 31) / sizeof(T)) { - throw std::invalid_argument("Too much memory to allocate."); - } - size_t bytes = count * sizeof(T); - bytes += size_t{31}; - bytes &= ~size_t{31}; - T *result = (T *)std::aligned_alloc(32, bytes); - if (result == nullptr) { - throw std::invalid_argument("Failed to allocate."); - } - return result; -} - template static uint32_t max_u32_buf_untagged(const MonotonicArena &buf) { uint32_t result = 0; @@ -571,18 +555,18 @@ Circuit MutableCircuit::to_validated_circuit() const { result.num_info_tag_chars = info_tag_chars.size(); // Copy qubit and bit data into contiguous memory blocks. - result.op_types = aligned_alloc_32_else_throw(op_types.size()); - result.qqq0 = aligned_alloc_32_else_throw(qqq0.size()); - result.qqq1 = aligned_alloc_32_else_throw(qqq1.size()); - result.qqq2 = aligned_alloc_32_else_throw(qqq2.size()); - result.qq0 = aligned_alloc_32_else_throw(qq0.size()); - result.qq1 = aligned_alloc_32_else_throw(qq1.size()); - result.q0 = aligned_alloc_32_else_throw(q0.size()); - result.bc = aligned_alloc_32_else_throw(bc.size()); - result.b0 = aligned_alloc_32_else_throw(b0.size()); - result.angles = aligned_alloc_32_else_throw(angles.size()); - result.info_tag_end_offsets = aligned_alloc_32_else_throw(info_tag_end_offsets.size()); - result.info_tag_chars = aligned_alloc_32_else_throw(info_tag_chars.size()); + result.op_types = aligned_alloc_else_throw(op_types.size()); + result.qqq0 = aligned_alloc_else_throw(qqq0.size()); + result.qqq1 = aligned_alloc_else_throw(qqq1.size()); + result.qqq2 = aligned_alloc_else_throw(qqq2.size()); + result.qq0 = aligned_alloc_else_throw(qq0.size()); + result.qq1 = aligned_alloc_else_throw(qq1.size()); + result.q0 = aligned_alloc_else_throw(q0.size()); + result.bc = aligned_alloc_else_throw(bc.size()); + result.b0 = aligned_alloc_else_throw(b0.size()); + result.angles = aligned_alloc_else_throw(angles.size()); + result.info_tag_end_offsets = aligned_alloc_else_throw(info_tag_end_offsets.size()); + result.info_tag_chars = aligned_alloc_else_throw(info_tag_chars.size()); op_types.memcpy_into(result.op_types); qqq0.memcpy_into(result.qqq0); qqq1.memcpy_into(result.qqq1); diff --git a/kickmix/src/kickmix/circuit/op_type.h b/kickmix/src/kickmix/circuit/op_type.h index 7e53c13068..65560d5bc9 100644 --- a/kickmix/src/kickmix/circuit/op_type.h +++ b/kickmix/src/kickmix/circuit/op_type.h @@ -1,8 +1,10 @@ #ifndef KICKMIX_CIRCUIT_OP_TYPE_H #define KICKMIX_CIRCUIT_OP_TYPE_H +#include #include #include +#include namespace kickmix { diff --git a/kickmix/src/kickmix/circuit/register_data.h b/kickmix/src/kickmix/circuit/register_data.h index 109610812f..fe2338d7af 100644 --- a/kickmix/src/kickmix/circuit/register_data.h +++ b/kickmix/src/kickmix/circuit/register_data.h @@ -1,6 +1,8 @@ #ifndef KICKMIX_REGISTER_DATA_H #define KICKMIX_REGISTER_DATA_H +#include +#include #include #include "kickmix/id/qubit_or_bit.h" diff --git a/kickmix/src/kickmix/cmd/main_count_operations_sampled.cc b/kickmix/src/kickmix/cmd/main_count_operations_sampled.cc index e8b21726a4..78ef4ecbe7 100644 --- a/kickmix/src/kickmix/cmd/main_count_operations_sampled.cc +++ b/kickmix/src/kickmix/cmd/main_count_operations_sampled.cc @@ -1,3 +1,4 @@ +#include #include #include diff --git a/kickmix/src/kickmix/cmd/main_sample.cc b/kickmix/src/kickmix/cmd/main_sample.cc index 462cb3ee13..53999a8306 100644 --- a/kickmix/src/kickmix/cmd/main_sample.cc +++ b/kickmix/src/kickmix/cmd/main_sample.cc @@ -1,3 +1,4 @@ +#include #include #include diff --git a/kickmix/src/kickmix/id/array_xz.cc b/kickmix/src/kickmix/id/array_xz.cc index 1e09967a65..9d5b11d7fc 100644 --- a/kickmix/src/kickmix/id/array_xz.cc +++ b/kickmix/src/kickmix/id/array_xz.cc @@ -3,11 +3,13 @@ #include #include +#include "kickmix/mem/util.h" + using namespace kickmix; array_xz array_xz::alloc_noinit(size_t count, QXZTypeTag8 tag) { array_xz result; - result.items = (QubitOrXZBitOrXZBool *)malloc(count * sizeof(uint32_t)); + result.items = alloc_else_throw(count); result.count = count; result.common_type = tag; return result; @@ -29,7 +31,7 @@ array_xz &array_xz::operator=(array_xz &&other) noexcept { return *this; } if (items != nullptr) { - free(items); + free_aligned(items); } items = other.items; count = other.count; @@ -41,7 +43,7 @@ array_xz &array_xz::operator=(array_xz &&other) noexcept { array_xz::~array_xz() { if (items != nullptr) { - free(items); + free_aligned(items); } } @@ -74,7 +76,7 @@ std::ostream &kickmix::operator<<(std::ostream &out, const array_xz &v) { array_xz array_xz::copy_of_concat(const stride_span_xz &lhs, const stride_span_xz &rhs) { array_xz result; size_t n = lhs.size() + rhs.size(); - result.items = (QubitOrXZBitOrXZBool *)malloc(n * sizeof(QubitOrXZBitOrXZBool)); + result.items = alloc_else_throw(n); result.count = n; // Copy data. diff --git a/kickmix/src/kickmix/id/array_z.cc b/kickmix/src/kickmix/id/array_z.cc index f0f6ac12dc..c649cc473b 100644 --- a/kickmix/src/kickmix/id/array_z.cc +++ b/kickmix/src/kickmix/id/array_z.cc @@ -3,11 +3,13 @@ #include #include +#include "kickmix/mem/util.h" + using namespace kickmix; array_z array_z::alloc_noinit(size_t count, QXZTypeTag8 tag) { array_z result; - result.items = (QubitOrBitOrBool *)malloc(count * sizeof(uint32_t)); + result.items = alloc_else_throw(count); result.count = count; result.common_type = tag; return result; @@ -24,7 +26,7 @@ array_z &array_z::operator=(array_z &&other) noexcept { return *this; } if (items != nullptr) { - free(items); + free_aligned(items); } items = other.items; count = other.count; @@ -36,7 +38,7 @@ array_z &array_z::operator=(array_z &&other) noexcept { array_z::~array_z() { if (items != nullptr) { - free(items); + free_aligned(items); } } diff --git a/kickmix/src/kickmix/id/bit_id.h b/kickmix/src/kickmix/id/bit_id.h index 9cb380ef03..52f239f329 100644 --- a/kickmix/src/kickmix/id/bit_id.h +++ b/kickmix/src/kickmix/id/bit_id.h @@ -3,6 +3,8 @@ #include #include +#include +#include #include "id_tagging.h" diff --git a/kickmix/src/kickmix/id/qubit_id.h b/kickmix/src/kickmix/id/qubit_id.h index eb9ae27a51..40f58ecbae 100644 --- a/kickmix/src/kickmix/id/qubit_id.h +++ b/kickmix/src/kickmix/id/qubit_id.h @@ -3,6 +3,8 @@ #include #include +#include +#include #include "id_tagging.h" diff --git a/kickmix/src/kickmix/id/register_id.h b/kickmix/src/kickmix/id/register_id.h index f954a9d8d1..5f5bc645f2 100644 --- a/kickmix/src/kickmix/id/register_id.h +++ b/kickmix/src/kickmix/id/register_id.h @@ -3,6 +3,7 @@ #include #include +#include namespace kickmix { struct RegisterId { diff --git a/kickmix/src/kickmix/id/stride_span_xz.h b/kickmix/src/kickmix/id/stride_span_xz.h index 30deaa9f85..19ef28f0ef 100644 --- a/kickmix/src/kickmix/id/stride_span_xz.h +++ b/kickmix/src/kickmix/id/stride_span_xz.h @@ -1,7 +1,11 @@ #ifndef KICKGEN_QC_STRIDE_SPAN_XZ_H #define KICKGEN_QC_STRIDE_SPAN_XZ_H +#include +#include #include +#include +#include #include "kickmix/mem/stride_span.h" #include "qubit_or_bit_or_bool.h" diff --git a/kickmix/src/kickmix/id/stride_span_z.h b/kickmix/src/kickmix/id/stride_span_z.h index 6b369770cb..f477bf81d4 100644 --- a/kickmix/src/kickmix/id/stride_span_z.h +++ b/kickmix/src/kickmix/id/stride_span_z.h @@ -1,7 +1,14 @@ #ifndef KICKGEN_QC_STRIDE_SPAN_H #define KICKGEN_QC_STRIDE_SPAN_H +#include +#include +#include #include +#include +#include +#include +#include #include "kickmix/mem/stride_span.h" #include "qubit_or_bit.h" diff --git a/kickmix/src/kickmix/id/xbool.h b/kickmix/src/kickmix/id/xbool.h index 1232dbaf68..8d3c26673b 100644 --- a/kickmix/src/kickmix/id/xbool.h +++ b/kickmix/src/kickmix/id/xbool.h @@ -3,6 +3,7 @@ #include #include +#include #include "id_tagging.h" diff --git a/kickmix/src/kickmix/main.cc b/kickmix/src/kickmix/main.cc index 02115b39fc..c35d59f903 100644 --- a/kickmix/src/kickmix/main.cc +++ b/kickmix/src/kickmix/main.cc @@ -1,5 +1,6 @@ #include #include +#include #include "cmd/main_util.h" diff --git a/kickmix/src/kickmix/mem/monotonic_arena.cc b/kickmix/src/kickmix/mem/monotonic_arena.cc index 54a35888ef..f726cc7f3b 100644 --- a/kickmix/src/kickmix/mem/monotonic_arena.cc +++ b/kickmix/src/kickmix/mem/monotonic_arena.cc @@ -1,15 +1,21 @@ #include "kickmix/mem/monotonic_arena.h" +#include + +#include "util.h" + +using namespace kickmix; + static void dealloc(kickmix::MonotonicArena_Helper &buf) { for (const auto &e : buf.frozen_allocations) { - free(e.data()); + free_aligned(e.data()); } buf.frozen_allocations.clear(); buf.cur_length_used = 0; buf.total_frozen_length_used = 0; buf.cur_length_cap = 0; if (buf.cur != nullptr) { - free(buf.cur); + free_aligned(buf.cur); buf.cur = nullptr; } } @@ -52,7 +58,7 @@ kickmix::MonotonicArena_Helper::~MonotonicArena_Helper() { void kickmix::MonotonicArena_Helper::clear() { for (const auto &e : frozen_allocations) { - free(e.data()); + free_aligned(e.data()); } frozen_allocations.clear(); cur_length_used = 0; @@ -76,7 +82,7 @@ char *kickmix::MonotonicArena_Helper::grab_writeable(size_t length, size_t alloc frozen_allocations.push_back(std::span{cur, cur_length_used}); total_frozen_length_used += cur_length_used; } else { - free(cur); + free_aligned(cur); } } @@ -86,13 +92,11 @@ char *kickmix::MonotonicArena_Helper::grab_writeable(size_t length, size_t alloc new_size <<= 1; new_size += alloc_alignment - 1; new_size &= ~size_t{alloc_alignment - 1}; - cur = (char *)aligned_alloc(alloc_alignment, new_size); + cur = nullptr; cur_length_used = 0; + cur_length_cap = 0; + cur = (char *)aligned_alloc_bytes_else_throw(alloc_alignment, new_size); cur_length_cap = new_size; - if (cur == nullptr) { - cur_length_cap = 0; - throw std::invalid_argument("Failed to allocate"); - } } char *result = cur + cur_length_used; diff --git a/kickmix/src/kickmix/mem/stride_span.h b/kickmix/src/kickmix/mem/stride_span.h index cc205abc39..68cf9dcbd9 100644 --- a/kickmix/src/kickmix/mem/stride_span.h +++ b/kickmix/src/kickmix/mem/stride_span.h @@ -6,6 +6,8 @@ #include #include #include +#include +#include #include #include "kickmix/build/iota.h" diff --git a/kickmix/src/kickmix/mem/util.h b/kickmix/src/kickmix/mem/util.h index 16a7fca9ac..9743d59adf 100644 --- a/kickmix/src/kickmix/mem/util.h +++ b/kickmix/src/kickmix/mem/util.h @@ -3,12 +3,76 @@ #include #include +#include +#include +#if defined(_MSC_VER) || defined(_WIN32) +#include +#endif namespace kickmix { uint32_t max_u32(const uint32_t *items, size_t num_items); void memcpy_repeat(void *dst, const void *src, size_t src_len, size_t repetitions); +inline void *aligned_alloc_bytes_else_throw(size_t alignment, size_t bytes) { + size_t eff_alignment = alignment < sizeof(void *) ? sizeof(void *) : alignment; + size_t align_mask = eff_alignment - size_t{1}; + if (bytes == 0) { + return nullptr; + } + if (bytes > SIZE_MAX - eff_alignment) { + throw std::runtime_error("Failed to allocate."); + } + bytes += align_mask; + bytes &= ~align_mask; + +#if defined(_MSC_VER) || defined(_WIN32) + void *result = _aligned_malloc(bytes, eff_alignment); +#elif defined(__APPLE__) + void *result = nullptr; + if (posix_memalign(&result, eff_alignment, bytes) != 0) { + result = nullptr; + } +#else + void *result = std::aligned_alloc(eff_alignment, bytes); +#endif + + if (result == nullptr) { + throw std::runtime_error("Failed to allocate."); + } + return result; +} + +template +static T *aligned_alloc_else_throw(size_t count) { + static_assert(alignment > 0, "Alignment must be a power of two."); + static_assert((alignment & (alignment - size_t{1})) == 0, "Alignment must be a power of two."); + static_assert(alignment <= 1024, "Alignment exceeded 1024."); + constexpr size_t eff_alignment = alignment < sizeof(void *) ? sizeof(void *) : alignment; + + if (count == 0) { + return nullptr; + } + if (count > (SIZE_MAX - eff_alignment) / sizeof(T)) { + throw std::runtime_error("Failed to allocate."); + } + return (T *)aligned_alloc_bytes_else_throw(eff_alignment, count * sizeof(T)); +} + +template +static T *alloc_else_throw(size_t count) { + return aligned_alloc_else_throw(count); +} + +template +inline void free_aligned(T *items) noexcept { +#if defined(_MSC_VER) || defined(_WIN32) + _aligned_free((void *)items); +#else + free((void *)items); +#endif +} + } // namespace kickmix #endif diff --git a/kickmix/src/kickmix/mem/util.test.cc b/kickmix/src/kickmix/mem/util.test.cc new file mode 100644 index 0000000000..1c18c4e5ae --- /dev/null +++ b/kickmix/src/kickmix/mem/util.test.cc @@ -0,0 +1,59 @@ +#include "util.h" + +#include "gtest/gtest.h" + +#include "kickmix/mem/util.h" + +using namespace kickmix; + +TEST(mem_util, max_u32) { + std::vector items; + for (size_t k = 0; k < 100; k++) { + items.push_back(k * k % 255); + } + ASSERT_EQ(max_u32(items.data(), 100), 246); + ASSERT_EQ(max_u32(items.data(), 10), 81); + ASSERT_EQ(max_u32(items.data() + 20, 10), 229); + ASSERT_EQ(max_u32(items.data(), 0), 0); + ASSERT_EQ(max_u32(items.data(), 1), 0); + ASSERT_EQ(max_u32(items.data() + 1, 1), 1); + ASSERT_EQ(max_u32(items.data() + 2, 1), 4); +} + +TEST(mem_util, memcpy_repeat) { + std::vector items; + for (size_t k = 0; k < 100; k++) { + items.push_back(k); + } + std::vector dst; + dst.resize(5); + memcpy_repeat(dst.data(), items.data() + 3, 1 * sizeof(uint32_t), 5); + ASSERT_EQ(dst, (std::vector{3, 3, 3, 3, 3})); + + dst.resize(10); + memcpy_repeat(dst.data(), items.data() + 3, 2 * sizeof(uint32_t), 5); + ASSERT_EQ(dst, (std::vector{3, 4, 3, 4, 3, 4, 3, 4, 3, 4})); + + dst.resize(items.size() * 64); + memcpy_repeat(dst.data(), items.data(), items.size() * sizeof(uint32_t), 64); + for (size_t k = 0; k < dst.size(); k++) { + ASSERT_EQ(dst[k], items[k % items.size()]); + } +} + +TEST(mem_util, aligned_alloc_else_throw) { + ASSERT_THROW({ (aligned_alloc_else_throw(SIZE_MAX)); }, std::runtime_error); + + uint32_t *v = aligned_alloc_else_throw(5); + ASSERT_EQ((uintptr_t)v & 31, 0); + v[4] = 4; + v[0] = 0; + // Allocation gets rounded up to a multiple of 16, so that it's safe to use simd values. + // This implies v[7] must exist because v[8] would be at the 32-byte aligned boundary. + v[7] = 7; + + ASSERT_EQ(v[4], 4); + ASSERT_EQ(v[0], 0); + ASSERT_EQ(v[7], 7); + free_aligned(v); +} diff --git a/kickmix/src/kickmix/py/circuit/circuit.pybind.cc b/kickmix/src/kickmix/py/circuit/circuit.pybind.cc index cc9cefc2c2..f6cb3d06cb 100644 --- a/kickmix/src/kickmix/py/circuit/circuit.pybind.cc +++ b/kickmix/src/kickmix/py/circuit/circuit.pybind.cc @@ -315,7 +315,7 @@ void kickmix::register_circuit_methods(pybind11::class_ &c_circuit) { return Circuit(pybind11::cast(content)); } auto data = pybind11::cast(pybind11::bytes(content)); - file.reset(tmpfile()); + file.reset(open_tmpfile_else_throw()); fwrite_else_throw(data.data(), data.size(), file.get()); rewind(file.get()); } else { @@ -380,7 +380,7 @@ void kickmix::register_circuit_methods(pybind11::class_ &c_circuit) { "Unrecognized format '" + std::string(format) + "'. Expected 'kmx' or 'kmb'."); } if (pybind11::hasattr(path, "write")) { - std::unique_ptr tmp(tmpfile(), &fclose); + std::unique_ptr tmp(open_tmpfile_else_throw(), &fclose); if (format == "kmx") { self.write_kmx_to(tmp.get()); } else { diff --git a/kickmix/src/kickmix/py/interactive_simulator.pybind.cc b/kickmix/src/kickmix/py/interactive_simulator.pybind.cc index 88dfef298f..5b3a1fda11 100644 --- a/kickmix/src/kickmix/py/interactive_simulator.pybind.cc +++ b/kickmix/src/kickmix/py/interactive_simulator.pybind.cc @@ -31,18 +31,14 @@ static std::mt19937_64 make_rng(const pybind11::object &seed) { throw pybind11::type_error(ss.str()); } std::array seed_words{}; - if (!seed.equal(pybind11::cast(0))) { - Py_ssize_t num_bytes = PyLong_AsNativeBytes( - seed.ptr(), - seed_words.data(), - sizeof(seed_words), - Py_ASNATIVEBYTES_LITTLE_ENDIAN | Py_ASNATIVEBYTES_REJECT_NEGATIVE | Py_ASNATIVEBYTES_UNSIGNED_BUFFER); - if (num_bytes < 0 || static_cast(num_bytes) > sizeof(seed_words)) { - PyErr_Clear(); - std::stringstream ss; - ss << "seed=" << pybind11::repr(seed) << " not in range(2**256)"; - throw std::invalid_argument(ss.str()); - } + try { + pybind11::bytes b = seed.attr("to_bytes")(sizeof(seed_words), "little"); + memcpy(seed_words.data(), PyBytes_AsString(b.ptr()), sizeof(seed_words)); + } catch (pybind11::error_already_set &) { + PyErr_Clear(); + std::stringstream ss; + ss << "seed=" << pybind11::repr(seed) << " not in range(2**256)"; + throw std::invalid_argument(ss.str()); } std::seed_seq seq(seed_words.begin(), seed_words.end()); return std::mt19937_64(seq); @@ -88,24 +84,19 @@ void int_obj_to_bytes_with_max_bits( } size_t max_bytes = (max_bits + 7) / 8; - if (int_obj.equal(pybind11::cast(0))) { - // Need this special case because PyLong_AsNativeBytes claims 0 takes 8 bytes to store rather than 0. - memset(bytes_buffer, 0, max_bytes); - return; - } - - Py_ssize_t non_zero_bytes = PyLong_AsNativeBytes( - int_obj.ptr(), - bytes_buffer, - max_bytes, - Py_ASNATIVEBYTES_LITTLE_ENDIAN | Py_ASNATIVEBYTES_REJECT_NEGATIVE | Py_ASNATIVEBYTES_UNSIGNED_BUFFER); - bool has_extra = false; - if (max_bits % 8) { - uint8_t mask = 0xFF >> (8 - (max_bits % 8)); - uint8_t b = bytes_buffer[max_bytes - 1]; - has_extra = (b & mask) != b; + bool has_err = false; + try { + pybind11::bytes b = int_obj.attr("to_bytes")(max_bytes, "little"); + memcpy(bytes_buffer, PyBytes_AsString(b.ptr()), max_bytes); + if (max_bits % 8) { + uint8_t mask = 0xFF >> (8 - (max_bits % 8)); + uint8_t last_byte = bytes_buffer[max_bytes - 1]; + has_err = (last_byte & mask) != last_byte; + } + } catch (pybind11::error_already_set &) { + PyErr_Clear(); + has_err = true; } - bool has_err = non_zero_bytes < 0 || (size_t)non_zero_bytes > max_bytes || has_extra; if (has_err) { std::stringstream ss; ss << value_name << "=" << pybind11::repr(int_obj); @@ -125,11 +116,8 @@ pybind11::object int_obj_from_bytes_with_max_bits(uint8_t *bytes_buffer, size_t uint8_t mask = 0xFF >> (8 - (max_bits % 8)); bytes_buffer[bytes_len - 1] &= mask; } - PyObject *py_long = PyLong_FromUnsignedNativeBytes(bytes_buffer, bytes_len, Py_ASNATIVEBYTES_LITTLE_ENDIAN); - if (!py_long) { - throw pybind11::error_already_set(); - } - return pybind11::reinterpret_steal(py_long); + pybind11::bytes b(reinterpret_cast(bytes_buffer), bytes_len); + return pybind11::reinterpret_borrow((PyObject *)&PyLong_Type).attr("from_bytes")(b, "little"); } PyInteractiveSimulator::PyInteractiveSimulator( diff --git a/kickmix/src/kickmix/py/operation_statistics.pybind.cc b/kickmix/src/kickmix/py/operation_statistics.pybind.cc index f4dde74db2..c2c7028efd 100644 --- a/kickmix/src/kickmix/py/operation_statistics.pybind.cc +++ b/kickmix/src/kickmix/py/operation_statistics.pybind.cc @@ -1,10 +1,14 @@ #include "kickmix/py/operation_statistics.pybind.h" #include +#include #include #include #include #include +#include +#include +#include #include #include diff --git a/kickmix/src/kickmix/py/util.pybind.h b/kickmix/src/kickmix/py/util.pybind.h index 80f38b2e3c..7c6b9d6f3f 100644 --- a/kickmix/src/kickmix/py/util.pybind.h +++ b/kickmix/src/kickmix/py/util.pybind.h @@ -1,6 +1,7 @@ #ifndef KICKGEN_PYBIND_UTIL_H #define KICKGEN_PYBIND_UTIL_H +#include #include namespace kickmix_py { diff --git a/kickmix/src/kickmix/py/val/array.pybind.cc b/kickmix/src/kickmix/py/val/array.pybind.cc index c5df73a74c..dcfef2ae0d 100644 --- a/kickmix/src/kickmix/py/val/array.pybind.cc +++ b/kickmix/src/kickmix/py/val/array.pybind.cc @@ -1,5 +1,6 @@ #include "kickmix/py/val/array.pybind.h" +#include #include #include #include diff --git a/kickmix/src/kickmix/py/val/broadcast_resolver.pybind.h b/kickmix/src/kickmix/py/val/broadcast_resolver.pybind.h index a4a26f3387..452c10cc8b 100644 --- a/kickmix/src/kickmix/py/val/broadcast_resolver.pybind.h +++ b/kickmix/src/kickmix/py/val/broadcast_resolver.pybind.h @@ -1,6 +1,9 @@ #ifndef KICKGEN_PYBIND_BROADCAST_RESOLVER_H #define KICKGEN_PYBIND_BROADCAST_RESOLVER_H +#include +#include + #include "kickmix/py/val/converted_array.pybind.h" namespace kickmix_py { diff --git a/kickmix/src/kickmix/py/val/converted_array.pybind.cc b/kickmix/src/kickmix/py/val/converted_array.pybind.cc index e1de464500..554a7c8918 100644 --- a/kickmix/src/kickmix/py/val/converted_array.pybind.cc +++ b/kickmix/src/kickmix/py/val/converted_array.pybind.cc @@ -11,7 +11,7 @@ using namespace kickmix_py; ConvertedArrayXZ::~ConvertedArrayXZ() { if (free_on_destruct != nullptr) { - free(free_on_destruct); + free_aligned(free_on_destruct); free_on_destruct = nullptr; } } @@ -21,24 +21,6 @@ static const QubitOrXZBitOrXZBool GLOBAL_FALSE = false; static const QubitOrXZBitOrXZBool GLOBAL_PLUS = XBool(false); static const QubitOrXZBitOrXZBool GLOBAL_MINUS = XBool(true); -template -static T *aligned_alloc_32_else_throw(size_t count) { - if (count == 0) { - return nullptr; - } - if (count > (SIZE_MAX - 31) / sizeof(T)) { - throw std::invalid_argument("Too much memory to allocate."); - } - size_t bytes = count * sizeof(T); - bytes += size_t{31}; - bytes &= ~size_t{31}; - T *result = (T *)std::aligned_alloc(32, bytes); - if (result == nullptr) { - throw std::invalid_argument("Failed to allocate."); - } - return result; -} - ConvertedArrayXZ ConvertedArrayXZ::from_obj_expecting_list( const pybind11::handle &obj, const char *context_value_name) { PyObject *obj_ptr = obj.ptr(); @@ -51,7 +33,7 @@ ConvertedArrayXZ ConvertedArrayXZ::from_obj_expecting_list( // Treat list-like objects as km.array. if (pybind11::hasattr(obj, "__len__") && type_ptr != &PyUnicode_Type && type_ptr != &PyBytes_Type) { size_t n = pybind11::len(obj); - QubitOrXZBitOrXZBool *out = aligned_alloc_32_else_throw(n); + QubitOrXZBitOrXZBool *out = aligned_alloc_else_throw(n); ConvertedArrayXZ result(stride_span_xz(out, 1, n, QXZTypeTag8::MAY_BE_MIXED), out, false); for (const auto &e : obj) { *out++ = obj_to_qubit_or_xzbit_or_xzbool(e, "item"); @@ -134,7 +116,7 @@ ConvertedArrayXZ ConvertedArrayXZ::from_obj(const pybind11::handle &obj, const c // Treat list-like objects as kmarrays. if (pybind11::hasattr(obj, "__len__") && type_ptr != &PyUnicode_Type && type_ptr != &PyBytes_Type) { size_t n = pybind11::len(obj); - QubitOrXZBitOrXZBool *out = aligned_alloc_32_else_throw(n); + QubitOrXZBitOrXZBool *out = aligned_alloc_else_throw(n); ConvertedArrayXZ result(stride_span_xz(out, 1, n, QXZTypeTag8::MAY_BE_MIXED), out, false); for (const auto &e : obj) { *out++ = obj_to_qubit_or_xzbit_or_xzbool(e, "item"); @@ -166,7 +148,7 @@ ConvertedArrayXZ ConvertedArrayXZ::from_obj_or_int( ss << ", 1 << n)"; throw std::invalid_argument(ss.str()); } - QubitOrXZBitOrXZBool *out = aligned_alloc_32_else_throw(int_num_bits); + QubitOrXZBitOrXZBool *out = aligned_alloc_else_throw(int_num_bits); if (int_num_bits <= 64) { uint64_t u = pybind11::cast(val); for (size_t k = 0; k < int_num_bits; k++) { diff --git a/kickmix/src/kickmix/py/val/converted_array.pybind.h b/kickmix/src/kickmix/py/val/converted_array.pybind.h index 7c4b1b9e9a..6ebc7a3d79 100644 --- a/kickmix/src/kickmix/py/val/converted_array.pybind.h +++ b/kickmix/src/kickmix/py/val/converted_array.pybind.h @@ -4,6 +4,7 @@ #include #include "kickmix/id/stride_span_xz.h" +#include "kickmix/mem/util.h" namespace kickmix_py { @@ -28,7 +29,7 @@ struct ConvertedArrayXZ { ConvertedArrayXZ &operator=(const ConvertedArrayXZ &) = delete; ConvertedArrayXZ &operator=(ConvertedArrayXZ &&other) noexcept { if (free_on_destruct != nullptr) { - free(free_on_destruct); + free_aligned(free_on_destruct); } span = other.span; free_on_destruct = other.free_on_destruct; diff --git a/kickmix/src/kickmix/sim/sim.h b/kickmix/src/kickmix/sim/sim.h index 8d7facc23a..adbacdb20e 100644 --- a/kickmix/src/kickmix/sim/sim.h +++ b/kickmix/src/kickmix/sim/sim.h @@ -1,10 +1,13 @@ #ifndef KICKMIX_SIM_H #define KICKMIX_SIM_H +#include +#include #include #include #include #include +#include #include #include "kickmix/circuit/circuit.h" diff --git a/kickmix/src/kickmix/simd/b128_polyfill.h b/kickmix/src/kickmix/simd/b128_polyfill.h index 76e00e3288..e96adba414 100644 --- a/kickmix/src/kickmix/simd/b128_polyfill.h +++ b/kickmix/src/kickmix/simd/b128_polyfill.h @@ -2,6 +2,8 @@ #define KICKMIX_SIMD_128_H #include +#include +#include #include #include diff --git a/kickmix/src/kickmix/simd/b128_sse.h b/kickmix/src/kickmix/simd/b128_sse.h index 6ef5366ab7..2821464869 100644 --- a/kickmix/src/kickmix/simd/b128_sse.h +++ b/kickmix/src/kickmix/simd/b128_sse.h @@ -2,6 +2,9 @@ #define KICKMIX_SIMD_128_SSE_H #include +#include +#include +#include #include #include #include diff --git a/kickmix/src/kickmix/simd/b256_avx.h b/kickmix/src/kickmix/simd/b256_avx.h index fda637b1d2..5f115d5672 100644 --- a/kickmix/src/kickmix/simd/b256_avx.h +++ b/kickmix/src/kickmix/simd/b256_avx.h @@ -2,6 +2,9 @@ #define KICKMIX_SIMD_256_AVX_H #include +#include +#include +#include #include #ifdef __AVX__ #include diff --git a/kickmix/src/kickmix/simd/b256_polyfill.h b/kickmix/src/kickmix/simd/b256_polyfill.h index 941a0fdc20..8e38c8052f 100644 --- a/kickmix/src/kickmix/simd/b256_polyfill.h +++ b/kickmix/src/kickmix/simd/b256_polyfill.h @@ -2,6 +2,8 @@ #define KICKMIX_SIMD_256_H #include +#include +#include #include #include diff --git a/kickmix/src/kickmix/simd/b512_avx.h b/kickmix/src/kickmix/simd/b512_avx.h index b8409fbc4a..a65a4a2e3c 100644 --- a/kickmix/src/kickmix/simd/b512_avx.h +++ b/kickmix/src/kickmix/simd/b512_avx.h @@ -2,6 +2,9 @@ #define KICKMIX_SIMD_512_AVX_H #include +#include +#include +#include #include #include #include diff --git a/kickmix/src/kickmix/simd/b512_polyfill.h b/kickmix/src/kickmix/simd/b512_polyfill.h index e9dd462300..b69dd1bdbc 100644 --- a/kickmix/src/kickmix/simd/b512_polyfill.h +++ b/kickmix/src/kickmix/simd/b512_polyfill.h @@ -2,6 +2,8 @@ #define KICKMIX_SIMD_512_H #include +#include +#include #include #include diff --git a/kickmix/src/kickmix/simd/b64_polyfill.h b/kickmix/src/kickmix/simd/b64_polyfill.h index a850604cd5..e9fc246c91 100644 --- a/kickmix/src/kickmix/simd/b64_polyfill.h +++ b/kickmix/src/kickmix/simd/b64_polyfill.h @@ -2,8 +2,11 @@ #define KICKMIX_SIMD_64_H #include +#include +#include #include #include +#include namespace kickmix { diff --git a/kickmix/src/kickmix/util/arg_parse.h b/kickmix/src/kickmix/util/arg_parse.h index 0a08655ef8..98751e5a40 100644 --- a/kickmix/src/kickmix/util/arg_parse.h +++ b/kickmix/src/kickmix/util/arg_parse.h @@ -1,8 +1,14 @@ #ifndef _KICKMIX_ARG_PARSE_H #define _KICKMIX_ARG_PARSE_H +#include +#include +#include #include #include +#include +#include +#include #include namespace kickmix { diff --git a/kickmix/src/kickmix/util/binary_file_tools.cc b/kickmix/src/kickmix/util/binary_file_tools.cc index e025aeada9..0d1f159d22 100644 --- a/kickmix/src/kickmix/util/binary_file_tools.cc +++ b/kickmix/src/kickmix/util/binary_file_tools.cc @@ -1,5 +1,6 @@ #include "binary_file_tools.h" +#include #include #include @@ -15,7 +16,7 @@ void kickmix::write_u32_block_be(FILE *file, const uint32_t *data, size_t count) memcpy(buf.data(), data, n * sizeof(uint32_t)); data += n; for (size_t k2 = 0; k2 < n; k2++) { - buf[k2] = __builtin_bswap32(buf[k2]); + buf[k2] = bswap32(buf[k2]); } fwrite_else_throw(buf.data(), n * sizeof(uint32_t), file); } diff --git a/kickmix/src/kickmix/util/binary_file_tools.h b/kickmix/src/kickmix/util/binary_file_tools.h index 3f08d5b0a5..291c48f398 100644 --- a/kickmix/src/kickmix/util/binary_file_tools.h +++ b/kickmix/src/kickmix/util/binary_file_tools.h @@ -4,10 +4,36 @@ #include #include #include +#include #include namespace kickmix { +inline uint32_t bswap32(uint32_t val) { +#if defined(_MSC_VER) + return _byteswap_ulong(val); +#else + return __builtin_bswap32(val); +#endif +} + +inline uint64_t bswap64(uint64_t val) { +#if defined(_MSC_VER) + return _byteswap_uint64(val); +#else + return __builtin_bswap64(val); +#endif +} + +inline FILE *open_tmpfile_else_throw() { + FILE *f = tmpfile(); + f = tmpfile(); + if (f == nullptr) { + throw std::runtime_error("Failed to create temporary file."); + } + return f; +} + inline void fread_else_throw(void *ptr, size_t size, FILE *file) { if (size && fread(ptr, size, 1, file) != 1) { throw std::invalid_argument("Failed to read from file."); @@ -21,14 +47,14 @@ inline void fwrite_else_throw(const void *ptr, size_t size, FILE *file) { inline void write_u32_be(FILE *file, uint32_t val) { if constexpr (std::endian::native == std::endian::big) { - val = __builtin_bswap32(val); + val = bswap32(val); } fwrite_else_throw(&val, sizeof(uint32_t), file); } inline void write_u64_be(FILE *file, uint64_t val) { if constexpr (std::endian::native == std::endian::big) { - val = __builtin_bswap64(val); + val = bswap64(val); } fwrite_else_throw(&val, sizeof(uint64_t), file); } diff --git a/kickmix/src/kickmix/util/chacha.h b/kickmix/src/kickmix/util/chacha.h index 92bdd46ac5..82a1bb8f8c 100644 --- a/kickmix/src/kickmix/util/chacha.h +++ b/kickmix/src/kickmix/util/chacha.h @@ -62,7 +62,12 @@ struct ChaCha { buffer_idx = 16; } - __attribute__((noinline)) void refill() { +#if defined(_MSC_VER) + __declspec(noinline) +#else + __attribute__((noinline)) +#endif + void refill() { TWord x0 = state[0]; TWord x1 = state[1]; TWord x2 = state[2]; diff --git a/kickmix/src/kickmix/util/fixed_precision_angle_128.cc b/kickmix/src/kickmix/util/fixed_precision_angle_128.cc index 8645f04354..492b3d63d9 100644 --- a/kickmix/src/kickmix/util/fixed_precision_angle_128.cc +++ b/kickmix/src/kickmix/util/fixed_precision_angle_128.cc @@ -8,6 +8,8 @@ #include #include +#include "kickmix/util/word_ops.h" + using namespace kickmix; uint8_t FixedPrecisionAngle128::clifford_hierarchy_level() const { @@ -116,9 +118,19 @@ FixedPrecisionAngle128 FixedPrecisionAngle128::from_rounded_decimal_half_turns( auto step_digit = [&](uint64_t d) { uint64_t rem = d; for (int i = 7; i >= 0; i--) { +#if defined(__SIZEOF_INT128__) unsigned __int128 cur = (static_cast(rem) << 64) | acc[i]; acc[i] = static_cast(cur / 10); rem = static_cast(cur % 10); +#else + uint64_t hi = (rem << 32) | (acc[i] >> 32); + uint64_t q_hi = hi / 10; + uint64_t r_hi = hi % 10; + uint64_t lo = (r_hi << 32) | (acc[i] & 0xFFFFFFFFULL); + uint64_t q_lo = lo / 10; + rem = lo % 10; + acc[i] = (q_hi << 32) | q_lo; +#endif } }; @@ -167,20 +179,21 @@ inline void to_half_turns_decimal_helper(const FixedPrecisionAngle128 &angle, co } while (w0 != 0 || w1 != 0) { - unsigned __int128 low_prod = static_cast(w0) * 10; - unsigned __int128 high_prod = static_cast(w1) * 10 + (low_prod >> 64); - uint64_t digit = static_cast(high_prod >> 64); + uint64_t carry = 0; + uint64_t digit = 0; + full_fma64(w0, 10, 0, 0, &w0, &carry); + full_fma64(w1, 10, carry, 0, &w1, &digit); callback('0' + digit); - w0 = static_cast(low_prod); - w1 = static_cast(high_prod); } } FixedPrecisionAngle128 FixedPrecisionAngle128::operator*(uint64_t factor) const { - unsigned __int128 cur = (static_cast(words[1]) << 64) | words[0]; - cur *= factor; - auto w0 = (uint64_t)(cur & UINT64_MAX); - auto w1 = (uint64_t)(cur >> 64); + uint64_t w0 = 0; + uint64_t carry = 0; + uint64_t w1 = 0; + uint64_t unused_high = 0; + full_fma64(words[0], factor, 0, 0, &w0, &carry); + full_fma64(words[1], factor, carry, 0, &w1, &unused_high); return {w0, w1}; } diff --git a/kickmix/src/kickmix/util/fixed_precision_angle_128.h b/kickmix/src/kickmix/util/fixed_precision_angle_128.h index d51f0d6202..96eec0ecc7 100644 --- a/kickmix/src/kickmix/util/fixed_precision_angle_128.h +++ b/kickmix/src/kickmix/util/fixed_precision_angle_128.h @@ -1,7 +1,9 @@ #ifndef KICKMIX_UTIL_FIXED_PRECISION_ANGLE_H #define KICKMIX_UTIL_FIXED_PRECISION_ANGLE_H +#include #include +#include #include #include #include diff --git a/kickmix/src/kickmix/util/fixed_width_int.cc b/kickmix/src/kickmix/util/fixed_width_int.cc index 296aa83d36..50a2300a31 100644 --- a/kickmix/src/kickmix/util/fixed_width_int.cc +++ b/kickmix/src/kickmix/util/fixed_width_int.cc @@ -1,5 +1,7 @@ #include "kickmix/util/fixed_width_int.h" +#include +#include #include #include #include @@ -7,7 +9,6 @@ #include #include #include -#include #include "kickmix/util/word_ops.h" @@ -1192,23 +1193,6 @@ FixedWidthInt &FixedWidthInt::operator*=(const FixedWidthInt &value) { return *this; } -inline void full_fma64( - uint64_t factor1, uint64_t factor2, uint64_t offset1, uint64_t offset2, uint64_t *out_low, uint64_t *out_high) { -#if defined(__GNUC__) || defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wpedantic" -#endif - - auto result = factor1 * (unsigned __int128)factor2 + (unsigned __int128)offset1 + offset2; - -#if defined(__GNUC__) || defined(__clang__) -#pragma GCC diagnostic pop -#endif - - *out_high = static_cast(result >> 64); - *out_low = static_cast(result); -} - FixedWidthInt &FixedWidthInt::operator*=(uint64_t value) { uint64_t carry = 0; for (size_t k = 0; k < num_words; k++) { diff --git a/kickmix/src/kickmix/util/fixed_width_int.h b/kickmix/src/kickmix/util/fixed_width_int.h index 51c4b5077e..7057645bae 100644 --- a/kickmix/src/kickmix/util/fixed_width_int.h +++ b/kickmix/src/kickmix/util/fixed_width_int.h @@ -1,10 +1,15 @@ #ifndef _KICKMIX_BIG_INT_H #define _KICKMIX_BIG_INT_H +#include #include #include +#include #include +#include #include +#include +#include namespace kickmix { @@ -130,7 +135,7 @@ struct FixedWidthInt { if (!(num_bits & 63)) { return false; } - uint64_t mask = ~(-static_cast(1) << (num_bits & 63)); + uint64_t mask = (uint64_t{1} << (num_bits & 63)) - 1; auto old_word = words[num_words - 1]; words[num_words - 1] = old_word & mask; return old_word != words[num_words - 1]; diff --git a/kickmix/src/kickmix/util/gf2_matrix.cc b/kickmix/src/kickmix/util/gf2_matrix.cc index c099651f82..3625612eb5 100644 --- a/kickmix/src/kickmix/util/gf2_matrix.cc +++ b/kickmix/src/kickmix/util/gf2_matrix.cc @@ -1,5 +1,6 @@ #include "kickmix/util/gf2_matrix.h" +#include #include #include #include diff --git a/kickmix/src/kickmix/util/gf2_poly.cc b/kickmix/src/kickmix/util/gf2_poly.cc index 0c79fbc261..323319c63b 100644 --- a/kickmix/src/kickmix/util/gf2_poly.cc +++ b/kickmix/src/kickmix/util/gf2_poly.cc @@ -1,5 +1,6 @@ #include "kickmix/util/gf2_poly.h" +#include #include #include #include diff --git a/kickmix/src/kickmix/util/mod_int.cc b/kickmix/src/kickmix/util/mod_int.cc index de3b3acf25..6d6af13a5c 100644 --- a/kickmix/src/kickmix/util/mod_int.cc +++ b/kickmix/src/kickmix/util/mod_int.cc @@ -1,5 +1,6 @@ #include "mod_int.h" +#include #include #include #include diff --git a/kickmix/src/kickmix/util/mod_rand.cc b/kickmix/src/kickmix/util/mod_rand.cc index 1a82de8745..57e0216430 100644 --- a/kickmix/src/kickmix/util/mod_rand.cc +++ b/kickmix/src/kickmix/util/mod_rand.cc @@ -1,5 +1,6 @@ #include "kickmix/util/mod_rand.h" +#include #include #include diff --git a/kickmix/src/kickmix/util/word_ops.h b/kickmix/src/kickmix/util/word_ops.h index e548365f10..b0cab29ebc 100644 --- a/kickmix/src/kickmix/util/word_ops.h +++ b/kickmix/src/kickmix/util/word_ops.h @@ -50,6 +50,44 @@ inline uint64_t extract_bits_u64(uint64_t value, uint64_t mask) { #endif } +/// Computes factor1 * factor2 + offset1 + offset2 as a 128-bit unsigned integer. +inline void full_fma64( + uint64_t factor1, uint64_t factor2, uint64_t offset1, uint64_t offset2, uint64_t *out_low, uint64_t *out_high) { +#if defined(__SIZEOF_INT128__) +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" +#endif + auto result = factor1 * (unsigned __int128)factor2 + (unsigned __int128)offset1 + offset2; +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic pop +#endif + *out_high = static_cast(result >> 64); + *out_low = static_cast(result); +#else + uint64_t a_lo = factor1 & 0xFFFFFFFFULL; + uint64_t a_hi = factor1 >> 32; + uint64_t b_lo = factor2 & 0xFFFFFFFFULL; + uint64_t b_hi = factor2 >> 32; + + uint64_t p0 = a_lo * b_lo; + uint64_t p1 = a_lo * b_hi; + uint64_t p2 = a_hi * b_lo; + uint64_t p3 = a_hi * b_hi; + + uint64_t mid = (p0 >> 32) + (p1 & 0xFFFFFFFFULL) + (p2 & 0xFFFFFFFFULL); + uint64_t lo = (p0 & 0xFFFFFFFFULL) | (mid << 32); + uint64_t hi = p3 + (p1 >> 32) + (p2 >> 32) + (mid >> 32); + + unsigned long long sum1 = 0; + bool c1 = add_carry_u64(false, lo, offset1, &sum1); + unsigned long long sum2 = 0; + bool c2 = add_carry_u64(false, sum1, offset2, &sum2); + *out_low = sum2; + *out_high = hi + c1 + c2; +#endif +} + } // namespace kickmix #endif