diff --git a/arm_compute/core/Coordinates.h b/arm_compute/core/Coordinates.h index 71576b749a..3dcd656f9d 100644 --- a/arm_compute/core/Coordinates.h +++ b/arm_compute/core/Coordinates.h @@ -49,6 +49,16 @@ class Coordinates : public Dimensions constexpr Coordinates(Ts... coords) : Dimensions{coords...} { } + + /** Constructor to initialize the coordinates from a vector. + * + * @param[in] coords Vector containing the values to initialize the dimensions. + */ + template + constexpr Coordinates(std::vector coords) : Dimensions(coords) + { + } + /** Allow instances of this class to be copy constructed */ constexpr Coordinates(const Coordinates &) = default; /** Allow instances of this class to be copied */ diff --git a/arm_compute/core/Dimensions.h b/arm_compute/core/Dimensions.h index ba4a1b1d64..afcb61f3a7 100644 --- a/arm_compute/core/Dimensions.h +++ b/arm_compute/core/Dimensions.h @@ -58,6 +58,15 @@ class Dimensions { } + /** Constructor to initialize the tensor shape with a list of dim values. + * + * @param[in] dims Vector of values to initialize the dimensions. + */ + explicit Dimensions(std::vector dims) : _id(), _num_dimensions{dims.size()} + { + std::copy_n(dims.begin(), std::min(num_max_dimensions, dims.size()), _id.begin()); + } + /** Allow instances of this class to be copy constructed */ Dimensions(const Dimensions &) = default; diff --git a/arm_compute/core/Helpers.h b/arm_compute/core/Helpers.h index 73728ed5a3..979c169949 100644 --- a/arm_compute/core/Helpers.h +++ b/arm_compute/core/Helpers.h @@ -34,6 +34,7 @@ #include "arm_compute/core/Types.h" #include "arm_compute/core/Validate.h" #include "arm_compute/core/Window.h" +#include "arm_compute/core/SparseTensor.h" #include #include @@ -126,6 +127,42 @@ class Iterator std::array _dims; }; +class SparseIterator +{ +public: + /** Create an iterator for the metadata and allocation contained in the SparseTensor + * + * @param[in] tensor A reference to the tensor to associate to the iterator. + */ + SparseIterator(const SparseTensor &tensor); + + /** Returns true if there is at least one more element to iterate */ + bool has_next() const; + + /** Move to the next non-zero element */ + void next(); + + /** Get the coordinates of the current non-zero element */ + Coordinates coordinates() const; + + /** Get the value of the current non-zero element */ + const uint8_t *value() const; + + /** Reset iterator to the beginning */ + void reset(); + + /** Get current index (nth non-zero) */ + size_t index() const; + + /** Get the number of non-zero elements that are left to iterate */ + size_t num_left() const; + +private: + const SparseTensor &_tensor; + size_t _index; + size_t _nnz; +}; + /** Iterate through the passed window, automatically adjusting the iterators and calling the lambda_functino for each element. * It passes the x and y positions to the lambda_function for each iteration * diff --git a/arm_compute/core/Helpers.inl b/arm_compute/core/Helpers.inl index d18809fc6b..5ce0fc6047 100644 --- a/arm_compute/core/Helpers.inl +++ b/arm_compute/core/Helpers.inl @@ -101,6 +101,7 @@ inline Iterator::Iterator(const ITensor *tensor, const Window &win) : Iterator() { ARM_COMPUTE_ERROR_ON(tensor == nullptr); ARM_COMPUTE_ERROR_ON(tensor->info() == nullptr); + ARM_COMPUTE_ERROR_ON_MSG(tensor->info()->is_sparse(), "Sparse tensors are not supported by Iterators. Use a SparseIterator instead"); initialize(tensor->info()->num_dimensions(), tensor->info()->strides_in_bytes(), tensor->buffer(), tensor->info()->offset_first_element_in_bytes(), win); @@ -169,6 +170,45 @@ inline void Iterator::reset(const size_t dimension) } } +inline SparseIterator::SparseIterator(const SparseTensor &tensor) : _tensor(tensor), _index(0), _nnz(tensor.nnz()) +{ +} + +inline bool SparseIterator::has_next() const +{ + return _index < _nnz; +} + +inline void SparseIterator::next() +{ + ++_index; +} + +inline Coordinates SparseIterator::coordinates() const +{ + return _tensor.get_coordinates(_index); +} + +inline const uint8_t *SparseIterator::value() const +{ + return _tensor.get_value(coordinates()); +} + +inline void SparseIterator::reset() +{ + _index = 0; +} + +inline size_t SparseIterator::index() const +{ + return _index; +} + +inline size_t SparseIterator::num_left() const +{ + return _nnz - _index; +} + inline Coordinates index2coords(const TensorShape &shape, int index) { int num_elements = shape.total_size(); diff --git a/arm_compute/core/IReducibleTensor.h b/arm_compute/core/IReducibleTensor.h new file mode 100644 index 0000000000..b99c5311ed --- /dev/null +++ b/arm_compute/core/IReducibleTensor.h @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2025 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef ACL_ARM_COMPUTE_CORE_IREDUCIBLETENSOR_H +#define ACL_ARM_COMPUTE_CORE_IREDUCIBLETENSOR_H + +#include "arm_compute/core/SparseTensor.h" + +namespace arm_compute +{ +/** Forward declaration of COOTensor and CSRTensor class */ +class COOTensor; +class CSRTensor; + +/** Interface for all reducible tensors, i.e. all tensors that can be + * converted to a sparse representation. + */ +class IReducibleTensor +{ +public: + virtual ~IReducibleTensor() = default; + /** Convert a dense tensor to sparse tensor with specified sparse dimensions using the default + * sparse tensor representation: COO format. + * + * @param[in] dim sparse dimension + * + * @return A unique pointer to a SparseTensor object. + */ + virtual std::unique_ptr to_sparse(size_t dim) const = 0; + /** Convert a dense tensor to COO sparse tensor with specified sparse dimensions. + * + * @param[in] dim sparse dimension + * + * @return A unique pointer to a COOTensor object. + */ + virtual std::unique_ptr to_coo_sparse(size_t dim) const = 0; + /** Convert a dense tensor to COO sparse tensor with the default sparse dimension. + * For COO format, the number of sparse dimensions is equal to the total number of dimensions. + * + * @return A unique pointer to a COOTensor object. + */ + virtual std::unique_ptr to_coo_sparse() const = 0; + /** Convert a dense tensor to CSR sparse tensor with specified sparse dimensions. + * + * @param[in] dim sparse dimension + * + * @return A unique pointer to a CSRTensor object. + */ + virtual std::unique_ptr to_csr_sparse(size_t dim) const = 0; + /** Convert a dense tensor to CSR sparse tensor with the default sparse dimension. + * For CSR format, the number of sparse dimensions is equal to 2. + * + * @return A unique pointer to a CSRTensor object. + */ + virtual std::unique_ptr to_csr_sparse() const = 0; +}; +} + +#endif // ACL_ARM_COMPUTE_CORE_IREDUCIBLETENSOR_H diff --git a/arm_compute/core/ITensorInfo.h b/arm_compute/core/ITensorInfo.h index 239f200f7f..8bd32b073c 100644 --- a/arm_compute/core/ITensorInfo.h +++ b/arm_compute/core/ITensorInfo.h @@ -31,6 +31,7 @@ #include "arm_compute/core/Coordinates.h" #include "arm_compute/core/Strides.h" #include "arm_compute/core/TensorShape.h" +#include "arm_compute/core/TensorFormat.h" #include "arm_compute/core/Types.h" #include "arm_compute/core/utils/misc/Utility.h" @@ -105,6 +106,13 @@ class ITensorInfo : public misc::ICloneable * @return Reference to this ITensorInfo object */ virtual ITensorInfo &set_format(Format format) = 0; + /** Set the tensor format of an already initialized tensor. + * + * @param[in] tensor format to distinguish between dense and sparse tensors. + * + * @return Reference to this ITensorInfo object + */ + virtual ITensorInfo &set_tensor_format(TensorFormat tf) = 0; /** Set the shape of an already initialized tensor. * * @warning Changing the shape requires to recompute the strides and is @@ -261,6 +269,16 @@ class ITensorInfo : public misc::ICloneable * @return True if the tensor size can be changed. */ virtual bool is_resizable() const = 0; + /** Flag indicating whether the tensor is sparse. + * + * @return True if the tensor format different from TensorFormat::Dense. + */ + virtual bool is_sparse() const = 0; + /** Returns the format of the Tensor + * + * @return True if the tensor is an instance of SparseTensor. + */ + virtual TensorFormat tensor_format() const = 0; /** Set the tensor as dynamic/static * * @param[in] dynamic True if tensor is dynamic diff --git a/arm_compute/core/SparseTensor.h b/arm_compute/core/SparseTensor.h new file mode 100644 index 0000000000..36b1431bee --- /dev/null +++ b/arm_compute/core/SparseTensor.h @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2025 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef ACL_ARM_COMPUTE_RUNTIME_SPARSETENSOR_H +#define ACL_ARM_COMPUTE_RUNTIME_SPARSETENSOR_H + +#include "arm_compute/core/ITensor.h" +#include "arm_compute/core/Types.h" + +#include +#include + +typedef std::function predicate_t; + +namespace arm_compute +{ +/** Common base class for all sparse tensors */ +class SparseTensor : public ITensor +{ +public: + /** Prevent instances of this class to be constructed by the default constructor */ + SparseTensor() = delete; + /** Prevent instances of this class to be copy constructed */ + SparseTensor(const SparseTensor&) = delete; + /** Prevent instances of this class to be copied */ + SparseTensor& operator=(const SparseTensor&) = delete; + ~SparseTensor() = default; + + /** Returns the number of sparse dimensions */ + size_t sparse_dim() const; + /** Returns the number of dense dimensions */ + size_t dense_dim() const; + /** Returns the (total) number of dimensions */ + size_t dim() const; + /** Returns true if the tensor is hybrid (contains both sparse and dense dimensions) + * + * @note A sparse tensor is hybrid if it has at least one dense dimension. + */ + bool is_hybrid() const; + /** Returns the ratio of zero-valued elements to the total number of elements */ + float sparsity() const; + /** Returns the ratio of non-zero elements to the total number of elements */ + float density() const; + /** Returns the dense volume */ + uint32_t dense_volume(size_t sparse_dim) const; + /** Returns the number of non zero elements */ + virtual size_t nnz() const = 0; + /** Converts the sparse tensor to a dense tensor */ + virtual std::unique_ptr to_dense() = 0; + /** Returns the coordinates of the n-th (non-zero) element. + * + * @param nth The *zero-base* index of the element + * + * @return The coordinates of the element + */ + virtual Coordinates get_coordinates(size_t nth) const = 0; + /** Returns a pointer to the n-th (non-zero) element. If the element specified by + * the coordinates is zero, nullptr is returned. + * + * @param nth The *zero-base* index of the element + * + * @return The value of the element + * + * @note The value has size dense_volume(sparse_dim()). + */ + virtual const uint8_t *get_value(Coordinates coords) const = 0; + +protected: + SparseTensor(size_t dim, size_t sparse_dim); + + std::function make_is_nonzero_predicate(DataType dt) const; + bool has_non_zero_elements(uint8_t *arr, size_t len, size_t element_size, predicate_t is_non_zero) const; + void print_values(std::ostream &os, const uint8_t *data, size_t offset, size_t count) const; + +private: + size_t _total_dim; + size_t _sparse_dim; +}; +} + +#endif // ACL_ARM_COMPUTE_RUNTIME_SPARSETENSOR_H diff --git a/arm_compute/core/SubTensorInfo.h b/arm_compute/core/SubTensorInfo.h index df15c9dbc9..7a67788ae4 100644 --- a/arm_compute/core/SubTensorInfo.h +++ b/arm_compute/core/SubTensorInfo.h @@ -34,6 +34,7 @@ #include "arm_compute/core/Strides.h" #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/TensorShape.h" +#include "arm_compute/core/TensorFormat.h" #include #include @@ -200,6 +201,17 @@ class SubTensorInfo final : public ITensorInfo ARM_COMPUTE_ERROR_ON(_parent == nullptr); return _parent->is_resizable(); } + bool is_sparse() const override + { + ARM_COMPUTE_ERROR_ON(_parent == nullptr); + return _parent->is_sparse(); + } + ITensorInfo &set_tensor_format(TensorFormat tf) override; + TensorFormat tensor_format() const override + { + ARM_COMPUTE_ERROR_ON(_parent == nullptr); + return _parent->tensor_format(); + } ITensorInfo &set_dynamic(bool dynamic) override; bool is_dynamic() const override { diff --git a/arm_compute/core/TensorFormat.h b/arm_compute/core/TensorFormat.h new file mode 100644 index 0000000000..db6dec05d7 --- /dev/null +++ b/arm_compute/core/TensorFormat.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2025 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ACL_ARM_COMPUTE_RUNTIME_TENSOR_FORMAT_H +#define ACL_ARM_COMPUTE_RUNTIME_TENSOR_FORMAT_H + +enum class TensorFormat +{ + Dense = 0, + COO = 1, + CSR = 2, +}; + +#endif // ACL_ARM_COMPUTE_RUNTIME_TENSOR_FORMAT_H diff --git a/arm_compute/core/TensorInfo.h b/arm_compute/core/TensorInfo.h index 629139fddd..9ebe97c0c5 100644 --- a/arm_compute/core/TensorInfo.h +++ b/arm_compute/core/TensorInfo.h @@ -34,6 +34,7 @@ #include "arm_compute/core/Strides.h" #include "arm_compute/core/TensorShape.h" #include "arm_compute/core/Types.h" +#include "arm_compute/core/TensorFormat.h" #include "ITensorInfo.h" #include @@ -278,6 +279,19 @@ class TensorInfo final : public ITensorInfo { return _is_resizable; } + bool is_sparse() const override + { + return _tensor_format != TensorFormat::Dense; + } + TensorFormat tensor_format() const override + { + return _tensor_format; + } + ITensorInfo &set_tensor_format(TensorFormat tf) override + { + _tensor_format = tf; + return *this; + } ITensorInfo &set_dynamic(bool dynamic) override { std::fill(std::begin(_dims_state), std::end(_dims_state), @@ -362,6 +376,7 @@ class TensorInfo final : public ITensorInfo bool _are_values_constant; ITensorInfo::Id _id; bool _lock_paddings; + TensorFormat _tensor_format; }; /** Check whether two tensor info are equal. diff --git a/arm_compute/runtime/COOTensor.h b/arm_compute/runtime/COOTensor.h new file mode 100644 index 0000000000..66aec0d27a --- /dev/null +++ b/arm_compute/runtime/COOTensor.h @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2025 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef ACL_ARM_COMPUTE_RUNTIME_COOTENSOR_H +#define ACL_ARM_COMPUTE_RUNTIME_COOTENSOR_H + +#include "arm_compute/core/SparseTensor.h" +#include "arm_compute/runtime/SparseTensorAllocator.h" + +#include + +namespace arm_compute +{ +class COOTensor final : public SparseTensor, public IMemoryManageable +{ +public: + /** Prevent instances of this class to be move constructed */ + COOTensor(COOTensor &&) = delete; + /** Prevent instances of this class to be moved */ + COOTensor &operator=(COOTensor &&) = delete; + + /** Print the internal state of the COOTensor instance + * + * @param[in] os the output stream; std::cout set as default. + * + * @note It prints (on os stream) the vector of the indices with the format + * index: [idx_0, idx_1, ...], and the corresponding values with the format + * value: [val_0, val_1, ...]. + * @note This print function should overlap the one defined for ITensor. + */ + void print(std::ostream &os = std::cout) const; + + // Inherited methods overridden: + ITensorInfo *info() const override; + ITensorInfo *info() override; + uint8_t *buffer() const override; + size_t nnz() const override; + std::unique_ptr to_dense() override; + Coordinates get_coordinates(size_t nth) const override; + const uint8_t *get_value(Coordinates coords) const override; + void associate_memory_group(IMemoryGroup *memory_group) override; + +private: + /** Convert a dense tensor to sparse tensor with specified sparse dimensions using COO format. + * + * @param[in] tensor + * @param[in] sparse_dim Belongs to [1, tensor->info->num_dimensions()] + */ + COOTensor(const ITensor *tensor, size_t sparse_dim); + /** Convert a dense tensor to a *fully* sparse COOTensor. + * + * @param[in] tensor + * + * @note sparse_dim = tensor->info->num_dimensions() + */ + COOTensor(const ITensor *tensor); + + std::vector _indices; + mutable SparseTensorAllocator _allocator; /**< Instance of the basic CPU allocator.*/ + +friend class Tensor; +}; +} + +#endif // ACL_ARM_COMPUTE_RUNTIME_COOTENSOR_H diff --git a/arm_compute/runtime/CSRTensor.h b/arm_compute/runtime/CSRTensor.h new file mode 100644 index 0000000000..44e536718f --- /dev/null +++ b/arm_compute/runtime/CSRTensor.h @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2025 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef ACL_ARM_COMPUTE_RUNTIME_CSRTENSOR_H +#define ACL_ARM_COMPUTE_RUNTIME_CSRTENSOR_H + +#include "arm_compute/core/SparseTensor.h" +#include "arm_compute/runtime/SparseTensorAllocator.h" + +#include + +namespace arm_compute +{ +class CSRTensor final : public SparseTensor, public IMemoryManageable +{ +public: + /** Prevent instances of this class to be move constructed */ + CSRTensor(CSRTensor &&) = delete; + /** Prevent instances of this class to be moved */ + CSRTensor &operator=(CSRTensor &&) = delete; + + /** Print the internal state of the CSRTensor instance + * + * @param[in] os the output stream; std::cout set as default. + * + * @note It prints (on os stream) the two vectors of the indices with the format + * [row_idx_0, row_idx_1, ...] and [col_idx_0, col_idx_1, ...] + * @note This print function should overlap the one defined for ITensor. + */ + void print(std::ostream &os = std::cout) const; + + // Inherited methods overridden: + ITensorInfo *info() const override; + ITensorInfo *info() override; + uint8_t *buffer() const override; + size_t nnz() const override; + std::unique_ptr to_dense() override; + Coordinates get_coordinates(size_t nth) const override; + const uint8_t *get_value(Coordinates coords) const override; + void associate_memory_group(IMemoryGroup *memory_group) override; + +private: + /** The size of each index element */ + static constexpr size_t index_size = sizeof(int32_t); + + /** Convert a dense tensor to sparse tensor with specified sparse dimensions using COO format. + * + * @param[in] tensor + * @param[in] sparse_dim It should belong to [1, tensor->info->num_dimensions()] + */ + CSRTensor(const ITensor *tensor, size_t sparse_dim); + /** Convert a dense tensor to a *fully* sparse tensor. + * + * @note sparse_dim = tensor->info->num_dimensions(). + * If tensor->info->num_dimensions() > 2 an error is raised. + */ + CSRTensor(const ITensor *tensor); + + size_t _crow_bytes; /**< Row index size in bytes */ + size_t _col_bytes; /**< Column index size in bytes */ + // In the SparseTensorAllocator buffer, the memory is stored that way + // +---------------+---------------+----------... + // | Row offsets | Col indices | Values ... + // +---------------+---------------+----------... + mutable SparseTensorAllocator _allocator; /**< Instance of the basic CPU allocator.*/ + +friend class Tensor; +}; +} + +#endif // ACL_ARM_COMPUTE_RUNTIME_CSRTENSOR_H diff --git a/arm_compute/runtime/SparseTensorAllocator.h b/arm_compute/runtime/SparseTensorAllocator.h new file mode 100644 index 0000000000..a435c8c20d --- /dev/null +++ b/arm_compute/runtime/SparseTensorAllocator.h @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2025 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ACL_ARM_COMPUTE_RUNTIME_SPARSETENSORALLOCATOR_H +#define ACL_ARM_COMPUTE_RUNTIME_SPARSETENSORALLOCATOR_H + +/** @file + * @publicapi + */ + +#include "arm_compute/runtime/ITensorAllocator.h" +#include "arm_compute/runtime/Memory.h" +#include "arm_compute/runtime/MemoryGroup.h" + +#include +#include +#include + +namespace arm_compute +{ +// Forward declaration +class Coordinates; +class TensorInfo; + +/** Basic implementation of a CPU memory sparse tensor allocator. */ +class SparseTensorAllocator : public ITensorAllocator +{ +public: + /** Default constructor. + * + * @param[in] owner Memory manageable owner + */ + SparseTensorAllocator(IMemoryManageable *owner); + /** Default destructor */ + ~SparseTensorAllocator(); + /** Prevent instances of this class from being copied (As this class contains pointers) */ + SparseTensorAllocator(const SparseTensorAllocator &) = delete; + /** Prevent instances of this class from being copy assigned (As this class contains pointers) */ + SparseTensorAllocator &operator=(const SparseTensorAllocator &) = delete; + /** Allow instances of this class to be moved */ + SparseTensorAllocator(SparseTensorAllocator &&) noexcept; + /** Allow instances of this class to be moved */ + SparseTensorAllocator &operator=(SparseTensorAllocator &&) noexcept; + + /** Initializes the sparse tensor allocator with the number of non-zero elements. + * + * @param[in] input The input tensor information + * @param[in] values_bytes The bytes to be allocated for the values + * @param[in] indices_bytes The bytes to be allocated for the indices + * @param[in] alignment The alignment requirement + * + * @note ITensorAllocator's init methods should be disabled. Sparse tensors' memory allocator + * should be initialized with the number of non zero elements. + */ + void init(const TensorInfo &input, size_t values_bytes, size_t indices_bytes, size_t alignment = 0); + + /** Shares the same backing memory with another tensor allocator, while the tensor info might be different. + * In other words this can be used to create a sub-tensor from another tensor while sharing the same memory. + * + * @note SparseTensorAllocator have to be of the same specialized type. + * + * @param[in] allocator The allocator that owns the backing memory to be shared. Ownership becomes shared afterwards. + * @param[in] coords The starting coordinates of the new tensor inside the parent tensor. + * @param[in] sub_info The new tensor information (e.g. shape etc) + */ + void init(const SparseTensorAllocator &allocator, const Coordinates &coords, TensorInfo &sub_info); + + /** Returns the pointer to the allocated data. + * + * @return a pointer to the allocated data. + */ + uint8_t *data() const; + + /** Allocate size specified by TensorInfo of CPU memory. + * + * @note The tensor must not already be allocated when calling this function. + * + */ + void allocate() override; + + bool is_allocated() const override; + + /** Free allocated CPU memory. + * + * @note The sparse tensor must have been allocated when calling this function. + * + */ + void free() override; + /** Import an existing memory as a tensor's backing memory + * + * @warning size is expected to be compliant with total_size reported by ITensorInfo. + * @warning ownership of memory is not transferred. + * @warning tensor shouldn't be memory managed. + * @warning padding should be accounted by the client code. + * @warning memory must be writable in case of in-place operations + * @note buffer alignment will be checked to be compliant with alignment reported by ITensorInfo. + * + * @param[in] memory Raw memory pointer to be used as backing memory + * + * @return An error status + */ + Status import_memory(void *memory); + /** Associates the tensor with a memory group + * + * @param[in] associated_memory_group Memory group to associate the tensor with + */ + void set_associated_memory_group(IMemoryGroup *associated_memory_group); + /** Returns the size in bytes of the allocated memory + * + * @return The size in bytes of the allocated memory + */ + size_t size_bytes() const; + +protected: + /** No-op for CPU memory + * + * @return A pointer to the beginning of the tensor's allocation. + */ + uint8_t *lock() override; + + /** No-op for CPU memory. */ + void unlock() override; + +private: + IMemoryManageable *_owner; /**< Memory manageable object that owns the allocator */ + IMemoryGroup *_associated_memory_group; /**< Registered memory manager */ + Memory _memory; /**< CPU memory */ + size_t _values_bytes; /**< Size in bytes of the allocated values */ + size_t _indices_bytes; /**< Size in bytes of the allocated indices */ +}; +} // namespace arm_compute + +#endif // ACL_ARM_COMPUTE_RUNTIME_SPARSETENSORALLOCATOR_H diff --git a/arm_compute/runtime/Tensor.h b/arm_compute/runtime/Tensor.h index a28dd859fd..e10284c701 100644 --- a/arm_compute/runtime/Tensor.h +++ b/arm_compute/runtime/Tensor.h @@ -29,7 +29,10 @@ */ #include "arm_compute/core/ITensor.h" +#include "arm_compute/runtime/COOTensor.h" +#include "arm_compute/runtime/CSRTensor.h" #include "arm_compute/runtime/TensorAllocator.h" +#include "arm_compute/core/IReducibleTensor.h" #include @@ -38,7 +41,7 @@ namespace arm_compute class ITensorInfo; class IRuntimeContext; /** Basic implementation of the tensor interface */ -class Tensor : public ITensor, public IMemoryManageable +class Tensor : public ITensor, public IMemoryManageable, public IReducibleTensor { public: /** Constructor @@ -60,10 +63,15 @@ class Tensor : public ITensor, public IMemoryManageable TensorAllocator *allocator(); // Inherited methods overridden: - ITensorInfo *info() const override; - ITensorInfo *info() override; - uint8_t *buffer() const override; - void associate_memory_group(IMemoryGroup *memory_group) override; + ITensorInfo *info() const override; + ITensorInfo *info() override; + uint8_t *buffer() const override; + void associate_memory_group(IMemoryGroup *memory_group) override; + std::unique_ptr to_sparse(size_t dim) const override; + std::unique_ptr to_coo_sparse(size_t dim) const override; + std::unique_ptr to_coo_sparse() const override; + std::unique_ptr to_csr_sparse(size_t dim) const override; + std::unique_ptr to_csr_sparse() const override; private: mutable TensorAllocator _allocator; /**< Instance of the basic CPU allocator.*/ diff --git a/filelist.json b/filelist.json index 81bd5ef9c8..109fe14b34 100644 --- a/filelist.json +++ b/filelist.json @@ -21,6 +21,7 @@ "src/core/ITensorPack.cpp", "src/core/Rounding.cpp", "src/core/Size2D.cpp", + "src/core/SparseTensor.cpp", "src/core/SubTensorInfo.cpp", "src/core/TensorInfo.cpp", "src/core/Utils.cpp", @@ -67,8 +68,11 @@ "src/runtime/SchedulerFactory.cpp", "src/runtime/SchedulerUtils.cpp", "src/runtime/SubTensor.cpp", + "src/runtime/COOTensor.cpp", + "src/runtime/CSRTensor.cpp", "src/runtime/Tensor.cpp", "src/runtime/TensorAllocator.cpp", + "src/runtime/SparseTensorAllocator.cpp", "src/runtime/Utils.cpp", "src/runtime/CPP/ICPPSimpleFunction.cpp", "src/runtime/CPP/functions/CPPBoxWithNonMaximaSuppressionLimit.cpp", @@ -1631,7 +1635,7 @@ "src/runtime/experimental/operators/CpuMul.cpp", "src/runtime/experimental/operators/CpuQuantize.cpp", "src/runtime/experimental/operators/CpuSoftmax.cpp", - "src/runtime/experimental/operators/CpuPool2d.cpp", + "src/runtime/experimental/operators/CpuPool2d.cpp", "src/runtime/experimental/operators/CpuSub.cpp", "src/runtime/experimental/operators/CpuTranspose.cpp", "src/runtime/experimental/operators/CpuWinogradConv2d.cpp" @@ -1869,7 +1873,7 @@ "src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp16_mla_8x3VL/generic.cpp", "src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp32_mla_8x3VL/a64fx.cpp", "src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp32_mla_8x3VL/generic.cpp" - ] + ] } } }, diff --git a/src/BUILD.bazel b/src/BUILD.bazel index 89e632ddd4..2dc36b5bed 100644 --- a/src/BUILD.bazel +++ b/src/BUILD.bazel @@ -634,6 +634,7 @@ filegroup( "core/NEON/kernels/convolution/winograd/winograd_fp32.cpp", "core/Rounding.cpp", "core/Size2D.cpp", + "core/SparseTensor.cpp", "core/SubTensorInfo.cpp", "core/TensorInfo.cpp", "core/Utils.cpp", @@ -980,8 +981,11 @@ filegroup( "runtime/SchedulerFactory.cpp", "runtime/SchedulerUtils.cpp", "runtime/SubTensor.cpp", + "runtime/COOTensor.cpp", + "runtime/CSRTensor.cpp", "runtime/Tensor.cpp", "runtime/TensorAllocator.cpp", + "runtime/SparseTensorAllocator.cpp", "runtime/Utils.cpp", "runtime/experimental/low_level/CpuGemmAssemblyDispatch.cpp", "runtime/experimental/operators/CpuActivation.cpp", diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3a8815d836..9004270324 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -627,6 +627,7 @@ target_sources( core/NEON/kernels/convolution/winograd/winograd_fp32.cpp core/Rounding.cpp core/Size2D.cpp + core/SparseTensor.cpp core/SubTensorInfo.cpp core/TensorInfo.cpp core/Utils.cpp @@ -974,7 +975,10 @@ target_sources( runtime/SchedulerUtils.cpp runtime/SubTensor.cpp runtime/Tensor.cpp + runtime/COOTensor.cpp + runtime/CSRTensor.cpp runtime/TensorAllocator.cpp + runtime/SparseTensorAllocator.cpp runtime/Utils.cpp runtime/experimental/low_level/CpuGemmAssemblyDispatch.cpp runtime/experimental/operators/CpuActivation.cpp diff --git a/src/core/SparseTensor.cpp b/src/core/SparseTensor.cpp new file mode 100644 index 0000000000..a616c19f2d --- /dev/null +++ b/src/core/SparseTensor.cpp @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2025 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "arm_compute/core/SparseTensor.h" + +#include "arm_compute/core/Error.h" + +namespace arm_compute +{ +SparseTensor::SparseTensor(size_t dim, size_t sparse_dim) : _total_dim(dim), _sparse_dim(sparse_dim) +{ +} + +size_t SparseTensor::sparse_dim() const +{ + return _sparse_dim; +} + +size_t SparseTensor::dense_dim() const +{ + return _total_dim - _sparse_dim; +} + +float SparseTensor::sparsity() const +{ + return 1.0f - density(); +} + +float SparseTensor::density() const +{ + return static_cast(nnz()) / static_cast(info()->total_size()); +} + +size_t SparseTensor::dim() const +{ + return _total_dim; +} + +bool SparseTensor::is_hybrid() const +{ + return dense_dim() > 0; +} + +uint32_t SparseTensor::dense_volume(size_t sparse_dim) const +{ + const auto &ts = info()->tensor_shape(); + return std::accumulate(ts.begin() + sparse_dim, ts.end(), 1, std::multiplies()); +} + +bool SparseTensor::has_non_zero_elements(uint8_t *arr, size_t len, size_t element_size, predicate_t is_non_zero) const +{ + for(size_t i = 0; i < len; i += element_size) + { + if(is_non_zero(arr + i)) + { + return true; + } + } + return false; +} + +std::function SparseTensor::make_is_nonzero_predicate(DataType dt) const +{ + switch (dt) + { + case DataType::F32: + return [](const void *ptr) { + return *(static_cast(ptr)) != 0.0f; + }; + case DataType::F16: // raw bitwise comparison + case DataType::U16: + case DataType::QSYMM16: + case DataType::QASYMM16: + return [](const void *ptr) { + return *(static_cast(ptr)) != 0; + }; + case DataType::S32: + return [](const void *ptr) { + return *(static_cast(ptr)) != 0; + }; + case DataType::S16: + return [](const void *ptr) { + return *(static_cast(ptr)) != 0; + }; + case DataType::U32: + return [](const void *ptr) { + return *(static_cast(ptr)) != 0; + }; + case DataType::U8: + case DataType::QSYMM8: + case DataType::QASYMM8: + case DataType::QSYMM8_PER_CHANNEL: + return [](const void *ptr) { + return *(static_cast(ptr)) != 0; + }; + case DataType::S8: + case DataType::QASYMM8_SIGNED: + return [](const void *ptr) { + return *(static_cast(ptr)) != 0; + }; + default: + throw std::runtime_error("Unsupported DataType in make_is_nonzero_predicate()"); + } +} + +void SparseTensor::print_values(std::ostream &os, const uint8_t *data, size_t offset, size_t count) const +{ + const size_t element_size = info()->element_size(); + const uint8_t *block_ptr = data + offset * count * element_size; + + os << "["; + for(size_t j = 0; j < count; ++j) + { + const void *value_ptr = block_ptr + j * element_size; + + switch(info()->data_type()) + { + case DataType::U8: + os << static_cast(*reinterpret_cast(value_ptr)); + break; + case DataType::S8: + os << static_cast(*reinterpret_cast(value_ptr)); + break; + case DataType::U32: + os << *reinterpret_cast(value_ptr); + break; + case DataType::S32: + os << *reinterpret_cast(value_ptr); + break; + case DataType::F16: + os << static_cast(*reinterpret_cast(value_ptr)); + break; + case DataType::F32: + os << *reinterpret_cast(value_ptr); + break; + default: + os << ""; + } + + if(j < count - 1) os << ", "; + } + os << "]" << std::endl; +} +} // namespace arm_compute diff --git a/src/core/SubTensorInfo.cpp b/src/core/SubTensorInfo.cpp index ac25d725ed..8e0d23f77f 100644 --- a/src/core/SubTensorInfo.cpp +++ b/src/core/SubTensorInfo.cpp @@ -133,6 +133,12 @@ ITensorInfo &SubTensorInfo::set_tensor_dims_state(const TensorDimsState &state) return *this; } +ITensorInfo &SubTensorInfo::set_tensor_format(TensorFormat tf) +{ + ARM_COMPUTE_ERROR_ON(_parent == nullptr); + return _parent->set_tensor_format(tf); +} + ITensorInfo &SubTensorInfo::set_dynamic(bool dynamic) { if (dynamic) diff --git a/src/core/TensorInfo.cpp b/src/core/TensorInfo.cpp index 36e2772514..2e17c4bd7d 100644 --- a/src/core/TensorInfo.cpp +++ b/src/core/TensorInfo.cpp @@ -25,6 +25,7 @@ #include "arm_compute/core/Error.h" #include "arm_compute/core/Helpers.h" +#include "arm_compute/core/TensorFormat.h" #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/Validate.h" @@ -50,7 +51,8 @@ TensorInfo::TensorInfo() _data_layout(DataLayout::NCHW), _are_values_constant(true), _id(invalid_tensor_id), - _lock_paddings(false) + _lock_paddings(false), + _tensor_format(TensorFormat::Dense) { _dims_state.fill(ITensorInfo::get_static_state_value()); } @@ -73,6 +75,7 @@ TensorInfo::TensorInfo(const ITensorInfo &info) : TensorInfo() _are_values_constant = info.are_values_constant(); _id = info.id(); _lock_paddings = info.lock_paddings(); + _tensor_format = info.tensor_format(); } TensorInfo::TensorInfo(const TensorInfo &info) : TensorInfo() @@ -93,6 +96,7 @@ TensorInfo::TensorInfo(const TensorInfo &info) : TensorInfo() _are_values_constant = info.are_values_constant(); _id = info.id(); _lock_paddings = false; + _tensor_format = info.tensor_format(); } TensorInfo::TensorInfo(Format format) : TensorInfo(TensorShape(), format) { diff --git a/src/runtime/COOTensor.cpp b/src/runtime/COOTensor.cpp new file mode 100644 index 0000000000..bf03a4068d --- /dev/null +++ b/src/runtime/COOTensor.cpp @@ -0,0 +1,264 @@ +/* + * Copyright (c) 2025 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "arm_compute/runtime/COOTensor.h" + +#include "arm_compute/core/CoreTypes.h" +#include "arm_compute/core/Error.h" +#include "arm_compute/core/TensorFormat.h" +#include "arm_compute/runtime/Tensor.h" + +#include "src/core/helpers/Utils.h" + + +namespace arm_compute +{ +namespace +{ +TensorInfo coo_tensor_info(const ITensorInfo *src_info) +{ + return src_info->clone()->set_tensor_format(TensorFormat::COO); +} +} // namespace + +COOTensor::COOTensor(const ITensor *tensor, size_t sparse_dim) : SparseTensor(tensor->info()->num_dimensions(), sparse_dim), _indices(), _allocator(this) +{ + ARM_COMPUTE_ERROR_ON_NULLPTR(tensor); + const ITensorInfo *info = tensor->info(); + + ARM_COMPUTE_ERROR_ON_MSG(info->data_layout() != DataLayout::NCHW, "COOTensor only supports NCHW layout at the moment"); + ARM_COMPUTE_ERROR_ON_MSG(info->is_sparse(), "cannot create a COOTensor from a sparse tensor"); + ARM_COMPUTE_ERROR_ON_MSG_VAR(sparse_dim < 1 || sparse_dim > dim(), + "argument must be in [1,%zu] range. %zu is given", + dim(), sparse_dim); + + const uint8_t *data = tensor->buffer(); + const size_t dense_dims = dense_dim(); + const auto is_nonzero = make_is_nonzero_predicate(info->data_type()); + + std::vector sparse_shape(sparse_dim); + std::vector dense_shape(dense_dims); + for(size_t i = 0; i < sparse_dim; i++) + { + sparse_shape[i] = info->dimension(i); + } + for(size_t i = 0; i < dense_dims; i++) + { + dense_shape[i] = info->dimension(sparse_dim + i); + } + + std::vector temp_values; + + const size_t step = std::accumulate(dense_shape.begin(), dense_shape.end(), size_t(1), std::multiplies()); + const size_t max_iter = std::accumulate(sparse_shape.begin(), sparse_shape.end(), size_t(1), std::multiplies()); + const size_t element_size = info->element_size(); + const size_t slice_size = step * element_size; + + size_t value_byte_size = 0; + size_t indices_bytes = 0; + for(size_t i = 0; i < max_iter; i++) + { + const size_t offset = i * slice_size; + if(has_non_zero_elements(const_cast(data + offset), slice_size, element_size, is_nonzero)) + { + value_byte_size += slice_size; + indices_bytes += dim() * sizeof(int32_t); + } + } + + _allocator.init(coo_tensor_info(info), value_byte_size, indices_bytes); + _allocator.allocate(); + + for(size_t i = 0; i < max_iter; i++) + { + const size_t offset = i * slice_size; + if(!has_non_zero_elements(const_cast(data + offset), slice_size, element_size, is_nonzero)) + { + continue; + } + + // ----------------- + // TODO: I built this function in a similar way to numpy's unravel_index. + // Do you think it's worth creating a util function somewhere else? + // Also, it stores the coordinates in a reverse way, with the fastest + // changing dimension last. Probably it would be better to store them + // in a more natural order. + std::vector multi_index(dim(), std::int32_t{0}); + size_t remainder = i; + for(size_t sd = sparse_dim; sd-- > 0;) + { + multi_index[sd] = remainder % sparse_shape[sd]; + remainder /= sparse_shape[sd]; + } + // ----------------- + + _indices.push_back(Coordinates(multi_index)); + temp_values.insert(temp_values.end(), data + offset, data + offset + slice_size); + } + + if(!temp_values.empty()) + { + std::memcpy(_allocator.data(), temp_values.data(), temp_values.size()); + } +} + +COOTensor::COOTensor(const ITensor *tensor) : COOTensor(tensor, tensor->info()->num_dimensions()) +{ +} + +size_t COOTensor::nnz() const +{ + return _indices.size(); +} + +ITensorInfo *COOTensor::info() const +{ + return &_allocator.info(); +} + +ITensorInfo *COOTensor::info() +{ + return &_allocator.info(); +} + +uint8_t *COOTensor::buffer() const +{ + return _allocator.data(); +} + +std::unique_ptr COOTensor::to_dense() +{ + ARM_COMPUTE_ERROR_ON_MSG(info()->data_layout() != DataLayout::NCHW, "COOTensor only supports NCHW layout at the moment"); + + std::unique_ptr tensor = std::make_unique(); + tensor->allocator()->init(info()->clone()->set_tensor_format(TensorFormat::Dense)); + tensor->allocator()->allocate(); + + const size_t element_size = info()->element_size(); + const size_t total_size = info()->total_size(); + const size_t dense_vol = dense_volume(sparse_dim()); + const size_t first_elem_offset = info()->offset_first_element_in_bytes(); + + std::memset(tensor->buffer() + first_elem_offset, 0, total_size); + + if(nnz() == 0) + { + return tensor; + } + + for(size_t i = 0; i < _indices.size(); ++i) + { + const Coordinates &c = _indices[i]; + const uint8_t *block_ptr = buffer() + i * dense_vol * element_size; + + size_t final_offset = 0; + for(size_t d = 0; d < sparse_dim(); ++d) + { + final_offset += c[d] * dense_volume(d+1); + } + final_offset *= element_size; + + for(size_t j = 0; j < dense_vol; ++j) + { + const void *value_ptr = block_ptr + j * element_size; + uint8_t *base_ptr = tensor->buffer() + final_offset + j * element_size; + + std::memcpy(base_ptr, value_ptr, element_size); + } + } + + return tensor; +} + +Coordinates COOTensor::get_coordinates(size_t nth) const +{ + ARM_COMPUTE_ERROR_ON_MSG(nth >= nnz(), "Invalid index"); + + return _indices[nth]; +} + +const uint8_t *COOTensor::get_value(Coordinates coords) const +{ + ARM_COMPUTE_ERROR_ON_MSG(coords.num_dimensions() != info()->num_dimensions(), "Invalid coordinate dimension"); + for(size_t i = 0; i < coords.num_dimensions(); ++i) + { + ARM_COMPUTE_ERROR_ON_MSG(static_cast(coords[i]) >= info()->tensor_shape()[i], "Invalid coordinates shape"); + } + + const uint8_t *data = static_cast(buffer()); + const size_t dense_vol = dense_volume(sparse_dim()); + + for(size_t i = 0; i < _indices.size(); ++i) + { + const Coordinates &c = _indices[i]; + bool match = false; + + for(size_t d = 0; d < coords.num_dimensions(); ++d) + { + if(c[d] == coords[d]) + { + match = true; + } + } + if(match) + { + return data + i * dense_vol * info()->element_size(); + } + } + + // This coordinates point to a zero value + return nullptr; +} + +void COOTensor::associate_memory_group(IMemoryGroup *memory_group) +{ + _allocator.set_associated_memory_group(memory_group); +} + +#ifdef ARM_COMPUTE_ASSERTS_ENABLED +void COOTensor::print(std::ostream &os) const +{ + const uint8_t *data = static_cast(buffer()); + + if(_indices.empty()) + { + os << "index: [] values: []" << std::endl; + return; + } + + for(size_t i = 0; i < _indices.size(); ++i) + { + const Coordinates &coord = _indices[i]; + os << "index: ["; + for (size_t j = 0; j < coord.num_dimensions(); ++j) + { + os << coord[j]; + if (j < coord.num_dimensions() - 1) os << ", "; + } + os << "] values: "; + print_values(os, data, i, dense_volume(sparse_dim())); + } +} +#endif // ARM_COMPUTE_ASSERTS_ENABLED + +} // namespace arm_compute diff --git a/src/runtime/CSRTensor.cpp b/src/runtime/CSRTensor.cpp new file mode 100644 index 0000000000..035022a61a --- /dev/null +++ b/src/runtime/CSRTensor.cpp @@ -0,0 +1,272 @@ +/* + * Copyright (c) 2025 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "arm_compute/runtime/COOTensor.h" + +#include "arm_compute/core/CoreTypes.h" +#include "arm_compute/core/Error.h" +#include "arm_compute/core/TensorFormat.h" +#include "arm_compute/runtime/Tensor.h" + +#include "src/core/helpers/Utils.h" + +#include + +namespace arm_compute +{ + +namespace +{ +TensorInfo csr_tensor_info(const ITensorInfo *src_info) +{ + return src_info->clone()->set_tensor_format(TensorFormat::CSR); +} +} // namespace + +CSRTensor::CSRTensor(const ITensor *tensor, size_t sparse_dim) : SparseTensor(tensor->info()->num_dimensions(), 2), _crow_bytes(), _col_bytes(), _allocator(this) +{ + ARM_COMPUTE_UNUSED(sparse_dim); + ARM_COMPUTE_ERROR_ON_NULLPTR(tensor); + const auto *info = tensor->info(); + + // As of now, CSRTensor only supports 2D tensors with NCHW layout. + ARM_COMPUTE_ERROR_ON_MSG(info->data_layout() != DataLayout::NCHW, "CSRTensor only supports NCHW layout at the moment"); + ARM_COMPUTE_ERROR_ON_MSG(info->is_sparse(), "cannot create a CSRTensor from a sparse tensor"); + ARM_COMPUTE_ERROR_ON_MSG(dim() != 2, "CSRTensor only supports 2D tensors at the moment"); + + const int32_t rows = info->dimension(0); + const int32_t cols = info->dimension(1); + const int32_t element_size = info->element_size(); + const int32_t row_size_bytes = cols * element_size; + const auto is_nonzero = make_is_nonzero_predicate(info->data_type()); + const uint8_t *data = tensor->buffer() + info->offset_first_element_in_bytes(); + size_t value_byte_size = 0; + + _crow_bytes = index_size; // The first row index is always a 0 + _col_bytes = 0; + + for(int32_t row = 0; row < rows; ++row) + { + const int32_t row_offset = row * row_size_bytes; + _crow_bytes += index_size; + for(int32_t col = 0; col < cols; ++col) + { + const int32_t element_offset = row_offset + col * element_size; + if(is_nonzero(data + element_offset)) + { + _col_bytes += index_size; + value_byte_size += element_size * dense_volume(sparse_dim); + } + } + } + + _allocator.init(csr_tensor_info(info), value_byte_size, _crow_bytes + _col_bytes); + _allocator.allocate(); + + uint8_t *_row_offsets = _allocator.data(); + uint8_t *_col_indices = _allocator.data() + _crow_bytes; + uint8_t *_values = _allocator.data() + _crow_bytes + _col_bytes; + size_t num_non_zero = 0; + int32_t last_row = 0; + int32_t col_index = 0; + + std::memcpy(_row_offsets, &last_row, index_size); + + for(int32_t row = 0; row < rows; ++row) + { + const int32_t row_offset = row * row_size_bytes; + int32_t non_zero_row_elements = 0; + for(int32_t col = 0; col < cols; ++col) + { + const size_t element_offset = row_offset + col * element_size; + if(is_nonzero(data + element_offset)) + { + std::memcpy(_col_indices + col_index * index_size, &col, index_size); + std::memcpy(_values + num_non_zero * element_size, data + element_offset, element_size); + non_zero_row_elements++; + num_non_zero++; + col_index++; + } + } + + last_row += non_zero_row_elements; + std::memcpy(_row_offsets + (row + 1) * index_size, &last_row, index_size); + } +} + +CSRTensor::CSRTensor(const ITensor *tensor) : CSRTensor(tensor, 2) +{ +} + +size_t CSRTensor::nnz() const +{ + return static_cast(_col_bytes / index_size); +} + +ITensorInfo *CSRTensor::info() const +{ + return &_allocator.info(); +} + +ITensorInfo *CSRTensor::info() +{ + return &_allocator.info(); +} + +uint8_t *CSRTensor::buffer() const +{ + return _allocator.data(); +} + +Coordinates CSRTensor::get_coordinates(size_t nth) const +{ + ARM_COMPUTE_ERROR_ON_MSG(nth >= nnz(), "Invalid index"); + + const uint8_t *row_indices = _allocator.data(); + const uint8_t *col_indices = _allocator.data() + _crow_bytes; + size_t low = 0; + size_t high = (_crow_bytes / index_size) - 1; + + while(low < high) + { + const size_t mid = (low + high) / 2; + if(*reinterpret_cast(row_indices + (mid + 1) * index_size) <= static_cast(nth)) + { + low = mid + 1; + } + else + { + high = mid; + } + } + + return Coordinates{ low , *reinterpret_cast(col_indices + (nth * index_size)) }; +} + +const uint8_t *CSRTensor::get_value(Coordinates coords) const +{ + ARM_COMPUTE_ERROR_ON_MSG(coords.num_dimensions() != info()->num_dimensions(), "Invalid coordinate dimension"); + for(size_t i = 0; i < coords.num_dimensions(); ++i) + { + ARM_COMPUTE_ERROR_ON_MSG(static_cast(coords[i]) >= info()->tensor_shape()[i], "Invalid coordinates shape"); + } + + const uint8_t *row_indices = _allocator.data(); + const uint8_t *col_indices = _allocator.data() + _crow_bytes; + const uint8_t *values = _allocator.data() + _crow_bytes + _col_bytes; + const int32_t row = coords[0]; + const int32_t col = coords[1]; + + const int32_t start = *reinterpret_cast(row_indices + row * index_size); + const int32_t end = *reinterpret_cast(row_indices + (row + 1) * index_size); + + for(int32_t i = start; i < end; ++i) + { + if(*reinterpret_cast(col_indices + (i * index_size)) == col) + { + return values + i * info()->element_size(); + } + } + + // This coordinates point to a zero value + return nullptr; +} + +std::unique_ptr CSRTensor::to_dense() +{ + ARM_COMPUTE_ERROR_ON_MSG(info()->data_layout() != DataLayout::NCHW, "CSRTensor only supports NCHW layout at the moment"); + + auto tensor = std::make_unique(); + tensor->allocator()->init(info()->clone()->set_tensor_format(TensorFormat::Dense)); + tensor->allocator()->allocate(); + + const size_t first_elem_offset = info()->offset_first_element_in_bytes(); + uint8_t *data = tensor->buffer() + first_elem_offset; + const size_t element_size = info()->element_size(); + const size_t total_size = info()->total_size(); + const uint8_t *row_offsets = _allocator.data(); + const uint8_t *col_indices = _allocator.data() + _crow_bytes; + const uint8_t *values = _allocator.data() + _crow_bytes + _col_bytes; + size_t element = 0; + + std::memset(data, 0, total_size); + + for(size_t i = 0, j = 1, row = 0; j < _crow_bytes / index_size; ++i, ++j, ++row) + { + const int32_t current_col = *reinterpret_cast(row_offsets + (i * index_size)); + const int32_t next_col = *reinterpret_cast(row_offsets + (j * index_size)); + + if(current_col == next_col) + { + continue; + } + + for(size_t current = static_cast(current_col); current < static_cast(next_col); ++current) + { + const size_t col = *reinterpret_cast(col_indices + (current * index_size)); + const uint8_t *value_ptr = values + (element * element_size); + + std::memcpy(data + (row * info()->dimension(1) + col) * element_size, value_ptr, element_size); + element++; + } + } + + return tensor; +} + +void CSRTensor::associate_memory_group(IMemoryGroup *memory_group) +{ + _allocator.set_associated_memory_group(memory_group); +} + +#ifdef ARM_COMPUTE_ASSERTS_ENABLED +void CSRTensor::print(std::ostream &os) const +{ + const uint8_t *_row_offsets = _allocator.data(); + const uint8_t *_col_indices = _allocator.data() + _crow_bytes; + const uint8_t *values = _allocator.data() + _crow_bytes + _col_bytes; + + os << "r_offsets: ["; + for(size_t i = 0; i < _crow_bytes / index_size; ++i) + { + os << *reinterpret_cast(_row_offsets + (i * index_size)); + if (i < _crow_bytes / index_size - 1) + { + os << ", "; + } + } + os << "] cols: ["; + for(size_t i = 0; i < _col_bytes / index_size; ++i) + { + os << *reinterpret_cast(_col_indices + (i * index_size)); + if (i < _col_bytes / index_size - 1) + { + os << ", "; + } + } + os << "] values: "; + print_values(os, values, 0, nnz()); +} +#endif // ARM_COMPUTE_ASSERTS_ENABLED + +} // namespace arm_compute diff --git a/src/runtime/SparseTensorAllocator.cpp b/src/runtime/SparseTensorAllocator.cpp new file mode 100644 index 0000000000..4c404c5edc --- /dev/null +++ b/src/runtime/SparseTensorAllocator.cpp @@ -0,0 +1,177 @@ +/* + * Copyright (c) 2025 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "arm_compute/runtime/SparseTensorAllocator.h" + +#include "arm_compute/core/Coordinates.h" +#include "arm_compute/core/Error.h" +#include "arm_compute/core/TensorInfo.h" +#include "arm_compute/runtime/MemoryGroup.h" +#include "arm_compute/runtime/MemoryRegion.h" + +#include + +namespace arm_compute +{ +SparseTensorAllocator::SparseTensorAllocator(IMemoryManageable *owner) : _owner(owner), _associated_memory_group(nullptr), _memory(), _values_bytes(0), _indices_bytes(0) +{ +} + +SparseTensorAllocator::~SparseTensorAllocator() +{ + info().set_is_resizable(true); +} + +SparseTensorAllocator::SparseTensorAllocator(SparseTensorAllocator &&o) noexcept + : ITensorAllocator(std::move(o)), + _owner(o._owner), + _associated_memory_group(o._associated_memory_group), + _memory(std::move(o._memory)), + _values_bytes(o._values_bytes), + _indices_bytes(o._indices_bytes) +{ + o._owner = nullptr; + o._associated_memory_group = nullptr; + o._memory = Memory(); + o._values_bytes = 0; + o._indices_bytes = 0; +} + +SparseTensorAllocator &SparseTensorAllocator::operator=(SparseTensorAllocator &&o) noexcept +{ + if (&o != this) + { + _owner = o._owner; + o._owner = nullptr; + + _associated_memory_group = o._associated_memory_group; + o._associated_memory_group = nullptr; + + _memory = std::move(o._memory); + o._memory = Memory(); + + _values_bytes = o._values_bytes; + _values_bytes = 0; + + _indices_bytes = o._indices_bytes; + _indices_bytes = 0; + + ITensorAllocator::operator=(std::move(o)); + } + return *this; +} + +void SparseTensorAllocator::init(const TensorInfo &input, size_t values_bytes, size_t indices_bytes, size_t alignment) +{ + ITensorAllocator::init(input, alignment); + _values_bytes = values_bytes; + _indices_bytes = indices_bytes; +} + +void SparseTensorAllocator::init(const SparseTensorAllocator &allocator, const Coordinates &coords, TensorInfo &sub_info) +{ + // Get parent info + const TensorInfo parent_info = allocator.info(); + + // Copy pointer to buffer + _memory = Memory(allocator._memory.region()); + + // Init tensor info with new dimensions + size_t total_size = + parent_info.offset_element_in_bytes(coords) + sub_info.total_size() - sub_info.offset_first_element_in_bytes(); + sub_info.init(sub_info.tensor_shape(), sub_info.format(), parent_info.strides_in_bytes(), + parent_info.offset_element_in_bytes(coords), total_size); + + // Set TensorInfo + ITensorAllocator::init(sub_info); +} + +uint8_t *SparseTensorAllocator::data() const +{ + return (_memory.region() == nullptr) ? nullptr : reinterpret_cast(_memory.region()->buffer()); +} + +void SparseTensorAllocator::allocate() +{ + // Align to 64-byte boundaries by default if alignment is not specified + const size_t alignment_to_use = (alignment() != 0) ? alignment() : 64; + const size_t size = size_bytes(); + + if (_associated_memory_group == nullptr) + { + _memory.set_owned_region(std::make_unique(size, alignment_to_use)); + } + else + { + _associated_memory_group->finalize_memory(_owner, _memory, size, alignment_to_use); + } + info().set_is_resizable(false); +} + +void SparseTensorAllocator::free() +{ + _memory.set_region(nullptr); + info().set_is_resizable(true); +} + +bool SparseTensorAllocator::is_allocated() const +{ + return _memory.region() != nullptr; +} + +Status SparseTensorAllocator::import_memory(void *memory) +{ + ARM_COMPUTE_RETURN_ERROR_ON(memory == nullptr); + ARM_COMPUTE_RETURN_ERROR_ON(_associated_memory_group != nullptr); + ARM_COMPUTE_RETURN_ERROR_ON(alignment() != 0 && !arm_compute::utility::check_aligned(memory, alignment())); + + _memory.set_owned_region(std::make_unique(memory, info().total_size())); + info().set_is_resizable(false); + + return Status{}; +} + +void SparseTensorAllocator::set_associated_memory_group(IMemoryGroup *associated_memory_group) +{ + ARM_COMPUTE_ERROR_ON(associated_memory_group == nullptr); + ARM_COMPUTE_ERROR_ON(_associated_memory_group != nullptr && _associated_memory_group != associated_memory_group); + ARM_COMPUTE_ERROR_ON(_memory.region() != nullptr && _memory.region()->buffer() != nullptr); + + _associated_memory_group = associated_memory_group; +} + +size_t SparseTensorAllocator::size_bytes() const +{ + return _values_bytes + _indices_bytes; +} + +uint8_t *SparseTensorAllocator::lock() +{ + ARM_COMPUTE_ERROR_ON(_memory.region() == nullptr); + return reinterpret_cast(_memory.region()->buffer()); +} + +void SparseTensorAllocator::unlock() +{ +} +} // namespace arm_compute diff --git a/src/runtime/Tensor.cpp b/src/runtime/Tensor.cpp index f17e323694..b4c5e26279 100644 --- a/src/runtime/Tensor.cpp +++ b/src/runtime/Tensor.cpp @@ -49,6 +49,31 @@ TensorAllocator *Tensor::allocator() return &_allocator; } +std::unique_ptr Tensor::to_sparse(size_t dim) const +{ + return to_coo_sparse(dim); +} + +std::unique_ptr Tensor::to_coo_sparse(size_t dim) const +{ + return std::unique_ptr(new COOTensor(this, dim)); +} + +std::unique_ptr Tensor::to_coo_sparse() const +{ + return std::unique_ptr(new COOTensor(this)); +} + +std::unique_ptr Tensor::to_csr_sparse(size_t dim) const +{ + return std::unique_ptr(new CSRTensor(this, dim)); +} + +std::unique_ptr Tensor::to_csr_sparse() const +{ + return std::unique_ptr(new CSRTensor(this)); +} + void Tensor::associate_memory_group(IMemoryGroup *memory_group) { _allocator.set_associated_memory_group(memory_group); diff --git a/tests/AssetsLibrary.h b/tests/AssetsLibrary.h index dedad5227f..41e330d9ef 100644 --- a/tests/AssetsLibrary.h +++ b/tests/AssetsLibrary.h @@ -41,6 +41,7 @@ #include #include #include +#include #include #include #include @@ -358,6 +359,27 @@ class AssetsLibrary final template void fill_tensor_uniform(T &&tensor, std::random_device::result_type seed_offset, D low, D high) const; + /** Fill a tensor with zeros and randomly modify a portion of its elements + * + * This function sets all elements of the tensor to zero and then modifies + * approximately @p modification_ratio fraction of them with random values + * uniformly sampled from the range [1, 255]. + * + * The modified elements are selected randomly across the entire tensor. + * + * @tparam T Type of the tensor (e.g., Tensor, ITensor). + * + * @param[in, out] tensor Tensor to be filled and modified. Must be allocated. + * @param[in] modification_ratio Fraction [0,1] of elements to randomly assign a non-zero value. + * + * @note The tensor will be completely filled with zeros, and then @p modification_ratio * total_elements + * positions will be overwritten with random values. The rest remain zero. + * For floating point types, random values are drawn from a uniform real distribution. + * For integral types, a uniform integer distribution is used. + */ + template + void fill_tensor_sparse_random(T &&tensor, float modification_ratio) const; + /** Fill a tensor with uniform distribution across the specified range * * @param[in, out] tensor To be filled tensor. @@ -1018,6 +1040,48 @@ void AssetsLibrary::fill_tensor_uniform(T &&tensor, std::random_device::result_t } } +template +void AssetsLibrary::fill_tensor_sparse_random(T &&tensor, float modification_ratio) const +{ + if(modification_ratio < 0.0f) modification_ratio = 0.0f; + if(modification_ratio > 1.0f) modification_ratio = 1.0f; + + const unsigned min = 1; + const unsigned max = 255; + const size_t num_elements = tensor.shape().total_size(); + const size_t num_mod = static_cast(modification_ratio * num_elements); + + std::vector indices(num_elements); + std::iota(indices.begin(), indices.end(), 0); + std::shuffle(indices.begin(), indices.end(), std::mt19937{std::random_device{}()}); + indices.resize(num_mod); + + std::mt19937 rng{std::random_device{}()}; + + if(tensor.data_type() == DataType::F32) + { + std::vector zero_filled(num_elements, float{0}); + std::uniform_real_distribution dist_real(static_cast(min), static_cast(max)); + for(size_t idx : indices) + { + zero_filled[idx] = static_cast(dist_real(rng)); + } + fill_static_values(std::forward(tensor), zero_filled); + } + else + { + // Signed types will interpret numbers >= 128 as negative; + // for our purposes, it doesn't matter, as long as they are different from 0. + std::vector zero_filled(num_elements, unsigned{0}); + std::uniform_int_distribution dist_int(min, max); + for(size_t idx : indices) + { + zero_filled[idx] = static_cast(dist_int(rng)); + } + fill_static_values(std::forward(tensor), zero_filled); + } +} + template void AssetsLibrary::fill_layer_data(T &&tensor, std::string name) const { diff --git a/tests/validation/UNIT/SparseTensor.cpp b/tests/validation/UNIT/SparseTensor.cpp new file mode 100644 index 0000000000..e26e8d4c0a --- /dev/null +++ b/tests/validation/UNIT/SparseTensor.cpp @@ -0,0 +1,206 @@ +/* + * Copyright (c) 2025 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "arm_compute/core/TensorFormat.h" +#include "arm_compute/core/Types.h" +#include "arm_compute/runtime/Tensor.h" +#include "arm_compute/runtime/COOTensor.h" +#include "tests/framework/Asserts.h" +#include "tests/framework/Macros.h" +#include "tests/framework/datasets/Datasets.h" +#include "tests/validation/Validation.h" +#include "tests/validation/Helpers.h" + +#include "tests/NEON/Accessor.h" +#include "tests/NEON/Helper.h" + +#include + +namespace arm_compute +{ +namespace +{ +bool are_values_equal(const uint8_t *a, const uint8_t *b, DataType dt, size_t element_size) +{ + if(dt == DataType::F32) + { + float va = *reinterpret_cast(a); + float vb = *reinterpret_cast(b); + if(std::fabs(va - vb) > 0e-5f) + { + return false; + } + } else + { + if(std::memcmp(a, b, element_size) != 0) + { + return false; + } + } + + return true; +} + +bool tensors_are_equal(const test::Accessor &a, const test::Accessor &b) +{ + if(a.shape() != b.shape() || a.data_type() != b.data_type()) + return false; + + const size_t element_size = a.element_size(); + Window window; + window.use_tensor_dimensions(a.shape()); + + bool equal = true; + + execute_window_loop(window, [&](const Coordinates &id) + { + const uint8_t *a_value = static_cast(a(id)); + const uint8_t *b_value = static_cast(b(id)); + + equal = are_values_equal(a_value, b_value, a.data_type(), element_size); + }); + + return equal; +} +} // namespace + +namespace test +{ +namespace validation +{ +TEST_SUITE(UNIT) +TEST_SUITE(SparseTensor) + +// clang-format off +/** Validates TensorInfo Autopadding */ +DATA_TEST_CASE(ConvertCOOTensorToDense, framework::DatasetMode::ALL, combine( + framework::dataset::make("TensorShape", { + TensorShape(8U), + TensorShape(3U, 3U), + TensorShape(2U, 5U, 5U), + TensorShape(4U, 2U, 2U, 9U)}), + framework::dataset::make("TensorType", { + DataType::U8, + DataType::S8, + DataType::U32, + DataType::S32, + DataType::F16, + DataType::F32}) + ), shape, type) +{ + const auto t_info = TensorInfo(shape, 1, type, DataLayout::NCHW); + auto t = create_tensor(t_info); + auto t_zero = create_tensor(t_info); + + t.allocator()->allocate(); + library->fill_tensor_sparse_random(Accessor(t), 0.2); + + t_zero.allocator()->allocate(); + library->fill_static_values(Accessor(t_zero), std::vector(shape.total_size(), 0)); + + for(size_t sparse_dim = 1; sparse_dim <= shape.num_dimensions(); sparse_dim++) + { + auto st = t.to_coo_sparse(sparse_dim); + bool is_sparse = st->info()->is_sparse(); + bool is_coo = st->info()->tensor_format() == TensorFormat::COO; + size_t dense_dim = shape.num_dimensions() - sparse_dim; + size_t is_hybrid = dense_dim > 0; + auto td = st->to_dense(); + + ARM_COMPUTE_EXPECT(is_sparse, framework::LogLevel::ERRORS); + ARM_COMPUTE_EXPECT(is_coo, framework::LogLevel::ERRORS); + ARM_COMPUTE_EXPECT(st->sparse_dim() == sparse_dim, framework::LogLevel::ERRORS); + ARM_COMPUTE_EXPECT(st->dense_dim() == dense_dim, framework::LogLevel::ERRORS); + ARM_COMPUTE_EXPECT(st->is_hybrid() == is_hybrid, framework::LogLevel::ERRORS); + ARM_COMPUTE_EXPECT(tensors_are_equal(Accessor(t), Accessor(*td)), framework::LogLevel::ERRORS); + + auto st_zero = t_zero.to_coo_sparse(sparse_dim); + auto td_zero = st_zero->to_dense(); + ARM_COMPUTE_EXPECT(tensors_are_equal(Accessor(t_zero), Accessor(*td_zero)), framework::LogLevel::ERRORS); + } +} +// clang-format on +// *INDENT-ON* + +// clang-format off +/** Validates TensorInfo Autopadding */ +DATA_TEST_CASE(ConvertCSRTensorToDense, framework::DatasetMode::ALL, combine( + framework::dataset::make("TensorShape", { + TensorShape(8U), + TensorShape(3U, 3U), + TensorShape(2U, 5U, 5U), + TensorShape(4U, 2U, 2U, 9U)}), + framework::dataset::make("TensorType", { + DataType::U8, + DataType::S8, + DataType::U32, + DataType::S32, + DataType::F16, + DataType::F32}) + ), shape, type) +{ + // Currently, CSRTensor only supports 2D tensors + if(shape.num_dimensions() < 2) + { + return; + } + const TensorShape tensor_shape(shape[0], shape[1]); + + const auto t_info = TensorInfo(tensor_shape, 1, type, DataLayout::NCHW); + auto t = create_tensor(t_info); + auto t_zero = create_tensor(t_info); + + t.allocator()->allocate(); + library->fill_tensor_sparse_random(Accessor(t), 0.2); + + t_zero.allocator()->allocate(); + library->fill_static_values(Accessor(t_zero), std::vector(tensor_shape.total_size(), 0)); + + auto st = t.to_csr_sparse(); + auto td = st->to_dense(); + bool is_sparse = st->info()->is_sparse(); + bool is_csr = st->info()->tensor_format() == TensorFormat::CSR; + size_t sparse_dim = tensor_shape.num_dimensions(); + size_t dense_dim = tensor_shape.num_dimensions() - sparse_dim; + size_t is_hybrid = dense_dim > 0; + + ARM_COMPUTE_EXPECT(is_sparse, framework::LogLevel::ERRORS); + ARM_COMPUTE_EXPECT(is_csr, framework::LogLevel::ERRORS); + ARM_COMPUTE_EXPECT(st->sparse_dim() == sparse_dim, framework::LogLevel::ERRORS); + ARM_COMPUTE_EXPECT(st->dense_dim() == dense_dim, framework::LogLevel::ERRORS); + ARM_COMPUTE_EXPECT(st->is_hybrid() == is_hybrid, framework::LogLevel::ERRORS); + ARM_COMPUTE_EXPECT(tensors_are_equal(Accessor(t), Accessor(*td)), framework::LogLevel::ERRORS); + + auto st_zero = t_zero.to_coo_sparse(sparse_dim); + auto td_zero = st_zero->to_dense(); + ARM_COMPUTE_EXPECT(tensors_are_equal(Accessor(t_zero), Accessor(*td_zero)), framework::LogLevel::ERRORS); +} +// clang-format on +// *INDENT-ON* + +TEST_SUITE_END() // SparseTensor +TEST_SUITE_END() // UNIT + +} // namespace validation +} // namespace test +} // namespace arm_compute