CUDNN Frontend API  8.3.0
cudnn_frontend_Tensor.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20  * DEALINGS IN THE SOFTWARE.
21  */
22 
23 #pragma once
24 
25 #include <algorithm>
26 #include <array>
27 #include <functional>
28 #include <memory>
29 #include <sstream>
30 #include <utility>
31 
32 #include "cudnn_frontend_utils.h"
33 
34 namespace cudnn_frontend {
35 
52 class Tensor_v8 : public BackendDescriptor {
53  public:
54  friend class TensorBuilder_v8;
55  std::string
56  describe() const override {
57  std::stringstream ss;
58  char sep = ' ';
59  ss << "CUDNN_BACKEND_TENSOR_DESCRIPTOR :"
60  << " Datatype: " << to_string(data_type) << " Id: " << std::to_string(id) << " Alignment: " << alignment
61  << " nDims " << nDims << " VectorCount: " << vectorCount << " vectorDimension " << vectorDimension;
62  ss << " Dim [";
63  for (auto i = 0; i < nDims; i++) {
64  ss << sep << btensor_dimA[i];
65  sep = ',';
66  }
67  ss << " ] Str [";
68  for (auto i = 0; i < nDims; i++) {
69  ss << sep << btensor_strA[i];
70  sep = ',';
71  }
72  ss << "]";
73  ss << " isVirtual: " << std::to_string(isVirtual) << " isByValue: " << std::to_string(isByValue);
74 #if (CUDNN_VERSION >= 8300)
75  ss << " reorder_type: " << reorder_type;
76 #endif
77  return ss.str();
78  }
79 
80  int64_t
82  int64_t count = vectorCount;
83  for (auto i = 0; i < nDims; i++) {
84  count = count * btensor_dimA[i];
85  }
86  return count;
87  };
88 
89  int64_t
91  return nDims;
92  }
93 
94  int64_t const *
95  getDimArray() const {
96  return btensor_dimA;
97  }
98 
99  int64_t const *
100  getStrideArray() const {
101  return btensor_strA;
102  }
103 
104  int64_t
105  getDataType() const {
106  return static_cast<int64_t>(data_type);
107  }
108 
109  Tensor_v8(Tensor_v8 &&from) = default;
110  Tensor_v8 &
111  operator=(Tensor_v8 &&) = default;
112 
113  ~Tensor_v8() = default;
114 
115  private:
116  Tensor_v8() = default;
117  Tensor_v8(Tensor_v8 const &) = delete;
118  Tensor_v8 &
119  operator=(Tensor_v8 const &) = delete;
120 
121  cudnnDataType_t data_type = CUDNN_DATA_FLOAT;
122  int64_t btensor_dimA[CUDNN_DIM_MAX + 1] = {-1};
123  int64_t btensor_strA[CUDNN_DIM_MAX + 1] = {-1};
124  int64_t id = -1;
125  int64_t alignment = -1;
126  int64_t nDims = -1;
128  int64_t vectorDimension = -1;
129  int64_t vectorCount = 1;
130  bool isVirtual = false;
131  bool isByValue = false;
132 #if (CUDNN_VERSION >= 8300)
133  cudnnBackendTensorReordering_t reorder_type = CUDNN_TENSOR_REORDERING_NONE;
134 #endif
135 };
136 
141  public:
146  auto
148  setDataType(cudnnDataType_t data_type_) -> TensorBuilder_v8 & {
149  m_tensor.data_type = data_type_;
150  return *this;
151  }
153  auto
154  setDim(int64_t ndim, int64_t const *dim) -> TensorBuilder_v8 & {
155  std::copy((dim), dim + ndim, m_tensor.btensor_dimA);
156  m_tensor.nDims = ndim;
157  return *this;
158  }
160  auto
161  setStrides(int64_t ndim, int64_t const *strides) -> TensorBuilder_v8 & {
162  std::copy(strides, strides + ndim, m_tensor.btensor_strA);
163  return *this;
164  }
166  auto
167  setId(int64_t id_) -> TensorBuilder_v8 & {
168  m_tensor.id = id_;
169  return *this;
170  }
172  auto
173  setAlignment(int64_t alignment_) -> TensorBuilder_v8 & {
174  m_tensor.alignment = alignment_;
175  return *this;
176  }
178  auto
179  setVirtual(bool virtual_ = true) -> TensorBuilder_v8 & {
180  m_tensor.isVirtual = virtual_;
181  return *this;
182  }
183  auto
184  setByValue(bool isByValue_ = true) -> TensorBuilder_v8 & {
185  m_tensor.isByValue = isByValue_;
186  return *this;
187  }
188  auto
189  setVectorCountAndDimension(int64_t vectorCount_, int64_t vectorDimension_) -> TensorBuilder_v8 & {
190  m_tensor.vectorCount = vectorCount_;
191  m_tensor.vectorDimension = vectorDimension_;
192  return *this;
193  }
194 
195 #if (CUDNN_VERSION >= 8300)
196  auto
197  setReorderType(cudnnBackendTensorReordering_t type_) -> TensorBuilder_v8 & {
198  m_tensor.reorder_type = type_;
199  return *this;
200  }
201 #endif
202 
204  Tensor_v8 &&
207  build() {
208  // Sanity check if non-default fields have been set correctly.
209  if (m_tensor.alignment <= 0) {
211  &m_tensor,
212  CUDNN_STATUS_BAD_PARAM,
213  "CUDNN_BACKEND_TENSOR_DESCRIPTOR: Check and Set the CUDNN_ATTR_TENSOR_BYTE_ALIGNMENT field");
214  return std::move(m_tensor);
215  }
216  if (m_tensor.id <= 0) {
218  &m_tensor,
219  CUDNN_STATUS_BAD_PARAM,
220  "CUDNN_BACKEND_TENSOR_DESCRIPTOR: Check and Set the CUDNN_ATTR_TENSOR_UNIQUE_ID as a valid value");
221  return std::move(m_tensor);
222  }
223  if (m_tensor.btensor_strA[0] <= 0) {
225  &m_tensor,
226  CUDNN_STATUS_BAD_PARAM,
227  "CUDNN_BACKEND_TENSOR_DESCRIPTOR: Check and Set the CUDNN_ATTR_TENSOR_STRIDES Correctly");
228  return std::move(m_tensor);
229  }
230  if (m_tensor.btensor_dimA[0] <= 0) {
232  &m_tensor,
233  CUDNN_STATUS_BAD_PARAM,
234  "CUDNN_BACKEND_TENSOR_DESCRIPTOR: Check and Set the CUDNN_ATTR_TENSOR_DIMENSIONS Correctly");
235  return std::move(m_tensor);
236  }
237  if (m_tensor.pointer != nullptr) {
239  CUDNN_STATUS_BAD_PARAM,
240  "CUDNN_BACKEND_TENSOR_DESCRIPTOR: Bad tensor created. The tensor already "
241  "seems to be pointing to something");
242  return std::move(m_tensor);
243  }
244 
245  // Create a descriptor. Memory allocation happens here.
246  auto status = m_tensor.initialize_managed_backend_pointer(CUDNN_BACKEND_TENSOR_DESCRIPTOR);
247  if (status != CUDNN_STATUS_SUCCESS) {
249  &m_tensor, status, "CUDNN_BACKEND_TENSOR_DESCRIPTOR: cudnnCreate Descriptor Failed");
250  return std::move(m_tensor);
251  }
252 
253  // Once Created lets set the descriptor parameters.
254  status = cudnnBackendSetAttribute(m_tensor.pointer->get_backend_descriptor(),
255  CUDNN_ATTR_TENSOR_DATA_TYPE,
256  CUDNN_TYPE_DATA_TYPE,
257  1,
258  &m_tensor.data_type);
259  if (status != CUDNN_STATUS_SUCCESS) {
261  &m_tensor, status, "CUDNN_BACKEND_TENSOR_DESCRIPTOR: SetAttribute CUDNN_ATTR_TENSOR_DATA_TYPE Failed");
262  return std::move(m_tensor);
263  }
264  status = cudnnBackendSetAttribute(m_tensor.pointer->get_backend_descriptor(),
265  CUDNN_ATTR_TENSOR_DIMENSIONS,
266  CUDNN_TYPE_INT64,
267  m_tensor.nDims,
268  m_tensor.btensor_dimA);
269  if (status != CUDNN_STATUS_SUCCESS) {
271  &m_tensor, status, "CUDNN_BACKEND_TENSOR_DESCRIPTOR: SetAttribute CUDNN_ATTR_TENSOR_DIMENSIONS Failed");
272  return std::move(m_tensor);
273  }
274  status = cudnnBackendSetAttribute(m_tensor.pointer->get_backend_descriptor(),
275  CUDNN_ATTR_TENSOR_STRIDES,
276  CUDNN_TYPE_INT64,
277  m_tensor.nDims,
278  m_tensor.btensor_strA);
279  if (status != CUDNN_STATUS_SUCCESS) {
281  &m_tensor, status, "CUDNN_BACKEND_TENSOR_DESCRIPTOR: SetAttribute CUDNN_ATTR_TENSOR_STRIDES Failed");
282  return std::move(m_tensor);
283  }
284  status = cudnnBackendSetAttribute(
285  m_tensor.pointer->get_backend_descriptor(), CUDNN_ATTR_TENSOR_UNIQUE_ID, CUDNN_TYPE_INT64, 1, &m_tensor.id);
286  if (status != CUDNN_STATUS_SUCCESS) {
288  &m_tensor, status, "CUDNN_BACKEND_TENSOR_DESCRIPTOR: SetAttribute CUDNN_ATTR_TENSOR_UNIQUE_ID Failed");
289  return std::move(m_tensor);
290  }
291  cudnnBackendSetAttribute(m_tensor.pointer->get_backend_descriptor(),
292  CUDNN_ATTR_TENSOR_BYTE_ALIGNMENT,
293  CUDNN_TYPE_INT64,
294  1,
295  &m_tensor.alignment);
296  if (status != CUDNN_STATUS_SUCCESS) {
298  &m_tensor,
299  status,
300  "CUDNN_BACKEND_TENSOR_DESCRIPTOR: SetAttribute CUDNN_ATTR_TENSOR_BYTE_ALIGNMENT Failed");
301  return std::move(m_tensor);
302  }
303  if (m_tensor.isVirtual) {
304  cudnnBackendSetAttribute(m_tensor.pointer->get_backend_descriptor(),
305  CUDNN_ATTR_TENSOR_IS_VIRTUAL,
306  CUDNN_TYPE_BOOLEAN,
307  1,
308  &m_tensor.isVirtual);
309  if (status != CUDNN_STATUS_SUCCESS) {
311  &m_tensor,
312  status,
313  "CUDNN_BACKEND_TENSOR_DESCRIPTOR: SetAttribute CUDNN_ATTR_TENSOR_BYTE_ALIGNMENT Failed");
314  return std::move(m_tensor);
315  }
316  }
317  if (m_tensor.isByValue) {
318  cudnnBackendSetAttribute(m_tensor.pointer->get_backend_descriptor(),
319  CUDNN_ATTR_TENSOR_IS_BY_VALUE,
320  CUDNN_TYPE_BOOLEAN,
321  1,
322  &m_tensor.isByValue);
323  if (status != CUDNN_STATUS_SUCCESS) {
325  &m_tensor,
326  status,
327  "CUDNN_BACKEND_TENSOR_DESCRIPTOR: SetAttribute CUDNN_ATTR_TENSOR_IS_BY_VALUE Failed");
328  return std::move(m_tensor);
329  }
330  }
331 
332  if (m_tensor.vectorCount > 1) {
333  cudnnBackendSetAttribute(m_tensor.pointer->get_backend_descriptor(),
334  CUDNN_ATTR_TENSOR_VECTOR_COUNT,
335  CUDNN_TYPE_INT64,
336  1,
337  &m_tensor.vectorCount);
338  if (status != CUDNN_STATUS_SUCCESS) {
340  &m_tensor,
341  status,
342  "CUDNN_BACKEND_TENSOR_DESCRIPTOR: SetAttribute CUDNN_ATTR_TENSOR_VECTOR_COUNT Failed");
343  return std::move(m_tensor);
344  }
345  }
346  if (m_tensor.vectorDimension >= 0) {
347  cudnnBackendSetAttribute(m_tensor.pointer->get_backend_descriptor(),
348  CUDNN_ATTR_TENSOR_VECTORIZED_DIMENSION,
349  CUDNN_TYPE_INT64,
350  1,
351  &m_tensor.vectorDimension);
352  if (status != CUDNN_STATUS_SUCCESS) {
354  &m_tensor,
355  status,
356  "CUDNN_BACKEND_TENSOR_DESCRIPTOR: SetAttribute CUDNN_ATTR_TENSOR_VECTORIZED_DIMENSION Failed");
357  return std::move(m_tensor);
358  }
359  }
360 
361  // Set the reorder_type
362 #if (CUDNN_VERSION >= 8300)
363  if (m_tensor.reorder_type != CUDNN_TENSOR_REORDERING_NONE) {
364  cudnnBackendSetAttribute(m_tensor.pointer->get_backend_descriptor(),
365  CUDNN_ATTR_TENSOR_REORDERING_MODE,
366  CUDNN_TYPE_TENSOR_REORDERING_MODE,
367  1,
368  &m_tensor.reorder_type);
369  if (status != CUDNN_STATUS_SUCCESS) {
371  &m_tensor,
372  status,
373  "CUDNN_BACKEND_TENSOR_DESCRIPTOR: SetAttribute CUDNN_ATTR_TENSOR_REORDERING_MODE Failed");
374  return std::move(m_tensor);
375  }
376  }
377 #endif
378  // Finalizing the descriptor
379  status = cudnnBackendFinalize(m_tensor.pointer->get_backend_descriptor());
380  if (status != CUDNN_STATUS_SUCCESS) {
381  set_error_and_throw_exception(&m_tensor, status, "CUDNN_BACKEND_TENSOR_DESCRIPTOR cudnnFinalize failed");
382  return std::move(m_tensor);
383  }
384  getLogger() << "[cudnn_frontend] " << m_tensor << std::endl;
385  return std::move(m_tensor);
386  }
387 
388  explicit TensorBuilder_v8() = default;
389  ~TensorBuilder_v8() = default;
390  TensorBuilder_v8(TensorBuilder_v8 &&) = delete;
391  TensorBuilder_v8(TensorBuilder_v8 const &) = delete;
393  operator=(TensorBuilder_v8 const &) = delete;
394 
395  private:
397 };
398 
399 }
bool isByValue
Whether it is an intermediate tensor of an op graph.
auto setVirtual(bool virtual_=true) -> TensorBuilder_v8 &
Set Alignment of the tensor.
ConditionalStreamer & getLogger()
static void set_error_and_throw_exception(BackendDescriptor const *desc, cudnnStatus_t status, const char *message)
NLOHMANN_BASIC_JSON_TPL_DECLARATION std::string to_string(const NLOHMANN_BASIC_JSON_TPL &j)
user-defined to_string function for JSON values
Definition: json.hpp:25855
int64_t btensor_strA[CUDNN_DIM_MAX+1]
n, g, c, d, h, w
Tensor_v8 & operator=(Tensor_v8 &&)=default
auto setByValue(bool isByValue_=true) -> TensorBuilder_v8 &
int64_t alignment
Unique id of the tensor.
bool isVirtual
What is the vectorization count (4 or 32)
auto setAlignment(int64_t alignment_) -> TensorBuilder_v8 &
Set Alignment of the tensor.
int64_t const * getDimArray() const
int64_t vectorCount
Which dimension of the tensor is vectorized (Generally the c dim)
int64_t const * getStrideArray() const
int64_t nDims
Certain engine config expect minimum alignment of 16B.
int64_t btensor_dimA[CUDNN_DIM_MAX+1]
Datatype of the elements.
std::string describe() const override
Return a string describing the backend Descriptor.
auto setVectorCountAndDimension(int64_t vectorCount_, int64_t vectorDimension_) -> TensorBuilder_v8 &
auto setStrides(int64_t ndim, int64_t const *strides) -> TensorBuilder_v8 &
Set Strides of the tensor.
static std::string to_string(cudnnDataType_t type)
int64_t vectorDimension
Number of Dimensions of the tensor.
auto setId(int64_t id_) -> TensorBuilder_v8 &
Set Unique Id of the tensor.
auto setDim(int64_t ndim, int64_t const *dim) -> TensorBuilder_v8 &
Set Dimensions of the tensor.
auto setDataType(cudnnDataType_t data_type_) -> TensorBuilder_v8 &
Set Datatype for the Tensor_v8.
cudnnStatus_t status
Shared pointer of the OpaqueBackendPointer.