CUDNN Frontend API  8.3.0
cudnn_frontend_Engine.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20  * DEALINGS IN THE SOFTWARE.
21  */
22 
23 #pragma once
24 
25 #include <algorithm>
26 #include <array>
27 #include <functional>
28 #include <memory>
29 #include <sstream>
30 #include <utility>
31 #include <vector>
32 
33 #include <cudnn.h>
34 #include <cudnn_backend.h>
35 
37 #include "cudnn_frontend_utils.h"
38 
39 namespace cudnn_frontend {
40 
52 class Engine_v8 : public BackendDescriptor {
53  private:
54  Engine_v8() = default;
55  Engine_v8(Engine_v8 const &) = delete;
56  Engine_v8 &
57  operator=(Engine_v8 const &) = delete;
58 
62  class Knob {
63  public:
64  Knob(cudnnBackendKnobType_t type_, int64_t max, int64_t min, int64_t stride_)
65  : knobType(type_), maxValue(max), minValue(min), stride(stride_) {}
66 
67  std::string
68  describe() const {
69  std::stringstream ss;
70  ss << "Knob:" << knobType;
71  ss << " Min: " << minValue;
72  ss << " Max: " << maxValue;
73  ss << " Stride: " << stride;
74  return ss.str();
75  }
76 
77  void
78  setChoice(uint64_t val_) {
79  choice = val_;
80  }
81 
82  int64_t
83  getChoice() const {
84  return choice;
85  }
86 
87  cudnnBackendKnobType_t
88  getKnobType() const {
89  return knobType;
90  }
91 
92  int64_t
93  getMinValue() const {
94  return minValue;
95  }
96 
97  int64_t
98  getMaxValue() const {
99  return minValue;
100  }
101 
102  int64_t
103  getStride() const {
104  return stride;
105  }
106 
107  private:
108  cudnnBackendKnobType_t knobType = CUDNN_KNOB_TYPE_COUNTS;
109  int64_t maxValue = 0, minValue = 0, stride = 0;
110  int64_t choice = -1;
111  };
112 
114  int64_t idx = -1;
115  int64_t numKnobs = 0;
116  std::array<ManagedOpaqueDescriptor, CUDNN_KNOB_TYPE_COUNTS> bKnobs = {};
117  std::vector<Knob> knobs;
118  std::string opGraphTag;
119 
121  void
123  cudnnStatus_t status;
124  for (auto i = 0; i < numKnobs; i++) {
125  auto bKnob = bKnobs[i]->get_backend_descriptor();
126  cudnnBackendKnobType_t type;
127  int64_t maxValue, minValue, stride, elemCount;
128  status =
129  cudnnBackendGetAttribute(bKnob, CUDNN_ATTR_KNOB_INFO_TYPE, CUDNN_TYPE_KNOB_TYPE, 1, &elemCount, &type);
130  if (status != CUDNN_STATUS_SUCCESS) {
132  status,
133  "CUDNN_BACKEND_ENGINE_DESCRIPTOR: CUDNN_BACKEND_KNOB_INFO_DESCRIPTOR "
134  "GetAttribute CUDNN_ATTR_KNOB_INFO_TYPE failed");
135  }
136  status = cudnnBackendGetAttribute(
137  bKnob, CUDNN_ATTR_KNOB_INFO_MAXIMUM_VALUE, CUDNN_TYPE_INT64, 1, &elemCount, &maxValue);
138  if (status != CUDNN_STATUS_SUCCESS) {
140  status,
141  "CUDNN_BACKEND_ENGINE_DESCRIPTOR: CUDNN_BACKEND_KNOB_INFO_DESCRIPTOR "
142  "GetAttribute CUDNN_ATTR_KNOB_INFO_MAXIMUM_VALUE Failed");
143  }
144  status = cudnnBackendGetAttribute(
145  bKnob, CUDNN_ATTR_KNOB_INFO_MINIMUM_VALUE, CUDNN_TYPE_INT64, 1, &elemCount, &minValue);
146  if (status != CUDNN_STATUS_SUCCESS) {
148  status,
149  "CUDNN_BACKEND_ENGINE_DESCRIPTOR: CUDNN_BACKEND_KNOB_INFO_DESCRIPTOR "
150  "GetAttribute CUDNN_ATTR_KNOB_INFO_MINIMUM_VALUE Failed");
151  }
152  status =
153  cudnnBackendGetAttribute(bKnob, CUDNN_ATTR_KNOB_INFO_STRIDE, CUDNN_TYPE_INT64, 1, &elemCount, &stride);
154  if (status != CUDNN_STATUS_SUCCESS) {
156  status,
157  "CUDNN_BACKEND_ENGINE_DESCRIPTOR: CUDNN_BACKEND_KNOB_INFO_DESCRIPTOR "
158  "GetAttribute CUDNN_ATTR_KNOB_INFO_STRIDE Failed");
159  }
160  knobs.emplace_back(Knob(type, maxValue, minValue, stride));
161  }
162  }
163 
164  public:
165  friend class EngineBuilder_v8;
166  std::string
167  describe() const override {
168  std::stringstream ss;
169  ss << "CUDNN_BACKEND_ENGINE_DESCRIPTOR :";
170  ss << " ID: " << idx;
171  ss << " Has " << numKnobs << " knobs";
172  return ss.str();
173  }
175  : BackendDescriptor(from.get_desc(), from.get_status(), from.get_error()),
176  opGraph(from.opGraph),
177  idx(from.idx),
178  opGraphTag(from.opGraphTag) {
179  cudnnStatus_t status;
180  for (uint64_t i = 0; i < bKnobs.size(); i++) {
181  bKnobs[i] = make_shared_backend_pointer(CUDNN_BACKEND_KNOB_INFO_DESCRIPTOR);
182  if (bKnobs[i]->is_good() == false) {
183  status = bKnobs[i]->get_status();
185  this,
186  status,
187  "CUDNN_BACKEND_ENGINE_DESCRIPTOR: CUDNN_BACKEND_KNOB_INFO_DESCRIPTOR cudnnCreate Failed");
188  }
189  }
190 
191  std::array<cudnnBackendDescriptor_t, CUDNN_KNOB_TYPE_COUNTS> bKnobs_ =
192  {};
193  for (std::uint32_t i = 0; i < bKnobs.size(); i++) {
194  bKnobs_[i] = bKnobs[i]->get_backend_descriptor();
195  }
196  status = cudnnBackendGetAttribute(pointer->get_backend_descriptor(),
197  CUDNN_ATTR_ENGINE_KNOB_INFO,
198  CUDNN_TYPE_BACKEND_DESCRIPTOR,
199  CUDNN_KNOB_TYPE_COUNTS,
200  &numKnobs,
201  bKnobs_.data());
202  if (status != CUDNN_STATUS_SUCCESS) {
204  this, status, "CUDNN_BACKEND_ENGINE_DESCRIPTOR: GetAttribute CUDNN_ATTR_ENGINE_KNOB_INFO Query Failed");
205  }
206  buildKnobs();
207  }
208  Engine_v8 &
209  operator=(Engine_v8 &&) = default;
210  ~Engine_v8() = default;
211 
212  std::string const &
213  getTag() const {
214  return opGraphTag;
215  }
216 
218  std::vector<Knob> &
220  return knobs;
221  }
222 
224  std::vector<Knob> const &
226  return knobs;
227  }
228 
229  bool knobs_set() const {
230  bool is_knob_set = false;
231  for (auto i = 0; i < numKnobs; i++) {
232  if(knobs[i].getChoice() != -1) {
233  is_knob_set = true;
234  break;
235  }
236  }
237  return is_knob_set;
238  }
239 };
240 
245  public:
250  auto
253  m_engine.opGraph = opGraph_.get_desc();
254  m_engine.opGraphTag = opGraph_.getTag();
255  return *this;
256  }
257 
259  auto
261  m_engine.opGraph = desc_;
262  return *this;
263  }
265  auto
267  m_engine.idx = idx_;
268  return *this;
269  }
272  Engine_v8 &&
275  build() {
276  if (m_engine.idx < 0) {
278  &m_engine,
279  CUDNN_STATUS_BAD_PARAM,
280  "CUDNN_BACKEND_ENGINE_DESCRIPTOR: Check and Set the CUDNN_ATTR_ENGINE_GLOBAL_INDEX to valid value");
281  return std::move(m_engine);
282  }
283  if (m_engine.opGraph == nullptr) {
285  &m_engine,
286  CUDNN_STATUS_BAD_PARAM,
287  "CUDNN_BACKEND_ENGINE_DESCRIPTOR: Check and Set CUDNN_ATTR_ENGINE_OPERATION_GRAPH to valid value");
288  return std::move(m_engine);
289  }
290 
291  // Create a descriptor. Memory allocation happens here.
292  auto status = m_engine.initialize_managed_backend_pointer(CUDNN_BACKEND_ENGINE_DESCRIPTOR);
293  if (status != CUDNN_STATUS_SUCCESS) {
295  &m_engine, status, "CUDNN_BACKEND_ENGINE_DESCRIPTOR: cudnnCreate Descriptor Failed");
296  return std::move(m_engine);
297  }
298 
299  status = cudnnBackendSetAttribute(m_engine.pointer->get_backend_descriptor(),
300  CUDNN_ATTR_ENGINE_OPERATION_GRAPH,
301  CUDNN_TYPE_BACKEND_DESCRIPTOR,
302  1,
303  &(m_engine.opGraph->get_backend_descriptor()));
304  if (status != CUDNN_STATUS_SUCCESS) {
306  &m_engine,
307  status,
308  "CUDNN_BACKEND_ENGINE_DESCRIPTOR: SetAttribute CUDNN_ATTR_ENGINE_OPERATION_GRAPH Failed");
309  return std::move(m_engine);
310  }
311 
312  status = cudnnBackendSetAttribute(m_engine.pointer->get_backend_descriptor(),
313  CUDNN_ATTR_ENGINE_GLOBAL_INDEX,
314  CUDNN_TYPE_INT64,
315  1,
316  &m_engine.idx);
317  if (status != CUDNN_STATUS_SUCCESS) {
319  &m_engine,
320  status,
321  "CUDNN_BACKEND_ENGINE_DESCRIPTOR: SetAttribute CUDNN_ATTR_ENGINE_GLOBAL_INDEX Failed");
322  return std::move(m_engine);
323  }
324 
325  // Finalizing the descriptor
326  status = cudnnBackendFinalize(m_engine.pointer->get_backend_descriptor());
327  if (status != CUDNN_STATUS_SUCCESS) {
328  set_error_and_throw_exception(&m_engine, status, "CUDNN_BACKEND_ENGINE_DESCRIPTOR: cudnnFinalize Failed");
329  return std::move(m_engine);
330  }
331 
332  getLogger() << "[cudnn_frontend] " << m_engine << std::endl;
333  return std::move(m_engine);
334  }
335 
336  explicit EngineBuilder_v8() = default;
337  ~EngineBuilder_v8() = default;
338  EngineBuilder_v8(EngineBuilder_v8 &&) = delete;
339  EngineBuilder_v8(EngineBuilder_v8 const &) = delete;
341  operator=(EngineBuilder_v8 const &) = delete;
342 
343  private:
345 };
346 }
int64_t choice
Choice set by the user.
Engine_v8 & operator=(Engine_v8 const &)=delete
auto setOperationGraph(ManagedOpaqueDescriptor desc_) -> EngineBuilder_v8 &
Set operationGraph for the engine.
ConditionalStreamer & getLogger()
int64_t stride
min, max and stride of the knob value
static void set_error_and_throw_exception(BackendDescriptor const *desc, cudnnStatus_t status, const char *message)
cudnnBackendKnobType_t getKnobType() const
auto setGlobalEngineIdx(int64_t idx_) -> EngineBuilder_v8 &
Set engine index for the engine.
std::string const & getTag() const
int64_t idx
Global Index of the engine for the given operationGraph.
static ManagedOpaqueDescriptor make_shared_backend_pointer(cudnnBackendDescriptorType_t type)
void buildKnobs()
Called from the constructor builds the internal knobs vector.
ManagedOpaqueDescriptor opGraph
ManagedOpaqueDescriptor get_desc() const
Returns a copy of underlying managed descriptor.
std::array< ManagedOpaqueDescriptor, CUDNN_KNOB_TYPE_COUNTS > bKnobs
Opaque pointer to the backend knobs.
auto setOperationGraph(OperationGraph_v8 const &opGraph_) -> EngineBuilder_v8 &
Set operationGraph for the engine.
cudnnStatus_t get_status() const
Current status of the descriptor.
std::vector< Knob > const & getFinalizedKnobs() const
Returns a final vector of knobs. Used in EngineConfigBuilder.
std::vector< Knob > & getSupportedKnobs()
Returns a vector of knobs to the user for modification.
std::shared_ptr< OpaqueBackendPointer > ManagedOpaqueDescriptor
std::string describe() const override
Return a string describing the backend Descriptor.
const char * get_error() const
Diagonistic error message if any.
Knob(cudnnBackendKnobType_t type_, int64_t max, int64_t min, int64_t stride_)
cudnnStatus_t status
Shared pointer of the OpaqueBackendPointer.
int64_t numKnobs
Count of the backend knobs in the engine.