Loading...
Searching...
No Matches
VirtualArray.h
Go to the documentation of this file.
1// BSD 3-Clause License; see https://github.com/scikit-hep/awkward-1.0/blob/main/LICENSE
2
3#ifndef AWKWARD_VIRTUALARRAY_H_
4#define AWKWARD_VIRTUALARRAY_H_
5
6#include <string>
7#include <memory>
8#include <vector>
9
10#include "awkward/common.h"
11#include "awkward/Slice.h"
12#include "awkward/Content.h"
15
16namespace awkward {
21 public:
23 VirtualForm(bool has_identities,
24 const util::Parameters& parameters,
25 const FormKey& form_key,
26 const FormPtr& form,
27 bool has_length);
28
29 bool
30 has_form() const;
31
32 const FormPtr
33 form() const;
34
35 bool
36 has_length() const;
37
38 const TypePtr
39 type(const util::TypeStrs& typestrs) const override;
40
41 void
42 tojson_part(ToJson& builder, bool verbose) const override;
43
44 const FormPtr
45 shallow_copy() const override;
46
47 const FormPtr
48 with_form_key(const FormKey& form_key) const override;
49
50 const std::string
51 purelist_parameter(const std::string& key) const override;
52
53 bool
54 purelist_isregular() const override;
55
56 int64_t
57 purelist_depth() const override;
58
59 bool
60 dimension_optiontype() const override;
61
62 const std::pair<int64_t, int64_t>
63 minmax_depth() const override;
64
65 const std::pair<bool, int64_t>
66 branch_depth() const override;
67
68 int64_t
69 numfields() const override;
70
71 int64_t
72 fieldindex(const std::string& key) const override;
73
74 const std::string
75 key(int64_t fieldindex) const override;
76
77 bool
78 haskey(const std::string& key) const override;
79
80 const std::vector<std::string>
81 keys() const override;
82
83 bool
84 istuple() const override;
85
86 bool
87 equal(const FormPtr& other,
88 bool check_identities,
89 bool check_parameters,
90 bool check_form_key,
91 bool compatibility_check) const override;
92
93 const FormPtr
94 getitem_field(const std::string& key) const override;
95
96 const FormPtr
97 getitem_fields(const std::vector<std::string>& keys) const override;
98
99 private:
100 const FormPtr form_;
101 bool has_length_;
102 };
103
110 public:
123 VirtualArray(const IdentitiesPtr& identities,
124 const util::Parameters& parameters,
125 const ArrayGeneratorPtr& generator,
126 const ArrayCachePtr& cache,
127 const std::string& cache_key,
128 const kernel::lib ptr_lib = kernel::lib::cpu);
129
132 VirtualArray(const IdentitiesPtr& identities,
133 const util::Parameters& parameters,
134 const ArrayGeneratorPtr& generator,
135 const ArrayCachePtr& cache,
136 const kernel::lib ptr_lib = kernel::lib::cpu);
137
141 generator() const;
142
145 const ArrayCachePtr
146 cache() const;
147
148 const kernel::lib
149 ptr_lib() const;
150
155 const ContentPtr
156 peek_array() const;
157
161 const ContentPtr
162 array() const;
163
165 const std::string
166 cache_key() const;
167
169 const std::string
170 classname() const override;
171
172 void
173 setidentities() override;
174
175 void
176 setidentities(const IdentitiesPtr& identities) override;
177
178 const TypePtr
179 type(const util::TypeStrs& typestrs) const override;
180
181 const FormPtr
182 form(bool materialize) const override;
183
185 kernels() const override;
186
187 void
188 caches(std::vector<ArrayCachePtr>& out) const override;
189
190 const std::string
191 tostring_part(const std::string& indent,
192 const std::string& pre,
193 const std::string& post) const override;
194
195 void
196 tojson_part(ToJson& builder, bool include_beginendlist) const override;
197
202 void
203 nbytes_part(std::map<size_t, int64_t>& largest) const override;
204
205 int64_t
206 length() const override;
207
208 const ContentPtr
209 shallow_copy() const override;
210
211 const ContentPtr
212 deep_copy(bool copyarrays,
213 bool copyindexes,
214 bool copyidentities) const override;
215
216 void
217 check_for_iteration() const override;
218
219 const ContentPtr
220 getitem_nothing() const override;
221
222 const ContentPtr
223 getitem_at(int64_t at) const override;
224
225 const ContentPtr
226 getitem_at_nowrap(int64_t at) const override;
227
228 const ContentPtr
229 getitem_range(int64_t start, int64_t stop) const override;
230
231 const ContentPtr
232 getitem_range_nowrap(int64_t start, int64_t stop) const override;
233
234 const ContentPtr
235 getitem_field(const std::string& key) const override;
236
237 const ContentPtr
238 getitem_field(const std::string& key,
239 const Slice& only_fields) const override;
240
241 const ContentPtr
242 getitem_fields(const std::vector<std::string>& keys) const override;
243
244 const ContentPtr
245 getitem_fields(const std::vector<std::string>& keys,
246 const Slice& only_fields) const override;
247
248 const ContentPtr
250 const Slice& tail,
251 const Index64& advanced) const override;
252
253 const ContentPtr
254 getitem_next_jagged(const Index64& slicestarts,
255 const Index64& slicestops,
256 const SliceItemPtr& slicecontent,
257 const Slice& tail) const override;
258
259 const ContentPtr
260 carry(const Index64& carry, bool allow_lazy) const override;
261
262 int64_t
263 purelist_depth() const override;
264
265 const std::pair<int64_t, int64_t>
266 minmax_depth() const override;
267
268 const std::pair<bool, int64_t>
269 branch_depth() const override;
270
271 int64_t
272 numfields() const override;
273
274 int64_t
275 fieldindex(const std::string& key) const override;
276
277 const std::string
278 key(int64_t fieldindex) const override;
279
280 bool
281 haskey(const std::string& key) const override;
282
283 const std::vector<std::string>
284 keys() const override;
285
286 bool
287 istuple() const override;
288
289 // operations
290 const std::string
291 validityerror(const std::string& path) const override;
292
293 const ContentPtr
294 shallow_simplify() const override;
295
296 const ContentPtr
297 num(int64_t axis, int64_t depth) const override;
298
299 const std::pair<Index64, ContentPtr>
300 offsets_and_flattened(int64_t axis, int64_t depth) const override;
301
302 bool
303 mergeable(const ContentPtr& other, bool mergebool) const override;
304
305 bool
306 referentially_equal(const ContentPtr& other) const override;
307
308 const ContentPtr
309 mergemany(const ContentPtrVec& others) const override;
310
311 const SliceItemPtr
312 asslice() const override;
313
314 const ContentPtr
315 fillna(const ContentPtr& value) const override;
316
317 const ContentPtr
318 rpad(int64_t target, int64_t axis, int64_t depth) const override;
319
320 const ContentPtr
321 rpad_and_clip(int64_t target,
322 int64_t axis,
323 int64_t depth) const override;
324
325 const ContentPtr
326 reduce_next(const Reducer& reducer,
327 int64_t negaxis,
328 const Index64& starts,
329 const Index64& shifts,
330 const Index64& parents,
331 int64_t outlength,
332 bool mask,
333 bool keepdims) const override;
334
335 const ContentPtr
336 sort_next(int64_t negaxis,
337 const Index64& starts,
338 const Index64& parents,
339 int64_t outlength,
340 bool ascending,
341 bool stable) const override;
342
343 const ContentPtr
344 argsort_next(int64_t negaxis,
345 const Index64& starts,
346 const Index64& shifts,
347 const Index64& parents,
348 int64_t outlength,
349 bool ascending,
350 bool stable) const override;
351
352
353 const ContentPtr
354 localindex(int64_t axis, int64_t depth) const override;
355
356 const ContentPtr
357 combinations(int64_t n,
358 bool replacement,
359 const util::RecordLookupPtr& recordlookup,
360 const util::Parameters& parameters,
361 int64_t axis,
362 int64_t depth) const override;
363
364 const ContentPtr
365 getitem(const Slice& where) const override;
366
367 const ContentPtr
369 const Slice& tail,
370 const Index64& advanced) const override;
371
372 const ContentPtr
374 const Slice& tail,
375 const Index64& advanced) const override;
376
377 const ContentPtr
379 const Slice& tail,
380 const Index64& advanced) const override;
381
382 const ContentPtr
384 const Slice& tail,
385 const Index64& advanced) const override;
386
387 const ContentPtr
389 const Slice& tail,
390 const Index64& advanced) const override;
391
392 const ContentPtr
394 const Slice& tail,
395 const Index64& advanced) const override;
396
397 const ContentPtr
398 getitem_next_jagged(const Index64& slicestarts,
399 const Index64& slicestops,
400 const SliceArray64& slicecontent,
401 const Slice& tail) const override;
402
403 const ContentPtr
404 getitem_next_jagged(const Index64& slicestarts,
405 const Index64& slicestops,
406 const SliceMissing64& slicecontent,
407 const Slice& tail) const override;
408
409 const ContentPtr
410 getitem_next_jagged(const Index64& slicestarts,
411 const Index64& slicestops,
412 const SliceJagged64& slicecontent,
413 const Slice& tail) const override;
414
415 const ContentPtr
416 copy_to(kernel::lib ptr_lib) const override;
417
418 const ContentPtr
419 numbers_to_type(const std::string& name) const override;
420
422 bool
423 is_unique() const override;
424
426 const ContentPtr
427 unique() const override;
428
430 bool
431 is_subrange_equal(const Index64& start, const Index64& stop) const override;
432
433 private:
435 const util::Parameters
436 forward_parameters() const;
437
440 void
441 set_cache_depths_from(const VirtualArray* original);
442
445 void
446 set_cache_depths_from(const FormPtr& form);
447
448 void
449 add_to_cache_depths(int64_t delta);
450
452 const ArrayGeneratorPtr generator_;
454 const ArrayCachePtr cache_;
456 const std::string cache_key_;
458 const kernel::lib ptr_lib_;
459
462 std::vector<int64_t> cache_depths_;
463 };
464
465}
466
467#endif // AWKWARD_VIRTUALARRAY_H_
Abstract superclass of all array node types (flat hierarchy). Any Content can be nested within any ot...
Definition: Content.h:276
Abstract superclass of all array node forms, which expresses the nesting structure without any large ...
Definition: Content.h:39
A contiguous, one-dimensional array of integers used to represent data structures,...
Definition: Index.h:82
Abstract class for all reducer algorithms.
Definition: Reducer.h:20
Represents an array of integers in a slice (possibly converted from an array of booleans).
Definition: Slice.h:229
Represents an integer in a tuple of slices passed to __getitem__ in Python.
Definition: Slice.h:58
Represents a single string in a slice tuple, indicating that a RecordArray should be replaced by one ...
Definition: Slice.h:340
Definition: Slice.h:384
Represents an array of nested lists, where the content may be SliceArrayOf, SliceMissingOf,...
Definition: Slice.h:515
Represents a SliceArrayOf, SliceMissingOf, or SliceJaggedOf with missing values: None (no equivalent ...
Definition: Slice.h:435
Represents a Python slice object (usual syntax: array[start:stop:step]).
Definition: Slice.h:93
A sequence of SliceItem objects representing a tuple passed to Python's __getitem__.
Definition: Slice.h:585
Definition: json.h:21
Represents an array that can be generated on demand.
Definition: VirtualArray.h:109
int64_t purelist_depth() const override
The list-depth of this array, not counting any contained within a RecordArray.
const ContentPtr argsort_next(int64_t negaxis, const Index64 &starts, const Index64 &shifts, const Index64 &parents, int64_t outlength, bool ascending, bool stable) const override
This array sorted indices.
VirtualArray(const IdentitiesPtr &identities, const util::Parameters &parameters, const ArrayGeneratorPtr &generator, const ArrayCachePtr &cache, const std::string &cache_key, const kernel::lib ptr_lib=kernel::lib::cpu)
Creates a VirtualArray from a full set of parameters.
void caches(std::vector< ArrayCachePtr > &out) const override
Accumulates all the unique #ArrayCache objects from nested VirtualArray nodes. (Uniqueness is determi...
const ContentPtr getitem_fields(const std::vector< std::string > &keys) const override
This array with the first nested RecordArray replaced by a RecordArray of a given subset of keys.
const ContentPtr getitem_next_jagged(const Index64 &slicestarts, const Index64 &slicestops, const SliceArray64 &slicecontent, const Slice &tail) const override
Internal function that propagates a jagged array (array with irregular-length dimensions) slice throu...
const ContentPtr getitem_at(int64_t at) const override
Returns the element at a given position in the array, handling negative indexing and bounds-checking ...
void setidentities() override
Assign a surrogate index of Identities to this array (in-place).
const std::string cache_key() const
The key this VirtualArray will use when filling a cache.
const ContentPtr getitem_field(const std::string &key, const Slice &only_fields) const override
const ContentPtr getitem_next(const SliceArray64 &array, const Slice &tail, const Index64 &advanced) const override
Internal function that propagates a generic getitem request through one axis (including advanced inde...
const ContentPtr unique() const override
Returns an array where all components are unique.
const ContentPtr sort_next(int64_t negaxis, const Index64 &starts, const Index64 &parents, int64_t outlength, bool ascending, bool stable) const override
This array sorted.
void tojson_part(ToJson &builder, bool include_beginendlist) const override
Internal function to produce a JSON representation one node at a time.
const ContentPtr getitem_fields(const std::vector< std::string > &keys, const Slice &only_fields) const override
int64_t length() const override
The number of elements in the array.
const ContentPtr deep_copy(bool copyarrays, bool copyindexes, bool copyidentities) const override
Copies this node and all nodes hierarchically nested within it, optionally copying the associated arr...
const std::string validityerror(const std::string &path) const override
Returns an error message if this array is invalid; otherwise, returns an empty string.
const ContentPtr getitem_next(const SliceAt &at, const Slice &tail, const Index64 &advanced) const override
Internal function that propagates a generic getitem request through one axis (including advanced inde...
bool istuple() const override
Returns true if the outermost RecordArray is a tuple.
const kernel::lib ptr_lib() const
const ContentPtr getitem_next(const SliceField &field, const Slice &tail, const Index64 &advanced) const override
Internal function that propagates a generic getitem request through one axis (including advanced inde...
const ContentPtr array() const
Ensures that the array is generated and returns it.
const ContentPtr getitem_nothing() const override
Internal function to get an empty slice (with the correct type).
kernel::lib kernels() const override
Returns the kernel library enum for all nested ptr_lib within the array's tree structure....
const ContentPtr carry(const Index64 &carry, bool allow_lazy) const override
Returns an array of the same type with elements filtered, rearranged, and possibly duplicated by the ...
const ContentPtr combinations(int64_t n, bool replacement, const util::RecordLookupPtr &recordlookup, const util::Parameters &parameters, int64_t axis, int64_t depth) const override
Tuples or records of all n-tuple combinations of list items at some axis depth.
void setidentities(const IdentitiesPtr &identities) override
Assign a specified set of Identities to this array (in-place).
const ContentPtr localindex(int64_t axis, int64_t depth) const override
A (possibly nested) array of integers indicating the positions of elements within each nested list.
const ContentPtr getitem_range_nowrap(int64_t start, int64_t stop) const override
Subinterval of this array, without handling negative indexing or bounds-checking.
const std::pair< Index64, ContentPtr > offsets_and_flattened(int64_t axis, int64_t depth) const override
Returns (a) an offsets Index and (b) a flattened version of the array at some axis depth.
const ContentPtr getitem(const Slice &where) const override
Entry point for general slicing: Slice represents a tuple of SliceItem nodes applying to each level o...
int64_t fieldindex(const std::string &key) const override
The position of a tuple or record key name if this array contains a RecordArray.
void check_for_iteration() const override
Performs up-front validity checks on an array so that they don't have to be checked in getitem_at_now...
const std::vector< std::string > keys() const override
A list of RecordArray keys or an empty list if this array does not contain a RecordArray.
const ContentPtr getitem_field(const std::string &key) const override
This array with the first nested RecordArray replaced by the field at key.
VirtualArray(const IdentitiesPtr &identities, const util::Parameters &parameters, const ArrayGeneratorPtr &generator, const ArrayCachePtr &cache, const kernel::lib ptr_lib=kernel::lib::cpu)
Creates a VirtualArray with an automatically assigned cache_key (unique per process).
const std::pair< bool, int64_t > branch_depth() const override
Returns (a) whether the list-depth of this array "branches," or differs when followed through differe...
const ContentPtr rpad(int64_t target, int64_t axis, int64_t depth) const override
If axis = 0, returns a view of this array padded on the right with None values to have a minimum leng...
const ContentPtr getitem_next(const SliceItemPtr &head, const Slice &tail, const Index64 &advanced) const override
Internal function that propagates a generic getitem request through one axis (including advanced inde...
const ContentPtr getitem_at_nowrap(int64_t at) const override
Returns the element at a given position in the array, without handling negative indexing or bounds-ch...
void nbytes_part(std::map< size_t, int64_t > &largest) const override
Internal function used to calculate nbytes.
const ContentPtr mergemany(const ContentPtrVec &others) const override
Returns an array with this and the others concatenated (in order, this first, others last).
const std::string key(int64_t fieldindex) const override
The record name associated with a given field index or the tuple index as a string (e....
const ArrayGeneratorPtr generator() const
Function that materializes the array and possibly checks it against an expected Form.
const ContentPtr shallow_copy() const override
Copies this node without copying any nodes hierarchically nested within it or any array/index/identit...
const std::pair< int64_t, int64_t > minmax_depth() const override
Returns (a) the minimum list-depth and (b) the maximum list-depth of the array, which can differ if t...
bool is_unique() const override
Returns 'true' if all components of the array are unique.
const FormPtr form(bool materialize) const override
Low-level Form describing all the features of this array except the actual data buffers (Index,...
const SliceItemPtr asslice() const override
Converts this array into a SliceItem that can be used in getitem.
const std::string tostring_part(const std::string &indent, const std::string &pre, const std::string &post) const override
Internal function to build an output string for tostring.
const ContentPtr rpad_and_clip(int64_t target, int64_t axis, int64_t depth) const override
If axis = 0, returns a view of this array padded on the right.
const ContentPtr getitem_next(const SliceJagged64 &jagged, const Slice &tail, const Index64 &advanced) const override
Internal function that propagates a generic getitem request through one axis (including advanced inde...
const ContentPtr getitem_range(int64_t start, int64_t stop) const override
Subinterval of this array, handling negative indexing and bounds-checking like Python.
bool is_subrange_equal(const Index64 &start, const Index64 &stop) const override
Returns 'true' if subranges are equal.
const ContentPtr getitem_next_jagged(const Index64 &slicestarts, const Index64 &slicestops, const SliceItemPtr &slicecontent, const Slice &tail) const override
Internal function that propagates a jagged array (array with irregular-length dimensions) slice throu...
const ContentPtr fillna(const ContentPtr &value) const override
Returns this array with None values replaced by a given value.
int64_t numfields() const override
The number of fields in the first nested tuple or records or -1 if this array does not contain a Reco...
bool referentially_equal(const ContentPtr &other) const override
Returns true if this array has all the same buffers and parameters as other; false otherwise.
const TypePtr type(const util::TypeStrs &typestrs) const override
High-level Type describing this array.
const ContentPtr num(int64_t axis, int64_t depth) const override
The length of this array (as a NumpyArray scalar) if axis = 0 or the lengths of subarrays (as an arra...
const ContentPtr getitem_next(const SliceFields &fields, const Slice &tail, const Index64 &advanced) const override
Internal function that propagates a generic getitem request through one axis (including advanced inde...
const ArrayCachePtr cache() const
Temporary storage for materialized arrays to avoid calling the generator more than necessary....
bool mergeable(const ContentPtr &other, bool mergebool) const override
Returns true if this array can be merged with the other; false otherwise.
bool haskey(const std::string &key) const override
Returns true if the array contains a RecordArray with the specified key; false otherwise.
const ContentPtr getitem_next_jagged(const Index64 &slicestarts, const Index64 &slicestops, const SliceJagged64 &slicecontent, const Slice &tail) const override
Internal function that propagates a jagged array (array with irregular-length dimensions) slice throu...
const ContentPtr peek_array() const
Returns the array if it exists in the cache; nullptr otherwise.
const ContentPtr getitem_next_jagged(const Index64 &slicestarts, const Index64 &slicestops, const SliceMissing64 &slicecontent, const Slice &tail) const override
Internal function that propagates a jagged array (array with irregular-length dimensions) slice throu...
const ContentPtr getitem_next(const SliceRange &range, const Slice &tail, const Index64 &advanced) const override
Internal function that propagates a generic getitem request through one axis (including advanced inde...
const ContentPtr shallow_simplify() const override
Returns an equivalent array simplified at one level only using simplify_optiontype if an option-type ...
const ContentPtr reduce_next(const Reducer &reducer, int64_t negaxis, const Index64 &starts, const Index64 &shifts, const Index64 &parents, int64_t outlength, bool mask, bool keepdims) const override
This array with one axis removed by applying a Reducer (e.g. "sum", "max", "any", "all).
const ContentPtr numbers_to_type(const std::string &name) const override
Change the leaf types to 'totype'.
const std::string classname() const override
User-friendly name of this class: "VirtualArray".
const ContentPtr copy_to(kernel::lib ptr_lib) const override
Recursively copies components of the array from main memory to a GPU (if ptr_lib == kernel::lib::cuda...
Form describing VirtualArray.
Definition: VirtualArray.h:20
int64_t purelist_depth() const override
The list-depth of this array, not counting any contained within a RecordForm.
const FormPtr with_form_key(const FormKey &form_key) const override
Copies this node, adding or replacing a form_key.
VirtualForm(bool has_identities, const util::Parameters &parameters, const FormKey &form_key, const FormPtr &form, bool has_length)
Creates a VirtualForm. See VirtualArray for documentation.
bool has_length() const
bool equal(const FormPtr &other, bool check_identities, bool check_parameters, bool check_form_key, bool compatibility_check) const override
Returns true if this Form is equal to the other Form; false otherwise.
bool istuple() const override
Returns true if the outermost RecordArray is a tuple.
const FormPtr shallow_copy() const override
Copies this node without copying any nodes hierarchically nested within it.
void tojson_part(ToJson &builder, bool verbose) const override
Internal function to produce a JSON representation one node at a time.
const FormPtr getitem_fields(const std::vector< std::string > &keys) const override
Returns the Form that would result from a fields-slice.
int64_t fieldindex(const std::string &key) const override
The position of a tuple or record key name if this array contains a RecordForm.
const std::vector< std::string > keys() const override
A list of RecordArray keys or an empty list if this array does not contain a RecordArray.
const std::pair< bool, int64_t > branch_depth() const override
Returns (a) whether the list-depth of this array "branches," or differs when followed through differe...
const std::string key(int64_t fieldindex) const override
The record name associated with a given field index or the tuple index as a string (e....
const std::pair< int64_t, int64_t > minmax_depth() const override
Returns (a) the minimum list-depth and (b) the maximum list-depth of the array, which can differ if t...
const FormPtr getitem_field(const std::string &key) const override
Returns the Form that would result from a field-slice.
bool has_form() const
int64_t numfields() const override
The number of fields in the first nested tuple or records or -1 if this array does not contain a Reco...
const TypePtr type(const util::TypeStrs &typestrs) const override
High-level Type describing this Form.
const FormPtr form() const
const std::string purelist_parameter(const std::string &key) const override
The parameter associated with key at the first level that has a non-null value, descending only as de...
bool haskey(const std::string &key) const override
Returns true if the array contains a RecordForm with the specified key; false otherwise.
bool purelist_isregular() const override
Returns true if all nested lists down to the first RecordForm are RegularForm nodes; false otherwise.
bool dimension_optiontype() const override
Returns true if this dimension has option-type; false otherwise.
#define LIBAWKWARD_EXPORT_SYMBOL
Definition: common.h:45
lib
Definition: kernel-dispatch.h:20
std::map< std::string, std::string > Parameters
Definition: util.h:165
std::shared_ptr< RecordLookup > RecordLookupPtr
Definition: util.h:130
std::map< std::string, std::string > TypeStrs
Definition: util.h:215
Definition: BitMaskedArray.h:15
std::shared_ptr< std::string > FormKey
Definition: Content.h:19
std::shared_ptr< ArrayGenerator > ArrayGeneratorPtr
Definition: ArrayGenerator.h:14
std::vector< std::shared_ptr< Content > > ContentPtrVec
Definition: Content.h:16
std::shared_ptr< SliceItem > SliceItemPtr
Definition: Slice.h:15
std::shared_ptr< Content > ContentPtr
Definition: Content.h:15
std::shared_ptr< Form > FormPtr
Definition: Content.h:18
std::shared_ptr< Type > TypePtr
Definition: Content.h:23
std::shared_ptr< ArrayCache > ArrayCachePtr
Definition: Content.h:21
std::shared_ptr< Identities > IdentitiesPtr
Definition: Identities.h:16