001/*- 002 ******************************************************************************* 003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd. 004 * All rights reserved. This program and the accompanying materials 005 * are made available under the terms of the Eclipse Public License v1.0 006 * which accompanies this distribution, and is available at 007 * http://www.eclipse.org/legal/epl-v10.html 008 * 009 * Contributors: 010 * Peter Chang - initial API and implementation and/or initial documentation 011 *******************************************************************************/ 012 013package org.eclipse.january.dataset; 014 015import java.io.Serializable; 016import java.lang.annotation.Annotation; 017import java.lang.reflect.Array; 018import java.lang.reflect.Field; 019import java.util.ArrayList; 020import java.util.Arrays; 021import java.util.Collection; 022import java.util.concurrent.ConcurrentMap; 023import java.util.concurrent.ConcurrentHashMap; 024import java.util.List; 025import java.util.Map; 026 027import org.eclipse.january.DatasetException; 028import org.eclipse.january.MetadataException; 029import org.eclipse.january.metadata.Dirtiable; 030import org.eclipse.january.metadata.ErrorMetadata; 031import org.eclipse.january.metadata.IMetadata; 032import org.eclipse.january.metadata.MetadataFactory; 033import org.eclipse.january.metadata.MetadataType; 034import org.eclipse.january.metadata.Reshapeable; 035import org.eclipse.january.metadata.Sliceable; 036import org.eclipse.january.metadata.Transposable; 037import org.slf4j.Logger; 038import org.slf4j.LoggerFactory; 039 040/** 041 * Common base for both lazy and normal dataset implementations 042 */ 043public abstract class LazyDatasetBase implements ILazyDataset, Serializable { 044 045 private static final long serialVersionUID = 767926846438976050L; 046 047 private static final Logger logger = LoggerFactory.getLogger(LazyDatasetBase.class); 048 049 transient private boolean dirty = true; // indicate dirty state of metadata 050 protected String name = ""; 051 052 /** 053 * The shape or dimensions of the dataset 054 */ 055 protected int[] shape; 056 057 protected ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> metadata = null; 058 059 /** 060 * @return type of dataset item 061 */ 062 abstract public int getDType(); 063 064 @Override 065 public LazyDatasetBase clone() { 066 return null; 067 } 068 069 @Override 070 public boolean equals(Object obj) { 071 if (this == obj) { 072 return true; 073 } 074 if (obj == null) { 075 return false; 076 } 077 if (!getClass().equals(obj.getClass())) { 078 return false; 079 } 080 081 LazyDatasetBase other = (LazyDatasetBase) obj; 082 if (getDType() != other.getDType()) { 083 return false; 084 } 085 if (getElementsPerItem() != other.getElementsPerItem()) { 086 return false; 087 } 088 if (!Arrays.equals(shape, other.shape)) { 089 return false; 090 } 091 return true; 092 } 093 094 @Override 095 public int hashCode() { 096 int hash = getDType() * 17 + getElementsPerItem(); 097 int rank = shape.length; 098 for (int i = 0; i < rank; i++) { 099 hash = hash*17 + shape[i]; 100 } 101 return hash; 102 } 103 104 @Override 105 public String getName() { 106 return name; 107 } 108 109 @Override 110 public void setName(String name) { 111 this.name = name; 112 } 113 114 @Override 115 public int[] getShape() { 116 return shape.clone(); 117 } 118 119 @Override 120 public int getRank() { 121 return shape.length; 122 } 123 124 /** 125 * This method allows anything that dirties the dataset to clear various metadata values 126 * so that the other methods can work correctly. 127 * @since 2.1 128 */ 129 public void setDirty() { 130 dirty = true; 131 } 132 133 protected void checkSliceND(SliceND slice) { 134 if (slice != null) { 135 int[] source = slice.getSourceShape(); 136 boolean fail = false; 137 if (slice.isExpanded()) { 138 fail = shape.length != source.length; 139 if (!fail) { 140 for (int i = 0; i < shape.length; i++) { 141 if (shape[i] > source[i]) { 142 fail = true; 143 break; 144 } 145 } 146 } 147 } else { 148 fail = !Arrays.equals(shape, source); 149 } 150 if (fail) { 151 throw new IllegalArgumentException("Slice's shape must match dataset's shape"); 152 } 153 } 154 } 155 156 /** 157 * Find first sub-interface of (or class that directly implements) MetadataType 158 * @param clazz metadata type 159 * @return sub-interface 160 * @exception IllegalArgumentException when given class is {@link MetadataType} or an anonymous sub-class of it 161 */ 162 @SuppressWarnings("unchecked") 163 public static Class<? extends MetadataType> findMetadataTypeSubInterfaces(Class<? extends MetadataType> clazz) { 164 if (clazz.equals(MetadataType.class)) { 165 throw new IllegalArgumentException("Cannot accept MetadataType"); 166 } 167 168 if (clazz.isInterface()) { 169 return clazz; 170 } 171 172 if (clazz.isAnonymousClass()) { // special case 173 Class<?> s = clazz.getSuperclass(); 174 if (!s.equals(Object.class)) { 175 // only use super class if it is not an anonymous class of an interface 176 clazz = (Class<? extends MetadataType>) s; 177 } 178 } 179 180 for (Class<?> c : clazz.getInterfaces()) { 181 if (c.equals(MetadataType.class)) { 182 if (clazz.isAnonymousClass()) { 183 throw new IllegalArgumentException("Cannot accept anonymous subclasses of MetadataType"); 184 } 185 return clazz; 186 } 187 if (MetadataType.class.isAssignableFrom(c)) { 188 return (Class<? extends MetadataType>) c; 189 } 190 } 191 192 Class<?> c = clazz.getSuperclass(); // Naughty: someone has sub-classed a metadata class 193 if (c != null) { 194 return findMetadataTypeSubInterfaces((Class<? extends MetadataType>) c); 195 } 196 197 logger.error("Somehow the search for metadata type interface ended in a bad place"); 198 assert false; // should not be able to get here!!! 199 return null; 200 } 201 202 @Override 203 public void setMetadata(MetadataType metadata) { 204 addMetadata(metadata, true); 205 } 206 207 @Override 208 public void addMetadata(MetadataType metadata) { 209 addMetadata(metadata, false); 210 } 211 212 private synchronized void addMetadata(MetadataType metadata, boolean clear) { 213 if (metadata == null) { 214 return; 215 } 216 217 if (this.metadata == null) { 218 this.metadata = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>(); 219 } 220 221 Class<? extends MetadataType> clazz = findMetadataTypeSubInterfaces(metadata.getClass()); 222 if (!this.metadata.containsKey(clazz)) { 223 this.metadata.put(clazz, new ArrayList<MetadataType>()); 224 } else if (clear) { 225 this.metadata.get(clazz).clear(); 226 } 227 this.metadata.get(clazz).add(metadata); 228 229 // add for special case of sub-interfaces of IMetadata 230 if (!IMetadata.class.equals(clazz) && IMetadata.class.isAssignableFrom(clazz)) { 231 clazz = IMetadata.class; 232 if (!this.metadata.containsKey(clazz)) { 233 this.metadata.put(clazz, new ArrayList<MetadataType>()); 234 } else if (clear) { 235 this.metadata.get(clazz).clear(); 236 } 237 this.metadata.get(clazz).add(metadata); 238 } 239 } 240 241 @Override 242 @Deprecated 243 public synchronized IMetadata getMetadata() { 244 return getFirstMetadata(IMetadata.class); 245 } 246 247 @SuppressWarnings("unchecked") 248 @Override 249 public synchronized <T extends MetadataType> List<T> getMetadata(Class<T> clazz) throws MetadataException { 250 if (metadata == null) { 251 dirty = false; 252 return null; 253 } 254 255 if (dirty) { 256 dirtyMetadata(); 257 dirty = false; 258 } 259 260 if (clazz == null) { 261 List<T> all = new ArrayList<>(); 262 for (Class<? extends MetadataType> c : metadata.keySet()) { 263 all.addAll((Collection<T>) metadata.get(c)); 264 } 265 return all; 266 } 267 268 return (List<T>) metadata.get(findMetadataTypeSubInterfaces(clazz)); 269 } 270 271 @Override 272 public synchronized <T extends MetadataType> T getFirstMetadata(Class<T> clazz) { 273 try { 274 List<T> ml = getMetadata(clazz); 275 if (ml == null) { 276 return null; 277 } 278 for (T t : ml) { 279 if (clazz.isInstance(t)) { 280 return t; 281 } 282 } 283 } catch (Exception e) { 284 logger.error("Get metadata failed!",e); 285 } 286 287 return null; 288 } 289 290 @Override 291 public synchronized void clearMetadata(Class<? extends MetadataType> clazz) { 292 if (metadata == null) { 293 return; 294 } 295 296 if (clazz == null) { 297 metadata.clear(); 298 return; 299 } 300 301 List<MetadataType> list = metadata.get(findMetadataTypeSubInterfaces(clazz)); 302 if( list != null) { 303 list.clear(); 304 } 305 } 306 307 /** 308 * @return copy of metadata 309 * @since 2.0 310 */ 311 protected synchronized ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata() { 312 return copyMetadata(metadata); 313 } 314 315 /** 316 * @param metadata type 317 * @return copy of metadata of given type 318 * @since 2.0 319 */ 320 protected static ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> metadata) { 321 if (metadata == null) { 322 return null; 323 } 324 325 ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>> map = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>(); 326 copyMetadata(metadata, map); 327 return map; 328 } 329 330 private static void copyMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> inMetadata, 331 Map<Class<? extends MetadataType>, List<MetadataType>> outMetadata) { 332 for (Class<? extends MetadataType> c : inMetadata.keySet()) { 333 List<MetadataType> l = inMetadata.get(c); 334 List<MetadataType> nl = new ArrayList<MetadataType>(l.size()); 335 outMetadata.put(c, nl); 336 for (MetadataType m : l) { 337 if (m == null || isMetadataDirty(m)) { // skip dirty metadata 338 continue; 339 } 340 nl.add(m.clone()); 341 } 342 } 343 } 344 345 protected void restoreMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> oldMetadata) { 346 copyMetadata(oldMetadata, metadata); 347 } 348 349 /** 350 * @param a dataset 351 * @param clone if true, copy metadata 352 * @return copy of metadata 353 * @since 2.2 354 */ 355 protected static ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> getMetadataMap(ILazyDataset a, boolean clone) { 356 List<MetadataType> all = null; 357 try { 358 all = a.getMetadata(null); 359 } catch (Exception e) { 360 } 361 if (all == null) { 362 return null; 363 } 364 365 ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> map = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>(); 366 367 for (MetadataType m : all) { 368 if (m == null || isMetadataDirty(m)) { // skip dirty metadata 369 continue; 370 } 371 Class<? extends MetadataType> c = findMetadataTypeSubInterfaces(m.getClass()); 372 List<MetadataType> l = map.get(c); 373 if (l == null) { 374 l = new ArrayList<MetadataType>(); 375 map.put(c, l); 376 } 377 if (clone) { 378 m = m.clone(); 379 } 380 l.add(m); 381 } 382 return map; 383 } 384 385 private static boolean isMetadataDirty(MetadataType m) { 386 Class<? extends MetadataType> c = m.getClass(); 387 for (Field f : c.getDeclaredFields()) { 388 if (f.isAnnotationPresent(Dirtiable.class)) { 389 Class<?> t = f.getType(); 390 if (t.equals(boolean.class) || t.equals(Boolean.class)) { 391 try { 392 f.setAccessible(true); 393 Object o = f.get(m); 394 if (o.equals(true)) { 395 return true; 396 } 397 } catch (Exception e) { 398 logger.debug("Could not retrieve value of dirty variable: {}", c.getCanonicalName(), e); 399 } 400 } 401 } 402 } 403 404 return false; 405 } 406 407 interface MetadatasetAnnotationOperation { 408 /** 409 * Process value of given field 410 * <p> 411 * When the field is not a container then the returned value 412 * may replace the old value 413 * @param f given field 414 * @param o value of field 415 * @return transformed field 416 */ 417 Object processField(Field f, Object o); 418 419 /** 420 * @return annotated class 421 */ 422 Class<? extends Annotation> getAnnClass(); 423 424 /** 425 * @param axis 426 * @return number of dimensions to insert or remove 427 */ 428 int change(int axis); 429 430 /** 431 * 432 * @return rank or -1 to match 433 */ 434 int getNewRank(); 435 436 /** 437 * Run on given lazy dataset 438 * @param lz 439 * @return 440 */ 441 ILazyDataset run(ILazyDataset lz); 442 } 443 444 class MdsSlice implements MetadatasetAnnotationOperation { 445 private boolean asView; 446 private SliceND slice; 447 private int[] oShape; 448 private long oSize; 449 450 public MdsSlice(boolean asView, SliceND slice) { 451 this.asView = asView; 452 this.slice = slice; 453 oShape = slice.getSourceShape(); 454 oSize = ShapeUtils.calcLongSize(oShape); 455 } 456 457 @Override 458 public Object processField(Field field, Object o) { 459 return o; 460 } 461 462 @Override 463 public Class<? extends Annotation> getAnnClass() { 464 return Sliceable.class; 465 } 466 467 @Override 468 public int change(int axis) { 469 return 0; 470 } 471 472 @Override 473 public int getNewRank() { 474 return -1; 475 } 476 477 @Override 478 public ILazyDataset run(ILazyDataset lz) { 479 int rank = lz.getRank(); 480 if (slice.getStart().length != rank) { 481 throw new IllegalArgumentException("Slice rank does not match dataset!"); 482 } 483 484 int[] shape = lz.getShape(); 485 SliceND nslice; 486 if (!ShapeUtils.areShapesBroadcastCompatible(oShape, shape)) { 487 nslice = new SliceND(shape); 488 for (int i = 0; i < rank; i++) { 489 int s = shape[i]; 490 int os = oShape[i]; 491 if (s >= os) { 492 nslice.setSlice(i, 0, os, 1); 493 } else if (s == 1) { 494 nslice.setSlice(i, 0, 1, 1); 495 } else { 496 throw new IllegalArgumentException("Sliceable dataset has non-unit dimension less than host!"); 497 } 498 } 499 lz = lz.getSliceView(nslice); 500 shape = nslice.getShape(); 501 } 502 if (lz.getSize() == oSize && Arrays.equals(shape, oShape)) { 503 nslice = slice; 504 } else { 505 nslice = slice.clone(); 506 for (int i = 0; i < rank; i++) { 507 int s = shape[i]; 508 if (s >= oShape[i]) { 509 continue; 510 } else if (s == 1) { 511 nslice.setSlice(i, 0, 1, 1); 512 } else { 513 throw new IllegalArgumentException("Sliceable dataset has non-unit dimension less than host!"); 514 } 515 } 516 nslice.updateSourceShape(shape); 517 } 518 519 if (asView || (lz instanceof IDataset)) { 520 return lz.getSliceView(nslice); 521 } 522 try { 523 return lz.getSlice(nslice); 524 } catch (DatasetException e) { 525 logger.error("Could not slice dataset in metadata", e); 526 return null; 527 } 528 } 529 } 530 531 class MdsReshape implements MetadatasetAnnotationOperation { 532 private boolean matchRank; 533 private int[] oldShape; 534 private int[] newShape; 535 boolean onesOnly; 536 int[] differences; 537 538 /* 539 * if only ones then record differences (insertions and deletions) 540 * 541 * if shape changing, find broadcasted dimensions and disallow 542 * merging that include those dimensions 543 */ 544 public MdsReshape(final int[] oldShape, final int[] newShape) { 545 this.oldShape = oldShape; 546 this.newShape = newShape; 547 differences = null; 548 } 549 550 @Override 551 public Object processField(Field field, Object o) { 552 Annotation a = field.getAnnotation(Reshapeable.class); 553 if (a != null) { // cannot be null 554 matchRank = ((Reshapeable) a).matchRank(); 555 } 556 return o; 557 } 558 559 @Override 560 public Class<? extends Annotation> getAnnClass() { 561 return Reshapeable.class; 562 } 563 564 @Override 565 public int change(int axis) { 566 if (matchRank) { 567 if (differences == null) { 568 init(); 569 } 570 571 if (onesOnly) { 572 return differences == null ? 0 : differences[axis]; 573 } 574 throw new UnsupportedOperationException("TODO support other shape operations"); 575 } 576 return 0; 577 } 578 579 @Override 580 public int getNewRank() { 581 return matchRank ? newShape.length : -1; 582 } 583 584 private void init() { 585 int or = oldShape.length - 1; 586 int nr = newShape.length - 1; 587 if (or < 0 || nr < 0) { // zero-rank shapes 588 onesOnly = true; 589 differences = new int[1]; 590 differences[0] = or < 0 ? nr + 1 : or + 1; 591 return; 592 } 593 onesOnly = ShapeUtils.differsByOnes(oldShape, newShape); 594 int ob = 0; 595 int nb = 0; 596 if (onesOnly) { 597 differences = ShapeUtils.calcShapePadding(oldShape, newShape); 598 } else { 599 differences = new int[or + 2]; 600 if (matchRank) { 601 logger.error("Combining dimensions is currently not supported"); 602 throw new IllegalArgumentException("Combining dimensions is currently not supported"); 603 } 604 // work out mapping: contiguous dimensions can be grouped or split 605 while (ob <= or && nb <= nr) { 606 int ol = oldShape[ob]; 607 while (ol == 1 && ol <= or) { 608 ob++; 609 ol = oldShape[ob]; 610 } 611 int oe = ob + 1; 612 int nl = newShape[nb]; 613 while (nl == 1 && nl <= nr) { 614 nb++; 615 nl = newShape[nb]; 616 } 617 int ne = nb + 1; 618 if (ol < nl) { 619 differences[ob] = 1; 620 do { // case where new shape combines several dimensions into one dimension 621 if (oe == (or + 1)) { 622 break; 623 } 624 differences[oe] = 1; 625 ol *= oldShape[oe++]; 626 } while (ol < nl); 627 differences[oe - 1] = oe - ob; // signal end with difference 628 if (nl != ol) { 629 logger.error("Single dimension is incompatible with subshape"); 630 throw new IllegalArgumentException("Single dimension is incompatible with subshape"); 631 } 632 } else if (ol > nl) { 633 do { // case where new shape spreads single dimension over several dimensions 634 if (ne == (nr + 1)) { 635 break; 636 } 637 nl *= newShape[ne++]; 638 } while (nl < ol); 639 if (nl != ol) { 640 logger.error("Subshape is incompatible with single dimension"); 641 throw new IllegalArgumentException("Subshape is incompatible with single dimension"); 642 } 643 } 644 645 ob = oe; 646 nb = ne; 647 } 648 } 649 } 650 651 @Override 652 public ILazyDataset run(ILazyDataset lz) { 653 if (differences == null) { 654 init(); 655 } 656 657 int[] lshape = lz.getShape(); 658 if (Arrays.equals(newShape, lshape)) { 659 return lz; 660 } 661 int or = lshape.length; 662 int nr = newShape.length; 663 int[] nshape; 664 if (onesOnly) { 665 nshape = ShapeUtils.padShape(differences, nr, lshape); 666 } else { 667 nshape = new int[nr]; 668 boolean[] broadcast = new boolean[or]; 669 for (int ob = 0; ob < or; ob++) { 670 broadcast[ob] = oldShape[ob] != 1 && lshape[ob] == 1; 671 } 672 int osize = lz.getSize(); 673 674 // cannot do 3x5x... to 15x... if metadata is broadcasting (i.e. 1x5x...) 675 int ob = 0; 676 int nsize = 1; 677 for (int i = 0; i < nr; i++) { 678 if (ob < or && broadcast[ob]) { 679 if (differences[ob] != 0) { 680 logger.error("Metadata contains a broadcast axis which cannot be reshaped"); 681 throw new IllegalArgumentException("Metadata contains a broadcast axis which cannot be reshaped"); 682 } 683 nshape[i] = 1; 684 } else { 685 nshape[i] = nsize < osize ? newShape[i] : 1; 686 } 687 nsize *= nshape[i]; 688 ob++; 689 } 690 } 691 692 ILazyDataset nlz; 693 if (lz instanceof Dataset) { 694 nlz = ((Dataset) lz).reshape(nshape); 695 } else { 696 nlz = lz.getSliceView(); 697 nlz.setShape(nshape); 698 } 699 return nlz; 700 } 701 } 702 703 class MdsTranspose implements MetadatasetAnnotationOperation { 704 int[] map; 705 706 public MdsTranspose(final int[] axesMap) { 707 map = axesMap; 708 } 709 710 @SuppressWarnings({ "rawtypes", "unchecked" }) 711 @Override 712 public Object processField(Field f, Object o) { 713 // reorder arrays and lists according the axes map 714 if (o.getClass().isArray()) { 715 int l = Array.getLength(o); 716 if (l == map.length) { 717 Object narray = Array.newInstance(o.getClass().getComponentType(), l); 718 for (int i = 0; i < l; i++) { 719 Array.set(narray, i, Array.get(o, map[i])); 720 } 721 for (int i = 0; i < l; i++) { 722 Array.set(o, i, Array.get(narray, i)); 723 } 724 } 725 } else if (o instanceof List<?>) { 726 List list = (List) o; 727 int l = list.size(); 728 if (l == map.length) { 729 Object narray = Array.newInstance(o.getClass().getComponentType(), l); 730 for (int i = 0; i < l; i++) { 731 Array.set(narray, i, list.get(map[i])); 732 } 733 list.clear(); 734 for (int i = 0; i < l; i++) { 735 list.add(Array.get(narray, i)); 736 } 737 } 738 } 739 return o; 740 } 741 742 @Override 743 public Class<? extends Annotation> getAnnClass() { 744 return Transposable.class; 745 } 746 747 @Override 748 public int change(int axis) { 749 return 0; 750 } 751 752 @Override 753 public int getNewRank() { 754 return -1; 755 } 756 757 @Override 758 public ILazyDataset run(ILazyDataset lz) { 759 return lz.getTransposedView(map); 760 } 761 } 762 763 class MdsDirty implements MetadatasetAnnotationOperation { 764 765 @Override 766 public Object processField(Field f, Object o) { 767 // throw exception if not boolean??? 768 Class<?> t = f.getType(); 769 if (t.equals(boolean.class) || t.equals(Boolean.class)) { 770 if (o.equals(false)) { 771 o = true; 772 } 773 } 774 return o; 775 } 776 777 @Override 778 public Class<? extends Annotation> getAnnClass() { 779 return Dirtiable.class; 780 } 781 782 @Override 783 public int change(int axis) { 784 return 0; 785 } 786 787 @Override 788 public int getNewRank() { 789 return -1; 790 } 791 792 @Override 793 public ILazyDataset run(ILazyDataset lz) { 794 return lz; 795 } 796 } 797 798 /** 799 * Slice all datasets in metadata that are annotated by @Sliceable. Call this on the new sliced 800 * dataset after cloning the metadata 801 * @param asView if true then just a view 802 * @param slice an n-D slice 803 */ 804 protected void sliceMetadata(boolean asView, final SliceND slice) { 805 processAnnotatedMetadata(new MdsSlice(asView, slice)); 806 } 807 808 /** 809 * Reshape all datasets in metadata that are annotated by @Reshapeable. Call this when squeezing 810 * or setting the shape 811 * @param oldShape old shape 812 * @param newShape new shape 813 */ 814 protected void reshapeMetadata(final int[] oldShape, final int[] newShape) { 815 processAnnotatedMetadata(new MdsReshape(oldShape, newShape)); 816 } 817 818 /** 819 * Transpose all datasets in metadata that are annotated by @Transposable. Call this on the transposed 820 * dataset after cloning the metadata 821 * @param axesMap if zero length then axes order reversed 822 */ 823 protected void transposeMetadata(final int[] axesMap) { 824 processAnnotatedMetadata(new MdsTranspose(axesMap)); 825 } 826 827 /** 828 * Dirty metadata that are annotated by @Dirtiable. Call this when the dataset has been modified 829 * @since 2.0 830 */ 831 protected void dirtyMetadata() { 832 processAnnotatedMetadata(new MdsDirty()); 833 } 834 835 @SuppressWarnings("unchecked") 836 private void processAnnotatedMetadata(MetadatasetAnnotationOperation op) { 837 if (metadata == null) 838 return; 839 840 for (List<MetadataType> l : metadata.values()) { 841 for (MetadataType m : l) { 842 if (m == null) { 843 continue; 844 } 845 846 Class<? extends MetadataType> mc = m.getClass(); 847 do { // iterate over super-classes 848 processClass(op, m, mc); 849 Class<?> sclazz = mc.getSuperclass(); 850 if (!MetadataType.class.isAssignableFrom(sclazz)) { 851 break; 852 } 853 mc = (Class<? extends MetadataType>) sclazz; 854 } while (true); 855 } 856 } 857 } 858 859 @SuppressWarnings({ "unchecked", "rawtypes" }) 860 private static void processClass(MetadatasetAnnotationOperation op, MetadataType m, Class<? extends MetadataType> mc) { 861 for (Field f : mc.getDeclaredFields()) { 862 if (!f.isAnnotationPresent(op.getAnnClass())) 863 continue; 864 865 try { 866 f.setAccessible(true); 867 Object o = f.get(m); 868 if (o == null) { 869 continue; 870 } 871 872 Object no = op.processField(f, o); 873 if (no != o) { 874 f.set(m, no); 875 continue; 876 } 877 Object r = null; 878 if (o instanceof ILazyDataset) { 879 try { 880 f.set(m, op.run((ILazyDataset) o)); 881 } catch (Exception e) { 882 logger.error("Problem processing " + o, e); 883 throw e; 884 } 885 } else if (o.getClass().isArray()) { 886 int l = Array.getLength(o); 887 888 for (int i = 0; r == null && i < l; i++) { 889 r = Array.get(o, i); 890 } 891 int n = op.getNewRank(); 892 if (r == null) { 893 if (n < 0 || n != l) { // all nulls be need to match rank as necessary 894 f.set(m, Array.newInstance(o.getClass().getComponentType(), n < 0 ? l : n)); 895 } 896 continue; 897 } 898 if (n < 0) { 899 n = l; 900 } 901 Object narray = Array.newInstance(r.getClass(), n); 902 for (int i = 0, si = 0, di = 0; di < n && si < l; i++) { 903 int c = op.change(i); 904 if (c == 0) { 905 Array.set(narray, di++, processObject(op, Array.get(o, si++))); 906 } else if (c > 0) { 907 di += c; // add nulls by skipping forward in destination array 908 } else if (c < 0) { 909 si -= c; // remove dimensions by skipping forward in source array 910 } 911 } 912 if (n == l) { 913 for (int i = 0; i < l; i++) { 914 Array.set(o, i, Array.get(narray, i)); 915 } 916 } else { 917 f.set(m, narray); 918 } 919 } else if (o instanceof List<?>) { 920 List list = (List) o; 921 int l = list.size(); 922 923 for (int i = 0; r == null && i < l; i++) { 924 r = list.get(i); 925 } 926 int n = op.getNewRank(); 927 if (r == null) { 928 if (n < 0 || n != l) { // all nulls be need to match rank as necessary 929 list.clear(); 930 for (int i = 0, imax = n < 0 ? l : n; i < imax; i++) { 931 list.add(null); 932 } 933 } 934 continue; 935 } 936 937 if (n < 0) { 938 n = l; 939 } 940 Object narray = Array.newInstance(r.getClass(), n); 941 for (int i = 0, si = 0, di = 0; i < l && si < l; i++) { 942 int c = op.change(i); 943 if (c == 0) { 944 Array.set(narray, di++, processObject(op, list.get(si++))); 945 } else if (c > 0) { 946 di += c; // add nulls by skipping forward in destination array 947 } else if (c < 0) { 948 si -= c; // remove dimensions by skipping forward in source array 949 } 950 } 951 list.clear(); 952 for (int i = 0; i < n; i++) { 953 list.add(Array.get(narray, i)); 954 } 955 } else if (o instanceof Map<?,?>) { 956 Map map = (Map) o; 957 for (Object k : map.keySet()) { 958 map.put(k, processObject(op, map.get(k))); 959 } 960 } 961 } catch (Exception e) { 962 logger.error("Problem occurred when processing metadata of class {}: {}", mc.getCanonicalName(), e); 963 throw new RuntimeException(e); 964 } 965 } 966 } 967 968 @SuppressWarnings({ "unchecked", "rawtypes" }) 969 private static Object processObject(MetadatasetAnnotationOperation op, Object o) throws Exception { 970 if (o == null) { 971 return o; 972 } 973 974 if (o instanceof ILazyDataset) { 975 try { 976 return op.run((ILazyDataset) o); 977 } catch (Exception e) { 978 logger.error("Problem processing " + o, e); 979 throw e; 980 } 981 } else if (o.getClass().isArray()) { 982 int l = Array.getLength(o); 983 for (int i = 0; i < l; i++) { 984 Array.set(o, i, processObject(op, Array.get(o, i))); 985 } 986 } else if (o instanceof List<?>) { 987 List list = (List) o; 988 for (int i = 0, imax = list.size(); i < imax; i++) { 989 list.set(i, processObject(op, list.get(i))); 990 } 991 } else if (o instanceof Map<?,?>) { 992 Map map = (Map) o; 993 for (Object k : map.keySet()) { 994 map.put(k, processObject(op, map.get(k))); 995 } 996 } 997 return o; 998 } 999 1000 protected ILazyDataset createFromSerializable(Serializable blob, boolean keepLazy) { 1001 ILazyDataset d = null; 1002 if (blob instanceof ILazyDataset) { 1003 d = (ILazyDataset) blob; 1004 if (d instanceof IDataset) { 1005 Dataset ed = DatasetUtils.convertToDataset((IDataset) d); 1006 int is = ed.getElementsPerItem(); 1007 if (is != 1 && is != getElementsPerItem()) { 1008 throw new IllegalArgumentException("Dataset has incompatible number of elements with this dataset"); 1009 } 1010 d = ed.cast(is == 1 ? DoubleDataset.class: CompoundDoubleDataset.class); 1011 } else if (!keepLazy) { 1012 final int is = getElementsPerItem(); 1013 try { 1014 d = DatasetUtils.cast(is == 1 ? DoubleDataset.class: CompoundDoubleDataset.class, d.getSlice()); 1015 } catch (DatasetException e) { 1016 logger.error("Could not get data from lazy dataset", e); 1017 return null; 1018 } 1019 } 1020 } else { 1021 final int is = getElementsPerItem(); 1022 if (is == 1) { 1023 d = DatasetFactory.createFromObject(DoubleDataset.class, blob); 1024 } else { 1025 try { 1026 d = DatasetFactory.createFromObject(is, CompoundDoubleDataset.class, blob); 1027 } catch (IllegalArgumentException e) { // if only single value supplied try again 1028 d = DatasetFactory.createFromObject(DoubleDataset.class, blob); 1029 } 1030 } 1031 if (d.getSize() == getSize() && !Arrays.equals(d.getShape(), shape)) { 1032 d.setShape(shape.clone()); 1033 } 1034 } 1035 List<int[]> s = BroadcastUtils.broadcastShapesToMax(shape, d.getShape()); 1036 d.setShape(s.get(0)); 1037 1038 return d; 1039 } 1040 1041 @Override 1042 public void setErrors(Serializable errors) { 1043 if (shape == null) { 1044 throw new IllegalArgumentException("Cannot set errors for null dataset"); 1045 } 1046 if (errors == null) { 1047 clearMetadata(ErrorMetadata.class); 1048 return; 1049 } 1050 if (errors == this) { 1051 logger.warn("Ignoring setting error to itself as this will lead to infinite recursion"); 1052 return; 1053 } 1054 1055 ILazyDataset errorData = createFromSerializable(errors, true); 1056 1057 ErrorMetadata emd = getErrorMetadata(); 1058 if (emd == null) { 1059 try { 1060 emd = MetadataFactory.createMetadata(ErrorMetadata.class); 1061 setMetadata(emd); 1062 } catch (MetadataException me) { 1063 logger.error("Could not create metadata", me); 1064 } 1065 } 1066 emd.setError(errorData); 1067 } 1068 1069 protected ErrorMetadata getErrorMetadata() { 1070 try { 1071 List<ErrorMetadata> el = getMetadata(ErrorMetadata.class); 1072 if (el != null && !el.isEmpty()) { 1073 return el.get(0); 1074 } 1075 } catch (Exception e) { 1076 } 1077 return null; 1078 } 1079 1080 @Override 1081 public ILazyDataset getErrors() { 1082 ErrorMetadata emd = getErrorMetadata(); 1083 return emd == null ? null : emd.getError(); 1084 } 1085 1086 @Override 1087 public boolean hasErrors() { 1088 return LazyDatasetBase.this.getErrors() != null; 1089 } 1090 1091 /** 1092 * Check permutation axes 1093 * @param shape to use 1094 * @param axes if zero length then axes order reversed 1095 * @return cleaned up copy of axes or null if trivial 1096 */ 1097 public static int[] checkPermutatedAxes(int[] shape, int... axes) { 1098 int rank = shape == null ? 0 : shape.length; 1099 1100 if (axes == null || axes.length == 0) { 1101 axes = new int[rank]; 1102 for (int i = 0; i < rank; i++) { 1103 axes[i] = rank - 1 - i; 1104 } 1105 } else { 1106 axes = axes.clone(); 1107 } 1108 1109 if (axes.length != rank) { 1110 logger.error("axis permutation has length {} that does not match dataset's rank {}", axes.length, rank); 1111 throw new IllegalArgumentException("axis permutation does not match shape of dataset"); 1112 } 1113 1114 // check all permutation values are within bounds 1115 for (int i = 0; i < rank; i++) { 1116 axes[i] = ShapeUtils.checkAxis(rank, axes[i]); 1117 } 1118 1119 // check for a valid permutation (is this an unnecessary restriction?) 1120 int[] perm = axes.clone(); 1121 Arrays.sort(perm); 1122 1123 for (int i = 0; i < rank; i++) { 1124 if (perm[i] != i) { 1125 logger.error("axis permutation is not valid: it does not contain complete set of axes"); 1126 throw new IllegalArgumentException("axis permutation does not contain complete set of axes"); 1127 } 1128 } 1129 1130 if (Arrays.equals(axes, perm)) { 1131 return null; // signal identity or trivial permutation 1132 } 1133 1134 return axes; 1135 } 1136}