001/*-
002 *******************************************************************************
003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd.
004 * All rights reserved. This program and the accompanying materials
005 * are made available under the terms of the Eclipse Public License v1.0
006 * which accompanies this distribution, and is available at
007 * http://www.eclipse.org/legal/epl-v10.html
008 *
009 * Contributors:
010 *    Peter Chang - initial API and implementation and/or initial documentation
011 *******************************************************************************/
012
013package org.eclipse.january.dataset;
014
015import java.io.Serializable;
016import java.lang.annotation.Annotation;
017import java.lang.reflect.Array;
018import java.lang.reflect.Field;
019import java.util.ArrayList;
020import java.util.Arrays;
021import java.util.Collection;
022import java.util.concurrent.ConcurrentMap;
023import java.util.concurrent.ConcurrentHashMap;
024import java.util.List;
025import java.util.Map;
026
027import org.eclipse.january.DatasetException;
028import org.eclipse.january.MetadataException;
029import org.eclipse.january.metadata.Dirtiable;
030import org.eclipse.january.metadata.ErrorMetadata;
031import org.eclipse.january.metadata.IMetadata;
032import org.eclipse.january.metadata.MetadataFactory;
033import org.eclipse.january.metadata.MetadataType;
034import org.eclipse.january.metadata.Reshapeable;
035import org.eclipse.january.metadata.Sliceable;
036import org.eclipse.january.metadata.Transposable;
037import org.slf4j.Logger;
038import org.slf4j.LoggerFactory;
039
040/**
041 * Common base for both lazy and normal dataset implementations
042 */
043public abstract class LazyDatasetBase implements ILazyDataset, Serializable {
044
045        private static final long serialVersionUID = 767926846438976050L;
046
047        protected static final Logger logger = LoggerFactory.getLogger(LazyDatasetBase.class);
048
049        protected static boolean catchExceptions;
050        
051        static {
052                /**
053                 * Boolean to set to true if running jython scripts that utilise ScisoftPy in IDE
054                 */
055                catchExceptions = Boolean.getBoolean("run.in.eclipse");
056        }
057
058        transient private boolean dirty = true; // indicate dirty state of metadata
059        protected String name = "";
060
061        /**
062         * The shape or dimensions of the dataset
063         */
064        protected int[] shape;
065
066        protected ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> metadata = null;
067
068        /**
069         * @return type of dataset item
070         */
071        abstract public int getDType();
072
073        @Override
074        public Class<?> getElementClass() {
075                return DTypeUtils.getElementClass(getDType());
076        }
077
078        @Override
079        public LazyDatasetBase clone() {
080                return null;
081        }
082
083        @Override
084        public boolean equals(Object obj) {
085                if (this == obj) {
086                        return true;
087                }
088                if (obj == null) {
089                        return false;
090                }
091                if (!getClass().equals(obj.getClass())) {
092                        return false;
093                }
094        
095                LazyDatasetBase other = (LazyDatasetBase) obj;
096                if (getDType() != other.getDType()) {
097                        return false;
098                }
099                if (getElementsPerItem() != other.getElementsPerItem()) {
100                        return false;
101                }
102                if (!Arrays.equals(shape, other.shape)) {
103                        return false;
104                }
105                return true;
106        }
107
108        @Override
109        public int hashCode() {
110                int hash = getDType() * 17 + getElementsPerItem();
111                int rank = shape.length;
112                for (int i = 0; i < rank; i++) {
113                        hash = hash*17 + shape[i];
114                }
115                return hash;
116        }
117
118        @Override
119        public String getName() {
120                return name;
121        }
122
123        @Override
124        public void setName(String name) {
125                this.name = name;
126        }
127
128        @Override
129        public int[] getShape() {
130                return shape.clone();
131        }
132
133        @Override
134        public int getRank() {
135                return shape.length;
136        }
137
138        /**
139         * This method allows anything that dirties the dataset to clear various metadata values
140         * so that the other methods can work correctly.
141         */
142        public void setDirty() {
143                dirty = true;
144        }
145
146        /**
147         * Find first sub-interface of (or class that directly implements) MetadataType
148         * @param clazz
149         * @return sub-interface
150         * @exception IllegalArgumentException when given class is {@link MetadataType} or an anonymous sub-class of it
151         */
152        @SuppressWarnings("unchecked")
153        public static Class<? extends MetadataType> findMetadataTypeSubInterfaces(Class<? extends MetadataType> clazz) {
154                if (clazz.equals(MetadataType.class)) {
155                        throw new IllegalArgumentException("Cannot accept MetadataType");
156                }
157
158                if (clazz.isInterface()) {
159                        return clazz;
160                }
161
162                if (clazz.isAnonymousClass()) { // special case
163                        Class<?> s = clazz.getSuperclass();
164                        if (!s.equals(Object.class)) {
165                                // only use super class if it is not an anonymous class of an interface
166                                clazz = (Class<? extends MetadataType>) s;
167                        }
168                }
169
170                for (Class<?> c : clazz.getInterfaces()) {
171                        if (c.equals(MetadataType.class)) {
172                                if (clazz.isAnonymousClass()) {
173                                        throw new IllegalArgumentException("Cannot accept anonymous subclasses of MetadataType");
174                                }
175                                return clazz;
176                        }
177                        if (MetadataType.class.isAssignableFrom(c)) {
178                                return (Class<? extends MetadataType>) c;
179                        }
180                }
181
182                Class<?> c = clazz.getSuperclass(); // Naughty: someone has sub-classed a metadata class
183                if (c != null) {
184                        return findMetadataTypeSubInterfaces((Class<? extends MetadataType>) c);
185                }
186
187                logger.error("Somehow the search for metadata type interface ended in a bad place");
188                assert false; // should not be able to get here!!!
189                return null;
190        }
191
192        @Override
193        public void setMetadata(MetadataType metadata) {
194                addMetadata(metadata, true);
195        }
196
197        @Override
198        public void addMetadata(MetadataType metadata) {
199                addMetadata(metadata, false);
200        }
201
202        private synchronized void addMetadata(MetadataType metadata, boolean clear) {
203                if (metadata == null)
204                        return;
205
206                if (this.metadata == null) {
207                        this.metadata = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>();
208                }
209
210                Class<? extends MetadataType> clazz = findMetadataTypeSubInterfaces(metadata.getClass());
211                if (!this.metadata.containsKey(clazz)) {
212                        this.metadata.put(clazz, new ArrayList<MetadataType>());
213                } else if (clear) {
214                        this.metadata.get(clazz).clear();
215                }
216                this.metadata.get(clazz).add(metadata);
217
218                // add for special case of sub-interfaces of IMetadata
219                if (!IMetadata.class.equals(clazz) && IMetadata.class.isAssignableFrom(clazz)) {
220                        clazz = IMetadata.class;
221                        if (!this.metadata.containsKey(clazz)) {
222                                this.metadata.put(clazz, new ArrayList<MetadataType>());
223                        } else if (clear) {
224                                this.metadata.get(clazz).clear();
225                        }
226                        this.metadata.get(clazz).add(metadata);
227                }
228        }
229
230        @Override
231        @Deprecated
232        public synchronized IMetadata getMetadata() {
233                return getFirstMetadata(IMetadata.class);
234        }
235
236        @SuppressWarnings("unchecked")
237        @Override
238        public synchronized <S extends MetadataType, T extends S> List<S> getMetadata(Class<T> clazz) throws MetadataException {
239                if (metadata == null)
240                        return null;
241
242                if (dirty) {
243                        dirtyMetadata();
244                        dirty = false;
245                }
246
247                if (clazz == null) {
248                        List<S> all = new ArrayList<S>();
249                        for (Class<? extends MetadataType> c : metadata.keySet()) {
250                                all.addAll((Collection<S>) metadata.get(c));
251                        }
252                        return all;
253                }
254
255                return (List<S>) metadata.get(findMetadataTypeSubInterfaces(clazz));
256        }
257
258        @Override
259        public synchronized <S extends MetadataType, T extends S> S getFirstMetadata(Class<T> clazz) {
260                try {
261                        List<S> ml = getMetadata(clazz);
262                        if (ml == null) return null;
263                        for (S t : ml) {
264                                if (clazz.isInstance(t)) return t;
265                        }
266                } catch (Exception e) {
267                        logger.error("Get metadata failed!",e);
268                }
269
270                return null;
271        }
272
273        @Override
274        public synchronized void clearMetadata(Class<? extends MetadataType> clazz) {
275                if (metadata == null)
276                        return;
277
278                if (clazz == null) {
279                        metadata.clear();
280                        return;
281                }
282
283                List<MetadataType> list = metadata.get(findMetadataTypeSubInterfaces(clazz));
284                if( list != null) {
285                        list.clear();
286                }
287        }
288
289        /**
290         * @since 2.0
291         */
292        protected synchronized ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata() {
293                return copyMetadata(metadata);
294        }
295
296        /**
297         * @since 2.0
298         */
299        protected static ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> metadata) {
300                if (metadata == null)
301                        return null;
302
303                ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>> map = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>();
304
305                for (Class<? extends MetadataType> c : metadata.keySet()) {
306                        List<MetadataType> l = metadata.get(c);
307                        List<MetadataType> nl = new ArrayList<MetadataType>(l.size());
308                        map.put(c, nl);
309                        for (MetadataType m : l) {
310                                nl.add(m.clone());
311                        }
312                }
313                return map;
314        }
315
316        interface MetadatasetAnnotationOperation {
317                /**
318                 * Process value of given field
319                 * <p>
320                 * When the field is not a container then the returned value
321                 * may replace the old value
322                 * @param f given field
323                 * @param o value of field
324                 * @return transformed field
325                 */
326                Object processField(Field f, Object o);
327
328                /**
329                 * @return annotated class
330                 */
331                Class<? extends Annotation> getAnnClass();
332
333                /**
334                 * @param axis
335                 * @return number of dimensions to insert or remove
336                 */
337                int change(int axis);
338
339                /**
340                 * 
341                 * @return rank or -1 to match
342                 */
343                int getNewRank();
344
345                /**
346                 * Run on given lazy dataset
347                 * @param lz
348                 * @return 
349                 */
350                ILazyDataset run(ILazyDataset lz);
351        }
352
353        class MdsSlice implements MetadatasetAnnotationOperation {
354                private boolean asView;
355                private int[] start;
356                private int[] stop;
357                private int[] step;
358                private int[] oShape;
359                private long oSize;
360
361                public MdsSlice(boolean asView, final int[] start, final int[] stop, final int[] step, final int[] oShape) {
362                        this.asView = asView;
363                        this.start = start;
364                        this.stop = stop;
365                        this.step = step;
366                        this.oShape = oShape;
367                        oSize = ShapeUtils.calcLongSize(oShape);
368                }
369
370                @Override
371                public Object processField(Field field, Object o) {
372                        return o;
373                }
374
375                @Override
376                public Class<? extends Annotation> getAnnClass() {
377                        return Sliceable.class;
378                }
379
380                @Override
381                public int change(int axis) {
382                        return 0;
383                }
384
385                @Override
386                public int getNewRank() {
387                        return -1;
388                }
389
390                @Override
391                public ILazyDataset run(ILazyDataset lz) {
392                        int rank = lz.getRank();
393                        if (start.length != rank) {
394                                throw new IllegalArgumentException("Slice dimensions do not match dataset!");
395                        }
396
397                        int[] shape = lz.getShape();
398                        int[] stt;
399                        int[] stp;
400                        int[] ste;
401                        if (lz.getSize() == oSize) {
402                                stt = start;
403                                stp = stop;
404                                ste = step;
405                        } else {
406                                stt = start.clone();
407                                stp = stop.clone();
408                                ste = step.clone();
409                                for (int i = 0; i < rank; i++) {
410                                        if (shape[i] >= oShape[i]) continue;
411                                        if (shape[i] == 1) {
412                                                stt[i] = 0;
413                                                stp[i] = 1;
414                                                ste[1] = 1;
415                                        } else {
416                                                throw new IllegalArgumentException("Sliceable dataset has invalid size!");
417                                        }
418                                }
419                        }
420
421                        if (asView || (lz instanceof IDataset))
422                                return lz.getSliceView(stt, stp, ste);
423                        try {
424                                return lz.getSlice(stt, stp, ste);
425                        } catch (DatasetException e) {
426                                logger.error("Could not slice dataset in metadata", e);
427                                return null;
428                        }
429                }
430        }
431
432        class MdsReshape implements MetadatasetAnnotationOperation {
433                private boolean matchRank;
434                private int[] oldShape;
435                private int[] newShape;
436                boolean onesOnly;
437                int[] differences;
438
439                /*
440                 * if only ones then record differences (insertions and deletions)
441                 * 
442                 * if shape changing, find broadcasted dimensions and disallow
443                 * merging that include those dimensions
444                 */
445                public MdsReshape(final int[] oldShape, final int[] newShape) {
446                        this.oldShape = oldShape;
447                        this.newShape = newShape;
448                        differences = null;
449                }
450
451                @Override
452                public Object processField(Field field, Object o) {
453                        Annotation a = field.getAnnotation(Reshapeable.class);
454                        if (a != null) { // cannot be null
455                                matchRank = ((Reshapeable) a).matchRank();
456                        }
457                        return o;
458                }
459
460                @Override
461                public Class<? extends Annotation> getAnnClass() {
462                        return Reshapeable.class;
463                }       
464
465                @Override
466                public int change(int axis) {
467                        if (matchRank) {
468                                if (differences == null)
469                                        init();
470
471                                if (onesOnly) {
472                                        return differences[axis];
473                                }
474                                throw new UnsupportedOperationException("TODO support other shape operations");
475                        }
476                        return 0;
477                }
478
479                @Override
480                public int getNewRank() {
481                        return matchRank ? newShape.length : -1;
482                }
483
484                private void init() {
485                        int or = oldShape.length - 1;
486                        int nr = newShape.length - 1;
487                        if (or < 0 || nr < 0) { // zero-rank shapes
488                                onesOnly = true;
489                                differences = new int[1];
490                                differences[0] = or < 0 ? nr + 1 : or + 1;
491                                return;
492                        }
493                        int ob = 0;
494                        int nb = 0;
495                        onesOnly = true;
496                        do {
497                                while (oldShape[ob] == 1 && ob < or) {
498                                        ob++; // next non-unit dimension
499                                }
500                                while (newShape[nb] == 1 && nb < nr) {
501                                        nb++;
502                                }
503                                if (oldShape[ob++] != newShape[nb++]) {
504                                        onesOnly = false;
505                                        break;
506                                }
507                        } while (ob <= or && nb <= nr);
508
509                        ob = 0;
510                        nb = 0;
511                        differences = new int[or + 2];
512                        if (onesOnly) {
513                                // work out unit dimensions removed from or add to old
514                                int j = 0;
515                                do {
516                                        if (oldShape[ob] != 1 && newShape[nb] != 1) {
517                                                ob++;
518                                                nb++;
519                                        } else {
520                                                while (oldShape[ob] == 1 && ob < or) {
521                                                        ob++;
522                                                        differences[j]--;
523                                                }
524                                                while (newShape[nb] == 1 && nb < nr) {
525                                                        nb++;
526                                                        differences[j]++;
527                                                }
528                                        }
529                                        j++;
530                                } while (ob <= or && nb <= nr && j <= or);
531                                while (ob <= or && oldShape[ob] == 1) {
532                                        ob++;
533                                        differences[j]--;
534                                }
535                                while (nb <= nr && newShape[nb] == 1) {
536                                        nb++;
537                                        differences[j]++;
538                                }
539                        } else {
540                                if (matchRank) {
541                                        logger.error("Combining dimensions is currently not supported");
542                                        throw new IllegalArgumentException("Combining dimensions is currently not supported");
543                                }
544                                // work out mapping: contiguous dimensions can be grouped or split
545                                while (ob <= or && nb <= nr) {
546                                        int ol = oldShape[ob];
547                                        while (ol == 1 && ol <= or) {
548                                                ob++;
549                                                ol = oldShape[ob];
550                                        }
551                                        int oe = ob + 1;
552                                        int nl = newShape[nb];
553                                        while (nl == 1 && nl <= nr) {
554                                                nb++;
555                                                nl = newShape[nb];
556                                        }
557                                        int ne = nb + 1;
558                                        if (ol < nl) {
559                                                differences[ob] = 1;
560                                                do { // case where new shape combines several dimensions into one dimension
561                                                        if (oe == (or + 1)) {
562                                                                break;
563                                                        }
564                                                        differences[oe] = 1;
565                                                        ol *= oldShape[oe++];
566                                                } while (ol < nl);
567                                                differences[oe - 1] = oe - ob; // signal end with difference
568                                                if (nl != ol) {
569                                                        logger.error("Single dimension is incompatible with subshape");
570                                                        throw new IllegalArgumentException("Single dimension is incompatible with subshape");
571                                                }
572                                        } else if (ol > nl) {
573                                                do { // case where new shape spreads single dimension over several dimensions
574                                                        if (ne == (nr + 1)) {
575                                                                break;
576                                                        }
577                                                        nl *= newShape[ne++];
578                                                } while (nl < ol);
579                                                if (nl != ol) {
580                                                        logger.error("Subshape is incompatible with single dimension");
581                                                        throw new IllegalArgumentException("Subshape is incompatible with single dimension");
582                                                }
583
584                                        }
585
586                                        ob = oe;
587                                        nb = ne;
588                                }
589
590                        }
591                }
592
593                @Override
594                public ILazyDataset run(ILazyDataset lz) {
595                        if (differences == null)
596                                init();
597
598                        int[] lshape = lz.getShape();
599                        if (Arrays.equals(newShape, lshape)) {
600                                return lz;
601                        }
602                        int or = lz.getRank();
603                        int nr = newShape.length;
604                        int[] nshape = new int[nr];
605                        Arrays.fill(nshape, 1);
606                        if (onesOnly) {
607                                // ignore omit removed dimensions
608                                for (int i = 0, si = 0, di = 0; i < (or+1) && si <= or && di < nr; i++) {
609                                        int c = differences[i];
610                                        if (c == 0) {
611                                                nshape[di++] = lshape[si++];
612                                        } else if (c > 0) {
613                                                while (c-- > 0 && di < nr) {
614                                                        di++;
615                                                }
616                                        } else if (c < 0) {
617                                                si -= c; // remove dimensions by skipping forward in source array
618                                        }
619                                }
620                        } else {
621                                boolean[] broadcast = new boolean[or];
622                                for (int ob = 0; ob < or; ob++) {
623                                        broadcast[ob] = oldShape[ob] != 1 && lshape[ob] == 1;
624                                }
625                                int osize = lz.getSize();
626
627                                // cannot do 3x5x... to 15x... if metadata is broadcasting (i.e. 1x5x...)
628                                int ob = 0;
629                                int nsize = 1;
630                                for (int i = 0; i < nr; i++) {
631                                        if (ob < or && broadcast[ob]) {
632                                                if (differences[ob] != 0) {
633                                                        logger.error("Metadata contains a broadcast axis which cannot be reshaped");
634                                                        throw new IllegalArgumentException("Metadata contains a broadcast axis which cannot be reshaped");
635                                                }
636                                        } else {
637                                                nshape[i] = nsize < osize ? newShape[i] : 1;
638                                        }
639                                        nsize *= nshape[i];
640                                        ob++;
641                                }
642                        }
643
644                        ILazyDataset nlz = lz.getSliceView();
645                        if (lz instanceof Dataset) {
646                                nlz = ((Dataset) lz).reshape(nshape);
647                        } else {
648                                nlz = lz.getSliceView();
649                                nlz.setShape(nshape);
650                        }
651                        return nlz;
652                }
653        }
654
655        class MdsTranspose implements MetadatasetAnnotationOperation {
656                int[] map;
657
658                public MdsTranspose(final int[] axesMap) {
659                        map = axesMap;
660                }
661
662                @SuppressWarnings({ "rawtypes", "unchecked" })
663                @Override
664                public Object processField(Field f, Object o) {
665                        // reorder arrays and lists according the axes map
666                        if (o.getClass().isArray()) {
667                                int l = Array.getLength(o);
668                                if (l == map.length) {
669                                        Object narray = Array.newInstance(o.getClass().getComponentType(), l);
670                                        for (int i = 0; i < l; i++) {
671                                                Array.set(narray, i, Array.get(o, map[i]));
672                                        }
673                                        for (int i = 0; i < l; i++) {
674                                                Array.set(o, i, Array.get(narray, i));
675                                        }
676                                }
677                        } else if (o instanceof List<?>) {
678                                List list = (List) o;
679                                int l = list.size();
680                                if (l == map.length) {
681                                        Object narray = Array.newInstance(o.getClass().getComponentType(), l);
682                                        for (int i = 0; i < l; i++) {
683                                                Array.set(narray, i, list.get(map[i]));
684                                        }
685                                        list.clear();
686                                        for (int i = 0; i < l; i++) {
687                                                list.add(Array.get(narray, i));
688                                        }
689                                }
690                        }
691                        return o;
692                }
693
694                @Override
695                public Class<? extends Annotation> getAnnClass() {
696                        return Transposable.class;
697                }
698
699                @Override
700                public int change(int axis) {
701                        return 0;
702                }
703
704                @Override
705                public int getNewRank() {
706                        return -1;
707                }
708
709                @Override
710                public ILazyDataset run(ILazyDataset lz) {
711                        return lz.getTransposedView(map);
712                }
713        }
714
715        class MdsDirty implements MetadatasetAnnotationOperation {
716
717                @Override
718                public Object processField(Field f, Object o) {
719                        // throw exception if not boolean???
720                        Class<?> t = f.getType();
721                        if (t.equals(boolean.class) || t.equals(Boolean.class)) {
722                                if (o.equals(false)) {
723                                        o = true;
724                                }
725                        }
726                        return o;
727                }
728
729                @Override
730                public Class<? extends Annotation> getAnnClass() {
731                        return Dirtiable.class;
732                }
733
734                @Override
735                public int change(int axis) {
736                        return 0;
737                }
738
739                @Override
740                public int getNewRank() {
741                        return -1;
742                }
743
744                @Override
745                public ILazyDataset run(ILazyDataset lz) {
746                        return lz;
747                }
748        }
749
750        /**
751         * Slice all datasets in metadata that are annotated by @Sliceable. Call this on the new sliced
752         * dataset after cloning the metadata
753         * @param asView if true then just a view
754         * @param slice
755         */
756        protected void sliceMetadata(boolean asView, final SliceND slice) {
757                processAnnotatedMetadata(new MdsSlice(asView, slice.getStart(), slice.getStop(), slice.getStep(), slice.getSourceShape()), true);
758        }
759
760        /**
761         * Reshape all datasets in metadata that are annotated by @Reshapeable. Call this when squeezing
762         * or setting the shape
763         * 
764         * @param newShape
765         */
766        protected void reshapeMetadata(final int[] oldShape, final int[] newShape) {
767                processAnnotatedMetadata(new MdsReshape(oldShape, newShape), true);
768        }
769
770        /**
771         * Transpose all datasets in metadata that are annotated by @Transposable. Call this on the transposed
772         * dataset after cloning the metadata
773         * @param axesMap
774         */
775        protected void transposeMetadata(final int[] axesMap) {
776                processAnnotatedMetadata(new MdsTranspose(axesMap), true);
777        }
778
779        /**
780         * Dirty metadata that are annotated by @Dirtiable. Call this when the dataset has been modified
781         * @since 2.0
782         */
783        protected void dirtyMetadata() {
784                processAnnotatedMetadata(new MdsDirty(), true);
785        }
786
787        @SuppressWarnings("unchecked")
788        private void processAnnotatedMetadata(MetadatasetAnnotationOperation op, boolean throwException) {
789                if (metadata == null)
790                        return;
791
792                for (Class<? extends MetadataType> c : metadata.keySet()) {
793                        for (MetadataType m : metadata.get(c)) {
794                                if (m == null)
795                                        continue;
796
797                                Class<? extends MetadataType> mc = m.getClass();
798                                do { // iterate over super-classes
799                                        processClass(op, m, mc, throwException);
800                                        Class<?> sclazz = mc.getSuperclass();
801                                        if (!MetadataType.class.isAssignableFrom(sclazz))
802                                                break;
803                                        mc = (Class<? extends MetadataType>) sclazz;
804                                } while (true);
805                        }
806                }
807        }
808
809        @SuppressWarnings({ "unchecked", "rawtypes" })
810        private static void processClass(MetadatasetAnnotationOperation op, MetadataType m, Class<? extends MetadataType> mc, boolean throwException) {
811                for (Field f : mc.getDeclaredFields()) {
812                        if (!f.isAnnotationPresent(op.getAnnClass()))
813                                continue;
814
815                        try {
816                                f.setAccessible(true);
817                                Object o = f.get(m);
818                                if (o == null)
819                                        continue;
820
821                                Object no = op.processField(f, o);
822                                if (no != o) {
823                                        f.set(m, no);
824                                        continue;
825                                }
826                                Object r = null;
827                                if (o instanceof ILazyDataset) {
828                                        try {
829                                                f.set(m, op.run((ILazyDataset) o));
830                                        } catch (Exception e) {
831                                                logger.error("Problem processing " + o, e);
832                                                if (!catchExceptions)
833                                                        throw e;
834                                        }
835                                } else if (o.getClass().isArray()) {
836                                        int l = Array.getLength(o);
837                                        if (l <= 0)
838                                                continue;
839
840                                        for (int i = 0; r == null && i < l; i++) {
841                                                r = Array.get(o, i);
842                                        }
843                                        int n = op.getNewRank();
844                                        if (r == null) {
845                                                if (n < 0 || n != l) { // all nulls be need to match rank as necessary
846                                                        f.set(m, Array.newInstance(o.getClass().getComponentType(), n < 0 ? l : n));
847                                                }
848                                                continue;
849                                        }
850                                        if (n < 0)
851                                                n = l;
852                                        Object narray = Array.newInstance(r.getClass(), n);
853                                        for (int i = 0, si = 0, di = 0; di < n && si < l; i++) {
854                                                int c = op.change(i);
855                                                if (c == 0) {
856                                                        Array.set(narray, di++, processObject(op, Array.get(o, si++)));
857                                                } else if (c > 0) {
858                                                        di += c; // add nulls by skipping forward in destination array
859                                                } else if (c < 0) {
860                                                        si -= c; // remove dimensions by skipping forward in source array
861                                                }
862                                        }
863                                        if (n == l) {
864                                                for (int i = 0; i < l; i++) {
865                                                        Array.set(o, i, Array.get(narray, i));
866                                                }
867                                        } else {
868                                                f.set(m, narray);
869                                        }
870                                } else if (o instanceof List<?>) {
871                                        List list = (List) o;
872                                        int l = list.size();
873                                        if (l <= 0)
874                                                continue;
875
876                                        for (int i = 0; r == null && i < l; i++) {
877                                                r = list.get(i);
878                                        }
879                                        int n = op.getNewRank();
880                                        if (r == null) {
881                                                if (n < 0 || n != l) { // all nulls be need to match rank as necessary
882                                                        list.clear();
883                                                        for (int i = 0, imax = n < 0 ? l : n; i < imax; i++) {
884                                                                list.add(null);
885                                                        }
886                                                }
887                                                continue;
888                                        }
889
890                                        if (n < 0)
891                                                n = l;
892                                        Object narray = Array.newInstance(r.getClass(), n);
893                                        for (int i = 0, si = 0, di = 0; i < l && si < l; i++) {
894                                                int c = op.change(i);
895                                                if (c == 0) {
896                                                        Array.set(narray, di++, processObject(op, list.get(si++)));
897                                                } else if (c > 0) {
898                                                        di += c; // add nulls by skipping forward in destination array
899                                                } else if (c < 0) {
900                                                        si -= c; // remove dimensions by skipping forward in source array
901                                                }
902                                        }
903                                        list.clear();
904                                        for (int i = 0; i < n; i++) {
905                                                list.add(Array.get(narray, i));
906                                        }
907                                } else if (o instanceof Map<?,?>) {
908                                        Map map = (Map) o;
909                                        for (Object k : map.keySet()) {
910                                                map.put(k, processObject(op, map.get(k)));
911                                        }
912                                }
913                        } catch (Exception e) {
914                                logger.error("Problem occurred when processing metadata of class {}: {}", mc.getCanonicalName(), e);
915                                if (throwException)
916                                        throw new RuntimeException(e);
917                        }
918                }
919        }
920
921        @SuppressWarnings({ "unchecked", "rawtypes" })
922        private static Object processObject(MetadatasetAnnotationOperation op, Object o) throws Exception {
923                if (o == null)
924                        return o;
925
926                if (o instanceof ILazyDataset) {
927                        try {
928                                return op.run((ILazyDataset) o);
929                        } catch (Exception e) {
930                                logger.error("Problem processing " + o, e);
931                                if (!catchExceptions)
932                                        throw e;
933                        }
934                } else if (o.getClass().isArray()) {
935                        int l = Array.getLength(o);
936                        for (int i = 0; i < l; i++) {
937                                Array.set(o, i, processObject(op, Array.get(o, i)));
938                        }
939                } else if (o instanceof List<?>) {
940                        List list = (List) o;
941                        for (int i = 0, imax = list.size(); i < imax; i++) {
942                                list.set(i, processObject(op, list.get(i)));
943                        }
944                } else if (o instanceof Map<?,?>) {
945                        Map map = (Map) o;
946                        for (Object k : map.keySet()) {
947                                map.put(k, processObject(op, map.get(k)));
948                        }
949                }
950                return o;
951        }
952
953        protected void restoreMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> oldMetadata) {
954                for (Class<? extends MetadataType> mc : oldMetadata.keySet()) {
955                        metadata.put(mc, oldMetadata.get(mc));
956                }
957        }
958
959        @SuppressWarnings("deprecation")
960        protected ILazyDataset createFromSerializable(Serializable blob, boolean keepLazy) {
961                ILazyDataset d = null;
962                if (blob instanceof ILazyDataset) {
963                        d = (ILazyDataset) blob;
964                        if (d instanceof IDataset) {
965                                Dataset ed = DatasetUtils.convertToDataset((IDataset) d);
966                                int is = ed.getElementsPerItem();
967                                if (is != 1 && is != getElementsPerItem()) {
968                                        throw new IllegalArgumentException("Dataset has incompatible number of elements with this dataset");
969                                }
970                                d = ed.cast(is == 1 ? Dataset.FLOAT64 : Dataset.ARRAYFLOAT64);
971                        } else if (!keepLazy) {
972                                final int is = getElementsPerItem();
973                                try {
974                                        d = DatasetUtils.cast(d.getSlice(), is == 1 ? Dataset.FLOAT64 : Dataset.ARRAYFLOAT64);
975                                } catch (DatasetException e) {
976                                        logger.error("Could not get data from lazy dataset", e);
977                                        return null;
978                                }
979                        }
980                } else {
981                        final int is = getElementsPerItem();
982                        if (is == 1) {
983                                d = DatasetFactory.createFromObject(Dataset.FLOAT64, blob);
984                        } else {
985                                try {
986                                        d = DatasetFactory.createFromObject(is, Dataset.ARRAYFLOAT64, blob);
987                                } catch (IllegalArgumentException e) { // if only single value supplied try again
988                                        d = DatasetFactory.createFromObject(Dataset.FLOAT64, blob);
989                                }
990                        }
991                        if (d.getSize() == getSize() && !Arrays.equals(d.getShape(), shape)) {
992                                d.setShape(shape.clone());
993                        }
994                }
995                List<int[]> s = BroadcastUtils.broadcastShapesToMax(shape, d.getShape());
996                d.setShape(s.get(0));
997
998                return d;
999        }
1000
1001        @Override
1002        public void setErrors(Serializable errors) {
1003                if (shape == null) {
1004                        throw new IllegalArgumentException("Cannot set errors for null dataset");
1005                }
1006                if (errors == null) {
1007                        clearMetadata(ErrorMetadata.class);
1008                        return;
1009                }
1010                if (errors == this) {
1011                        logger.warn("Ignoring setting error to itself as this will lead to infinite recursion");
1012                        return;
1013                }
1014
1015                ILazyDataset errorData = createFromSerializable(errors, true);
1016
1017                ErrorMetadata emd = getErrorMetadata();
1018                if (emd == null) {
1019                        try {
1020                                emd = MetadataFactory.createMetadata(ErrorMetadata.class);
1021                                setMetadata(emd);
1022                        } catch (MetadataException me) {
1023                                logger.error("Could not create metadata", me);
1024                        }
1025                }
1026                emd.setError(errorData);
1027        }
1028
1029        protected ErrorMetadata getErrorMetadata() {
1030                try {
1031                        List<ErrorMetadata> el = getMetadata(ErrorMetadata.class);
1032                        if (el != null && !el.isEmpty()) {
1033                                 return el.get(0);
1034                        }
1035                } catch (Exception e) {
1036                }
1037                return null;
1038        }
1039
1040        @Override
1041        public ILazyDataset getErrors() {
1042                ErrorMetadata emd = getErrorMetadata();
1043                return emd == null ? null : emd.getError();
1044        }
1045
1046        @Override
1047        public boolean hasErrors() {
1048                return LazyDatasetBase.this.getErrors() != null;
1049        }
1050
1051        /**
1052         * Check permutation axes
1053         * @param shape
1054         * @param axes
1055         * @return cleaned up axes or null if trivial
1056         */
1057        public static int[] checkPermutatedAxes(int[] shape, int... axes) {
1058                int rank = shape.length;
1059        
1060                if (axes == null || axes.length == 0) {
1061                        axes = new int[rank];
1062                        for (int i = 0; i < rank; i++) {
1063                                axes[i] = rank - 1 - i;
1064                        }
1065                }
1066
1067                if (axes.length != rank) {
1068                        logger.error("axis permutation has length {} that does not match dataset's rank {}", axes.length, rank);
1069                        throw new IllegalArgumentException("axis permutation does not match shape of dataset");
1070                }
1071        
1072                // check all permutation values are within bounds
1073                for (int d : axes) {
1074                        if (d < 0 || d >= rank) {
1075                                logger.error("axis permutation contains element {} outside rank of dataset", d);
1076                                throw new IllegalArgumentException("axis permutation contains element outside rank of dataset");
1077                        }
1078                }
1079        
1080                // check for a valid permutation (is this an unnecessary restriction?)
1081                int[] perm = axes.clone();
1082                Arrays.sort(perm);
1083
1084                for (int i = 0; i < rank; i++) {
1085                        if (perm[i] != i) {
1086                                logger.error("axis permutation is not valid: it does not contain complete set of axes");
1087                                throw new IllegalArgumentException("axis permutation does not contain complete set of axes");   
1088                        }
1089                }
1090
1091                if (Arrays.equals(axes, perm))
1092                        return null; // signal identity or trivial permutation
1093
1094                return axes;
1095        }
1096}