001/*-
002 *******************************************************************************
003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd.
004 * All rights reserved. This program and the accompanying materials
005 * are made available under the terms of the Eclipse Public License v1.0
006 * which accompanies this distribution, and is available at
007 * http://www.eclipse.org/legal/epl-v10.html
008 *
009 * Contributors:
010 *    Peter Chang - initial API and implementation and/or initial documentation
011 *******************************************************************************/
012
013package org.eclipse.january.dataset;
014
015import java.io.Serializable;
016import java.lang.reflect.Array;
017import java.text.Format;
018import java.util.Arrays;
019import java.util.List;
020
021import org.eclipse.january.DatasetException;
022import org.eclipse.january.IMonitor;
023import org.eclipse.january.MetadataException;
024import org.eclipse.january.metadata.ErrorMetadata;
025import org.eclipse.january.metadata.MetadataFactory;
026import org.eclipse.january.metadata.StatisticsMetadata;
027import org.eclipse.january.metadata.internal.ErrorMetadataImpl;
028import org.eclipse.january.metadata.internal.StatisticsMetadataImpl;
029
030/**
031 * Generic container class for data 
032 * <p>
033 * Each subclass has an array of primitive types, elements of this array are grouped or
034 * compounded to make items 
035 * <p>
036 * Data items can be boolean, integer, float, complex float, vector float, etc
037 */
038public abstract class AbstractDataset extends LazyDatasetBase implements Dataset {
039        // pin UID to base class
040        private static final long serialVersionUID = Dataset.serialVersionUID;
041
042        protected int size; // number of items
043
044        transient protected AbstractDataset base; // is null when not a view
045        protected int[] stride; // can be null for row-major, contiguous datasets
046        protected int offset;
047
048        /**
049         * The data itself, held in a 1D array, but the object will wrap it to appear as possessing as many dimensions as
050         * wanted
051         */
052        protected Serializable odata = null;
053
054        /**
055         * Set aliased data as base data
056         */
057        abstract protected void setData();
058
059        /**
060         * Constructor required for serialisation.
061         */
062        public AbstractDataset() {
063        }
064
065        @Override
066        public synchronized Dataset synchronizedCopy() {
067                return clone();
068        }
069
070        @Override
071        public int hashCode() {
072                return getStats().getHash(shape);
073        }
074
075        @Override
076        abstract public AbstractDataset clone();
077
078        protected Format stringFormat = null;
079
080        @Override
081        public void setStringFormat(Format format) {
082                stringFormat = format;
083        }
084
085        @Override
086        public Dataset copy(final int dtype) {
087                return copy(DTypeUtils.getInterface(dtype));
088        }
089
090        @Override
091        public <T extends Dataset> T copy(Class<T> clazz) {
092                return DatasetUtils.copy(clazz, this);
093        }
094
095        @Override
096        public Dataset cast(final int dtype) {
097                return cast(DTypeUtils.getInterface(dtype));
098        }
099
100        @SuppressWarnings("unchecked")
101        @Override
102        public <T extends Dataset> T cast(Class<T> clazz) {
103                if (getClass().equals(clazz)) {
104                        return (T) this;
105                }
106                return DatasetUtils.cast(clazz, this);
107        }
108
109        /**
110         * {@inheritDoc Dataset#cast(int, Class, boolean)}
111         * @since 2.3
112         */
113        @SuppressWarnings("unchecked")
114        @Override
115        public <T extends Dataset> T cast(final int isize, Class<T> clazz, final boolean repeat) {
116                if (getClass().equals(clazz) && getElementsPerItem() == isize) {
117                        return (T) this;
118                }
119                return DatasetUtils.cast(isize, clazz, this, repeat);
120        }
121
122        @Override
123        public Dataset cast(final boolean repeat, final int dtype, final int isize) {
124                return cast(isize, DTypeUtils.getInterface(dtype), repeat);
125        }
126
127        @Override
128        abstract public AbstractDataset getView(boolean deepCopyMetadata);
129
130        /**
131         * Copy fields from original to view
132         * @param orig
133         * @param view
134         * @param clone if true, then clone everything but bulk data
135         * @param cloneMetadata if true, clone metadata
136         */
137        protected static void copyToView(Dataset orig, AbstractDataset view, boolean clone, boolean cloneMetadata) {
138                view.name = orig.getName();
139                view.size = orig.getSize();
140                view.odata = orig.getBuffer();
141                view.offset = orig.getOffset();
142                view.base = orig instanceof AbstractDataset ? ((AbstractDataset) orig).base : null;
143
144                if (clone) {
145                        view.shape = orig.getShapeRef() == null ? null : orig.getShape();
146                        view.stride = orig instanceof AbstractDataset && ((AbstractDataset) orig).stride != null ?
147                                        ((AbstractDataset) orig).stride.clone() : null;
148                } else {
149                        view.shape = orig.getShapeRef();
150                        view.stride = orig instanceof AbstractDataset ? ((AbstractDataset) orig).stride : null;
151                }
152
153                view.metadata = getMetadataMap(orig, cloneMetadata);
154                int odtype = orig.getDType();
155                int vdtype = view.getDType();
156                if (odtype != vdtype) {
157                        view.setDirty();
158                }
159        }
160
161        @Override
162        public IntegerDataset getIndices() {
163                final IntegerDataset ret = DatasetUtils.indices(shape);
164                if (getName() != null) {
165                        ret.setName("Indices of " + getName());
166                }
167                return ret;
168        }
169
170        @Override
171        public Dataset getTransposedView(int... axes) {
172                axes = checkPermutatedAxes(shape, axes);
173
174                AbstractDataset t = getView(true);
175                if (axes == null || getRank() == 1)
176                        return t;
177
178                int rank = shape.length;
179                int[] tstride = new int[rank];
180                int[] toffset = new int[1];
181                int[] nshape = createStrides(new SliceND(shape), this, tstride, toffset);
182                int[] nstride = new int[rank];
183                for (int i = 0; i < rank; i++) {
184                        final int ax = axes[i];
185                        nstride[i] = tstride[ax];
186                        nshape[i] = shape[ax];
187                }
188                t.shape = nshape;
189                t.stride = nstride;
190                t.offset = toffset[0];
191                t.base = this;
192                t.setDirty();
193                t.transposeMetadata(axes);
194                return t;
195        }
196
197        @Override
198        public Dataset transpose(int... axes) {
199                Dataset t = getTransposedView(axes);
200                return t == null ? clone() : t.clone();
201        }
202
203        @Override
204        public Dataset swapAxes(int axis1, int axis2) {
205                int rank = shape.length;
206                axis1 = ShapeUtils.checkAxis(rank, axis1);
207                axis2 = ShapeUtils.checkAxis(rank, axis2);
208
209                if (rank == 1 || axis1 == axis2) {
210                        return this;
211                }
212
213                int[] axes = new int[rank];
214                for (int i = 0; i < rank; i++) {
215                        axes[i] = i;
216                }               
217
218                axes[axis1] = axis2;
219                axes[axis2] = axis1;
220                return getTransposedView(axes);
221        }
222
223        boolean isContiguous() {
224                if (stride == null)
225                        return true;
226
227                if (offset != 0)
228                        return false;
229
230                int s = getElementsPerItem();
231                for (int j = getRank() - 1; j >= 0; j--) {
232                        if (stride[j] != s) {
233                                return false;
234                        }
235                        s *= shape[j];
236                }
237
238                return true;
239        }
240
241        @Override
242        public Dataset flatten() {
243                if (!isContiguous()) { // need to make a copy if not contiguous
244                        return clone().flatten();
245                }
246                return reshape(size);
247        }
248
249        /**
250         * Fill dataset from object at depth dimension
251         * @param obj
252         * @param depth
253         * @param pos position
254         */
255        protected void fillData(Object obj, final int depth, final int[] pos) {
256                if (obj == null) {
257                        Class<?> c = InterfaceUtils.getElementClass(getClass());
258                        if (Float.class.equals(c)) {
259                                set(Float.NaN, pos);
260                        } else if (Float.class.equals(c)) {
261                                set(Double.NaN, pos);
262                        }
263                        return;
264                }
265
266                Class<?> clazz = obj.getClass();
267                if (obj instanceof List<?>) {
268                        List<?> jl = (List<?>) obj;
269                        int l = jl.size();
270                        for (int i = 0; i < l; i++) {
271                                Object lo = jl.get(i);
272                                fillData(lo, depth + 1, pos);
273                                pos[depth]++;
274                        }
275                        pos[depth] = 0;
276                } else if (clazz.isArray()) {
277                        int l = Array.getLength(obj);
278                        if (clazz.equals(odata.getClass())) {
279                                System.arraycopy(obj, 0, odata, get1DIndex(pos), l);
280                        } else if (clazz.getComponentType().isPrimitive()) {
281                                for (int i = 0; i < l; i++) {
282                                        set(Array.get(obj, i), pos);
283                                        pos[depth]++;
284                                }
285                                pos[depth] = 0;
286                        } else {
287                                for (int i = 0; i < l; i++) {
288                                        fillData(Array.get(obj, i), depth + 1, pos);
289                                        pos[depth]++;
290                                }
291                                pos[depth] = 0;
292                        }
293                } else if (obj instanceof IDataset) {
294                        boolean[] a = new boolean[shape.length];
295                        Arrays.fill(a, depth, a.length, true);
296                        setSlice(obj, getSliceIteratorFromAxes(pos, a));
297                } else {
298                        set(obj, pos);
299                }
300        }
301
302        @Override
303        public IndexIterator getIterator(final boolean withPosition) {
304                if (stride != null) {
305                        return base.getSize() == 1  ? (withPosition ? new PositionIterator(offset, shape) :
306                                new SingleItemIterator(offset, size)) : new StrideIterator(shape, stride, offset);
307                }
308                if (shape == null) {
309                        return new NullIterator(shape, shape);
310                }
311                
312                return withPosition ? new ContiguousIteratorWithPosition(shape, size) : new ContiguousIterator(size);
313        }
314
315        @Override
316        public IndexIterator getIterator() {
317                return getIterator(false);
318        }
319
320        @Override
321        public PositionIterator getPositionIterator(final int... axes) {
322                return new PositionIterator(shape, axes);
323        }
324
325        @Override
326        public IndexIterator getSliceIterator(final int[] start, final int[] stop, final int[] step) {
327                return getSliceIterator(new SliceND(shape, start, stop, step));
328        }
329
330        /**
331         * @param slice
332         * @return an slice iterator that operates like an IndexIterator
333         */
334        public IndexIterator getSliceIterator(SliceND slice) {
335                if (ShapeUtils.calcLongSize(slice.getShape()) == 0) {
336                        return new NullIterator(shape, slice.getShape());
337                }
338                if (stride != null) {
339                        return new StrideIterator(getElementsPerItem(), shape, stride, offset, slice);
340                }
341                return new SliceIterator(shape, size, slice);
342        }
343
344        @Override
345        public SliceIterator getSliceIteratorFromAxes(final int[] pos, boolean[] axes) {
346                int rank = shape.length;
347                int[] start;
348                int[] stop = new int[rank];
349                int[] step = new int[rank];
350
351                if (pos == null) {
352                        start = new int[rank];
353                } else if (pos.length == rank) {
354                        start = pos.clone();
355                } else {
356                        throw new IllegalArgumentException("pos array length is not equal to rank of dataset");
357                }
358                if (axes == null) {
359                        axes = new boolean[rank];
360                        Arrays.fill(axes, true);
361                } else if (axes.length != rank) {
362                        throw new IllegalArgumentException("axes array length is not equal to rank of dataset");
363                }
364
365                for (int i = 0; i < rank; i++) {
366                        if (axes[i]) {
367                                stop[i] = shape[i];
368                        } else {
369                                stop[i] = start[i] + 1;
370                        }
371                        step[i] = 1;
372                }
373                return (SliceIterator) getSliceIterator(start, stop, step);
374        }
375
376        @Override
377        public BooleanIterator getBooleanIterator(Dataset choice) {
378                return getBooleanIterator(choice, true);
379        }
380
381        @Override
382        public BooleanIterator getBooleanIterator(Dataset choice, boolean value) {
383                return BooleanIterator.createIterator(value, this, choice, this);
384        }
385
386        @Override
387        public Dataset getByBoolean(Dataset selection) {
388                checkCompatibility(selection);
389
390                final int length = ((Number) selection.sum()).intValue();
391                final int is = getElementsPerItem();
392                Dataset r = DatasetFactory.zeros(is, getClass(), length);
393                BooleanIterator biter = getBooleanIterator(selection);
394
395                int i = 0;
396                while (biter.hasNext()) {
397                        r.setObjectAbs(i, getObjectAbs(biter.index));
398                        i += is;
399                }
400                return r;
401        }
402
403        @Override
404        public Dataset getBy1DIndex(IntegerDataset index) {
405                final int is = getElementsPerItem();
406                final Dataset r = DatasetFactory.zeros(is, getClass(), index.getShapeRef());
407                final IntegerIterator iter = new IntegerIterator(index, size, is);
408
409                int i = 0;
410                while (iter.hasNext()) {
411                        r.setObjectAbs(i, getObjectAbs(iter.index));
412                        i += is;
413                }
414                return r;
415        }
416
417        @Override
418        public Dataset getByIndexes(final Object... indexes) {
419                final IntegersIterator iter = new IntegersIterator(shape, indexes);
420                final int is = getElementsPerItem();
421                final Dataset r = DatasetFactory.zeros(is, getClass(), iter.getShape());
422
423                final int[] pos = iter.getPos();
424                int i = 0;
425                while (iter.hasNext()) {
426                        r.setObjectAbs(i, getObject(pos));
427                        i += is;
428                }
429                return r;
430        }
431
432        @Override
433        public boolean hasFloatingPointElements() {
434                Class<?> cls = getElementClass();
435                return cls == Float.class || cls == Double.class;
436        }
437
438        @Override
439        public int getElementsPerItem() {
440                return DTypeUtils.getElementsPerItem(getDType());
441        }
442
443        @Override
444        public int getItemBytes() {
445                return DTypeUtils.getItemBytes(getDType(), getElementsPerItem());
446        }
447
448        @Override
449        public int getSize() {
450                return size;
451        }
452
453        @Override
454        public int[] getShape() {
455                // make a copy of the dimensions data, and put that out
456                if (shape == null) {
457                        logger.warn("Shape is null!!!");
458                        return new int[] {};
459                }
460                return shape.clone();
461        }
462
463        @Override
464        public int getRank() {
465                return shape == null ? 0 : shape.length;
466        }
467
468        @Override
469        public int getNbytes() {
470                return getSize() * getItemBytes();
471        }
472
473        /**
474         * Check for -1 placeholder in shape and replace if necessary
475         * @param shape
476         * @param size
477         */
478        private void checkShape(int[] shape, int size) {
479                int rank = shape.length;
480                int found = -1;
481                int nsize = 1;
482                for (int i = 0; i < rank; i++) {
483                        int d = shape[i];
484                        if (d == -1) {
485                                if (found == -1) {
486                                        found = i;
487                                } else {
488                                        logger.error("Can only have one -1 placeholder in shape");
489                                        throw new IllegalArgumentException("Can only have one -1 placeholder in shape");
490                                }
491                        } else {
492                                nsize *= d;
493                        }
494                }
495                if (found >= 0) {
496                        shape[found] = size/nsize;
497                } else if (nsize != size && !(rank == 0 && size == 0)) {
498                        logger.error("New shape is not same size as old shape");
499                        throw new IllegalArgumentException("New size is not same as the old size. Old size is "+size+" new size is "+nsize+" and shape is "+Arrays.toString(shape));
500                }
501        }
502
503        @Override
504        public void setShape(final int... shape) {
505                int[] nshape = shape.clone();
506                checkShape(nshape, size);
507                if (Arrays.equals(this.shape, nshape)) {
508                        return;
509                }
510
511                if (stride != null) {
512                        // the only compatible shapes are ones where new dimensions are factors of old dimensions
513                        // or are combined adjacent old dimensions 
514                        int[] oshape = this.shape;
515                        int orank = oshape.length;
516                        int nrank = nshape.length;
517                        int diff = nrank - orank;
518                        int[] nstride = new int[nrank];
519                        boolean ones = true;
520                        // work forwards for broadcasting cases
521                        for (int i = 0, j = 0; i < orank || j < nrank;) {
522                                if (j >= diff && i < orank && j < nrank && oshape[i] == nshape[j]) {
523                                        nstride[j++] = stride[i++];
524                                } else if (j < nrank && nshape[j] == 1) {
525                                        nstride[j++] = 0;
526                                } else if (i < orank && oshape[i] == 1) {
527                                        i++;
528                                } else {
529                                        if (j < nrank)
530                                                j++;
531                                        if (i < orank)
532                                                i++;
533                                        ones = false;
534                                }
535                        }
536                        if (!ones) { // not just ones differ in shapes
537                                int[] ostride = stride;
538                                int ob = 0;
539                                int oe = 1;
540                                int nb = 0;
541                                int ne = 1;
542                                while (ob < orank && nb < nrank) {
543                                        int ol = oshape[ob];
544                                        int nl = nshape[nb];
545                                        
546                                        if (nl < ol) { // find group of shape dimensions that form common size
547                                                do { // case where new shape spreads single dimension over several dimensions
548                                                        if (ne == nrank) {
549                                                                break;
550                                                        }
551                                                        nl *= nshape[ne++];
552                                                } while (nl < ol);
553                                                if (nl != ol) {
554                                                        logger.error("Subshape is incompatible with single dimension");
555                                                        throw new IllegalArgumentException("Subshape is incompatible with single dimension");
556                                                }
557                                                int on = ne - 1;
558                                                while (nshape[on] == 1) {
559                                                        on--;
560                                                }
561
562                                                nstride[on] = ostride[ob];
563                                                for (int n = on - 1; n >= nb; n--) {
564                                                        if (nshape[n] == 1)
565                                                                continue;
566
567                                                        nstride[n] = nshape[on] * nstride[on];
568                                                        on = n;
569                                                }
570                                        } else if (ol < nl) {
571                                                do { // case where new shape combines several dimensions into one dimension
572                                                        if (oe == orank) {
573                                                                break;
574                                                        }
575                                                        ol *= oshape[oe++];
576                                                } while (ol < nl);
577                                                if (nl != ol) {
578                                                        logger.error("Single dimension is incompatible with subshape");
579                                                        throw new IllegalArgumentException("Single dimension is incompatible with subshape");
580                                                }
581
582                                                int oo = oe - 1;
583                                                while (oshape[oo] == 1) {
584                                                        oo--;
585                                                }
586                                                int os = ostride[oo];
587                                                for (int o = oo - 1; o >= ob; o--) {
588                                                        if (oshape[o] == 1)
589                                                                continue;
590                                                        if (ostride[o] != oshape[oo] * ostride[oo]) {
591                                                                logger.error("Subshape cannot be a non-contiguous view");
592                                                                throw new IllegalArgumentException("Subshape cannot be a non-contiguous view");
593                                                        }
594                                                        oo = o;
595                                                }
596                                                nstride[nb] = os;
597                                        } else {
598                                                nstride[nb] = ostride[ob];
599                                        }
600
601                                        ob = oe++;
602                                        nb = ne++;
603                                }
604                        }
605        
606                        stride = nstride;
607                }
608
609                setDirty();
610                if (this.shape != null && metadata != null) {
611                        reshapeMetadata(this.shape, nshape);
612                }
613                this.shape = nshape;
614        }
615
616        @Override
617        public int[] getShapeRef() {
618                return shape;
619        }
620
621        @Override
622        public int getOffset() {
623                return offset;
624        }
625
626        @Override
627        public int[] getStrides() {
628                return stride;
629        }
630
631        @Override
632        public Serializable getBuffer() {
633                return odata;
634        }
635
636        @Override
637        public void overrideInternal(Serializable buffer, int... shape) {
638                if (buffer != null) {
639                        odata = buffer;
640                        setData();
641                        setDirty();
642                }
643        
644                if (shape != null) {
645                        this.shape = shape.clone();
646                        size = ShapeUtils.calcSize(this.shape);
647                }
648        }
649
650        /**
651         * Create a stride array from dataset
652         * @param a dataset
653         * @param offset output offset
654         * @return new strides
655         */
656        public static int[] createStrides(Dataset a, final int[] offset) {
657                return createStrides(a.getElementsPerItem(), a.getShapeRef(), a.getStrides(), a.getOffset(), offset);
658        }
659
660        /**
661         * Create a stride array from dataset
662         * @param isize
663         * @param shape
664         * @param oStride original stride
665         * @param oOffset original offset (only used if there is an original stride)
666         * @param offset output offset
667         * @return new strides
668         */
669        public static int[] createStrides(final int isize, final int[] shape, final int[] oStride, final int oOffset, final int[] offset) {
670                int rank = shape.length;
671                final int[] stride;
672                if (oStride == null) {
673                        offset[0] = 0;
674                        stride = new int[rank];
675                        int s = isize;
676                        for (int j = rank - 1; j >= 0; j--) {
677                                stride[j] = s;
678                                s *= shape[j];
679                        }
680                } else {
681                        offset[0] = oOffset;
682                        stride = oStride.clone();
683                }
684                return stride;
685        }
686
687        /**
688         * Create a stride array from slice information and a dataset
689         * @param slice
690         * @param a dataset
691         * @param stride output stride
692         * @param offset output offset
693         * @return new shape
694         */
695        public static int[] createStrides(final SliceND slice, final Dataset a, final int[] stride, final int[] offset) {
696                return createStrides(slice, a.getElementsPerItem(), a.getShapeRef(), a.getStrides(), a.getOffset(), stride, offset);
697        }
698
699        /**
700         * Create a stride array from slice and dataset information
701         * @param slice
702         * @param isize
703         * @param shape
704         * @param oStride original stride
705         * @param oOffset original offset (only used if there is an original stride)
706         * @param stride output stride
707         * @param offset output offset
708         * @return new shape
709         */
710        public static int[] createStrides(final SliceND slice, final int isize, final int[] shape, final int[] oStride, final int oOffset, final int[] stride, final int[] offset) {
711                int[] lstart = slice.getStart();
712                int[] lstep = slice.getStep();
713                int[] newShape = slice.getShape();
714                int rank = shape.length;
715
716                if (oStride == null) {
717                        int s = isize;
718                        offset[0] = 0;
719                        for (int j = rank - 1; j >= 0; j--) {
720                                stride[j] = s * lstep[j];
721                                offset[0] += s * lstart[j];
722                                s *= shape[j];
723                        }
724                } else {
725                        offset[0] = oOffset;
726                        for (int j = 0; j < rank; j++) {
727                                int s = oStride[j];
728                                stride[j] = lstep[j] * s;
729                                offset[0] += lstart[j] * s;
730                        }
731                }
732
733                return newShape;
734        }
735
736        @Override
737        public Dataset getBroadcastView(int... broadcastShape) {
738                AbstractDataset view = getView(true);
739                
740                if (!Arrays.equals(shape, broadcastShape)) {
741                        List<int[]> nShapes = BroadcastUtils.broadcastShapesToMax(broadcastShape, shape);
742                        view.setShape(nShapes.get(0));
743                        view.stride = BroadcastUtils.createBroadcastStrides(view, broadcastShape);
744                        view.base = this;
745                        view.shape = broadcastShape.clone();
746                        view.size = ShapeUtils.calcSize(broadcastShape);
747                        if (view.name == null || view.name.isEmpty()) {
748                                view.name = "Broadcast from " + Arrays.toString(shape);
749                        } else {
750                                view.name = "Broadcast of " + view.name + " from " + Arrays.toString(shape);
751                        }
752                }
753                return view;
754        }
755
756        @Override
757        public Dataset getSliceView(final int[] start, final int[] stop, final int[] step) {
758                return getSliceView(new SliceND(shape, start, stop, step));
759        }
760
761        @Override
762        public Dataset getSliceView(Slice... slice) {
763                if (slice == null || slice.length == 0) {
764                        return getView(true);
765                }
766
767                return getSliceView(new SliceND(shape, slice));
768        }
769
770        /**
771         * Get a slice of the dataset. The returned dataset is a view on a selection of items
772         * @param slice
773         * @return slice view
774         */
775        @Override
776        public Dataset getSliceView(SliceND slice) {
777                if (slice.isAll()) {
778                        return getView(true);
779                }
780
781                final int rank = shape.length;
782                int[] sStride = new int[rank];
783                int[] sOffset = new int[1];
784
785                int[] sShape = createStrides(slice, this, sStride, sOffset);
786        
787                AbstractDataset s = getView(false);
788                s.shape = sShape;
789                s.size = ShapeUtils.calcSize(sShape);
790                s.stride = sStride;
791                s.offset = sOffset[0];
792                s.base = this;
793
794                s.metadata = copyMetadata();
795                s.sliceMetadata(true, slice);
796
797                s.setDirty();
798                s.setName(name + BLOCK_OPEN + slice + BLOCK_CLOSE);
799
800                return s;
801        }
802
803        /**
804         * Get flattened view index of given position 
805         * @param pos
806         *            the integer array specifying the n-D position
807         * @return the index on the flattened dataset
808         */
809        private int getFlat1DIndex(final int[] pos) {
810                final int imax = pos.length;
811                if (imax == 0) {
812                        return 0;
813                }
814
815                return get1DIndexFromShape(pos);
816        }
817
818        /**
819         * @since 2.0
820         */
821        protected int getFirst1DIndex() {
822                if (shape == null) {
823                        throw new IllegalArgumentException("Cannot find an index from a null shape");
824                }
825                return stride == null ? 0 : offset;
826        }
827
828        @Override
829        public int get1DIndex(final int... n) {
830                if (n.length == 0 && shape.length == 0)
831                        return offset;
832
833                return stride == null ? get1DIndexFromShape(n) : get1DIndexFromStrides(n);
834        }
835
836        private static void throwAIOOBException(int i, int s, int d) {
837                throw new ArrayIndexOutOfBoundsException("Index (" + i + ") out of range [-" + s + "," + s
838                                + "] in dimension " + d);
839        }
840
841        /**
842         * @param i
843         * @return the index on the data array corresponding to that location
844         */
845        protected int get1DIndex(int i) {
846                if (shape == null) {
847                        throw new IllegalArgumentException("Cannot find an index from a null shape");
848                }
849                if (shape.length > 1) {
850                        logger.error("This dataset is not 1D but was addressed as such");
851                        throw new UnsupportedOperationException("This dataset is not 1D but was addressed as such");
852                }
853                if (i < 0) {
854                        i += shape[0];
855                }
856                if (i < 0 || i >= shape[0]) {
857                        throwAIOOBException(i, shape[0], 0);
858                }
859                return stride == null ? i : i*stride[0] + offset;
860        }
861
862        /**
863         * @param i
864         * @param j
865         * @return the index on the data array corresponding to that location
866         */
867        protected int get1DIndex(int i, int j) {
868                if (shape == null) {
869                        throw new IllegalArgumentException("Cannot find an index from a null shape");
870                }
871                if (shape.length != 2) {
872                        logger.error("This dataset is not 2D but was addressed as such");
873                        throw new UnsupportedOperationException("This dataset is not 2D but was addressed as such");
874                }
875                if (i < 0) {
876                        i += shape[0];
877                }
878                if (i < 0 || i >= shape[0]) {
879                        throwAIOOBException(i, shape[0], 0);
880                }
881                if (j < 0) {
882                        j += shape[1];
883                }
884                if (j < 0 || j >= shape[1]) {
885                        throwAIOOBException(i, shape[1], 1);
886                }
887                return stride == null ? i*shape[1] + j : i*stride[0] + j*stride[1] + offset;
888        }
889
890        protected int get1DIndexFromShape(final int[] n) {
891                return get1DIndexFromShape(shape, n);
892        }
893
894        protected static int get1DIndexFromShape(final int[] shape, final int[] n) {
895                if (shape == null) {
896                        throw new IllegalArgumentException("Cannot find an index from a null shape");
897                }
898                final int rank = shape.length;
899                if (rank != n.length) {
900                        String errMsg = String.format("Number of position values must be equal to rank of %d", rank);
901                        logger.error(errMsg);
902                        throw new IllegalArgumentException(errMsg);
903                }
904                int index = 0;
905                for (int i = 0; i < rank; i++) {
906                        final int si = shape[i];
907                        int ni = n[i];
908                        if (ni < 0) {
909                                ni += si;
910                        }
911                        if (ni < 0 || ni >= si) {
912                                throwAIOOBException(ni, si, i);
913                        }
914                        index = index * si + ni;
915                }
916
917                return index;
918        }
919
920        private int get1DIndexFromStrides(final int[] n) {
921                return get1DIndexFromStrides(shape, stride, offset, n);
922        }
923
924        private static int get1DIndexFromStrides(final int[] shape, final int[] stride, final int offset, final int[] n) {
925                if (shape == null) {
926                        throw new IllegalArgumentException("Cannot find an index from a null shape");
927                }
928                final int rank = shape.length;
929                if (rank != n.length) {
930                        String errMsg = String.format("Number of position values must be equal to rank of %d", rank);
931                        logger.error(errMsg);
932                        throw new IllegalArgumentException(errMsg);
933                }
934                int index = offset;
935                for (int i = 0; i < rank; i++) {
936                        final int st = stride[i];
937                        if (st != 0) { // not broadcasted
938                                final int si = shape[i];
939                                int ni = n[i];
940                                if (ni < 0) {
941                                        ni += si;
942                                }
943                                if (ni < 0 || ni >= si) {
944                                        throwAIOOBException(ni, si, i);
945                                }
946                                index += st * ni;
947                        }
948                }
949                return index;
950        }
951
952        @Override
953        public int[] getNDPosition(final int n) {
954                if (isIndexInRange(n)) {
955                        throw new IllegalArgumentException("Index provided " + n
956                                        + "is larger then the size of the containing array");
957                }
958
959                return stride == null ? ShapeUtils.getNDPositionFromShape(n, shape) : getNDPositionFromStrides(n);
960        }
961
962        private boolean isIndexInRange(final int n) {
963                if (stride == null) {
964                        return n >= size;
965                }
966                return n >= getBufferLength();
967        }
968
969        /**
970         * @return entire buffer length
971         */
972        abstract protected int getBufferLength();
973
974        private int[] getNDPositionFromStrides(int n) {
975                n -= offset;
976                int rank = shape.length;
977                if (rank == 1) {
978                        return new int[] { n / stride[0] };
979                }
980
981                int[] output = new int[rank];
982                int i = 0;
983                while (i != n) { // TODO find more efficient way than this exhaustive search
984                        int j = rank - 1;
985                        for (; j >= 0; j--) {
986                                output[j]++;
987                                i += stride[j];
988                                if (output[j] >= shape[j]) {
989                                        output[j] = 0;
990                                        i -= shape[j] * stride[j];
991                                } else {
992                                        break;
993                                }
994                        }
995                        if (j == -1) {
996                                logger.error("Index was not found in this strided dataset");
997                                throw new IllegalArgumentException("Index was not found in this strided dataset");
998                        }
999                }
1000
1001                return output;
1002        }
1003
1004        @Override
1005        public int checkAxis(int axis) {
1006                return ShapeUtils.checkAxis(shape.length, axis);
1007        }
1008
1009        @Deprecated
1010        protected static int checkAxis(int rank, int axis) {
1011                return ShapeUtils.checkAxis(rank, axis);
1012        }
1013
1014        protected static final char BLOCK_OPEN = '[';
1015        protected static final char BLOCK_CLOSE = ']';
1016
1017        @Override
1018        public String toString() {
1019                final int rank = shape == null ? 0 : shape.length;
1020                final StringBuilder out = new StringBuilder();
1021
1022                if (DTypeUtils.isDTypeElemental(getDType())) {
1023                        out.append("Dataset ");
1024                } else {
1025                        out.append("Compound dataset (");
1026                        out.append(getElementsPerItem());
1027                        out.append(") ");
1028                }
1029
1030                if (name != null && name.length() > 0) {
1031                        out.append("'");
1032                        out.append(name);
1033                        out.append("' has shape ");
1034                } else {
1035                        out.append("shape is ");
1036                }
1037
1038                out.append(BLOCK_OPEN);
1039                if (rank > 0) {
1040                        out.append(shape[0]);
1041                }
1042                for (int i = 1; i < rank; i++) {
1043                        out.append(", " + shape[i]);
1044                }
1045                out.append(BLOCK_CLOSE);
1046                return out.toString();
1047        }
1048
1049        @Override
1050        public String toString(boolean showData) {
1051                if (!showData) {
1052                        return toString();
1053                }
1054
1055                if (size == 0) {
1056                        return "[]";
1057                }
1058
1059                final int rank = shape == null ? 0 : shape.length;
1060                final StringBuilder out = new StringBuilder();
1061
1062                if (rank > 0) {
1063                        int[] pos = new int[rank];
1064                        final StringBuilder lead = new StringBuilder();
1065                        printBlocks(out, lead, 0, pos);
1066                } else {
1067                        out.append(getString());
1068                }
1069                return out.toString();
1070        }
1071
1072        /**
1073         * Limit to strings output via the toString() method
1074         */
1075        private static int maxStringLength = 120;
1076
1077        /**
1078         * Set maximum line length for toString() method
1079         * @param maxLineLength
1080         */
1081        public static void setMaxLineLength(int maxLineLength) {
1082                maxStringLength = maxLineLength;
1083        }
1084
1085        /**
1086         * @return maximum line length for toString() method
1087         */
1088        public static int getMaxLineLength() {
1089                return maxStringLength;
1090        }
1091
1092        /**
1093         * Limit to number of sub-blocks output via the toString() method
1094         */
1095        private static final int MAX_SUBBLOCKS = 6;
1096
1097        private final static String SEPARATOR = ",";
1098        private final static String SPACE = " ";
1099        private final static String ELLIPSIS = "...";
1100        private final static String NEWLINE = "\n";
1101
1102        /**
1103         * Make a line of output for last dimension of dataset
1104         * 
1105         * @param start
1106         * @return line
1107         */
1108        private StringBuilder makeLine(final int end, final int[] start) {
1109                StringBuilder line = new StringBuilder();
1110                final int[] pos;
1111                if (end >= start.length) {
1112                        pos = Arrays.copyOf(start, end + 1);
1113                } else {
1114                        pos = start;
1115                }
1116                pos[end] = 0;
1117                line.append(BLOCK_OPEN);
1118                line.append(getString(pos));
1119
1120                final int length = shape[end];
1121
1122                // trim elements printed if length exceed estimate of maximum elements
1123                int excess = length - maxStringLength / 3; // space + number + separator
1124                int midIndex = -1;
1125                if (excess > 0) {
1126                        int index = (length - excess) / 2;
1127                        for (int y = 1; y < index; y++) {
1128                                line.append(SEPARATOR + SPACE);
1129                                pos[end] = y;
1130                                line.append(getString(pos));
1131                        }
1132                        midIndex = line.length() + 2;
1133                        index = (length + excess) / 2;
1134                        for (int y = index; y < length; y++) {
1135                                line.append(SEPARATOR + SPACE);
1136                                pos[end] = y;
1137                                line.append(getString(pos));
1138                        }
1139                } else {
1140                        for (int y = 1; y < length; y++) {
1141                                line.append(SEPARATOR + SPACE);
1142                                pos[end] = y;
1143                                line.append(getString(pos));
1144                        }
1145                }
1146                line.append(BLOCK_CLOSE);
1147
1148                // trim string down to limit
1149                int lineLength = line.length();
1150                excess = lineLength - maxStringLength - ELLIPSIS.length() - 1;
1151                if (excess > 0) {
1152                        int index = (lineLength - excess) / 2;
1153                        if (midIndex > 0 && index > midIndex) {
1154                                index = midIndex;
1155                        } else {
1156                                index = line.lastIndexOf(SEPARATOR, index) + 2;
1157                        }
1158                        StringBuilder out = new StringBuilder(line.subSequence(0, index));
1159                        out.append(ELLIPSIS + SEPARATOR);
1160                        index = (lineLength + excess) / 2;
1161                        if (midIndex > 0 && index <= midIndex) {
1162                                index = midIndex - 1;
1163                        } else {
1164                                index = line.indexOf(SEPARATOR, index) + 1;
1165                        }
1166                        out.append(line.subSequence(index, lineLength));
1167                        return out;
1168                } else if (midIndex > 0) { // add ellipsis
1169                        StringBuilder out = new StringBuilder(line.subSequence(0, midIndex));
1170                        out.append(ELLIPSIS + SEPARATOR + SPACE);
1171                        out.append(line.subSequence(midIndex, lineLength));
1172                        return out;
1173                }
1174
1175                return line;
1176        }
1177
1178        /**
1179         * recursive method to print blocks
1180         */
1181        private void printBlocks(final StringBuilder out, final StringBuilder lead, final int level, final int[] pos) {
1182                if (out.length() > 0) {
1183                        char last = out.charAt(out.length() - 1);
1184                        if (last != BLOCK_OPEN) {
1185                                out.append(lead);
1186                        }
1187                }
1188                final int end = getRank() - 1;
1189                if (level != end) {
1190                        out.append(BLOCK_OPEN);
1191                        int length = shape[level];
1192
1193                        // first sub-block
1194                        pos[level] = 0;
1195                        StringBuilder newlead = new StringBuilder(lead);
1196                        newlead.append(SPACE);
1197                        printBlocks(out, newlead, level + 1, pos);
1198                        if (length < 2) { // escape
1199                                out.append(BLOCK_CLOSE);
1200                                return;
1201                        }
1202
1203                        out.append(SEPARATOR + NEWLINE);
1204                        for (int i = level + 1; i < end; i++) {
1205                                out.append(NEWLINE);
1206                        }
1207
1208                        // middle sub-blocks
1209                        if (length < MAX_SUBBLOCKS) {
1210                                for (int x = 1; x < length - 1; x++) {
1211                                        pos[level] = x;
1212                                        printBlocks(out, newlead, level + 1, pos);
1213                                        if (end <= level + 1) {
1214                                                out.append(SEPARATOR + NEWLINE);
1215                                        } else {
1216                                                out.append(SEPARATOR + NEWLINE + NEWLINE);
1217                                        }
1218                                }
1219                        } else {
1220                                final int excess = length - MAX_SUBBLOCKS;
1221                                int xmax = (length - excess) / 2;
1222                                for (int x = 1; x < xmax; x++) {
1223                                        pos[level] = x;
1224                                        printBlocks(out, newlead, level + 1, pos);
1225                                        if (end <= level + 1) {
1226                                                out.append(SEPARATOR + NEWLINE);
1227                                        } else {
1228                                                out.append(SEPARATOR + NEWLINE + NEWLINE);
1229                                        }
1230                                }
1231                                out.append(newlead);
1232                                out.append(ELLIPSIS + SEPARATOR + NEWLINE);
1233                                xmax = (length + excess) / 2;
1234                                for (int x = xmax; x < length - 1; x++) {
1235                                        pos[level] = x;
1236                                        printBlocks(out, newlead, level + 1, pos);
1237                                        if (end <= level + 1) {
1238                                                out.append(SEPARATOR + NEWLINE);
1239                                        } else {
1240                                                out.append(SEPARATOR + NEWLINE + NEWLINE);
1241                                        }
1242                                }
1243                        }
1244
1245                        // last sub-block
1246                        pos[level] = length - 1;
1247                        printBlocks(out, newlead, level + 1, pos);
1248                        out.append(BLOCK_CLOSE);
1249                } else {
1250                        out.append(makeLine(end, pos));
1251                }
1252        }
1253
1254        @Override
1255        public Dataset squeezeEnds() {
1256                return squeeze(true);
1257        }
1258
1259        @Override
1260        public Dataset squeeze() {
1261                return squeeze(false);
1262        }
1263
1264        @Override
1265        public Dataset squeeze(boolean onlyFromEnds) {
1266                final int[] tshape = ShapeUtils.squeezeShape(shape, onlyFromEnds);
1267                final int[] oshape = shape;
1268                if (stride == null) {
1269                        shape = tshape;
1270                } else {
1271                        int rank = shape.length;
1272                        int trank = tshape.length;
1273                        if (trank < rank) {
1274                                int[] tstride = new int[tshape.length];
1275                                if (onlyFromEnds) {
1276                                        for (int i = 0; i < rank; i++) {
1277                                                if (shape[i] != 1) {
1278                                                        for (int k = 0; k < trank; k++) {
1279                                                                tstride[k] = stride[i++];
1280                                                        }
1281                                                        break;
1282                                                }
1283                                        }
1284                                } else {
1285                                        int t = 0;
1286                                        for (int i = 0; i < rank; i++) {
1287                                                if (shape[i] != 1) {
1288                                                        tstride[t++] = stride[i];
1289                                                }
1290                                        }
1291                                }
1292                                shape = tshape;
1293                                stride = tstride;
1294                        }
1295                }
1296
1297                setDirty();
1298                reshapeMetadata(oshape, shape);
1299                return this;
1300        }
1301
1302        @Override
1303        public boolean isCompatibleWith(final ILazyDataset g) {
1304                return ShapeUtils.areShapesCompatible(shape, g.getShape());
1305        }
1306
1307        @Override
1308        public void checkCompatibility(final ILazyDataset g) throws IllegalArgumentException {
1309                ShapeUtils.checkCompatibility(this, g);
1310        }
1311
1312        @Override
1313        public Dataset reshape(final int... shape) {
1314                Dataset a;
1315                try {
1316                        a = getView(true);
1317                        a.setShape(shape);
1318                } catch (IllegalArgumentException e) {
1319                        a = clone();
1320                        a.setShape(shape);
1321                }
1322                return a;
1323        }
1324
1325        /**
1326         * @param start
1327         * @param stop
1328         * @param step
1329         * @return number of steps to take
1330         */
1331        protected static int calcSteps(final double start, final double stop, final double step) {
1332                return Math.max(0, (int) Math.ceil((stop - start) / step));
1333        }
1334
1335        @Override
1336        public boolean isComplex() {
1337                int type = getDType();
1338                return type == COMPLEX64 || type == COMPLEX128;
1339        }
1340
1341        @Override
1342        public Dataset getRealPart() {
1343                return this;
1344        }
1345
1346        @Override
1347        public Dataset getRealView() {
1348                return getView(true);
1349        }
1350
1351        @Override
1352        public Dataset getSlice(final int[] start, final int[] stop, final int[] step) {
1353                return getSlice(new SliceND(shape, start, stop, step));
1354        }
1355
1356        @Override
1357        public Dataset getSlice(Slice... slice) {
1358                return getSlice(new SliceND(shape, slice));
1359        }
1360
1361        @Override
1362        public Dataset getSlice(IMonitor monitor, Slice... slice) {
1363                return getSlice(slice);
1364        }
1365
1366        @Override
1367        public Dataset getSlice(IMonitor monitor, SliceND slice) {
1368                return getSlice(slice);
1369        }
1370
1371        @Override
1372        public Dataset getSlice(IMonitor monitor, int[] start, int[] stop, int[] step) {
1373                return getSlice(start, stop, step);
1374        }
1375
1376        /**
1377         * Get a slice of the dataset. The returned dataset is a copied selection of items
1378         * @param slice
1379         * @return The dataset of the sliced data
1380         */
1381        @Override
1382        public Dataset getSlice(final SliceND slice) {
1383                SliceIterator it = (SliceIterator) getSliceIterator(slice);
1384                AbstractDataset s = getSlice(it);
1385                s.metadata = copyMetadata();
1386                s.setDirty();
1387                s.sliceMetadata(true, slice);
1388                return s;
1389        }
1390
1391        /**
1392         * Get a slice of the dataset. The returned dataset is a copied selection of items
1393         * 
1394         * @param iterator Slice iterator
1395         * @return The dataset of the sliced data
1396         */
1397        abstract public AbstractDataset getSlice(final SliceIterator iterator);
1398
1399        @Override
1400        public Dataset setSlice(final Object obj, final SliceND slice) {
1401                Dataset ds;
1402                if (obj instanceof Dataset) {
1403                        ds = (Dataset) obj;
1404                } else if (obj instanceof IDataset) {
1405                        ds = DatasetUtils.convertToDataset((IDataset) obj);
1406                } else {
1407                        Class<? extends Dataset> dClass = getClass();
1408                        if (!BooleanDataset.class.equals(dClass)) {
1409                                dClass = DTypeUtils.getLargestDataset(dClass);
1410                        }
1411                        ds = DatasetFactory.createFromObject(getElementsPerItem(), dClass, obj);
1412                }
1413
1414                return setSlicedView(getSliceView(slice), ds);
1415        }
1416
1417        @Override
1418        public Dataset setSlice(final Object obj, final int[] start, final int[] stop, final int[] step) {
1419                return setSlice(obj, new SliceND(shape, start, stop, step));
1420        }
1421
1422        /**
1423         * Set a view of current dataset to given dataset with broadcasting
1424         * @param view
1425         * @param d
1426         * @return this dataset
1427         */
1428        abstract Dataset setSlicedView(Dataset view, Dataset d);
1429
1430        @Override
1431        public Dataset setSlice(Object obj, Slice... slice) {
1432                if (slice == null || slice.length == 0) {
1433                        return setSlice(obj, new SliceND(shape));
1434                }
1435                return setSlice(obj, new SliceND(shape, slice));
1436        }
1437
1438        @Override
1439        public boolean all() {
1440                return Comparisons.allTrue(this);
1441        }
1442
1443        @Override
1444        public BooleanDataset all(final int axis) {
1445                return Comparisons.allTrue(this, axis);
1446        }
1447
1448        @Override
1449        public boolean any() {
1450                return Comparisons.anyTrue(this);
1451        }
1452
1453        @Override
1454        public BooleanDataset any(final int axis) {
1455                return Comparisons.anyTrue(this, axis);
1456        }
1457
1458        @Override
1459        public Dataset ifloorDivide(final Object o) {
1460                return idivide(o).ifloor();
1461        }
1462
1463        @Override
1464        public double residual(final Object o) {
1465                return residual(o, null, false);
1466        }
1467
1468        @Override
1469        public double residual(final Object o, boolean ignoreNaNs) {
1470                return residual(o, null, ignoreNaNs);
1471        }
1472
1473        /**
1474         * @since 2.0
1475         */
1476        @SuppressWarnings("unchecked")
1477        protected StatisticsMetadata<Number> getStats() {
1478                StatisticsMetadata<Number> md = getFirstMetadata(StatisticsMetadata.class);
1479                if (md == null || md.isDirty(this)) {
1480                        md = new StatisticsMetadataImpl<Number>();
1481                        md.initialize(this);
1482                        setMetadata(md);
1483                }
1484                return md;
1485        }
1486
1487        /**
1488         * @since 2.0
1489         */
1490        @SuppressWarnings("unchecked")
1491        protected StatisticsMetadata<String> getStringStats() {
1492                StatisticsMetadata<String> md = getFirstMetadata(StatisticsMetadata.class);
1493                if (md == null || md.isDirty(this)) {
1494                        md = new StatisticsMetadataImpl<String>();
1495                        md.initialize(this);
1496                        setMetadata(md);
1497                }
1498                return md;
1499        }
1500
1501        @Override
1502        public Number max(boolean... ignoreInvalids) {
1503                return getStats().getMaximum(ignoreInvalids);
1504        }
1505
1506        @Override
1507        public Dataset max(int axis, boolean... ignoreInvalids) {
1508                return getStats().getMaximum(axis, ignoreInvalids);
1509        }
1510
1511        @Override
1512        public Dataset max(int[] axes, boolean... ignoreInvalids) {
1513                return getStats().getMaximum(axes, ignoreInvalids);
1514        }
1515
1516        @Override
1517        public Number min(boolean... ignoreInvalids) {
1518                return getStats().getMinimum(ignoreInvalids);
1519        }
1520
1521        @Override
1522        public Dataset min(int axis, boolean... ignoreInvalids) {
1523                return getStats().getMinimum(axis, ignoreInvalids);
1524        }
1525
1526        @Override
1527        public Dataset min(int[] axes, boolean... ignoreInvalids) {
1528                return getStats().getMinimum(axes, ignoreInvalids);
1529        }
1530
1531        @Override
1532        public int argMax(boolean... ignoreInvalids) {
1533                return getFlat1DIndex(maxPos(ignoreInvalids));
1534        }
1535
1536        /**
1537         * @since 2.0
1538         */
1539        @Override
1540        public IntegerDataset argMax(int axis, boolean... ignoreInvalids) {
1541                return (IntegerDataset) getStats().getArgMaximum(axis, ignoreInvalids);
1542        }
1543
1544        @Override
1545        public int argMin(boolean... ignoreInvalids) {
1546                return getFlat1DIndex(minPos(ignoreInvalids));
1547        }
1548
1549        /**
1550         * @since 2.0
1551         */
1552        @Override
1553        public IntegerDataset argMin(int axis, boolean... ignoreInvalids) {
1554                return (IntegerDataset) getStats().getArgMinimum(axis, ignoreInvalids);
1555        }
1556
1557        @Override
1558        public Number peakToPeak(boolean... ignoreInvalids) {
1559                return InterfaceUtils.fromDoubleToBiggestNumber(getClass(), max(ignoreInvalids).doubleValue() - min(ignoreInvalids).doubleValue());
1560        }
1561
1562        @Override
1563        public Dataset peakToPeak(int axis,  boolean... ignoreInvalids) {
1564                return Maths.subtract(max(axis, ignoreInvalids), min(axis, ignoreInvalids));
1565        }
1566
1567        @Override
1568        public Dataset peakToPeak(int[] axes,  boolean... ignoreInvalids) {
1569                return Maths.subtract(max(axes, ignoreInvalids), min(axes, ignoreInvalids));
1570        }
1571
1572        @Override
1573        public long count(boolean... ignoreInvalids) {
1574                return getStats().getCount(ignoreInvalids);
1575        }
1576
1577        @Override
1578        public Dataset count(int axis, boolean... ignoreInvalids) {
1579                return getStats().getCount(axis, ignoreInvalids);
1580        }
1581
1582        @Override
1583        public Dataset count(int[] axes, boolean... ignoreInvalids) {
1584                return getStats().getCount(axes, ignoreInvalids);
1585        }
1586
1587        @Override
1588        public Object sum(boolean... ignoreInvalids) {
1589                return InterfaceUtils.toBiggestNumber(getClass(), getStats().getSum(ignoreInvalids));
1590        }
1591
1592        @Override
1593        public Dataset sum(int axis, boolean... ignoreInvalids) {
1594                return getStats().getSum(axis, ignoreInvalids);
1595        }
1596
1597        @Override
1598        public Dataset sum(int[] axes, boolean... ignoreInvalids) {
1599                return getStats().getSum(axes, ignoreInvalids);
1600        }
1601
1602        @Override
1603        public Object product(boolean... ignoreInvalids) {
1604                return Stats.product(this, ignoreInvalids);
1605        }
1606
1607        @Override
1608        public Dataset product(int axis, boolean... ignoreInvalids) {
1609                return Stats.product(this, axis, ignoreInvalids);
1610        }
1611
1612        @Override
1613        public Dataset product(int[] axes, boolean... ignoreInvalids) {
1614                return Stats.product(this, axes, ignoreInvalids);
1615        }
1616
1617        @Override
1618        public Object mean(boolean... ignoreInvalids) {
1619                return getStats().getMean(ignoreInvalids);
1620        }
1621
1622        @Override
1623        public Dataset mean(int axis, boolean... ignoreInvalids) {
1624                return getStats().getMean(axis, ignoreInvalids);
1625        }
1626
1627        @Override
1628        public Dataset mean(int[] axes, boolean... ignoreInvalids) {
1629                return getStats().getMean(axes, ignoreInvalids);
1630        }
1631
1632        @Override
1633        public double variance() {
1634                return variance(false);
1635        }
1636
1637        @Override
1638        public double variance(boolean isWholePopulation, boolean... ignoreInvalids) {
1639                return getStats().getVariance(isWholePopulation, ignoreInvalids);
1640        }
1641
1642        @Override
1643        public Dataset variance(int axis) {
1644                return getStats().getVariance(axis, false);
1645        }
1646
1647        @Override
1648        public Dataset variance(int[] axes) {
1649                return getStats().getVariance(axes, false);
1650        }
1651
1652        @Override
1653        public Dataset variance(int axis, boolean isWholePopulation, boolean... ignoreInvalids) {
1654                return getStats().getVariance(axis, isWholePopulation, ignoreInvalids);
1655        }
1656
1657        @Override
1658        public Dataset variance(int[] axes, boolean isWholePopulation, boolean... ignoreInvalids) {
1659                return getStats().getVariance(axes, isWholePopulation, ignoreInvalids);
1660        }
1661
1662        @Override
1663        public double stdDeviation() {
1664                return Math.sqrt(variance());
1665        }
1666
1667        @Override
1668        public double stdDeviation(boolean isWholePopulation, boolean... ignoreInvalids) {
1669                return Math.sqrt(variance(isWholePopulation, ignoreInvalids));
1670        }
1671
1672        @Override
1673        public Dataset stdDeviation(int axis) {
1674                return Maths.sqrt(variance(axis, false));
1675        }
1676
1677        @Override
1678        public Dataset stdDeviation(int[] axes) {
1679                return Maths.sqrt(variance(axes, false));
1680        }
1681
1682        @Override
1683        public Dataset stdDeviation(int axis, boolean isWholePopulation, boolean... ignoreInvalids) {
1684                return Maths.sqrt(variance(axis, isWholePopulation, ignoreInvalids));
1685        }
1686
1687        @Override
1688        public Dataset stdDeviation(int[] axes, boolean isWholePopulation, boolean... ignoreInvalids) {
1689                return Maths.sqrt(variance(axes, isWholePopulation, ignoreInvalids));
1690        }
1691
1692        @Override
1693        public double rootMeanSquare(boolean... ignoreInvalids) {
1694                StatisticsMetadata<Number> stats = getStats();
1695                final double mean = stats.getMean(ignoreInvalids).doubleValue();
1696                final double var = stats.getVariance(true, ignoreInvalids);
1697                return Math.sqrt(var + mean * mean);
1698        }
1699
1700        @Override
1701        public Dataset rootMeanSquare(int axis, boolean... ignoreInvalids) {
1702                StatisticsMetadata<Number> stats = getStats();
1703                Dataset v = stats.getVariance(axis, true, ignoreInvalids);
1704                Dataset m = stats.getMean(axis, ignoreInvalids);
1705                Dataset result = Maths.multiply(m, m);
1706                return Maths.sqrt(result.iadd(v));
1707        }
1708
1709        @Override
1710        public Dataset rootMeanSquare(int[] axes, boolean... ignoreInvalids) {
1711                StatisticsMetadata<Number> stats = getStats();
1712                Dataset v = stats.getVariance(axes, true, ignoreInvalids);
1713                Dataset m = stats.getMean(axes, ignoreInvalids);
1714                Dataset result = Maths.multiply(m, m);
1715                return Maths.sqrt(result.iadd(v));
1716        }
1717
1718        /**
1719         * Set item from compatible dataset in a direct and speedy way. Remember to setDirty afterwards.
1720         * 
1721         * @param dindex
1722         * @param sindex
1723         * @param src
1724         *            is the source data buffer
1725         */
1726        protected abstract void setItemDirect(final int dindex, final int sindex, final Object src);
1727
1728        /**
1729         * @return error broadcasted to current shape
1730         */
1731        private Dataset getBroadcastedInternalError() {
1732                if (shape == null) {
1733                        throw new IllegalArgumentException("Cannot get error for null dataset");
1734                }
1735                ILazyDataset led = super.getErrors();
1736                if (led == null)
1737                        return null;
1738
1739                Dataset ed = null;
1740                try {
1741                        ed = DatasetUtils.sliceAndConvertLazyDataset(led);
1742                } catch (DatasetException e) {
1743                        logger.error("Could not get data from lazy dataset", e);
1744                }
1745                if (led != ed) {
1746                        setErrors(ed); // set back
1747                }
1748
1749                return ed.getBroadcastView(shape);
1750        }
1751
1752        @Override
1753        public Dataset getErrors() {
1754                Dataset ed = getBroadcastedInternalError();
1755                if (ed == null)
1756                        return null;
1757
1758                return ed;
1759        }
1760
1761        @Override
1762        public double getError() {
1763                Dataset ed = getBroadcastedInternalError();
1764                if (ed == null)
1765                        return 0;
1766
1767                return ed.getDouble();
1768        }
1769
1770        @Override
1771        public double getError(final int i) {
1772                Dataset ed = getBroadcastedInternalError();
1773                if (ed == null)
1774                        return 0;
1775
1776                return ed.getDouble(i);
1777        }
1778
1779        @Override
1780        public double getError(final int i, final int j) {
1781                Dataset ed = getBroadcastedInternalError();
1782                if (ed == null)
1783                        return 0;
1784
1785                return ed.getDouble(i, j);
1786        }
1787
1788        @Override
1789        public double getError(int... pos) {
1790                Dataset ed = getBroadcastedInternalError();
1791                if (ed == null)
1792                        return 0;
1793
1794                return ed.getDouble(pos);
1795        }
1796
1797        @Override
1798        public double[] getErrorArray(final int i) {
1799                Dataset ed = getBroadcastedInternalError();
1800                if (ed == null)
1801                        return null;
1802
1803                return new double[] {getError(i)};
1804        }
1805
1806        @Override
1807        public double[] getErrorArray(final int i, final int j) {
1808                Dataset ed = getBroadcastedInternalError();
1809                if (ed == null)
1810                        return null;
1811
1812                return new double[] {getError(i, j)};
1813        }
1814
1815        @Override
1816        public double[] getErrorArray(int... pos) {
1817                Dataset ed = getBroadcastedInternalError();
1818                if (ed == null)
1819                        return null;
1820
1821                return new double[] {getError(pos)};
1822        }
1823
1824        protected Dataset getInternalSquaredError() {
1825                Dataset sed = getErrorBuffer().getBroadcastView(shape);
1826                return sed;
1827        }
1828
1829        @Override
1830        public Dataset getErrorBuffer() {
1831                ErrorMetadata emd = getErrorMetadata();
1832                if (emd == null)
1833                        return null;
1834
1835                if (!(emd instanceof ErrorMetadataImpl)) {
1836                        ILazyDataset led = emd.getError();
1837                        Dataset ed;
1838                        try {
1839                                ed = DatasetUtils.sliceAndConvertLazyDataset(led);
1840                                emd = MetadataFactory.createMetadata(ErrorMetadata.class);
1841                                setMetadata(emd);
1842                                emd.setError(ed);
1843                        } catch (MetadataException me) {
1844                                logger.error("Could not create metadata", me);
1845                        } catch (DatasetException e) {
1846                                logger.error("Could not get data from lazy dataset", e);
1847                        }
1848                }
1849
1850                return ((ErrorMetadataImpl) emd).getSquaredError();
1851        }
1852
1853        /**
1854         * Set a copy of the buffer that backs the (squared) error data
1855         * @param buffer can be null, anything that can be used to create a DoubleDataset or CompoundDoubleDataset
1856         */
1857        @Override
1858        public void setErrorBuffer(Serializable buffer) {
1859                if (shape == null) {
1860                        throw new IllegalArgumentException("Cannot set error buffer for null dataset");
1861                }
1862                if (buffer == null) {
1863                        clearMetadata(ErrorMetadata.class);
1864                        return;
1865                }
1866
1867                IDataset d = (IDataset) createFromSerializable(buffer, false);
1868                ErrorMetadata emd = getErrorMetadata();
1869                if (!(emd instanceof ErrorMetadataImpl)) {
1870                        try {
1871                                emd = MetadataFactory.createMetadata(ErrorMetadata.class);
1872                                setMetadata(emd);
1873                        } catch (MetadataException me) {
1874                                logger.error("Could not create metadata", me);
1875                        }
1876                }
1877                ((ErrorMetadataImpl) emd).setSquaredError(d);
1878        }
1879}