001/*-
002 *******************************************************************************
003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd.
004 * All rights reserved. This program and the accompanying materials
005 * are made available under the terms of the Eclipse Public License v1.0
006 * which accompanies this distribution, and is available at
007 * http://www.eclipse.org/legal/epl-v10.html
008 *
009 * Contributors:
010 *    Peter Chang - initial API and implementation and/or initial documentation
011 *******************************************************************************/
012
013package org.eclipse.january.dataset;
014
015import java.io.Serializable;
016import java.lang.reflect.Array;
017import java.text.Format;
018import java.util.ArrayList;
019import java.util.Arrays;
020import java.util.List;
021import java.util.concurrent.ConcurrentHashMap;
022import java.util.concurrent.ConcurrentMap;
023
024import org.eclipse.january.DatasetException;
025import org.eclipse.january.IMonitor;
026import org.eclipse.january.MetadataException;
027import org.eclipse.january.metadata.ErrorMetadata;
028import org.eclipse.january.metadata.MetadataFactory;
029import org.eclipse.january.metadata.MetadataType;
030import org.eclipse.january.metadata.StatisticsMetadata;
031import org.eclipse.january.metadata.internal.ErrorMetadataImpl;
032import org.eclipse.january.metadata.internal.StatisticsMetadataImpl;
033
034/**
035 * Generic container class for data 
036 * <p/>
037 * Each subclass has an array of primitive types, elements of this array are grouped or
038 * compounded to make items 
039 * <p/>
040 * Data items can be boolean, integer, float, complex float, vector float, etc
041 */
042public abstract class AbstractDataset extends LazyDatasetBase implements Dataset {
043        // pin UID to base class
044        private static final long serialVersionUID = Dataset.serialVersionUID;
045
046        protected int size; // number of items
047
048        transient protected AbstractDataset base; // is null when not a view
049        protected int[] stride; // can be null for row-major, contiguous datasets
050        protected int offset;
051
052        /**
053         * The data itself, held in a 1D array, but the object will wrap it to appear as possessing as many dimensions as
054         * wanted
055         */
056        protected Serializable odata = null;
057
058        /**
059         * Set aliased data as base data
060         */
061        abstract protected void setData();
062
063        /**
064         * Constructor required for serialisation.
065         */
066        public AbstractDataset() {
067        }
068
069        @Override
070        public synchronized Dataset synchronizedCopy() {
071                return clone();
072        }
073
074        @Override
075        public boolean equals(Object obj) {
076                if (this == obj) {
077                        return true;
078                }
079                if (obj == null) {
080                        return false;
081                }
082                if (!getClass().equals(obj.getClass())) {
083                        if (getRank() == 0) // for zero-rank datasets
084                                return obj.equals(getObjectAbs(offset));
085                        return false;
086                }
087
088                Dataset other = (Dataset) obj;
089                if (getElementsPerItem() != other.getElementsPerItem())
090                        return false;
091                if (size != other.getSize())
092                        return false;
093                if (!Arrays.equals(shape, other.getShapeRef())) {
094                        return false;
095                }
096
097                return true;
098        }
099
100        @Override
101        public int hashCode() {
102                return getStats().getHash(shape);
103        }
104
105        @Override
106        abstract public AbstractDataset clone();
107
108        protected Format stringFormat = null;
109
110        @Override
111        public void setStringFormat(Format format) {
112                stringFormat = format;
113        }
114
115        @Override
116        public Dataset copy(final int dtype) {
117                if (getDType() == dtype) {
118                        return clone();
119                }
120                return DatasetUtils.copy(this, dtype);
121        }
122
123        @Override
124        public <T extends Dataset> T copy(Class<T> clazz) {
125                return DatasetUtils.copy(clazz, this);
126        }
127
128        @Override
129        public Dataset cast(final int dtype) {
130                if (getDType() == dtype) {
131                        return this;
132                }
133                return DatasetUtils.cast(this, dtype);
134        }
135
136        @Override
137        public <T extends Dataset> T cast(Class<T> clazz) {
138                return DatasetUtils.cast(clazz, this);
139        }
140
141        @Override
142        public Dataset cast(final boolean repeat, final int dtype, final int isize) {
143                if (getDType() == dtype && getElementsPerItem() == isize) {
144                        return this;
145                }
146                return DatasetUtils.cast(this, repeat, dtype, isize);
147        }
148
149        @Override
150        abstract public AbstractDataset getView(boolean deepCopyMetadata);
151
152        /**
153         * Copy fields from original to view
154         * @param orig
155         * @param view
156         * @param clone if true, then clone everything but bulk data
157         * @param cloneMetadata if true, clone metadata
158         */
159        protected static void copyToView(Dataset orig, AbstractDataset view, boolean clone, boolean cloneMetadata) {
160                view.name = orig.getName();
161                view.size = orig.getSize();
162                view.odata = orig.getBuffer();
163                view.offset = orig.getOffset();
164                view.base = orig instanceof AbstractDataset ? ((AbstractDataset) orig).base : null;
165
166                if (clone) {
167                        view.shape = orig.getShape();
168                        view.stride = orig instanceof AbstractDataset && ((AbstractDataset) orig).stride != null ?
169                                        ((AbstractDataset) orig).stride.clone() : null;
170                } else {
171                        view.shape = orig.getShapeRef();
172                        view.stride = orig instanceof AbstractDataset ? ((AbstractDataset) orig).stride : null;
173                }
174
175                view.metadata = getMetadataMap(orig, cloneMetadata);
176                int odtype = orig.getDType();
177                int vdtype = view.getDType();
178                if (odtype != vdtype) {
179                        view.setDirty();
180                }
181        }
182
183        /**
184         * @since 2.0
185         */
186        protected static ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> getMetadataMap(Dataset a, boolean clone) {
187                if (a == null)
188                        return null;
189
190                List<MetadataType> all = null;
191                try {
192                        all = a.getMetadata(null);
193                } catch (Exception e) {
194                }
195                if (all == null)
196                        return null;
197
198                ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> map = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>();
199
200                for (MetadataType m : all) {
201                        if (m == null) {
202                                continue;
203                        }
204                        Class<? extends MetadataType> c = findMetadataTypeSubInterfaces(m.getClass());
205                        List<MetadataType> l = map.get(c);
206                        if (l == null) {
207                                l = new ArrayList<MetadataType>();
208                                map.put(c, l);
209                        }
210                        if (clone)
211                                m = m.clone();
212                        l.add(m);
213                }
214                return map;
215        }
216
217        @Override
218        public IntegerDataset getIndices() {
219                final IntegerDataset ret = DatasetUtils.indices(shape);
220                if (getName() != null) {
221                        ret.setName("Indices of " + getName());
222                }
223                return ret;
224        }
225
226        @Override
227        public Dataset getTransposedView(int... axes) {
228                axes = checkPermutatedAxes(shape, axes);
229
230                AbstractDataset t = getView(true);
231                if (axes == null || getRank() == 1)
232                        return t;
233
234                int rank = shape.length;
235                int[] tstride = new int[rank];
236                int[] toffset = new int[1];
237                int[] nshape = createStrides(new SliceND(shape), this, tstride, toffset);
238                int[] nstride = new int[rank];
239                for (int i = 0; i < rank; i++) {
240                        final int ax = axes[i];
241                        nstride[i] = tstride[ax];
242                        nshape[i] = shape[ax];
243                }
244                t.shape = nshape;
245                t.stride = nstride;
246                t.offset = toffset[0];
247                t.base = this;
248                t.setDirty();
249                t.transposeMetadata(axes);
250                return t;
251        }
252
253        @Override
254        public Dataset transpose(int... axes) {
255                Dataset t = getTransposedView(axes);
256                return t == null ? clone() : t.clone();
257        }
258
259        @Override
260        public Dataset swapAxes(int axis1, int axis2) {
261                int rank = shape.length;
262                axis1 = ShapeUtils.checkAxis(rank, axis1);
263                axis2 = ShapeUtils.checkAxis(rank, axis2);
264
265                if (rank == 1 || axis1 == axis2) {
266                        return this;
267                }
268
269                int[] axes = new int[rank];
270                for (int i = 0; i < rank; i++) {
271                        axes[i] = i;
272                }               
273
274                axes[axis1] = axis2;
275                axes[axis2] = axis1;
276                return getTransposedView(axes);
277        }
278
279        private boolean isContiguous() {
280                if (stride == null)
281                        return true;
282
283                if (offset != 0)
284                        return false;
285
286                int s = getElementsPerItem();
287                for (int j = getRank() - 1; j >= 0; j--) {
288                        if (stride[j] != s) {
289                                return false;
290                        }
291                        s *= shape[j];
292                }
293
294                return true;
295        }
296
297        @Override
298        public Dataset flatten() {
299                if (!isContiguous()) { // need to make a copy if not contiguous
300                        return clone().flatten();
301                }
302                return reshape(size);
303        }
304
305        /**
306         * Fill dataset from object at depth dimension
307         * @param obj
308         * @param depth
309         * @param pos position
310         */
311        protected void fillData(Object obj, final int depth, final int[] pos) {
312                if (obj == null) {
313                        int dtype = getDType();
314                        if (dtype == FLOAT32)
315                                set(Float.NaN, pos);
316                        else if (dtype == FLOAT64)
317                                set(Double.NaN, pos);
318                        return;
319                }
320
321                Class<?> clazz = obj.getClass();
322                if (obj instanceof List<?>) {
323                        List<?> jl = (List<?>) obj;
324                        int l = jl.size();
325                        for (int i = 0; i < l; i++) {
326                                Object lo = jl.get(i);
327                                fillData(lo, depth + 1, pos);
328                                pos[depth]++;
329                        }
330                        pos[depth] = 0;
331                } else if (clazz.isArray()) {
332                        int l = Array.getLength(obj);
333                        if (clazz.equals(odata.getClass())) {
334                                System.arraycopy(obj, 0, odata, get1DIndex(pos), l);
335                        } else if (clazz.getComponentType().isPrimitive()) {
336                                for (int i = 0; i < l; i++) {
337                                        set(Array.get(obj, i), pos);
338                                        pos[depth]++;
339                                }
340                                pos[depth] = 0;
341                        } else {
342                                for (int i = 0; i < l; i++) {
343                                        fillData(Array.get(obj, i), depth + 1, pos);
344                                        pos[depth]++;
345                                }
346                                pos[depth] = 0;
347                        }
348                } else if (obj instanceof IDataset) {
349                        boolean[] a = new boolean[shape.length];
350                        for (int i = depth; i < a.length; i++)
351                                a[i] = true;
352                        setSlice(obj, getSliceIteratorFromAxes(pos, a));
353                } else {
354                        set(obj, pos);
355                }
356        }
357
358        @Override
359        public IndexIterator getIterator(final boolean withPosition) {
360                if (stride != null) {
361                        return base.getSize() == 1  ? (withPosition ? new PositionIterator(offset, shape) :
362                                new SingleItemIterator(offset, size)) : new StrideIterator(shape, stride, offset);
363                }
364                if (shape == null) {
365                        return new NullIterator(shape, shape);
366                }
367                
368                return withPosition ? new ContiguousIteratorWithPosition(shape, size) : new ContiguousIterator(size);
369        }
370
371        @Override
372        public IndexIterator getIterator() {
373                return getIterator(false);
374        }
375
376        @Override
377        public PositionIterator getPositionIterator(final int... axes) {
378                return new PositionIterator(shape, axes);
379        }
380
381        @Override
382        public IndexIterator getSliceIterator(final int[] start, final int[] stop, final int[] step) {
383                return getSliceIterator(new SliceND(shape, start, stop, step));
384        }
385
386        /**
387         * @param slice
388         * @return an slice iterator that operates like an IndexIterator
389         */
390        public IndexIterator getSliceIterator(SliceND slice) {
391                if (ShapeUtils.calcLongSize(slice.getShape()) == 0) {
392                        return new NullIterator(shape, slice.getShape());
393                }
394                if (stride != null) {
395                        return new StrideIterator(getElementsPerItem(), shape, stride, offset, slice);
396                }
397                return new SliceIterator(shape, size, slice);
398        }
399
400        @Override
401        public SliceIterator getSliceIteratorFromAxes(final int[] pos, boolean[] axes) {
402                int rank = shape.length;
403                int[] start;
404                int[] stop = new int[rank];
405                int[] step = new int[rank];
406
407                if (pos == null) {
408                        start = new int[rank];
409                } else if (pos.length == rank) {
410                        start = pos.clone();
411                } else {
412                        throw new IllegalArgumentException("pos array length is not equal to rank of dataset");
413                }
414                if (axes == null) {
415                        axes = new boolean[rank];
416                        Arrays.fill(axes, true);
417                } else if (axes.length != rank) {
418                        throw new IllegalArgumentException("axes array length is not equal to rank of dataset");
419                }
420
421                for (int i = 0; i < rank; i++) {
422                        if (axes[i]) {
423                                stop[i] = shape[i];
424                        } else {
425                                stop[i] = start[i] + 1;
426                        }
427                        step[i] = 1;
428                }
429                return (SliceIterator) getSliceIterator(start, stop, step);
430        }
431
432        @Override
433        public BooleanIterator getBooleanIterator(Dataset choice) {
434                return getBooleanIterator(choice, true);
435        }
436
437        @Override
438        public BooleanIterator getBooleanIterator(Dataset choice, boolean value) {
439                return BooleanIterator.createIterator(value, this, choice, this);
440        }
441
442        @Override
443        public Dataset getByBoolean(Dataset selection) {
444                checkCompatibility(selection);
445
446                final int length = ((Number) selection.sum()).intValue();
447                final int is = getElementsPerItem();
448                Dataset r = DatasetFactory.zeros(is, getClass(), length);
449                BooleanIterator biter = getBooleanIterator(selection);
450
451                int i = 0;
452                while (biter.hasNext()) {
453                        r.setObjectAbs(i, getObjectAbs(biter.index));
454                        i += is;
455                }
456                return r;
457        }
458
459        @Override
460        public Dataset getBy1DIndex(IntegerDataset index) {
461                final int is = getElementsPerItem();
462                final Dataset r = DatasetFactory.zeros(is, getClass(), index.getShape());
463                final IntegerIterator iter = new IntegerIterator(index, size, is);
464
465                int i = 0;
466                while (iter.hasNext()) {
467                        r.setObjectAbs(i, getObjectAbs(iter.index));
468                        i += is;
469                }
470                return r;
471        }
472
473        @Override
474        public Dataset getByIndexes(final Object... indexes) {
475                final IntegersIterator iter = new IntegersIterator(shape, indexes);
476                final int is = getElementsPerItem();
477                final Dataset r = DatasetFactory.zeros(is, getClass(), iter.getShape());
478
479                final int[] pos = iter.getPos();
480                int i = 0;
481                while (iter.hasNext()) {
482                        r.setObjectAbs(i, getObject(pos));
483                        i += is;
484                }
485                return r;
486        }
487
488        @Override
489        public Class<?> getElementClass() {
490                return DTypeUtils.getElementClass(getDType());
491        }
492
493        @Override
494        public boolean hasFloatingPointElements() {
495                Class<?> cls = getElementClass();
496                return cls == Float.class || cls == Double.class;
497        }
498
499        @Override
500        public int getElementsPerItem() {
501                return DTypeUtils.getElementsPerItem(getDType());
502        }
503
504        @Override
505        public int getItemBytes() {
506                return DTypeUtils.getItemBytes(getDType(), getElementsPerItem());
507        }
508
509        @Override
510        public String getName() {
511                return name;
512        }
513
514        @Override
515        public void setName(final String name) {
516                this.name = name;
517        }
518
519        @Override
520        public int getSize() {
521                return size;
522        }
523
524        @Override
525        public int[] getShape() {
526                // make a copy of the dimensions data, and put that out
527                if (shape == null) {
528                        logger.warn("Shape is null!!!");
529                        return new int[] {};
530                }
531                return shape.clone();
532        }
533
534        @Override
535        public int getRank() {
536                return shape == null ? 0 : shape.length;
537        }
538
539        @Override
540        public int getNbytes() {
541                return getSize() * getItemBytes();
542        }
543
544        /**
545         * Check for -1 placeholder in shape and replace if necessary
546         * @param shape
547         * @param size
548         */
549        private void checkShape(int[] shape, int size) {
550                int rank = shape.length;
551                int found = -1;
552                int nsize = 1;
553                for (int i = 0; i < rank; i++) {
554                        int d = shape[i];
555                        if (d == -1) {
556                                if (found == -1) {
557                                        found = i;
558                                } else {
559                                        logger.error("Can only have one -1 placeholder in shape");
560                                        throw new IllegalArgumentException("Can only have one -1 placeholder in shape");
561                                }
562                        } else {
563                                nsize *= d;
564                        }
565                }
566                if (found >= 0) {
567                        shape[found] = size/nsize;
568                } else if (nsize != size && !(rank == 0 && size == 0)) {
569                        logger.error("New shape is not same size as old shape");
570                        throw new IllegalArgumentException("New size is not same as the old size. Old size is "+size+" new size is "+nsize+" and shape is "+Arrays.toString(shape));
571                }
572        }
573
574        @Override
575        public void setShape(final int... shape) {
576                int[] nshape = shape.clone();
577                checkShape(nshape, size);
578                if (Arrays.equals(this.shape, nshape)) {
579                        return;
580                }
581
582                if (stride != null) {
583                        // the only compatible shapes are ones where new dimensions are factors of old dimensions
584                        // or are combined adjacent old dimensions 
585                        int[] oshape = this.shape;
586                        int orank = oshape.length;
587                        int nrank = nshape.length;
588                        int diff = nrank - orank;
589                        int[] nstride = new int[nrank];
590                        boolean ones = true;
591                        // work forwards for broadcasting cases
592                        for (int i = 0, j = 0; i < orank || j < nrank;) {
593                                if (j >= diff && i < orank && j < nrank && oshape[i] == nshape[j]) {
594                                        nstride[j++] = stride[i++];
595                                } else if (j < nrank && nshape[j] == 1) {
596                                        nstride[j++] = 0;
597                                } else if (i < orank && oshape[i] == 1) {
598                                        i++;
599                                } else {
600                                        if (j < nrank)
601                                                j++;
602                                        if (i < orank)
603                                                i++;
604                                        ones = false;
605                                }
606                        }
607                        if (!ones) { // not just ones differ in shapes
608                                int[] ostride = stride;
609                                int ob = 0;
610                                int oe = 1;
611                                int nb = 0;
612                                int ne = 1;
613                                while (ob < orank && nb < nrank) {
614                                        int ol = oshape[ob];
615                                        int nl = nshape[nb];
616                                        
617                                        if (nl < ol) { // find group of shape dimensions that form common size
618                                                do { // case where new shape spreads single dimension over several dimensions
619                                                        if (ne == nrank) {
620                                                                break;
621                                                        }
622                                                        nl *= nshape[ne++];
623                                                } while (nl < ol);
624                                                if (nl != ol) {
625                                                        logger.error("Subshape is incompatible with single dimension");
626                                                        throw new IllegalArgumentException("Subshape is incompatible with single dimension");
627                                                }
628                                                int on = ne - 1;
629                                                while (nshape[on] == 1) {
630                                                        on--;
631                                                }
632
633                                                nstride[on] = ostride[ob];
634                                                for (int n = on - 1; n >= nb; n--) {
635                                                        if (nshape[n] == 1)
636                                                                continue;
637
638                                                        nstride[n] = nshape[on] * nstride[on];
639                                                        on = n;
640                                                }
641                                        } else if (ol < nl) {
642                                                do { // case where new shape combines several dimensions into one dimension
643                                                        if (oe == orank) {
644                                                                break;
645                                                        }
646                                                        ol *= oshape[oe++];
647                                                } while (ol < nl);
648                                                if (nl != ol) {
649                                                        logger.error("Single dimension is incompatible with subshape");
650                                                        throw new IllegalArgumentException("Single dimension is incompatible with subshape");
651                                                }
652
653                                                int oo = oe - 1;
654                                                while (oshape[oo] == 1) {
655                                                        oo--;
656                                                }
657                                                int os = ostride[oo];
658                                                for (int o = oo - 1; o >= ob; o--) {
659                                                        if (oshape[o] == 1)
660                                                                continue;
661                                                        if (ostride[o] != oshape[oo] * ostride[oo]) {
662                                                                logger.error("Subshape cannot be a non-contiguous view");
663                                                                throw new IllegalArgumentException("Subshape cannot be a non-contiguous view");
664                                                        }
665                                                        oo = o;
666                                                }
667                                                nstride[nb] = os;
668                                        } else {
669                                                nstride[nb] = ostride[ob];
670                                        }
671
672                                        ob = oe++;
673                                        nb = ne++;
674                                }
675                        }
676        
677                        stride = nstride;
678                }
679
680                setDirty();
681                if (this.shape != null) {
682                        reshapeMetadata(this.shape, nshape);
683                }
684                this.shape = nshape;
685        }
686
687        @Override
688        public int[] getShapeRef() {
689                return shape;
690        }
691
692        @Override
693        public int getOffset() {
694                return offset;
695        }
696
697        @Override
698        public int[] getStrides() {
699                return stride;
700        }
701
702        @Override
703        public Serializable getBuffer() {
704                return odata;
705        }
706
707        @Override
708        public void overrideInternal(Serializable buffer, int... shape) {
709                if (buffer != null) {
710                        odata = buffer;
711                        setData();
712                        setDirty();
713                }
714        
715                if (shape != null) {
716                        this.shape = shape.clone();
717                        size = ShapeUtils.calcSize(this.shape);
718                }
719        }
720
721        /**
722         * Create a stride array from dataset
723         * @param a dataset
724         * @param offset output offset
725         * @return new strides
726         */
727        public static int[] createStrides(Dataset a, final int[] offset) {
728                return createStrides(a.getElementsPerItem(), a.getShapeRef(), a.getStrides(), a.getOffset(), offset);
729        }
730
731        /**
732         * Create a stride array from dataset
733         * @param isize
734         * @param shape
735         * @param oStride original stride
736         * @param oOffset original offset (only used if there is an original stride)
737         * @param offset output offset
738         * @return new strides
739         */
740        public static int[] createStrides(final int isize, final int[] shape, final int[] oStride, final int oOffset, final int[] offset) {
741                int rank = shape.length;
742                final int[] stride;
743                if (oStride == null) {
744                        offset[0] = 0;
745                        stride = new int[rank];
746                        int s = isize;
747                        for (int j = rank - 1; j >= 0; j--) {
748                                stride[j] = s;
749                                s *= shape[j];
750                        }
751                } else {
752                        offset[0] = oOffset;
753                        stride = oStride.clone();
754                }
755                return stride;
756        }
757
758        /**
759         * Create a stride array from slice information and a dataset
760         * @param slice
761         * @param a dataset
762         * @param stride output stride
763         * @param offset output offset
764         * @return new shape
765         */
766        public static int[] createStrides(final SliceND slice, final Dataset a, final int[] stride, final int[] offset) {
767                return createStrides(slice, a.getElementsPerItem(), a.getShapeRef(), a.getStrides(), a.getOffset(), stride, offset);
768        }
769
770        /**
771         * Create a stride array from slice and dataset information
772         * @param slice
773         * @param isize
774         * @param shape
775         * @param oStride original stride
776         * @param oOffset original offset (only used if there is an original stride)
777         * @param stride output stride
778         * @param offset output offset
779         * @return new shape
780         */
781        public static int[] createStrides(final SliceND slice, final int isize, final int[] shape, final int[] oStride, final int oOffset, final int[] stride, final int[] offset) {
782                int[] lstart = slice.getStart();
783                int[] lstep = slice.getStep();
784                int[] newShape = slice.getShape();
785                int rank = shape.length;
786
787                if (oStride == null) {
788                        int s = isize;
789                        offset[0] = 0;
790                        for (int j = rank - 1; j >= 0; j--) {
791                                stride[j] = s * lstep[j];
792                                offset[0] += s * lstart[j];
793                                s *= shape[j];
794                        }
795                } else {
796                        offset[0] = oOffset;
797                        for (int j = 0; j < rank; j++) {
798                                int s = oStride[j];
799                                stride[j] = lstep[j] * s;
800                                offset[0] += lstart[j] * s;
801                        }
802                }
803
804                return newShape;
805        }
806
807        @Override
808        public Dataset getBroadcastView(int... broadcastShape) {
809                AbstractDataset view = getView(true);
810                
811                if (!Arrays.equals(shape, broadcastShape)) {
812                        List<int[]> nShapes = BroadcastUtils.broadcastShapesToMax(broadcastShape, shape);
813                        view.setShape(nShapes.get(0));
814                        view.stride = BroadcastUtils.createBroadcastStrides(view, broadcastShape);
815                        view.base = this;
816                        view.shape = broadcastShape.clone();
817                        view.size = ShapeUtils.calcSize(broadcastShape);
818                        if (view.name == null || view.name.isEmpty()) {
819                                view.name = "Broadcast from " + Arrays.toString(shape);
820                        } else {
821                                view.name = "Broadcast of " + view.name + " from " + Arrays.toString(shape);
822                        }
823                }
824                return view;
825        }
826
827        @Override
828        public Dataset getSliceView(final int[] start, final int[] stop, final int[] step) {
829                return getSliceView(new SliceND(shape, start, stop, step));
830        }
831
832        @Override
833        public Dataset getSliceView(Slice... slice) {
834                if (slice == null || slice.length == 0) {
835                        return getView(true);
836                }
837
838                return getSliceView(new SliceND(shape, slice));
839        }
840
841        /**
842         * Get a slice of the dataset. The returned dataset is a view on a selection of items
843         * @param slice
844         * @return slice view
845         */
846        @Override
847        public Dataset getSliceView(SliceND slice) {
848                if (slice.isAll()) {
849                        return getView(true);
850                }
851
852                final int rank = shape.length;
853                int[] sStride = new int[rank];
854                int[] sOffset = new int[1];
855
856                int[] sShape = createStrides(slice, this, sStride, sOffset);
857        
858                AbstractDataset s = getView(false);
859                s.shape = sShape;
860                s.size = ShapeUtils.calcSize(sShape);
861                s.stride = sStride;
862                s.offset = sOffset[0];
863                s.base = this;
864
865                s.metadata = copyMetadata();
866                s.sliceMetadata(true, slice);
867
868                s.setDirty();
869                s.setName(name + BLOCK_OPEN + slice + BLOCK_CLOSE);
870
871                return s;
872        }
873
874        /**
875         * Get flattened view index of given position 
876         * @param pos
877         *            the integer array specifying the n-D position
878         * @return the index on the flattened dataset
879         */
880        private int getFlat1DIndex(final int[] pos) {
881                final int imax = pos.length;
882                if (imax == 0) {
883                        return 0;
884                }
885
886                return get1DIndexFromShape(pos);
887        }
888
889        /**
890         * @since 2.0
891         */
892        protected int getFirst1DIndex() {
893                if (shape == null) {
894                        throw new IllegalArgumentException("Cannot find an index from a null shape");
895                }
896                return stride == null ? 0 : offset;
897        }
898
899        @Override
900        public int get1DIndex(final int... n) {
901                if (n.length == 0 && shape.length == 0)
902                        return offset;
903
904                return stride == null ? get1DIndexFromShape(n) : get1DIndexFromStrides(n);
905        }
906
907        private static void throwAIOOBException(int i, int s, int d) {
908                throw new ArrayIndexOutOfBoundsException("Index (" + i + ") out of range [-" + s + "," + s
909                                + "] in dimension " + d);
910        }
911
912        /**
913         * @param i
914         * @return the index on the data array corresponding to that location
915         */
916        protected int get1DIndex(int i) {
917                if (shape == null) {
918                        throw new IllegalArgumentException("Cannot find an index from a null shape");
919                }
920                if (shape.length > 1) {
921                        logger.error("This dataset is not 1D but was addressed as such");
922                        throw new UnsupportedOperationException("This dataset is not 1D but was addressed as such");
923                }
924                if (i < 0) {
925                        i += shape[0];
926                }
927                if (i < 0 || i >= shape[0]) {
928                        throwAIOOBException(i, shape[0], 0);
929                }
930                return stride == null ? i : i*stride[0] + offset;
931        }
932
933        /**
934         * @param i
935         * @param j
936         * @return the index on the data array corresponding to that location
937         */
938        protected int get1DIndex(int i, int j) {
939                if (shape == null) {
940                        throw new IllegalArgumentException("Cannot find an index from a null shape");
941                }
942                if (shape.length != 2) {
943                        logger.error("This dataset is not 2D but was addressed as such");
944                        throw new UnsupportedOperationException("This dataset is not 2D but was addressed as such");
945                }
946                if (i < 0) {
947                        i += shape[0];
948                }
949                if (i < 0 || i >= shape[0]) {
950                        throwAIOOBException(i, shape[0], 0);
951                }
952                if (j < 0) {
953                        j += shape[1];
954                }
955                if (j < 0 || j >= shape[1]) {
956                        throwAIOOBException(i, shape[1], 1);
957                }
958                return stride == null ? i*shape[1] + j : i*stride[0] + j*stride[1] + offset;
959        }
960
961        protected int get1DIndexFromShape(final int[] n) {
962                return get1DIndexFromShape(shape, n);
963        }
964
965        protected static int get1DIndexFromShape(final int[] shape, final int[] n) {
966                if (shape == null) {
967                        throw new IllegalArgumentException("Cannot find an index from a null shape");
968                }
969                final int rank = shape.length;
970                if (rank != n.length) {
971                        String errMsg = String.format("Number of position values must be equal to rank of %d", rank);
972                        logger.error(errMsg);
973                        throw new IllegalArgumentException(errMsg);
974                }
975                int index = 0;
976                for (int i = 0; i < rank; i++) {
977                        final int si = shape[i];
978                        int ni = n[i];
979                        if (ni < 0) {
980                                ni += si;
981                        }
982                        if (ni < 0 || ni >= si) {
983                                throwAIOOBException(ni, si, i);
984                        }
985                        index = index * si + ni;
986                }
987
988                return index;
989        }
990
991        private int get1DIndexFromStrides(final int[] n) {
992                return get1DIndexFromStrides(shape, stride, offset, n);
993        }
994
995        private static int get1DIndexFromStrides(final int[] shape, final int[] stride, final int offset, final int[] n) {
996                if (shape == null) {
997                        throw new IllegalArgumentException("Cannot find an index from a null shape");
998                }
999                final int rank = shape.length;
1000                if (rank != n.length) {
1001                        String errMsg = String.format("Number of position values must be equal to rank of %d", rank);
1002                        logger.error(errMsg);
1003                        throw new IllegalArgumentException(errMsg);
1004                }
1005                int index = offset;
1006                for (int i = 0; i < rank; i++) {
1007                        final int st = stride[i];
1008                        if (st != 0) { // not broadcasted
1009                                final int si = shape[i];
1010                                int ni = n[i];
1011                                if (ni < 0) {
1012                                        ni += si;
1013                                }
1014                                if (ni < 0 || ni >= si) {
1015                                        throwAIOOBException(ni, si, i);
1016                                }
1017                                index += st * ni;
1018                        }
1019                }
1020                return index;
1021        }
1022
1023        @Override
1024        public int[] getNDPosition(final int n) {
1025                if (isIndexInRange(n)) {
1026                        throw new IllegalArgumentException("Index provided " + n
1027                                        + "is larger then the size of the containing array");
1028                }
1029
1030                return stride == null ? ShapeUtils.getNDPositionFromShape(n, shape) : getNDPositionFromStrides(n);
1031        }
1032
1033        private boolean isIndexInRange(final int n) {
1034                if (stride == null) {
1035                        return n >= size;
1036                }
1037                return n >= getBufferLength();
1038        }
1039
1040        /**
1041         * @return entire buffer length
1042         */
1043        abstract protected int getBufferLength();
1044
1045        private int[] getNDPositionFromStrides(int n) {
1046                n -= offset;
1047                int rank = shape.length;
1048                if (rank == 1) {
1049                        return new int[] { n / stride[0] };
1050                }
1051
1052                int[] output = new int[rank];
1053                int i = 0;
1054                while (i != n) { // TODO find more efficient way than this exhaustive search
1055                        int j = rank - 1;
1056                        for (; j >= 0; j--) {
1057                                output[j]++;
1058                                i += stride[j];
1059                                if (output[j] >= shape[j]) {
1060                                        output[j] = 0;
1061                                        i -= shape[j] * stride[j];
1062                                } else {
1063                                        break;
1064                                }
1065                        }
1066                        if (j == -1) {
1067                                logger.error("Index was not found in this strided dataset");
1068                                throw new IllegalArgumentException("Index was not found in this strided dataset");
1069                        }
1070                }
1071
1072                return output;
1073        }
1074
1075        @Override
1076        public int checkAxis(int axis) {
1077                return ShapeUtils.checkAxis(shape.length, axis);
1078        }
1079
1080        @Deprecated
1081        protected static int checkAxis(int rank, int axis) {
1082                return ShapeUtils.checkAxis(rank, axis);
1083        }
1084
1085        protected static final char BLOCK_OPEN = '[';
1086        protected static final char BLOCK_CLOSE = ']';
1087
1088        @Override
1089        public String toString() {
1090                final int rank = shape == null ? 0 : shape.length;
1091                final StringBuilder out = new StringBuilder();
1092
1093                if (DTypeUtils.isDTypeElemental(getDType())) {
1094                        out.append("Dataset ");
1095                } else {
1096                        out.append("Compound dataset (");
1097                        out.append(getElementsPerItem());
1098                        out.append(") ");
1099                }
1100
1101                if (name != null && name.length() > 0) {
1102                        out.append("'");
1103                        out.append(name);
1104                        out.append("' has shape ");
1105                } else {
1106                        out.append("shape is ");
1107                }
1108
1109                out.append(BLOCK_OPEN);
1110                if (rank > 0) {
1111                        out.append(shape[0]);
1112                }
1113                for (int i = 1; i < rank; i++) {
1114                        out.append(", " + shape[i]);
1115                }
1116                out.append(BLOCK_CLOSE);
1117                return out.toString();
1118        }
1119
1120        @Override
1121        public String toString(boolean showData) {
1122                if (!showData) {
1123                        return toString();
1124                }
1125
1126                if (size == 0) {
1127                        return "[]";
1128                }
1129
1130                final int rank = shape == null ? 0 : shape.length;
1131                final StringBuilder out = new StringBuilder();
1132
1133                if (rank > 0) {
1134                        int[] pos = new int[rank];
1135                        final StringBuilder lead = new StringBuilder();
1136                        printBlocks(out, lead, 0, pos);
1137                } else {
1138                        out.append(getString());
1139                }
1140                return out.toString();
1141        }
1142
1143        /**
1144         * Limit to strings output via the toString() method
1145         */
1146        private static int maxStringLength = 120;
1147
1148        /**
1149         * Set maximum line length for toString() method
1150         * @param maxLineLength
1151         */
1152        public static void setMaxLineLength(int maxLineLength) {
1153                maxStringLength = maxLineLength;
1154        }
1155
1156        /**
1157         * @return maximum line length for toString() method
1158         */
1159        public static int getMaxLineLength() {
1160                return maxStringLength;
1161        }
1162
1163        /**
1164         * Limit to number of sub-blocks output via the toString() method
1165         */
1166        private static final int MAX_SUBBLOCKS = 6;
1167
1168        private final static String SEPARATOR = ",";
1169        private final static String SPACE = " ";
1170        private final static String ELLIPSIS = "...";
1171        private final static String NEWLINE = "\n";
1172
1173        /**
1174         * Make a line of output for last dimension of dataset
1175         * 
1176         * @param start
1177         * @return line
1178         */
1179        private StringBuilder makeLine(final int end, final int[] start) {
1180                StringBuilder line = new StringBuilder();
1181                final int[] pos;
1182                if (end >= start.length) {
1183                        pos = Arrays.copyOf(start, end + 1);
1184                } else {
1185                        pos = start;
1186                }
1187                pos[end] = 0;
1188                line.append(BLOCK_OPEN);
1189                line.append(getString(pos));
1190
1191                final int length = shape[end];
1192
1193                // trim elements printed if length exceed estimate of maximum elements
1194                int excess = length - maxStringLength / 3; // space + number + separator
1195                int midIndex = -1;
1196                if (excess > 0) {
1197                        int index = (length - excess) / 2;
1198                        for (int y = 1; y < index; y++) {
1199                                line.append(SEPARATOR + SPACE);
1200                                pos[end] = y;
1201                                line.append(getString(pos));
1202                        }
1203                        midIndex = line.length() + 2;
1204                        index = (length + excess) / 2;
1205                        for (int y = index; y < length; y++) {
1206                                line.append(SEPARATOR + SPACE);
1207                                pos[end] = y;
1208                                line.append(getString(pos));
1209                        }
1210                } else {
1211                        for (int y = 1; y < length; y++) {
1212                                line.append(SEPARATOR + SPACE);
1213                                pos[end] = y;
1214                                line.append(getString(pos));
1215                        }
1216                }
1217                line.append(BLOCK_CLOSE);
1218
1219                // trim string down to limit
1220                int lineLength = line.length();
1221                excess = lineLength - maxStringLength - ELLIPSIS.length() - 1;
1222                if (excess > 0) {
1223                        int index = (lineLength - excess) / 2;
1224                        if (midIndex > 0 && index > midIndex) {
1225                                index = midIndex;
1226                        } else {
1227                                index = line.lastIndexOf(SEPARATOR, index) + 2;
1228                        }
1229                        StringBuilder out = new StringBuilder(line.subSequence(0, index));
1230                        out.append(ELLIPSIS + SEPARATOR);
1231                        index = (lineLength + excess) / 2;
1232                        if (midIndex > 0 && index <= midIndex) {
1233                                index = midIndex - 1;
1234                        } else {
1235                                index = line.indexOf(SEPARATOR, index) + 1;
1236                        }
1237                        out.append(line.subSequence(index, lineLength));
1238                        return out;
1239                } else if (midIndex > 0) { // add ellipsis
1240                        StringBuilder out = new StringBuilder(line.subSequence(0, midIndex));
1241                        out.append(ELLIPSIS + SEPARATOR + SPACE);
1242                        out.append(line.subSequence(midIndex, lineLength));
1243                        return out;
1244                }
1245
1246                return line;
1247        }
1248
1249        /**
1250         * recursive method to print blocks
1251         */
1252        private void printBlocks(final StringBuilder out, final StringBuilder lead, final int level, final int[] pos) {
1253                if (out.length() > 0) {
1254                        char last = out.charAt(out.length() - 1);
1255                        if (last != BLOCK_OPEN) {
1256                                out.append(lead);
1257                        }
1258                }
1259                final int end = getRank() - 1;
1260                if (level != end) {
1261                        out.append(BLOCK_OPEN);
1262                        int length = shape[level];
1263
1264                        // first sub-block
1265                        pos[level] = 0;
1266                        StringBuilder newlead = new StringBuilder(lead);
1267                        newlead.append(SPACE);
1268                        printBlocks(out, newlead, level + 1, pos);
1269                        if (length < 2) { // escape
1270                                out.append(BLOCK_CLOSE);
1271                                return;
1272                        }
1273
1274                        out.append(SEPARATOR + NEWLINE);
1275                        for (int i = level + 1; i < end; i++) {
1276                                out.append(NEWLINE);
1277                        }
1278
1279                        // middle sub-blocks
1280                        if (length < MAX_SUBBLOCKS) {
1281                                for (int x = 1; x < length - 1; x++) {
1282                                        pos[level] = x;
1283                                        printBlocks(out, newlead, level + 1, pos);
1284                                        if (end <= level + 1) {
1285                                                out.append(SEPARATOR + NEWLINE);
1286                                        } else {
1287                                                out.append(SEPARATOR + NEWLINE + NEWLINE);
1288                                        }
1289                                }
1290                        } else {
1291                                final int excess = length - MAX_SUBBLOCKS;
1292                                int xmax = (length - excess) / 2;
1293                                for (int x = 1; x < xmax; x++) {
1294                                        pos[level] = x;
1295                                        printBlocks(out, newlead, level + 1, pos);
1296                                        if (end <= level + 1) {
1297                                                out.append(SEPARATOR + NEWLINE);
1298                                        } else {
1299                                                out.append(SEPARATOR + NEWLINE + NEWLINE);
1300                                        }
1301                                }
1302                                out.append(newlead);
1303                                out.append(ELLIPSIS + SEPARATOR + NEWLINE);
1304                                xmax = (length + excess) / 2;
1305                                for (int x = xmax; x < length - 1; x++) {
1306                                        pos[level] = x;
1307                                        printBlocks(out, newlead, level + 1, pos);
1308                                        if (end <= level + 1) {
1309                                                out.append(SEPARATOR + NEWLINE);
1310                                        } else {
1311                                                out.append(SEPARATOR + NEWLINE + NEWLINE);
1312                                        }
1313                                }
1314                        }
1315
1316                        // last sub-block
1317                        pos[level] = length - 1;
1318                        printBlocks(out, newlead, level + 1, pos);
1319                        out.append(BLOCK_CLOSE);
1320                } else {
1321                        out.append(makeLine(end, pos));
1322                }
1323        }
1324
1325        @Override
1326        public Dataset squeezeEnds() {
1327                return squeeze(true);
1328        }
1329
1330        @Override
1331        public Dataset squeeze() {
1332                return squeeze(false);
1333        }
1334
1335        @Override
1336        public Dataset squeeze(boolean onlyFromEnds) {
1337                final int[] tshape = ShapeUtils.squeezeShape(shape, onlyFromEnds);
1338                final int[] oshape = shape;
1339                if (stride == null) {
1340                        shape = tshape;
1341                } else {
1342                        int rank = shape.length;
1343                        int trank = tshape.length;
1344                        if (trank < rank) {
1345                                int[] tstride = new int[tshape.length];
1346                                if (onlyFromEnds) {
1347                                        for (int i = 0; i < rank; i++) {
1348                                                if (shape[i] != 1) {
1349                                                        for (int k = 0; k < trank; k++) {
1350                                                                tstride[k] = stride[i++];
1351                                                        }
1352                                                        break;
1353                                                }
1354                                        }
1355                                } else {
1356                                        int t = 0;
1357                                        for (int i = 0; i < rank; i++) {
1358                                                if (shape[i] != 1) {
1359                                                        tstride[t++] = stride[i];
1360                                                }
1361                                        }
1362                                }
1363                                shape = tshape;
1364                                stride = tstride;
1365                        }
1366                }
1367
1368                setDirty();
1369                reshapeMetadata(oshape, shape);
1370                return this;
1371        }
1372
1373        @Override
1374        public boolean isCompatibleWith(final ILazyDataset g) {
1375                return ShapeUtils.areShapesCompatible(shape, g.getShape());
1376        }
1377
1378        @Override
1379        public void checkCompatibility(final ILazyDataset g) throws IllegalArgumentException {
1380                ShapeUtils.checkCompatibility(this, g);
1381        }
1382
1383        @Override
1384        public Dataset reshape(final int... shape) {
1385                Dataset a;
1386                try {
1387                        a = getView(true);
1388                        a.setShape(shape);
1389                } catch (IllegalArgumentException e) {
1390                        a = clone();
1391                        a.setShape(shape);
1392                }
1393                return a;
1394        }
1395
1396        /**
1397         * @param start
1398         * @param stop
1399         * @param step
1400         * @return number of steps to take
1401         */
1402        protected static int calcSteps(final double start, final double stop, final double step) {
1403                return Math.max(0, (int) Math.ceil((stop - start) / step));
1404        }
1405
1406        @Override
1407        public boolean isComplex() {
1408                int type = getDType();
1409                return type == COMPLEX64 || type == COMPLEX128;
1410        }
1411
1412        @Override
1413        public Dataset getRealPart() {
1414                return this;
1415        }
1416
1417        @Override
1418        public Dataset getRealView() {
1419                return getView(true);
1420        }
1421
1422        @Override
1423        public Dataset getSlice(final int[] start, final int[] stop, final int[] step) {
1424                return getSlice(new SliceND(shape, start, stop, step));
1425        }
1426
1427        @Override
1428        public Dataset getSlice(Slice... slice) {
1429                return getSlice(new SliceND(shape, slice));
1430        }
1431
1432        @Override
1433        public Dataset getSlice(IMonitor monitor, Slice... slice) {
1434                return getSlice(slice);
1435        }
1436
1437        @Override
1438        public Dataset getSlice(IMonitor monitor, SliceND slice) {
1439                return getSlice(slice);
1440        }
1441
1442        @Override
1443        public Dataset getSlice(IMonitor monitor, int[] start, int[] stop, int[] step) {
1444                return getSlice(start, stop, step);
1445        }
1446
1447        /**
1448         * Get a slice of the dataset. The returned dataset is a copied selection of items
1449         * @param slice
1450         * @return The dataset of the sliced data
1451         */
1452        @Override
1453        public Dataset getSlice(final SliceND slice) {
1454                SliceIterator it = (SliceIterator) getSliceIterator(slice);
1455                AbstractDataset s = getSlice(it);
1456                s.metadata = copyMetadata();
1457                s.setDirty();
1458                s.sliceMetadata(true, slice);
1459                return s;
1460        }
1461
1462        /**
1463         * Get a slice of the dataset. The returned dataset is a copied selection of items
1464         * 
1465         * @param iterator Slice iterator
1466         * @return The dataset of the sliced data
1467         */
1468        abstract public AbstractDataset getSlice(final SliceIterator iterator);
1469
1470        @SuppressWarnings("deprecation")
1471        @Override
1472        public Dataset setSlice(final Object obj, final SliceND slice) {
1473                Dataset ds;
1474                if (obj instanceof Dataset) {
1475                        ds = (Dataset) obj;
1476                } else if (obj instanceof IDataset) {
1477                        ds = DatasetUtils.convertToDataset((IDataset) obj);
1478                } else {
1479                        int dtype = getDType();
1480                        if (dtype != Dataset.BOOL) {
1481                                dtype = DTypeUtils.getLargestDType(dtype);
1482                        }
1483                        ds = DatasetFactory.createFromObject(getElementsPerItem(), dtype, obj);
1484                }
1485
1486                return setSlicedView(getSliceView(slice), ds);
1487        }
1488
1489        @Override
1490        public Dataset setSlice(final Object obj, final int[] start, final int[] stop, final int[] step) {
1491                return setSlice(obj, new SliceND(shape, start, stop, step));
1492        }
1493
1494        /**
1495         * Set a view of current dataset to given dataset with broadcasting
1496         * @param view
1497         * @param d
1498         * @return this dataset
1499         */
1500        abstract Dataset setSlicedView(Dataset view, Dataset d);
1501
1502        @Override
1503        public Dataset setSlice(Object obj, Slice... slice) {
1504                if (slice == null || slice.length == 0) {
1505                        return setSlice(obj, new SliceND(shape));
1506                }
1507                return setSlice(obj, new SliceND(shape, slice));
1508        }
1509
1510        @Override
1511        public boolean all() {
1512                return Comparisons.allTrue(this);
1513        }
1514
1515        @Override
1516        public BooleanDataset all(final int axis) {
1517                return Comparisons.allTrue(this, axis);
1518        }
1519
1520        @Override
1521        public boolean any() {
1522                return Comparisons.anyTrue(this);
1523        }
1524
1525        @Override
1526        public BooleanDataset any(final int axis) {
1527                return Comparisons.anyTrue(this, axis);
1528        }
1529
1530        @Override
1531        public Dataset ifloorDivide(final Object o) {
1532                return idivide(o).ifloor();
1533        }
1534
1535        @Override
1536        public double residual(final Object o) {
1537                return residual(o, null, false);
1538        }
1539
1540        @Override
1541        public double residual(final Object o, boolean ignoreNaNs) {
1542                return residual(o, null, ignoreNaNs);
1543        }
1544
1545        /**
1546         * @since 2.0
1547         */
1548        @SuppressWarnings("unchecked")
1549        protected StatisticsMetadata<Number> getStats() {
1550                StatisticsMetadata<Number> md = getFirstMetadata(StatisticsMetadata.class);
1551                if (md == null || md.isDirty()) {
1552                        md = new StatisticsMetadataImpl<Number>();
1553                        md.initialize(this);
1554                        setMetadata(md);
1555                }
1556                return md;
1557        }
1558
1559        /**
1560         * @since 2.0
1561         */
1562        @SuppressWarnings("unchecked")
1563        protected StatisticsMetadata<String> getStringStats() {
1564                StatisticsMetadata<String> md = getFirstMetadata(StatisticsMetadata.class);
1565                if (md == null || md.isDirty()) {
1566                        md = new StatisticsMetadataImpl<String>();
1567                        md.initialize(this);
1568                        setMetadata(md);
1569                }
1570                return md;
1571        }
1572
1573        @Override
1574        public Number max(boolean... ignoreInvalids) {
1575                return getStats().getMaximum(ignoreInvalids);
1576        }
1577
1578        @Override
1579        public Dataset max(int axis, boolean... ignoreInvalids) {
1580                return getStats().getMaximum(axis, ignoreInvalids);
1581        }
1582
1583        @Override
1584        public Number min(boolean... ignoreInvalids) {
1585                return getStats().getMinimum(ignoreInvalids);
1586        }
1587
1588        @Override
1589        public Dataset min(int axis, boolean... ignoreInvalids) {
1590                return getStats().getMinimum(axis, ignoreInvalids);
1591        }
1592
1593        @Override
1594        public int argMax(boolean... ignoreInvalids) {
1595                return getFlat1DIndex(maxPos(ignoreInvalids));
1596        }
1597
1598        /**
1599         * @since 2.0
1600         */
1601        @Override
1602        public IntegerDataset argMax(int axis, boolean... ignoreInvalids) {
1603                return (IntegerDataset) getStats().getArgMaximum(axis, ignoreInvalids);
1604        }
1605
1606        @Override
1607        public int argMin(boolean... ignoreInvalids) {
1608                return getFlat1DIndex(minPos(ignoreInvalids));
1609        }
1610
1611        /**
1612         * @since 2.0
1613         */
1614        @Override
1615        public IntegerDataset argMin(int axis, boolean... ignoreInvalids) {
1616                return (IntegerDataset) getStats().getArgMinimum(axis, ignoreInvalids);
1617        }
1618
1619        @Override
1620        public Number peakToPeak(boolean... ignoreInvalids) {
1621                return DTypeUtils.fromDoubleToBiggestNumber(max(ignoreInvalids).doubleValue() - min(ignoreInvalids).doubleValue(), getDType());
1622        }
1623
1624        @Override
1625        public Dataset peakToPeak(int axis,  boolean... ignoreInvalids) {
1626                return Maths.subtract(max(axis, ignoreInvalids), min(axis, ignoreInvalids));
1627        }
1628
1629
1630        @Override
1631        public long count(boolean... ignoreInvalids) {
1632                return getStats().getCount(ignoreInvalids);
1633        }
1634
1635        @Override
1636        public Dataset count(int axis, boolean... ignoreInvalids) {
1637                return getStats().getCount(axis, ignoreInvalids);
1638        }
1639
1640        @Override
1641        public Object sum(boolean... ignoreInvalids) {
1642                return getStats().getSum(ignoreInvalids);
1643        }
1644
1645        @Override
1646        public Dataset sum(int axis, boolean... ignoreInvalids) {
1647                return getStats().getSum(axis, ignoreInvalids);
1648        }
1649
1650        @Override
1651        public Object product(boolean... ignoreInvalids) {
1652                return Stats.product(this, ignoreInvalids);
1653        }
1654
1655        @Override
1656        public Dataset product(int axis, boolean... ignoreInvalids) {
1657                return Stats.product(this, axis, ignoreInvalids);
1658        }
1659
1660        @Override
1661        public Object mean(boolean... ignoreInvalids) {
1662                return getStats().getMean(ignoreInvalids);
1663        }
1664
1665        @Override
1666        public Dataset mean(int axis, boolean... ignoreInvalids) {
1667                return getStats().getMean(axis, ignoreInvalids);
1668        }
1669
1670        @Override
1671        public double variance() {
1672                return variance(false);
1673        }
1674
1675        @Override
1676        public double variance(boolean isWholePopulation, boolean... ignoreInvalids) {
1677                return getStats().getVariance(isWholePopulation, ignoreInvalids);
1678        }
1679
1680        @Override
1681        public Dataset variance(int axis) {
1682                return getStats().getVariance(axis, false);
1683        }
1684
1685        @Override
1686        public Dataset variance(int axis, boolean isWholePopulation, boolean... ignoreInvalids) {
1687                return getStats().getVariance(axis, isWholePopulation, ignoreInvalids);
1688        }
1689
1690        @Override
1691        public double stdDeviation() {
1692                return Math.sqrt(variance());
1693        }
1694
1695        @Override
1696        public double stdDeviation(boolean isWholePopulation, boolean... ignoreInvalids) {
1697                return Math.sqrt(variance(isWholePopulation, ignoreInvalids));
1698        }
1699
1700        @Override
1701        public Dataset stdDeviation(int axis) {
1702                return Maths.sqrt(variance(axis, false));
1703        }
1704
1705        @Override
1706        public Dataset stdDeviation(int axis, boolean isWholePopulation, boolean... ignoreInvalids) {
1707                return Maths.sqrt(variance(axis, isWholePopulation, ignoreInvalids));
1708        }
1709
1710        @Override
1711        public double rootMeanSquare(boolean... ignoreInvalids) {
1712                StatisticsMetadata<Number> stats = getStats();
1713                final double mean = stats.getMean(ignoreInvalids).doubleValue();
1714                final double var = stats.getVariance(true, ignoreInvalids);
1715                return Math.sqrt(var + mean * mean);
1716        }
1717
1718        @Override
1719        public Dataset rootMeanSquare(int axis, boolean... ignoreInvalids) {
1720                StatisticsMetadata<Number> stats = getStats();
1721                Dataset v = stats.getVariance(axis, true, ignoreInvalids);
1722                Dataset m = stats.getMean(axis, ignoreInvalids);
1723                Dataset result = Maths.multiply(m, m);
1724                return Maths.sqrt(result.iadd(v));
1725        }
1726
1727        /**
1728         * Set item from compatible dataset in a direct and speedy way. Remember to setDirty afterwards.
1729         * 
1730         * @param dindex
1731         * @param sindex
1732         * @param src
1733         *            is the source data buffer
1734         */
1735        protected abstract void setItemDirect(final int dindex, final int sindex, final Object src);
1736
1737        /**
1738         * @return error broadcasted to current shape
1739         */
1740        private Dataset getBroadcastedInternalError() {
1741                if (shape == null) {
1742                        throw new IllegalArgumentException("Cannot get error for null dataset");
1743                }
1744                ILazyDataset led = super.getErrors();
1745                if (led == null)
1746                        return null;
1747
1748                Dataset ed = null;
1749                try {
1750                        ed = DatasetUtils.sliceAndConvertLazyDataset(led);
1751                } catch (DatasetException e) {
1752                        logger.error("Could not get data from lazy dataset", e);
1753                }
1754                if (led != ed) {
1755                        setErrors(ed); // set back
1756                }
1757
1758                return ed.getBroadcastView(shape);
1759        }
1760
1761        @Override
1762        public Dataset getErrors() {
1763                Dataset ed = getBroadcastedInternalError();
1764                if (ed == null)
1765                        return null;
1766
1767                return ed;
1768        }
1769
1770        @Override
1771        public double getError() {
1772                Dataset ed = getBroadcastedInternalError();
1773                if (ed == null)
1774                        return 0;
1775
1776                return ed.getDouble();
1777        }
1778
1779        @Override
1780        public double getError(final int i) {
1781                Dataset ed = getBroadcastedInternalError();
1782                if (ed == null)
1783                        return 0;
1784
1785                return ed.getDouble(i);
1786        }
1787
1788        @Override
1789        public double getError(final int i, final int j) {
1790                Dataset ed = getBroadcastedInternalError();
1791                if (ed == null)
1792                        return 0;
1793
1794                return ed.getDouble(i, j);
1795        }
1796
1797        @Override
1798        public double getError(int... pos) {
1799                Dataset ed = getBroadcastedInternalError();
1800                if (ed == null)
1801                        return 0;
1802
1803                return ed.getDouble(pos);
1804        }
1805
1806        @Override
1807        public double[] getErrorArray(final int i) {
1808                Dataset ed = getBroadcastedInternalError();
1809                if (ed == null)
1810                        return null;
1811
1812                return new double[] {getError(i)};
1813        }
1814
1815        @Override
1816        public double[] getErrorArray(final int i, final int j) {
1817                Dataset ed = getBroadcastedInternalError();
1818                if (ed == null)
1819                        return null;
1820
1821                return new double[] {getError(i, j)};
1822        }
1823
1824        @Override
1825        public double[] getErrorArray(int... pos) {
1826                Dataset ed = getBroadcastedInternalError();
1827                if (ed == null)
1828                        return null;
1829
1830                return new double[] {getError(pos)};
1831        }
1832
1833        protected Dataset getInternalSquaredError() {
1834                Dataset sed = getErrorBuffer().getBroadcastView(shape);
1835                return sed;
1836        }
1837
1838        @Override
1839        public Dataset getErrorBuffer() {
1840                ErrorMetadata emd = getErrorMetadata();
1841                if (emd == null)
1842                        return null;
1843
1844                if (!(emd instanceof ErrorMetadataImpl)) {
1845                        ILazyDataset led = emd.getError();
1846                        Dataset ed;
1847                        try {
1848                                ed = DatasetUtils.sliceAndConvertLazyDataset(led);
1849                                emd = MetadataFactory.createMetadata(ErrorMetadata.class);
1850                                setMetadata(emd);
1851                                emd.setError(ed);
1852                        } catch (MetadataException me) {
1853                                logger.error("Could not create metadata", me);
1854                        } catch (DatasetException e) {
1855                                logger.error("Could not get data from lazy dataset", e);
1856                        }
1857                }
1858
1859                return ((ErrorMetadataImpl) emd).getSquaredError();
1860        }
1861
1862        /**
1863         * Set a copy of the buffer that backs the (squared) error data
1864         * @param buffer can be null, anything that can be used to create a DoubleDataset or CompoundDoubleDataset
1865         */
1866        @Override
1867        public void setErrorBuffer(Serializable buffer) {
1868                if (shape == null) {
1869                        throw new IllegalArgumentException("Cannot set error buffer for null dataset");
1870                }
1871                if (buffer == null) {
1872                        clearMetadata(ErrorMetadata.class);
1873                        return;
1874                }
1875
1876                IDataset d = (IDataset) createFromSerializable(buffer, false);
1877                ErrorMetadata emd = getErrorMetadata();
1878                if (!(emd instanceof ErrorMetadataImpl)) {
1879                        try {
1880                                emd = MetadataFactory.createMetadata(ErrorMetadata.class);
1881                                setMetadata(emd);
1882                        } catch (MetadataException me) {
1883                                logger.error("Could not create metadata", me);
1884                        }
1885                }
1886                ((ErrorMetadataImpl) emd).setSquaredError(d);
1887        }
1888}