001/*- 002 ******************************************************************************* 003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd. 004 * All rights reserved. This program and the accompanying materials 005 * are made available under the terms of the Eclipse Public License v1.0 006 * which accompanies this distribution, and is available at 007 * http://www.eclipse.org/legal/epl-v10.html 008 * 009 * Contributors: 010 * Peter Chang - initial API and implementation and/or initial documentation 011 *******************************************************************************/ 012 013package org.eclipse.january.dataset; 014 015import java.io.Serializable; 016import java.util.ArrayList; 017import java.util.List; 018 019import org.eclipse.january.DatasetException; 020import org.eclipse.january.IMonitor; 021 022/** 023 * Class to aggregate a set of lazy datasets and present them as a single lazy dataset where 024 * the first position value accesses the aggregation 025 */ 026public class AggregateDataset extends LazyDatasetBase implements ILazyDataset { 027 028 /** 029 * Update this when there are any serious changes to API 030 */ 031 private static final long serialVersionUID = -5523566223386837581L; 032 033 private ILazyDataset[] data = null; // array of lazy datasets 034 private int[] map = null; // map first dimension to index of dataset 035 private int[] offset = null; // cumulative first dimension lengths used as slice offsets 036 private int size; 037 private int dtype = -1; 038 private int isize; // number of elements per item 039 protected AggregateDataset base = null; 040 private int[] sliceStart = null; 041 private int[] sliceStep = null; 042 043 /** 044 * Calculate (possibly extended) shapes from given datasets 045 * @param extend if true, extend rank by one 046 * @param datasets 047 * @return array of shapes 048 */ 049 public static int[][] calcShapes(boolean extend, ILazyDataset... datasets) { 050 if (datasets.length == 0) 051 throw new IllegalArgumentException("No datasets given"); 052 053 int maxRank = -1; 054 for (ILazyDataset d : datasets) { 055 if (d == null) 056 throw new IllegalArgumentException("Null dataset given"); 057 058 int r = d.getRank(); 059 if (r > maxRank) 060 maxRank = r; 061 } 062 063 if (extend) 064 maxRank++; 065 066 int[][] shapes = new int[datasets.length][]; 067 for (int j = 0; j < datasets.length; j++) { 068 ILazyDataset d = datasets[j]; 069 int[] s = d.getShape(); 070 if (s.length < maxRank) { 071 int[] ns = new int[maxRank]; 072 int start = maxRank - s.length; 073 074 for (int i = 0; i < start; i++) { // prepend ones as necessary 075 ns[i] = 1; 076 } 077 for (int i = 0; i < s.length; i++) { 078 ns[i+start] = s[i]; 079 } 080 s = ns; 081 } 082 shapes[j] = s; 083 } 084 085 return shapes; 086 } 087 088 AggregateDataset(int itemSize, int[] shape, int dtype) { 089 isize = itemSize; 090 this.shape = shape.clone(); 091 try { 092 size = ShapeUtils.calcSize(shape); 093 } catch (IllegalArgumentException e) { 094 size = Integer.MAX_VALUE; // this indicates that the entire dataset cannot be read in! 095 } 096 this.dtype = dtype; 097 } 098 099 /** 100 * Create an aggregate dataset 101 * @param extend if true, extend rank by one 102 * @param datasets 103 */ 104 public AggregateDataset(boolean extend, ILazyDataset... datasets) { 105 106 final int[][] shapes = calcShapes(extend, datasets); 107 108 // check for same (sub-)shape 109 final int[] s = shapes[0]; 110 final int axis = extend ? -1 : 0; 111 for (int j = 1; j < shapes.length; j++) { 112 if (!ShapeUtils.areShapesCompatible(s, shapes[j], axis)) 113 throw new IllegalArgumentException("Dataset '" + datasets[j].getName() + "' has wrong shape"); 114 } 115 116 // set shapes of datasets 117 final int maxRank = s.length; 118 data = new ILazyDataset[datasets.length]; 119 isize = datasets[0].getElementsPerItem(); 120 for (int j = 0; j < datasets.length; j++) { 121 ILazyDataset d = datasets[j]; 122 int[] ds = d.getShape(); 123 if (ds.length < maxRank) { 124 d = d.clone(); 125 d.setShape(shapes[j]); 126 } 127 data[j] = d; 128 if (d.getElementsPerItem() != isize) { 129 throw new IllegalArgumentException("All datasets must have the same number of elements"); 130 } 131 } 132 133 // calculate new shape 134 shape = new int[maxRank]; 135 for (int i = 1; i < shape.length; i++) { 136 shape[i] = s[i]; 137 } 138 if (extend) { 139 shape[0] = data.length; 140 } else { 141 for (int j = 0; j < datasets.length; j++) { 142 shape[0] += shapes[j][0]; 143 } 144 } 145 try { 146 size = ShapeUtils.calcSize(shape); 147 } catch (IllegalArgumentException e) { 148 size = Integer.MAX_VALUE; // this indicates that the entire dataset cannot be read in! 149 } 150 151 // work out offsets from cumulative lengths 152 offset = new int[data.length]; 153 int cd = 0; 154 for (int i = 0; i < data.length; i++) { 155 offset[i] = cd; 156 cd += data[i].getShape()[0]; 157 } 158 159 // calculate mapping from aggregate dimension to dataset array index 160 map = new int[shape[0]]; 161 int k = 0; 162 for (int i = 0; i < data.length; i++) { 163 int jmax = data[i].getShape()[0]; 164 for (int j = 0; j < jmax; j++) 165 map[k++] = i; 166 } 167 168 for (ILazyDataset d : data) { 169 if (d instanceof LazyDatasetBase) { 170 dtype = DTypeUtils.getBestDType(dtype, ((LazyDatasetBase) d).getDType()); 171 } else { 172 dtype = DTypeUtils.getBestDType(dtype, DTypeUtils.getDTypeFromClass(d.getElementClass(), d.getElementsPerItem())); 173 } 174 } 175 176 for (ILazyDataset d : data) { 177 String n = d.getName(); 178 if (n != null) { 179 name = n; 180 break; 181 } 182 } 183 } 184 185 @Override 186 public Class<?> getElementClass() { 187 return DTypeUtils.getElementClass(dtype); 188 } 189 190 @Override 191 public int getElementsPerItem() { 192 return isize; 193 } 194 195 @Override 196 public int getDType() { 197 return dtype; 198 } 199 200 @Override 201 public int getSize() { 202 return size; 203 } 204 205 @Override 206 public void setShape(int... shape) { 207 throw new UnsupportedOperationException("Not implemented"); 208 } 209 210 @Override 211 public ILazyDataset squeezeEnds() { 212 throw new UnsupportedOperationException("Not implemented"); 213 } 214 215 @Override 216 public Dataset getSlice(int[] start, int[] stop, int[] step) throws DatasetException { 217 return getSlice(null, new SliceND(shape, start, stop, step)); 218 } 219 220 @Override 221 public Dataset getSlice(IMonitor monitor, int[] start, int[] stop, int[] step) throws DatasetException { 222 return getSlice(monitor, new SliceND(shape, start, stop, step)); 223 } 224 225 @Override 226 public Dataset getSlice(IMonitor monitor, SliceND slice) throws DatasetException { 227 int[] start = slice.getStart(); 228 int[] stop = slice.getStop(); 229 int[] step = slice.getStep(); 230 231 if (base != null) { 232 for (int i = 0; i < shape.length; i++) { 233 start[i] = sliceStart[i] + start[i] * sliceStep[i]; 234 stop[i] = sliceStart[i] + (stop[i] - 1) * sliceStep[i] + 1; 235 step[i] = step[i] * sliceStep[i]; 236 } 237 return base.getSlice(monitor, start, stop, step); 238 } 239 240 // convert first dimension's slice to individual slices per stored dataset 241 int fb = start[0]; 242 int fe = stop[0]; 243 int fs = step[0]; 244 245 List<Dataset> sliced = new ArrayList<Dataset>(); 246 int op = fb; 247 int p = op; 248 ILazyDataset od = data[map[op]]; 249 ILazyDataset nd; 250 while (p < fe) { 251 252 if (monitor!=null && monitor.isCancelled()) throw new DatasetException("Slice cancelled"); 253 nd = data[map[p]]; 254 if (nd != od) { 255 start[0] = op - offset[map[op]]; 256 stop[0] = p - offset[map[op]]; 257 Dataset a = DatasetUtils.convertToDataset(od.getSlice(monitor, start, stop, step)); 258 sliced.add(a.cast(dtype)); 259 260 od = nd; 261 op = p; 262 } 263 p += fs; 264 } 265 start[0] = op - offset[map[op]]; 266 stop[0] = p - offset[map[op]]; 267 Dataset a = DatasetUtils.convertToDataset(od.getSlice(monitor, start, stop, step)); 268 sliced.add(a.cast(dtype)); 269 270 Dataset d = DatasetUtils.concatenate(sliced.toArray(new Dataset[0]), 0); 271 d.setName(name); 272 return d; 273 } 274 275 @Override 276 public Dataset getSlice(Slice... slice) throws DatasetException { 277 return getSlice(null, slice); 278 } 279 280 @Override 281 public Dataset getSlice(SliceND slice) throws DatasetException { 282 return getSlice(null, slice); 283 } 284 285 @Override 286 public Dataset getSlice(IMonitor monitor, Slice... slice) throws DatasetException { 287 return getSlice(monitor, new SliceND(shape, slice)); 288 } 289 290 @Override 291 public AggregateDataset getSliceView(Slice... slice) { 292 return getSliceView(new SliceND(shape, slice)); 293 } 294 295 @Override 296 public AggregateDataset getSliceView(int[] start, int[] stop, int[] step) { 297 return getSliceView(new SliceND(shape, start, stop, step)); 298 } 299 300 @Override 301 public AggregateDataset getSliceView(SliceND slice) { 302 AggregateDataset lazy = new AggregateDataset(isize, slice.getShape(), dtype); 303 lazy.sliceStart = slice.getStart(); 304 lazy.sliceStep = slice.getStep(); 305 lazy.name = name + "[" + slice + "]"; 306 lazy.base = base == null ? this : base; 307 return lazy; 308 } 309 310 @Override 311 public AggregateDataset getTransposedView(int... axes) { 312 throw new UnsupportedOperationException("Not implemented"); 313 } 314 315 @Override 316 public AggregateDataset clone() { 317 throw new UnsupportedOperationException("Not implemented"); 318 } 319 320 @Override 321 public String toString() { 322 StringBuilder out = new StringBuilder(); 323 324 if (name != null && name.length() > 0) { 325 out.append("Aggregate dataset '"); 326 out.append(name); 327 out.append("' has shape ["); 328 } else { 329 out.append("Aggregate dataset shape is ["); 330 } 331 int rank = shape == null ? 0 : shape.length; 332 333 if (rank > 0 && shape[0] > 0) { 334 out.append(shape[0]); 335 } 336 for (int i = 1; i < rank; i++) { 337 out.append(", " + shape[i]); 338 } 339 out.append(']'); 340 341 return out.toString(); 342 } 343 344 @Override 345 public void setErrors(Serializable errors) { 346 throw new RuntimeException("setLazyErrors is unimplemented for "+getClass().getSimpleName()); 347 } 348 349 @Override 350 public ILazyDataset getErrors() { 351 return null; 352 } 353 354}