001/*- 002 ******************************************************************************* 003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd. 004 * All rights reserved. This program and the accompanying materials 005 * are made available under the terms of the Eclipse Public License v1.0 006 * which accompanies this distribution, and is available at 007 * http://www.eclipse.org/legal/epl-v10.html 008 * 009 * Contributors: 010 * Peter Chang - initial API and implementation and/or initial documentation 011 *******************************************************************************/ 012 013package org.eclipse.january.dataset; 014 015import java.io.Serializable; 016import java.lang.annotation.Annotation; 017import java.lang.reflect.Array; 018import java.lang.reflect.Field; 019import java.util.ArrayList; 020import java.util.Arrays; 021import java.util.Collection; 022import java.util.concurrent.ConcurrentMap; 023import java.util.concurrent.ConcurrentHashMap; 024import java.util.List; 025import java.util.Map; 026 027import org.eclipse.january.DatasetException; 028import org.eclipse.january.MetadataException; 029import org.eclipse.january.metadata.Dirtiable; 030import org.eclipse.january.metadata.ErrorMetadata; 031import org.eclipse.january.metadata.IMetadata; 032import org.eclipse.january.metadata.MetadataFactory; 033import org.eclipse.january.metadata.MetadataType; 034import org.eclipse.january.metadata.Reshapeable; 035import org.eclipse.january.metadata.Sliceable; 036import org.eclipse.january.metadata.Transposable; 037import org.slf4j.Logger; 038import org.slf4j.LoggerFactory; 039 040/** 041 * Common base for both lazy and normal dataset implementations 042 */ 043public abstract class LazyDatasetBase implements ILazyDataset, Serializable { 044 045 private static final long serialVersionUID = 767926846438976050L; 046 047 protected static final Logger logger = LoggerFactory.getLogger(LazyDatasetBase.class); 048 049 protected static boolean catchExceptions; 050 051 static { 052 /** 053 * Boolean to set to true if running jython scripts that utilise ScisoftPy in IDE 054 */ 055 try { 056 catchExceptions = Boolean.getBoolean("run.in.eclipse"); 057 } catch (SecurityException e) { 058 // set a default for when the security manager does not allow access to the requested key 059 catchExceptions = false; 060 } 061 } 062 063 transient private boolean dirty = true; // indicate dirty state of metadata 064 protected String name = ""; 065 066 /** 067 * The shape or dimensions of the dataset 068 */ 069 protected int[] shape; 070 071 protected ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> metadata = null; 072 073 /** 074 * @return type of dataset item 075 */ 076 abstract public int getDType(); 077 078 @Override 079 public Class<?> getElementClass() { 080 return DTypeUtils.getElementClass(getDType()); 081 } 082 083 @Override 084 public LazyDatasetBase clone() { 085 return null; 086 } 087 088 @Override 089 public boolean equals(Object obj) { 090 if (this == obj) { 091 return true; 092 } 093 if (obj == null) { 094 return false; 095 } 096 if (!getClass().equals(obj.getClass())) { 097 return false; 098 } 099 100 LazyDatasetBase other = (LazyDatasetBase) obj; 101 if (getDType() != other.getDType()) { 102 return false; 103 } 104 if (getElementsPerItem() != other.getElementsPerItem()) { 105 return false; 106 } 107 if (!Arrays.equals(shape, other.shape)) { 108 return false; 109 } 110 return true; 111 } 112 113 @Override 114 public int hashCode() { 115 int hash = getDType() * 17 + getElementsPerItem(); 116 int rank = shape.length; 117 for (int i = 0; i < rank; i++) { 118 hash = hash*17 + shape[i]; 119 } 120 return hash; 121 } 122 123 @Override 124 public String getName() { 125 return name; 126 } 127 128 @Override 129 public void setName(String name) { 130 this.name = name; 131 } 132 133 @Override 134 public int[] getShape() { 135 return shape.clone(); 136 } 137 138 @Override 139 public int getRank() { 140 return shape.length; 141 } 142 143 /** 144 * This method allows anything that dirties the dataset to clear various metadata values 145 * so that the other methods can work correctly. 146 * @since 2.1 147 */ 148 public void setDirty() { 149 dirty = true; 150 } 151 152 /** 153 * Find first sub-interface of (or class that directly implements) MetadataType 154 * @param clazz 155 * @return sub-interface 156 * @exception IllegalArgumentException when given class is {@link MetadataType} or an anonymous sub-class of it 157 */ 158 @SuppressWarnings("unchecked") 159 public static Class<? extends MetadataType> findMetadataTypeSubInterfaces(Class<? extends MetadataType> clazz) { 160 if (clazz.equals(MetadataType.class)) { 161 throw new IllegalArgumentException("Cannot accept MetadataType"); 162 } 163 164 if (clazz.isInterface()) { 165 return clazz; 166 } 167 168 if (clazz.isAnonymousClass()) { // special case 169 Class<?> s = clazz.getSuperclass(); 170 if (!s.equals(Object.class)) { 171 // only use super class if it is not an anonymous class of an interface 172 clazz = (Class<? extends MetadataType>) s; 173 } 174 } 175 176 for (Class<?> c : clazz.getInterfaces()) { 177 if (c.equals(MetadataType.class)) { 178 if (clazz.isAnonymousClass()) { 179 throw new IllegalArgumentException("Cannot accept anonymous subclasses of MetadataType"); 180 } 181 return clazz; 182 } 183 if (MetadataType.class.isAssignableFrom(c)) { 184 return (Class<? extends MetadataType>) c; 185 } 186 } 187 188 Class<?> c = clazz.getSuperclass(); // Naughty: someone has sub-classed a metadata class 189 if (c != null) { 190 return findMetadataTypeSubInterfaces((Class<? extends MetadataType>) c); 191 } 192 193 logger.error("Somehow the search for metadata type interface ended in a bad place"); 194 assert false; // should not be able to get here!!! 195 return null; 196 } 197 198 @Override 199 public void setMetadata(MetadataType metadata) { 200 addMetadata(metadata, true); 201 } 202 203 @Override 204 public void addMetadata(MetadataType metadata) { 205 addMetadata(metadata, false); 206 } 207 208 private synchronized void addMetadata(MetadataType metadata, boolean clear) { 209 if (metadata == null) { 210 return; 211 } 212 213 if (this.metadata == null) { 214 this.metadata = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>(); 215 } 216 217 Class<? extends MetadataType> clazz = findMetadataTypeSubInterfaces(metadata.getClass()); 218 if (!this.metadata.containsKey(clazz)) { 219 this.metadata.put(clazz, new ArrayList<MetadataType>()); 220 } else if (clear) { 221 this.metadata.get(clazz).clear(); 222 } 223 this.metadata.get(clazz).add(metadata); 224 225 // add for special case of sub-interfaces of IMetadata 226 if (!IMetadata.class.equals(clazz) && IMetadata.class.isAssignableFrom(clazz)) { 227 clazz = IMetadata.class; 228 if (!this.metadata.containsKey(clazz)) { 229 this.metadata.put(clazz, new ArrayList<MetadataType>()); 230 } else if (clear) { 231 this.metadata.get(clazz).clear(); 232 } 233 this.metadata.get(clazz).add(metadata); 234 } 235 } 236 237 @Override 238 @Deprecated 239 public synchronized IMetadata getMetadata() { 240 return getFirstMetadata(IMetadata.class); 241 } 242 243 @SuppressWarnings("unchecked") 244 @Override 245 public synchronized <S extends MetadataType, T extends S> List<S> getMetadata(Class<T> clazz) throws MetadataException { 246 if (metadata == null) { 247 dirty = false; 248 return null; 249 } 250 251 if (dirty) { 252 dirtyMetadata(); 253 dirty = false; 254 } 255 256 if (clazz == null) { 257 List<S> all = new ArrayList<S>(); 258 for (Class<? extends MetadataType> c : metadata.keySet()) { 259 all.addAll((Collection<S>) metadata.get(c)); 260 } 261 return all; 262 } 263 264 return (List<S>) metadata.get(findMetadataTypeSubInterfaces(clazz)); 265 } 266 267 @Override 268 public synchronized <S extends MetadataType, T extends S> S getFirstMetadata(Class<T> clazz) { 269 try { 270 List<S> ml = getMetadata(clazz); 271 if (ml == null) { 272 return null; 273 } 274 for (S t : ml) { 275 if (clazz.isInstance(t)) { 276 return t; 277 } 278 } 279 } catch (Exception e) { 280 logger.error("Get metadata failed!",e); 281 } 282 283 return null; 284 } 285 286 @Override 287 public synchronized void clearMetadata(Class<? extends MetadataType> clazz) { 288 if (metadata == null) { 289 return; 290 } 291 292 if (clazz == null) { 293 metadata.clear(); 294 return; 295 } 296 297 List<MetadataType> list = metadata.get(findMetadataTypeSubInterfaces(clazz)); 298 if( list != null) { 299 list.clear(); 300 } 301 } 302 303 /** 304 * @since 2.0 305 */ 306 protected synchronized ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata() { 307 return copyMetadata(metadata); 308 } 309 310 /** 311 * @since 2.0 312 */ 313 protected static ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> metadata) { 314 if (metadata == null) { 315 return null; 316 } 317 318 ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>> map = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>(); 319 copyMetadata(metadata, map); 320 return map; 321 } 322 323 private static void copyMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> inMetadata, 324 Map<Class<? extends MetadataType>, List<MetadataType>> outMetadata) { 325 for (Class<? extends MetadataType> c : inMetadata.keySet()) { 326 List<MetadataType> l = inMetadata.get(c); 327 List<MetadataType> nl = new ArrayList<MetadataType>(l.size()); 328 outMetadata.put(c, nl); 329 for (MetadataType m : l) { 330 if (m == null || isMetadataDirty(m)) { // skip dirty metadata 331 continue; 332 } 333 nl.add(m.clone()); 334 } 335 } 336 } 337 338 protected void restoreMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> oldMetadata) { 339 copyMetadata(oldMetadata, metadata); 340 } 341 342 /** 343 * @since 2.2 344 */ 345 protected static ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> getMetadataMap(ILazyDataset a, boolean clone) { 346 List<MetadataType> all = null; 347 try { 348 all = a.getMetadata(null); 349 } catch (Exception e) { 350 } 351 if (all == null) { 352 return null; 353 } 354 355 ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> map = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>(); 356 357 for (MetadataType m : all) { 358 if (m == null || isMetadataDirty(m)) { // skip dirty metadata 359 continue; 360 } 361 Class<? extends MetadataType> c = findMetadataTypeSubInterfaces(m.getClass()); 362 List<MetadataType> l = map.get(c); 363 if (l == null) { 364 l = new ArrayList<MetadataType>(); 365 map.put(c, l); 366 } 367 if (clone) { 368 m = m.clone(); 369 } 370 l.add(m); 371 } 372 return map; 373 } 374 375 private static boolean isMetadataDirty(MetadataType m) { 376 Class<? extends MetadataType> c = m.getClass(); 377 for (Field f : c.getDeclaredFields()) { 378 if (f.isAnnotationPresent(Dirtiable.class)) { 379 Class<?> t = f.getType(); 380 if (t.equals(boolean.class) || t.equals(Boolean.class)) { 381 try { 382 f.setAccessible(true); 383 Object o = f.get(m); 384 if (o.equals(true)) { 385 return true; 386 } 387 } catch (Exception e) { 388 logger.debug("Could not retrieve value of dirty variable: {}", c.getCanonicalName(), e); 389 } 390 } 391 } 392 } 393 394 return false; 395 } 396 397 interface MetadatasetAnnotationOperation { 398 /** 399 * Process value of given field 400 * <p> 401 * When the field is not a container then the returned value 402 * may replace the old value 403 * @param f given field 404 * @param o value of field 405 * @return transformed field 406 */ 407 Object processField(Field f, Object o); 408 409 /** 410 * @return annotated class 411 */ 412 Class<? extends Annotation> getAnnClass(); 413 414 /** 415 * @param axis 416 * @return number of dimensions to insert or remove 417 */ 418 int change(int axis); 419 420 /** 421 * 422 * @return rank or -1 to match 423 */ 424 int getNewRank(); 425 426 /** 427 * Run on given lazy dataset 428 * @param lz 429 * @return 430 */ 431 ILazyDataset run(ILazyDataset lz); 432 } 433 434 class MdsSlice implements MetadatasetAnnotationOperation { 435 private boolean asView; 436 private SliceND slice; 437 private int[] oShape; 438 private long oSize; 439 440 public MdsSlice(boolean asView, SliceND slice) { 441 this.asView = asView; 442 this.slice = slice; 443 oShape = slice.getSourceShape(); 444 oSize = ShapeUtils.calcLongSize(oShape); 445 } 446 447 @Override 448 public Object processField(Field field, Object o) { 449 return o; 450 } 451 452 @Override 453 public Class<? extends Annotation> getAnnClass() { 454 return Sliceable.class; 455 } 456 457 @Override 458 public int change(int axis) { 459 return 0; 460 } 461 462 @Override 463 public int getNewRank() { 464 return -1; 465 } 466 467 @Override 468 public ILazyDataset run(ILazyDataset lz) { 469 int rank = lz.getRank(); 470 if (slice.getStart().length != rank) { 471 throw new IllegalArgumentException("Slice rank does not match dataset!"); 472 } 473 474 int[] shape = lz.getShape(); 475 SliceND nslice; 476 if (!ShapeUtils.areShapesBroadcastCompatible(oShape, shape)) { 477 nslice = new SliceND(shape); 478 for (int i = 0; i < rank; i++) { 479 int s = shape[i]; 480 int os = oShape[i]; 481 if (s >= os) { 482 nslice.setSlice(i, 0, os, 1); 483 } else if (s == 1) { 484 nslice.setSlice(i, 0, 1, 1); 485 } else { 486 throw new IllegalArgumentException("Sliceable dataset has non-unit dimension less than host!"); 487 } 488 } 489 lz = lz.getSliceView(nslice); 490 } 491 if (lz.getSize() == oSize) { 492 nslice = slice; 493 } else { 494 nslice = slice.clone(); 495 for (int i = 0; i < rank; i++) { 496 int s = shape[i]; 497 if (s >= oShape[i]) { 498 continue; 499 } else if (s == 1) { 500 nslice.setSlice(i, 0, 1, 1); 501 } else { 502 throw new IllegalArgumentException("Sliceable dataset has non-unit dimension less than host!"); 503 } 504 } 505 } 506 507 if (asView || (lz instanceof IDataset)) { 508 return lz.getSliceView(nslice); 509 } 510 try { 511 return lz.getSlice(nslice); 512 } catch (DatasetException e) { 513 logger.error("Could not slice dataset in metadata", e); 514 return null; 515 } 516 } 517 } 518 519 class MdsReshape implements MetadatasetAnnotationOperation { 520 private boolean matchRank; 521 private int[] oldShape; 522 private int[] newShape; 523 boolean onesOnly; 524 int[] differences; 525 526 /* 527 * if only ones then record differences (insertions and deletions) 528 * 529 * if shape changing, find broadcasted dimensions and disallow 530 * merging that include those dimensions 531 */ 532 public MdsReshape(final int[] oldShape, final int[] newShape) { 533 this.oldShape = oldShape; 534 this.newShape = newShape; 535 differences = null; 536 } 537 538 @Override 539 public Object processField(Field field, Object o) { 540 Annotation a = field.getAnnotation(Reshapeable.class); 541 if (a != null) { // cannot be null 542 matchRank = ((Reshapeable) a).matchRank(); 543 } 544 return o; 545 } 546 547 @Override 548 public Class<? extends Annotation> getAnnClass() { 549 return Reshapeable.class; 550 } 551 552 @Override 553 public int change(int axis) { 554 if (matchRank) { 555 if (differences == null) { 556 init(); 557 } 558 559 if (onesOnly) { 560 return differences[axis]; 561 } 562 throw new UnsupportedOperationException("TODO support other shape operations"); 563 } 564 return 0; 565 } 566 567 @Override 568 public int getNewRank() { 569 return matchRank ? newShape.length : -1; 570 } 571 572 private void init() { 573 int or = oldShape.length - 1; 574 int nr = newShape.length - 1; 575 if (or < 0 || nr < 0) { // zero-rank shapes 576 onesOnly = true; 577 differences = new int[1]; 578 differences[0] = or < 0 ? nr + 1 : or + 1; 579 return; 580 } 581 int ob = 0; 582 int nb = 0; 583 onesOnly = true; 584 do { 585 while (oldShape[ob] == 1 && ob < or) { 586 ob++; // next non-unit dimension 587 } 588 while (newShape[nb] == 1 && nb < nr) { 589 nb++; 590 } 591 if (oldShape[ob++] != newShape[nb++]) { 592 onesOnly = false; 593 break; 594 } 595 } while (ob <= or && nb <= nr); 596 597 ob = 0; 598 nb = 0; 599 differences = new int[or + 2]; 600 if (onesOnly) { 601 // work out unit dimensions removed from or add to old 602 int j = 0; 603 do { 604 if (oldShape[ob] != 1 && newShape[nb] != 1) { 605 ob++; 606 nb++; 607 } else { 608 while (oldShape[ob] == 1 && ob < or) { 609 ob++; 610 differences[j]--; 611 } 612 while (newShape[nb] == 1 && nb < nr) { 613 nb++; 614 differences[j]++; 615 } 616 } 617 j++; 618 } while (ob <= or && nb <= nr && j <= or); 619 while (ob <= or && oldShape[ob] == 1) { 620 ob++; 621 differences[j]--; 622 } 623 while (nb <= nr && newShape[nb] == 1) { 624 nb++; 625 differences[j]++; 626 } 627 } else { 628 if (matchRank) { 629 logger.error("Combining dimensions is currently not supported"); 630 throw new IllegalArgumentException("Combining dimensions is currently not supported"); 631 } 632 // work out mapping: contiguous dimensions can be grouped or split 633 while (ob <= or && nb <= nr) { 634 int ol = oldShape[ob]; 635 while (ol == 1 && ol <= or) { 636 ob++; 637 ol = oldShape[ob]; 638 } 639 int oe = ob + 1; 640 int nl = newShape[nb]; 641 while (nl == 1 && nl <= nr) { 642 nb++; 643 nl = newShape[nb]; 644 } 645 int ne = nb + 1; 646 if (ol < nl) { 647 differences[ob] = 1; 648 do { // case where new shape combines several dimensions into one dimension 649 if (oe == (or + 1)) { 650 break; 651 } 652 differences[oe] = 1; 653 ol *= oldShape[oe++]; 654 } while (ol < nl); 655 differences[oe - 1] = oe - ob; // signal end with difference 656 if (nl != ol) { 657 logger.error("Single dimension is incompatible with subshape"); 658 throw new IllegalArgumentException("Single dimension is incompatible with subshape"); 659 } 660 } else if (ol > nl) { 661 do { // case where new shape spreads single dimension over several dimensions 662 if (ne == (nr + 1)) { 663 break; 664 } 665 nl *= newShape[ne++]; 666 } while (nl < ol); 667 if (nl != ol) { 668 logger.error("Subshape is incompatible with single dimension"); 669 throw new IllegalArgumentException("Subshape is incompatible with single dimension"); 670 } 671 672 } 673 674 ob = oe; 675 nb = ne; 676 } 677 678 } 679 } 680 681 @Override 682 public ILazyDataset run(ILazyDataset lz) { 683 if (differences == null) { 684 init(); 685 } 686 687 int[] lshape = lz.getShape(); 688 if (Arrays.equals(newShape, lshape)) { 689 return lz; 690 } 691 int or = lz.getRank(); 692 int nr = newShape.length; 693 int[] nshape = new int[nr]; 694 Arrays.fill(nshape, 1); 695 if (onesOnly) { 696 // ignore omit removed dimensions 697 for (int i = 0, si = 0, di = 0; i < (or+1) && si <= or && di < nr; i++) { 698 int c = differences[i]; 699 if (c == 0) { 700 nshape[di++] = lshape[si++]; 701 } else if (c > 0) { 702 while (c-- > 0 && di < nr) { 703 di++; 704 } 705 } else if (c < 0) { 706 si -= c; // remove dimensions by skipping forward in source array 707 } 708 } 709 } else { 710 boolean[] broadcast = new boolean[or]; 711 for (int ob = 0; ob < or; ob++) { 712 broadcast[ob] = oldShape[ob] != 1 && lshape[ob] == 1; 713 } 714 int osize = lz.getSize(); 715 716 // cannot do 3x5x... to 15x... if metadata is broadcasting (i.e. 1x5x...) 717 int ob = 0; 718 int nsize = 1; 719 for (int i = 0; i < nr; i++) { 720 if (ob < or && broadcast[ob]) { 721 if (differences[ob] != 0) { 722 logger.error("Metadata contains a broadcast axis which cannot be reshaped"); 723 throw new IllegalArgumentException("Metadata contains a broadcast axis which cannot be reshaped"); 724 } 725 } else { 726 nshape[i] = nsize < osize ? newShape[i] : 1; 727 } 728 nsize *= nshape[i]; 729 ob++; 730 } 731 } 732 733 ILazyDataset nlz = lz.getSliceView(); 734 if (lz instanceof Dataset) { 735 nlz = ((Dataset) lz).reshape(nshape); 736 } else { 737 nlz = lz.getSliceView(); 738 nlz.setShape(nshape); 739 } 740 return nlz; 741 } 742 } 743 744 class MdsTranspose implements MetadatasetAnnotationOperation { 745 int[] map; 746 747 public MdsTranspose(final int[] axesMap) { 748 map = axesMap; 749 } 750 751 @SuppressWarnings({ "rawtypes", "unchecked" }) 752 @Override 753 public Object processField(Field f, Object o) { 754 // reorder arrays and lists according the axes map 755 if (o.getClass().isArray()) { 756 int l = Array.getLength(o); 757 if (l == map.length) { 758 Object narray = Array.newInstance(o.getClass().getComponentType(), l); 759 for (int i = 0; i < l; i++) { 760 Array.set(narray, i, Array.get(o, map[i])); 761 } 762 for (int i = 0; i < l; i++) { 763 Array.set(o, i, Array.get(narray, i)); 764 } 765 } 766 } else if (o instanceof List<?>) { 767 List list = (List) o; 768 int l = list.size(); 769 if (l == map.length) { 770 Object narray = Array.newInstance(o.getClass().getComponentType(), l); 771 for (int i = 0; i < l; i++) { 772 Array.set(narray, i, list.get(map[i])); 773 } 774 list.clear(); 775 for (int i = 0; i < l; i++) { 776 list.add(Array.get(narray, i)); 777 } 778 } 779 } 780 return o; 781 } 782 783 @Override 784 public Class<? extends Annotation> getAnnClass() { 785 return Transposable.class; 786 } 787 788 @Override 789 public int change(int axis) { 790 return 0; 791 } 792 793 @Override 794 public int getNewRank() { 795 return -1; 796 } 797 798 @Override 799 public ILazyDataset run(ILazyDataset lz) { 800 return lz.getTransposedView(map); 801 } 802 } 803 804 class MdsDirty implements MetadatasetAnnotationOperation { 805 806 @Override 807 public Object processField(Field f, Object o) { 808 // throw exception if not boolean??? 809 Class<?> t = f.getType(); 810 if (t.equals(boolean.class) || t.equals(Boolean.class)) { 811 if (o.equals(false)) { 812 o = true; 813 } 814 } 815 return o; 816 } 817 818 @Override 819 public Class<? extends Annotation> getAnnClass() { 820 return Dirtiable.class; 821 } 822 823 @Override 824 public int change(int axis) { 825 return 0; 826 } 827 828 @Override 829 public int getNewRank() { 830 return -1; 831 } 832 833 @Override 834 public ILazyDataset run(ILazyDataset lz) { 835 return lz; 836 } 837 } 838 839 /** 840 * Slice all datasets in metadata that are annotated by @Sliceable. Call this on the new sliced 841 * dataset after cloning the metadata 842 * @param asView if true then just a view 843 * @param slice 844 */ 845 protected void sliceMetadata(boolean asView, final SliceND slice) { 846 processAnnotatedMetadata(new MdsSlice(asView, slice), true); 847 } 848 849 /** 850 * Reshape all datasets in metadata that are annotated by @Reshapeable. Call this when squeezing 851 * or setting the shape 852 * 853 * @param newShape 854 */ 855 protected void reshapeMetadata(final int[] oldShape, final int[] newShape) { 856 processAnnotatedMetadata(new MdsReshape(oldShape, newShape), true); 857 } 858 859 /** 860 * Transpose all datasets in metadata that are annotated by @Transposable. Call this on the transposed 861 * dataset after cloning the metadata 862 * @param axesMap 863 */ 864 protected void transposeMetadata(final int[] axesMap) { 865 processAnnotatedMetadata(new MdsTranspose(axesMap), true); 866 } 867 868 /** 869 * Dirty metadata that are annotated by @Dirtiable. Call this when the dataset has been modified 870 * @since 2.0 871 */ 872 protected void dirtyMetadata() { 873 processAnnotatedMetadata(new MdsDirty(), true); 874 } 875 876 @SuppressWarnings("unchecked") 877 private void processAnnotatedMetadata(MetadatasetAnnotationOperation op, boolean throwException) { 878 if (metadata == null) 879 return; 880 881 for (List<MetadataType> l : metadata.values()) { 882 for (MetadataType m : l) { 883 if (m == null) { 884 continue; 885 } 886 887 Class<? extends MetadataType> mc = m.getClass(); 888 do { // iterate over super-classes 889 processClass(op, m, mc, throwException); 890 Class<?> sclazz = mc.getSuperclass(); 891 if (!MetadataType.class.isAssignableFrom(sclazz)) { 892 break; 893 } 894 mc = (Class<? extends MetadataType>) sclazz; 895 } while (true); 896 } 897 } 898 } 899 900 @SuppressWarnings({ "unchecked", "rawtypes" }) 901 private static void processClass(MetadatasetAnnotationOperation op, MetadataType m, Class<? extends MetadataType> mc, boolean throwException) { 902 for (Field f : mc.getDeclaredFields()) { 903 if (!f.isAnnotationPresent(op.getAnnClass())) 904 continue; 905 906 try { 907 f.setAccessible(true); 908 Object o = f.get(m); 909 if (o == null) { 910 continue; 911 } 912 913 Object no = op.processField(f, o); 914 if (no != o) { 915 f.set(m, no); 916 continue; 917 } 918 Object r = null; 919 if (o instanceof ILazyDataset) { 920 try { 921 f.set(m, op.run((ILazyDataset) o)); 922 } catch (Exception e) { 923 logger.error("Problem processing " + o, e); 924 if (!catchExceptions) { 925 throw e; 926 } 927 } 928 } else if (o.getClass().isArray()) { 929 int l = Array.getLength(o); 930 if (l <= 0) { 931 continue; 932 } 933 934 for (int i = 0; r == null && i < l; i++) { 935 r = Array.get(o, i); 936 } 937 int n = op.getNewRank(); 938 if (r == null) { 939 if (n < 0 || n != l) { // all nulls be need to match rank as necessary 940 f.set(m, Array.newInstance(o.getClass().getComponentType(), n < 0 ? l : n)); 941 } 942 continue; 943 } 944 if (n < 0) { 945 n = l; 946 } 947 Object narray = Array.newInstance(r.getClass(), n); 948 for (int i = 0, si = 0, di = 0; di < n && si < l; i++) { 949 int c = op.change(i); 950 if (c == 0) { 951 Array.set(narray, di++, processObject(op, Array.get(o, si++))); 952 } else if (c > 0) { 953 di += c; // add nulls by skipping forward in destination array 954 } else if (c < 0) { 955 si -= c; // remove dimensions by skipping forward in source array 956 } 957 } 958 if (n == l) { 959 for (int i = 0; i < l; i++) { 960 Array.set(o, i, Array.get(narray, i)); 961 } 962 } else { 963 f.set(m, narray); 964 } 965 } else if (o instanceof List<?>) { 966 List list = (List) o; 967 int l = list.size(); 968 if (l <= 0) { 969 continue; 970 } 971 972 for (int i = 0; r == null && i < l; i++) { 973 r = list.get(i); 974 } 975 int n = op.getNewRank(); 976 if (r == null) { 977 if (n < 0 || n != l) { // all nulls be need to match rank as necessary 978 list.clear(); 979 for (int i = 0, imax = n < 0 ? l : n; i < imax; i++) { 980 list.add(null); 981 } 982 } 983 continue; 984 } 985 986 if (n < 0) { 987 n = l; 988 } 989 Object narray = Array.newInstance(r.getClass(), n); 990 for (int i = 0, si = 0, di = 0; i < l && si < l; i++) { 991 int c = op.change(i); 992 if (c == 0) { 993 Array.set(narray, di++, processObject(op, list.get(si++))); 994 } else if (c > 0) { 995 di += c; // add nulls by skipping forward in destination array 996 } else if (c < 0) { 997 si -= c; // remove dimensions by skipping forward in source array 998 } 999 } 1000 list.clear(); 1001 for (int i = 0; i < n; i++) { 1002 list.add(Array.get(narray, i)); 1003 } 1004 } else if (o instanceof Map<?,?>) { 1005 Map map = (Map) o; 1006 for (Object k : map.keySet()) { 1007 map.put(k, processObject(op, map.get(k))); 1008 } 1009 } 1010 } catch (Exception e) { 1011 logger.error("Problem occurred when processing metadata of class {}: {}", mc.getCanonicalName(), e); 1012 if (throwException) { 1013 throw new RuntimeException(e); 1014 } 1015 } 1016 } 1017 } 1018 1019 @SuppressWarnings({ "unchecked", "rawtypes" }) 1020 private static Object processObject(MetadatasetAnnotationOperation op, Object o) throws Exception { 1021 if (o == null) { 1022 return o; 1023 } 1024 1025 if (o instanceof ILazyDataset) { 1026 try { 1027 return op.run((ILazyDataset) o); 1028 } catch (Exception e) { 1029 logger.error("Problem processing " + o, e); 1030 if (!catchExceptions) { 1031 throw e; 1032 } 1033 } 1034 } else if (o.getClass().isArray()) { 1035 int l = Array.getLength(o); 1036 for (int i = 0; i < l; i++) { 1037 Array.set(o, i, processObject(op, Array.get(o, i))); 1038 } 1039 } else if (o instanceof List<?>) { 1040 List list = (List) o; 1041 for (int i = 0, imax = list.size(); i < imax; i++) { 1042 list.set(i, processObject(op, list.get(i))); 1043 } 1044 } else if (o instanceof Map<?,?>) { 1045 Map map = (Map) o; 1046 for (Object k : map.keySet()) { 1047 map.put(k, processObject(op, map.get(k))); 1048 } 1049 } 1050 return o; 1051 } 1052 1053 protected ILazyDataset createFromSerializable(Serializable blob, boolean keepLazy) { 1054 ILazyDataset d = null; 1055 if (blob instanceof ILazyDataset) { 1056 d = (ILazyDataset) blob; 1057 if (d instanceof IDataset) { 1058 Dataset ed = DatasetUtils.convertToDataset((IDataset) d); 1059 int is = ed.getElementsPerItem(); 1060 if (is != 1 && is != getElementsPerItem()) { 1061 throw new IllegalArgumentException("Dataset has incompatible number of elements with this dataset"); 1062 } 1063 d = ed.cast(is == 1 ? Dataset.FLOAT64 : Dataset.ARRAYFLOAT64); 1064 } else if (!keepLazy) { 1065 final int is = getElementsPerItem(); 1066 try { 1067 d = DatasetUtils.cast(d.getSlice(), is == 1 ? Dataset.FLOAT64 : Dataset.ARRAYFLOAT64); 1068 } catch (DatasetException e) { 1069 logger.error("Could not get data from lazy dataset", e); 1070 return null; 1071 } 1072 } 1073 } else { 1074 final int is = getElementsPerItem(); 1075 if (is == 1) { 1076 d = DatasetFactory.createFromObject(DoubleDataset.class, blob); 1077 } else { 1078 try { 1079 d = DatasetFactory.createFromObject(is, CompoundDoubleDataset.class, blob); 1080 } catch (IllegalArgumentException e) { // if only single value supplied try again 1081 d = DatasetFactory.createFromObject(DoubleDataset.class, blob); 1082 } 1083 } 1084 if (d.getSize() == getSize() && !Arrays.equals(d.getShape(), shape)) { 1085 d.setShape(shape.clone()); 1086 } 1087 } 1088 List<int[]> s = BroadcastUtils.broadcastShapesToMax(shape, d.getShape()); 1089 d.setShape(s.get(0)); 1090 1091 return d; 1092 } 1093 1094 @Override 1095 public void setErrors(Serializable errors) { 1096 if (shape == null) { 1097 throw new IllegalArgumentException("Cannot set errors for null dataset"); 1098 } 1099 if (errors == null) { 1100 clearMetadata(ErrorMetadata.class); 1101 return; 1102 } 1103 if (errors == this) { 1104 logger.warn("Ignoring setting error to itself as this will lead to infinite recursion"); 1105 return; 1106 } 1107 1108 ILazyDataset errorData = createFromSerializable(errors, true); 1109 1110 ErrorMetadata emd = getErrorMetadata(); 1111 if (emd == null) { 1112 try { 1113 emd = MetadataFactory.createMetadata(ErrorMetadata.class); 1114 setMetadata(emd); 1115 } catch (MetadataException me) { 1116 logger.error("Could not create metadata", me); 1117 } 1118 } 1119 emd.setError(errorData); 1120 } 1121 1122 protected ErrorMetadata getErrorMetadata() { 1123 try { 1124 List<ErrorMetadata> el = getMetadata(ErrorMetadata.class); 1125 if (el != null && !el.isEmpty()) { 1126 return el.get(0); 1127 } 1128 } catch (Exception e) { 1129 } 1130 return null; 1131 } 1132 1133 @Override 1134 public ILazyDataset getErrors() { 1135 ErrorMetadata emd = getErrorMetadata(); 1136 return emd == null ? null : emd.getError(); 1137 } 1138 1139 @Override 1140 public boolean hasErrors() { 1141 return LazyDatasetBase.this.getErrors() != null; 1142 } 1143 1144 /** 1145 * Check permutation axes 1146 * @param shape 1147 * @param axes 1148 * @return cleaned up axes or null if trivial 1149 */ 1150 public static int[] checkPermutatedAxes(int[] shape, int... axes) { 1151 int rank = shape == null ? 0 : shape.length; 1152 1153 if (axes == null || axes.length == 0) { 1154 axes = new int[rank]; 1155 for (int i = 0; i < rank; i++) { 1156 axes[i] = rank - 1 - i; 1157 } 1158 } 1159 1160 if (axes.length != rank) { 1161 logger.error("axis permutation has length {} that does not match dataset's rank {}", axes.length, rank); 1162 throw new IllegalArgumentException("axis permutation does not match shape of dataset"); 1163 } 1164 1165 // check all permutation values are within bounds 1166 for (int i = 0; i < rank; i++) { 1167 axes[i] = ShapeUtils.checkAxis(rank, axes[i]); 1168 } 1169 1170 // check for a valid permutation (is this an unnecessary restriction?) 1171 int[] perm = axes.clone(); 1172 Arrays.sort(perm); 1173 1174 for (int i = 0; i < rank; i++) { 1175 if (perm[i] != i) { 1176 logger.error("axis permutation is not valid: it does not contain complete set of axes"); 1177 throw new IllegalArgumentException("axis permutation does not contain complete set of axes"); 1178 } 1179 } 1180 1181 if (Arrays.equals(axes, perm)) { 1182 return null; // signal identity or trivial permutation 1183 } 1184 1185 return axes; 1186 } 1187}