001/*- 002 ******************************************************************************* 003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd. 004 * All rights reserved. This program and the accompanying materials 005 * are made available under the terms of the Eclipse Public License v1.0 006 * which accompanies this distribution, and is available at 007 * http://www.eclipse.org/legal/epl-v10.html 008 * 009 * Contributors: 010 * Peter Chang - initial API and implementation and/or initial documentation 011 *******************************************************************************/ 012 013package org.eclipse.january.dataset; 014 015import java.io.Serializable; 016import java.lang.annotation.Annotation; 017import java.lang.reflect.Array; 018import java.lang.reflect.Field; 019import java.util.ArrayList; 020import java.util.Arrays; 021import java.util.Collection; 022import java.util.concurrent.ConcurrentMap; 023import java.util.concurrent.ConcurrentHashMap; 024import java.util.List; 025import java.util.Map; 026 027import org.eclipse.january.DatasetException; 028import org.eclipse.january.MetadataException; 029import org.eclipse.january.metadata.Dirtiable; 030import org.eclipse.january.metadata.ErrorMetadata; 031import org.eclipse.january.metadata.IMetadata; 032import org.eclipse.january.metadata.MetadataFactory; 033import org.eclipse.january.metadata.MetadataType; 034import org.eclipse.january.metadata.Reshapeable; 035import org.eclipse.january.metadata.Sliceable; 036import org.eclipse.january.metadata.Transposable; 037import org.slf4j.Logger; 038import org.slf4j.LoggerFactory; 039 040/** 041 * Common base for both lazy and normal dataset implementations 042 */ 043public abstract class LazyDatasetBase implements ILazyDataset, Serializable { 044 045 private static final long serialVersionUID = 767926846438976050L; 046 047 protected static final Logger logger = LoggerFactory.getLogger(LazyDatasetBase.class); 048 049 protected static boolean catchExceptions; 050 051 static { 052 /** 053 * Boolean to set to true if running jython scripts that utilise ScisoftPy in IDE 054 */ 055 try { 056 catchExceptions = Boolean.getBoolean("run.in.eclipse"); 057 } catch (SecurityException e) { 058 // set a default for when the security manager does not allow access to the requested key 059 catchExceptions = false; 060 } 061 } 062 063 transient private boolean dirty = true; // indicate dirty state of metadata 064 protected String name = ""; 065 066 /** 067 * The shape or dimensions of the dataset 068 */ 069 protected int[] shape; 070 071 protected ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> metadata = null; 072 073 /** 074 * @return type of dataset item 075 */ 076 abstract public int getDType(); 077 078 @Override 079 public Class<?> getElementClass() { 080 return DTypeUtils.getElementClass(getDType()); 081 } 082 083 @Override 084 public LazyDatasetBase clone() { 085 return null; 086 } 087 088 @Override 089 public boolean equals(Object obj) { 090 if (this == obj) { 091 return true; 092 } 093 if (obj == null) { 094 return false; 095 } 096 if (!getClass().equals(obj.getClass())) { 097 return false; 098 } 099 100 LazyDatasetBase other = (LazyDatasetBase) obj; 101 if (getDType() != other.getDType()) { 102 return false; 103 } 104 if (getElementsPerItem() != other.getElementsPerItem()) { 105 return false; 106 } 107 if (!Arrays.equals(shape, other.shape)) { 108 return false; 109 } 110 return true; 111 } 112 113 @Override 114 public int hashCode() { 115 int hash = getDType() * 17 + getElementsPerItem(); 116 int rank = shape.length; 117 for (int i = 0; i < rank; i++) { 118 hash = hash*17 + shape[i]; 119 } 120 return hash; 121 } 122 123 @Override 124 public String getName() { 125 return name; 126 } 127 128 @Override 129 public void setName(String name) { 130 this.name = name; 131 } 132 133 @Override 134 public int[] getShape() { 135 return shape.clone(); 136 } 137 138 @Override 139 public int getRank() { 140 return shape.length; 141 } 142 143 /** 144 * This method allows anything that dirties the dataset to clear various metadata values 145 * so that the other methods can work correctly. 146 * @since 2.1 147 */ 148 public void setDirty() { 149 dirty = true; 150 } 151 152 /** 153 * Find first sub-interface of (or class that directly implements) MetadataType 154 * @param clazz 155 * @return sub-interface 156 * @exception IllegalArgumentException when given class is {@link MetadataType} or an anonymous sub-class of it 157 */ 158 @SuppressWarnings("unchecked") 159 public static Class<? extends MetadataType> findMetadataTypeSubInterfaces(Class<? extends MetadataType> clazz) { 160 if (clazz.equals(MetadataType.class)) { 161 throw new IllegalArgumentException("Cannot accept MetadataType"); 162 } 163 164 if (clazz.isInterface()) { 165 return clazz; 166 } 167 168 if (clazz.isAnonymousClass()) { // special case 169 Class<?> s = clazz.getSuperclass(); 170 if (!s.equals(Object.class)) { 171 // only use super class if it is not an anonymous class of an interface 172 clazz = (Class<? extends MetadataType>) s; 173 } 174 } 175 176 for (Class<?> c : clazz.getInterfaces()) { 177 if (c.equals(MetadataType.class)) { 178 if (clazz.isAnonymousClass()) { 179 throw new IllegalArgumentException("Cannot accept anonymous subclasses of MetadataType"); 180 } 181 return clazz; 182 } 183 if (MetadataType.class.isAssignableFrom(c)) { 184 return (Class<? extends MetadataType>) c; 185 } 186 } 187 188 Class<?> c = clazz.getSuperclass(); // Naughty: someone has sub-classed a metadata class 189 if (c != null) { 190 return findMetadataTypeSubInterfaces((Class<? extends MetadataType>) c); 191 } 192 193 logger.error("Somehow the search for metadata type interface ended in a bad place"); 194 assert false; // should not be able to get here!!! 195 return null; 196 } 197 198 @Override 199 public void setMetadata(MetadataType metadata) { 200 addMetadata(metadata, true); 201 } 202 203 @Override 204 public void addMetadata(MetadataType metadata) { 205 addMetadata(metadata, false); 206 } 207 208 private synchronized void addMetadata(MetadataType metadata, boolean clear) { 209 if (metadata == null) 210 return; 211 212 if (this.metadata == null) { 213 this.metadata = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>(); 214 } 215 216 Class<? extends MetadataType> clazz = findMetadataTypeSubInterfaces(metadata.getClass()); 217 if (!this.metadata.containsKey(clazz)) { 218 this.metadata.put(clazz, new ArrayList<MetadataType>()); 219 } else if (clear) { 220 this.metadata.get(clazz).clear(); 221 } 222 this.metadata.get(clazz).add(metadata); 223 224 // add for special case of sub-interfaces of IMetadata 225 if (!IMetadata.class.equals(clazz) && IMetadata.class.isAssignableFrom(clazz)) { 226 clazz = IMetadata.class; 227 if (!this.metadata.containsKey(clazz)) { 228 this.metadata.put(clazz, new ArrayList<MetadataType>()); 229 } else if (clear) { 230 this.metadata.get(clazz).clear(); 231 } 232 this.metadata.get(clazz).add(metadata); 233 } 234 } 235 236 @Override 237 @Deprecated 238 public synchronized IMetadata getMetadata() { 239 return getFirstMetadata(IMetadata.class); 240 } 241 242 @SuppressWarnings("unchecked") 243 @Override 244 public synchronized <S extends MetadataType, T extends S> List<S> getMetadata(Class<T> clazz) throws MetadataException { 245 if (metadata == null) { 246 dirty = false; 247 return null; 248 } 249 250 if (dirty) { 251 dirtyMetadata(); 252 dirty = false; 253 } 254 255 if (clazz == null) { 256 List<S> all = new ArrayList<S>(); 257 for (Class<? extends MetadataType> c : metadata.keySet()) { 258 all.addAll((Collection<S>) metadata.get(c)); 259 } 260 return all; 261 } 262 263 return (List<S>) metadata.get(findMetadataTypeSubInterfaces(clazz)); 264 } 265 266 @Override 267 public synchronized <S extends MetadataType, T extends S> S getFirstMetadata(Class<T> clazz) { 268 try { 269 List<S> ml = getMetadata(clazz); 270 if (ml == null) { 271 return null; 272 } 273 for (S t : ml) { 274 if (clazz.isInstance(t)) { 275 return t; 276 } 277 } 278 } catch (Exception e) { 279 logger.error("Get metadata failed!",e); 280 } 281 282 return null; 283 } 284 285 @Override 286 public synchronized void clearMetadata(Class<? extends MetadataType> clazz) { 287 if (metadata == null) 288 return; 289 290 if (clazz == null) { 291 metadata.clear(); 292 return; 293 } 294 295 List<MetadataType> list = metadata.get(findMetadataTypeSubInterfaces(clazz)); 296 if( list != null) { 297 list.clear(); 298 } 299 } 300 301 /** 302 * @since 2.0 303 */ 304 protected synchronized ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata() { 305 return copyMetadata(metadata); 306 } 307 308 /** 309 * @since 2.0 310 */ 311 protected static ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> metadata) { 312 if (metadata == null) 313 return null; 314 315 ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>> map = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>(); 316 317 for (Class<? extends MetadataType> c : metadata.keySet()) { 318 List<MetadataType> l = metadata.get(c); 319 List<MetadataType> nl = new ArrayList<MetadataType>(l.size()); 320 map.put(c, nl); 321 for (MetadataType m : l) { 322 nl.add(m.clone()); 323 } 324 } 325 return map; 326 } 327 328 interface MetadatasetAnnotationOperation { 329 /** 330 * Process value of given field 331 * <p> 332 * When the field is not a container then the returned value 333 * may replace the old value 334 * @param f given field 335 * @param o value of field 336 * @return transformed field 337 */ 338 Object processField(Field f, Object o); 339 340 /** 341 * @return annotated class 342 */ 343 Class<? extends Annotation> getAnnClass(); 344 345 /** 346 * @param axis 347 * @return number of dimensions to insert or remove 348 */ 349 int change(int axis); 350 351 /** 352 * 353 * @return rank or -1 to match 354 */ 355 int getNewRank(); 356 357 /** 358 * Run on given lazy dataset 359 * @param lz 360 * @return 361 */ 362 ILazyDataset run(ILazyDataset lz); 363 } 364 365 class MdsSlice implements MetadatasetAnnotationOperation { 366 private boolean asView; 367 private int[] start; 368 private int[] stop; 369 private int[] step; 370 private int[] oShape; 371 private long oSize; 372 373 public MdsSlice(boolean asView, final int[] start, final int[] stop, final int[] step, final int[] oShape) { 374 this.asView = asView; 375 this.start = start; 376 this.stop = stop; 377 this.step = step; 378 this.oShape = oShape; 379 oSize = ShapeUtils.calcLongSize(oShape); 380 } 381 382 @Override 383 public Object processField(Field field, Object o) { 384 return o; 385 } 386 387 @Override 388 public Class<? extends Annotation> getAnnClass() { 389 return Sliceable.class; 390 } 391 392 @Override 393 public int change(int axis) { 394 return 0; 395 } 396 397 @Override 398 public int getNewRank() { 399 return -1; 400 } 401 402 @Override 403 public ILazyDataset run(ILazyDataset lz) { 404 int rank = lz.getRank(); 405 if (start.length != rank) { 406 throw new IllegalArgumentException("Slice dimensions do not match dataset!"); 407 } 408 409 int[] shape = lz.getShape(); 410 int[] stt; 411 int[] stp; 412 int[] ste; 413 if (lz.getSize() == oSize) { 414 stt = start; 415 stp = stop; 416 ste = step; 417 } else { 418 stt = start.clone(); 419 stp = stop.clone(); 420 ste = step.clone(); 421 for (int i = 0; i < rank; i++) { 422 if (shape[i] >= oShape[i]) continue; 423 if (shape[i] == 1) { 424 stt[i] = 0; 425 stp[i] = 1; 426 ste[1] = 1; 427 } else { 428 throw new IllegalArgumentException("Sliceable dataset has invalid size!"); 429 } 430 } 431 } 432 433 if (asView || (lz instanceof IDataset)) 434 return lz.getSliceView(stt, stp, ste); 435 try { 436 return lz.getSlice(stt, stp, ste); 437 } catch (DatasetException e) { 438 logger.error("Could not slice dataset in metadata", e); 439 return null; 440 } 441 } 442 } 443 444 class MdsReshape implements MetadatasetAnnotationOperation { 445 private boolean matchRank; 446 private int[] oldShape; 447 private int[] newShape; 448 boolean onesOnly; 449 int[] differences; 450 451 /* 452 * if only ones then record differences (insertions and deletions) 453 * 454 * if shape changing, find broadcasted dimensions and disallow 455 * merging that include those dimensions 456 */ 457 public MdsReshape(final int[] oldShape, final int[] newShape) { 458 this.oldShape = oldShape; 459 this.newShape = newShape; 460 differences = null; 461 } 462 463 @Override 464 public Object processField(Field field, Object o) { 465 Annotation a = field.getAnnotation(Reshapeable.class); 466 if (a != null) { // cannot be null 467 matchRank = ((Reshapeable) a).matchRank(); 468 } 469 return o; 470 } 471 472 @Override 473 public Class<? extends Annotation> getAnnClass() { 474 return Reshapeable.class; 475 } 476 477 @Override 478 public int change(int axis) { 479 if (matchRank) { 480 if (differences == null) 481 init(); 482 483 if (onesOnly) { 484 return differences[axis]; 485 } 486 throw new UnsupportedOperationException("TODO support other shape operations"); 487 } 488 return 0; 489 } 490 491 @Override 492 public int getNewRank() { 493 return matchRank ? newShape.length : -1; 494 } 495 496 private void init() { 497 int or = oldShape.length - 1; 498 int nr = newShape.length - 1; 499 if (or < 0 || nr < 0) { // zero-rank shapes 500 onesOnly = true; 501 differences = new int[1]; 502 differences[0] = or < 0 ? nr + 1 : or + 1; 503 return; 504 } 505 int ob = 0; 506 int nb = 0; 507 onesOnly = true; 508 do { 509 while (oldShape[ob] == 1 && ob < or) { 510 ob++; // next non-unit dimension 511 } 512 while (newShape[nb] == 1 && nb < nr) { 513 nb++; 514 } 515 if (oldShape[ob++] != newShape[nb++]) { 516 onesOnly = false; 517 break; 518 } 519 } while (ob <= or && nb <= nr); 520 521 ob = 0; 522 nb = 0; 523 differences = new int[or + 2]; 524 if (onesOnly) { 525 // work out unit dimensions removed from or add to old 526 int j = 0; 527 do { 528 if (oldShape[ob] != 1 && newShape[nb] != 1) { 529 ob++; 530 nb++; 531 } else { 532 while (oldShape[ob] == 1 && ob < or) { 533 ob++; 534 differences[j]--; 535 } 536 while (newShape[nb] == 1 && nb < nr) { 537 nb++; 538 differences[j]++; 539 } 540 } 541 j++; 542 } while (ob <= or && nb <= nr && j <= or); 543 while (ob <= or && oldShape[ob] == 1) { 544 ob++; 545 differences[j]--; 546 } 547 while (nb <= nr && newShape[nb] == 1) { 548 nb++; 549 differences[j]++; 550 } 551 } else { 552 if (matchRank) { 553 logger.error("Combining dimensions is currently not supported"); 554 throw new IllegalArgumentException("Combining dimensions is currently not supported"); 555 } 556 // work out mapping: contiguous dimensions can be grouped or split 557 while (ob <= or && nb <= nr) { 558 int ol = oldShape[ob]; 559 while (ol == 1 && ol <= or) { 560 ob++; 561 ol = oldShape[ob]; 562 } 563 int oe = ob + 1; 564 int nl = newShape[nb]; 565 while (nl == 1 && nl <= nr) { 566 nb++; 567 nl = newShape[nb]; 568 } 569 int ne = nb + 1; 570 if (ol < nl) { 571 differences[ob] = 1; 572 do { // case where new shape combines several dimensions into one dimension 573 if (oe == (or + 1)) { 574 break; 575 } 576 differences[oe] = 1; 577 ol *= oldShape[oe++]; 578 } while (ol < nl); 579 differences[oe - 1] = oe - ob; // signal end with difference 580 if (nl != ol) { 581 logger.error("Single dimension is incompatible with subshape"); 582 throw new IllegalArgumentException("Single dimension is incompatible with subshape"); 583 } 584 } else if (ol > nl) { 585 do { // case where new shape spreads single dimension over several dimensions 586 if (ne == (nr + 1)) { 587 break; 588 } 589 nl *= newShape[ne++]; 590 } while (nl < ol); 591 if (nl != ol) { 592 logger.error("Subshape is incompatible with single dimension"); 593 throw new IllegalArgumentException("Subshape is incompatible with single dimension"); 594 } 595 596 } 597 598 ob = oe; 599 nb = ne; 600 } 601 602 } 603 } 604 605 @Override 606 public ILazyDataset run(ILazyDataset lz) { 607 if (differences == null) 608 init(); 609 610 int[] lshape = lz.getShape(); 611 if (Arrays.equals(newShape, lshape)) { 612 return lz; 613 } 614 int or = lz.getRank(); 615 int nr = newShape.length; 616 int[] nshape = new int[nr]; 617 Arrays.fill(nshape, 1); 618 if (onesOnly) { 619 // ignore omit removed dimensions 620 for (int i = 0, si = 0, di = 0; i < (or+1) && si <= or && di < nr; i++) { 621 int c = differences[i]; 622 if (c == 0) { 623 nshape[di++] = lshape[si++]; 624 } else if (c > 0) { 625 while (c-- > 0 && di < nr) { 626 di++; 627 } 628 } else if (c < 0) { 629 si -= c; // remove dimensions by skipping forward in source array 630 } 631 } 632 } else { 633 boolean[] broadcast = new boolean[or]; 634 for (int ob = 0; ob < or; ob++) { 635 broadcast[ob] = oldShape[ob] != 1 && lshape[ob] == 1; 636 } 637 int osize = lz.getSize(); 638 639 // cannot do 3x5x... to 15x... if metadata is broadcasting (i.e. 1x5x...) 640 int ob = 0; 641 int nsize = 1; 642 for (int i = 0; i < nr; i++) { 643 if (ob < or && broadcast[ob]) { 644 if (differences[ob] != 0) { 645 logger.error("Metadata contains a broadcast axis which cannot be reshaped"); 646 throw new IllegalArgumentException("Metadata contains a broadcast axis which cannot be reshaped"); 647 } 648 } else { 649 nshape[i] = nsize < osize ? newShape[i] : 1; 650 } 651 nsize *= nshape[i]; 652 ob++; 653 } 654 } 655 656 ILazyDataset nlz = lz.getSliceView(); 657 if (lz instanceof Dataset) { 658 nlz = ((Dataset) lz).reshape(nshape); 659 } else { 660 nlz = lz.getSliceView(); 661 nlz.setShape(nshape); 662 } 663 return nlz; 664 } 665 } 666 667 class MdsTranspose implements MetadatasetAnnotationOperation { 668 int[] map; 669 670 public MdsTranspose(final int[] axesMap) { 671 map = axesMap; 672 } 673 674 @SuppressWarnings({ "rawtypes", "unchecked" }) 675 @Override 676 public Object processField(Field f, Object o) { 677 // reorder arrays and lists according the axes map 678 if (o.getClass().isArray()) { 679 int l = Array.getLength(o); 680 if (l == map.length) { 681 Object narray = Array.newInstance(o.getClass().getComponentType(), l); 682 for (int i = 0; i < l; i++) { 683 Array.set(narray, i, Array.get(o, map[i])); 684 } 685 for (int i = 0; i < l; i++) { 686 Array.set(o, i, Array.get(narray, i)); 687 } 688 } 689 } else if (o instanceof List<?>) { 690 List list = (List) o; 691 int l = list.size(); 692 if (l == map.length) { 693 Object narray = Array.newInstance(o.getClass().getComponentType(), l); 694 for (int i = 0; i < l; i++) { 695 Array.set(narray, i, list.get(map[i])); 696 } 697 list.clear(); 698 for (int i = 0; i < l; i++) { 699 list.add(Array.get(narray, i)); 700 } 701 } 702 } 703 return o; 704 } 705 706 @Override 707 public Class<? extends Annotation> getAnnClass() { 708 return Transposable.class; 709 } 710 711 @Override 712 public int change(int axis) { 713 return 0; 714 } 715 716 @Override 717 public int getNewRank() { 718 return -1; 719 } 720 721 @Override 722 public ILazyDataset run(ILazyDataset lz) { 723 return lz.getTransposedView(map); 724 } 725 } 726 727 class MdsDirty implements MetadatasetAnnotationOperation { 728 729 @Override 730 public Object processField(Field f, Object o) { 731 // throw exception if not boolean??? 732 Class<?> t = f.getType(); 733 if (t.equals(boolean.class) || t.equals(Boolean.class)) { 734 if (o.equals(false)) { 735 o = true; 736 } 737 } 738 return o; 739 } 740 741 @Override 742 public Class<? extends Annotation> getAnnClass() { 743 return Dirtiable.class; 744 } 745 746 @Override 747 public int change(int axis) { 748 return 0; 749 } 750 751 @Override 752 public int getNewRank() { 753 return -1; 754 } 755 756 @Override 757 public ILazyDataset run(ILazyDataset lz) { 758 return lz; 759 } 760 } 761 762 /** 763 * Slice all datasets in metadata that are annotated by @Sliceable. Call this on the new sliced 764 * dataset after cloning the metadata 765 * @param asView if true then just a view 766 * @param slice 767 */ 768 protected void sliceMetadata(boolean asView, final SliceND slice) { 769 processAnnotatedMetadata(new MdsSlice(asView, slice.getStart(), slice.getStop(), slice.getStep(), slice.getSourceShape()), true); 770 } 771 772 /** 773 * Reshape all datasets in metadata that are annotated by @Reshapeable. Call this when squeezing 774 * or setting the shape 775 * 776 * @param newShape 777 */ 778 protected void reshapeMetadata(final int[] oldShape, final int[] newShape) { 779 processAnnotatedMetadata(new MdsReshape(oldShape, newShape), true); 780 } 781 782 /** 783 * Transpose all datasets in metadata that are annotated by @Transposable. Call this on the transposed 784 * dataset after cloning the metadata 785 * @param axesMap 786 */ 787 protected void transposeMetadata(final int[] axesMap) { 788 processAnnotatedMetadata(new MdsTranspose(axesMap), true); 789 } 790 791 /** 792 * Dirty metadata that are annotated by @Dirtiable. Call this when the dataset has been modified 793 * @since 2.0 794 */ 795 protected void dirtyMetadata() { 796 processAnnotatedMetadata(new MdsDirty(), true); 797 } 798 799 @SuppressWarnings("unchecked") 800 private void processAnnotatedMetadata(MetadatasetAnnotationOperation op, boolean throwException) { 801 if (metadata == null) 802 return; 803 804 for (Class<? extends MetadataType> c : metadata.keySet()) { 805 for (MetadataType m : metadata.get(c)) { 806 if (m == null) 807 continue; 808 809 Class<? extends MetadataType> mc = m.getClass(); 810 do { // iterate over super-classes 811 processClass(op, m, mc, throwException); 812 Class<?> sclazz = mc.getSuperclass(); 813 if (!MetadataType.class.isAssignableFrom(sclazz)) 814 break; 815 mc = (Class<? extends MetadataType>) sclazz; 816 } while (true); 817 } 818 } 819 } 820 821 @SuppressWarnings({ "unchecked", "rawtypes" }) 822 private static void processClass(MetadatasetAnnotationOperation op, MetadataType m, Class<? extends MetadataType> mc, boolean throwException) { 823 for (Field f : mc.getDeclaredFields()) { 824 if (!f.isAnnotationPresent(op.getAnnClass())) 825 continue; 826 827 try { 828 f.setAccessible(true); 829 Object o = f.get(m); 830 if (o == null) 831 continue; 832 833 Object no = op.processField(f, o); 834 if (no != o) { 835 f.set(m, no); 836 continue; 837 } 838 Object r = null; 839 if (o instanceof ILazyDataset) { 840 try { 841 f.set(m, op.run((ILazyDataset) o)); 842 } catch (Exception e) { 843 logger.error("Problem processing " + o, e); 844 if (!catchExceptions) 845 throw e; 846 } 847 } else if (o.getClass().isArray()) { 848 int l = Array.getLength(o); 849 if (l <= 0) 850 continue; 851 852 for (int i = 0; r == null && i < l; i++) { 853 r = Array.get(o, i); 854 } 855 int n = op.getNewRank(); 856 if (r == null) { 857 if (n < 0 || n != l) { // all nulls be need to match rank as necessary 858 f.set(m, Array.newInstance(o.getClass().getComponentType(), n < 0 ? l : n)); 859 } 860 continue; 861 } 862 if (n < 0) 863 n = l; 864 Object narray = Array.newInstance(r.getClass(), n); 865 for (int i = 0, si = 0, di = 0; di < n && si < l; i++) { 866 int c = op.change(i); 867 if (c == 0) { 868 Array.set(narray, di++, processObject(op, Array.get(o, si++))); 869 } else if (c > 0) { 870 di += c; // add nulls by skipping forward in destination array 871 } else if (c < 0) { 872 si -= c; // remove dimensions by skipping forward in source array 873 } 874 } 875 if (n == l) { 876 for (int i = 0; i < l; i++) { 877 Array.set(o, i, Array.get(narray, i)); 878 } 879 } else { 880 f.set(m, narray); 881 } 882 } else if (o instanceof List<?>) { 883 List list = (List) o; 884 int l = list.size(); 885 if (l <= 0) 886 continue; 887 888 for (int i = 0; r == null && i < l; i++) { 889 r = list.get(i); 890 } 891 int n = op.getNewRank(); 892 if (r == null) { 893 if (n < 0 || n != l) { // all nulls be need to match rank as necessary 894 list.clear(); 895 for (int i = 0, imax = n < 0 ? l : n; i < imax; i++) { 896 list.add(null); 897 } 898 } 899 continue; 900 } 901 902 if (n < 0) 903 n = l; 904 Object narray = Array.newInstance(r.getClass(), n); 905 for (int i = 0, si = 0, di = 0; i < l && si < l; i++) { 906 int c = op.change(i); 907 if (c == 0) { 908 Array.set(narray, di++, processObject(op, list.get(si++))); 909 } else if (c > 0) { 910 di += c; // add nulls by skipping forward in destination array 911 } else if (c < 0) { 912 si -= c; // remove dimensions by skipping forward in source array 913 } 914 } 915 list.clear(); 916 for (int i = 0; i < n; i++) { 917 list.add(Array.get(narray, i)); 918 } 919 } else if (o instanceof Map<?,?>) { 920 Map map = (Map) o; 921 for (Object k : map.keySet()) { 922 map.put(k, processObject(op, map.get(k))); 923 } 924 } 925 } catch (Exception e) { 926 logger.error("Problem occurred when processing metadata of class {}: {}", mc.getCanonicalName(), e); 927 if (throwException) 928 throw new RuntimeException(e); 929 } 930 } 931 } 932 933 @SuppressWarnings({ "unchecked", "rawtypes" }) 934 private static Object processObject(MetadatasetAnnotationOperation op, Object o) throws Exception { 935 if (o == null) 936 return o; 937 938 if (o instanceof ILazyDataset) { 939 try { 940 return op.run((ILazyDataset) o); 941 } catch (Exception e) { 942 logger.error("Problem processing " + o, e); 943 if (!catchExceptions) 944 throw e; 945 } 946 } else if (o.getClass().isArray()) { 947 int l = Array.getLength(o); 948 for (int i = 0; i < l; i++) { 949 Array.set(o, i, processObject(op, Array.get(o, i))); 950 } 951 } else if (o instanceof List<?>) { 952 List list = (List) o; 953 for (int i = 0, imax = list.size(); i < imax; i++) { 954 list.set(i, processObject(op, list.get(i))); 955 } 956 } else if (o instanceof Map<?,?>) { 957 Map map = (Map) o; 958 for (Object k : map.keySet()) { 959 map.put(k, processObject(op, map.get(k))); 960 } 961 } 962 return o; 963 } 964 965 protected void restoreMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> oldMetadata) { 966 for (Class<? extends MetadataType> mc : oldMetadata.keySet()) { 967 metadata.put(mc, oldMetadata.get(mc)); 968 } 969 } 970 971 protected ILazyDataset createFromSerializable(Serializable blob, boolean keepLazy) { 972 ILazyDataset d = null; 973 if (blob instanceof ILazyDataset) { 974 d = (ILazyDataset) blob; 975 if (d instanceof IDataset) { 976 Dataset ed = DatasetUtils.convertToDataset((IDataset) d); 977 int is = ed.getElementsPerItem(); 978 if (is != 1 && is != getElementsPerItem()) { 979 throw new IllegalArgumentException("Dataset has incompatible number of elements with this dataset"); 980 } 981 d = ed.cast(is == 1 ? Dataset.FLOAT64 : Dataset.ARRAYFLOAT64); 982 } else if (!keepLazy) { 983 final int is = getElementsPerItem(); 984 try { 985 d = DatasetUtils.cast(d.getSlice(), is == 1 ? Dataset.FLOAT64 : Dataset.ARRAYFLOAT64); 986 } catch (DatasetException e) { 987 logger.error("Could not get data from lazy dataset", e); 988 return null; 989 } 990 } 991 } else { 992 final int is = getElementsPerItem(); 993 if (is == 1) { 994 d = DatasetFactory.createFromObject(DoubleDataset.class, blob); 995 } else { 996 try { 997 d = DatasetFactory.createFromObject(is, CompoundDoubleDataset.class, blob); 998 } catch (IllegalArgumentException e) { // if only single value supplied try again 999 d = DatasetFactory.createFromObject(DoubleDataset.class, blob); 1000 } 1001 } 1002 if (d.getSize() == getSize() && !Arrays.equals(d.getShape(), shape)) { 1003 d.setShape(shape.clone()); 1004 } 1005 } 1006 List<int[]> s = BroadcastUtils.broadcastShapesToMax(shape, d.getShape()); 1007 d.setShape(s.get(0)); 1008 1009 return d; 1010 } 1011 1012 @Override 1013 public void setErrors(Serializable errors) { 1014 if (shape == null) { 1015 throw new IllegalArgumentException("Cannot set errors for null dataset"); 1016 } 1017 if (errors == null) { 1018 clearMetadata(ErrorMetadata.class); 1019 return; 1020 } 1021 if (errors == this) { 1022 logger.warn("Ignoring setting error to itself as this will lead to infinite recursion"); 1023 return; 1024 } 1025 1026 ILazyDataset errorData = createFromSerializable(errors, true); 1027 1028 ErrorMetadata emd = getErrorMetadata(); 1029 if (emd == null) { 1030 try { 1031 emd = MetadataFactory.createMetadata(ErrorMetadata.class); 1032 setMetadata(emd); 1033 } catch (MetadataException me) { 1034 logger.error("Could not create metadata", me); 1035 } 1036 } 1037 emd.setError(errorData); 1038 } 1039 1040 protected ErrorMetadata getErrorMetadata() { 1041 try { 1042 List<ErrorMetadata> el = getMetadata(ErrorMetadata.class); 1043 if (el != null && !el.isEmpty()) { 1044 return el.get(0); 1045 } 1046 } catch (Exception e) { 1047 } 1048 return null; 1049 } 1050 1051 @Override 1052 public ILazyDataset getErrors() { 1053 ErrorMetadata emd = getErrorMetadata(); 1054 return emd == null ? null : emd.getError(); 1055 } 1056 1057 @Override 1058 public boolean hasErrors() { 1059 return LazyDatasetBase.this.getErrors() != null; 1060 } 1061 1062 /** 1063 * Check permutation axes 1064 * @param shape 1065 * @param axes 1066 * @return cleaned up axes or null if trivial 1067 */ 1068 public static int[] checkPermutatedAxes(int[] shape, int... axes) { 1069 int rank = shape == null ? 0 : shape.length; 1070 1071 if (axes == null || axes.length == 0) { 1072 axes = new int[rank]; 1073 for (int i = 0; i < rank; i++) { 1074 axes[i] = rank - 1 - i; 1075 } 1076 } 1077 1078 if (axes.length != rank) { 1079 logger.error("axis permutation has length {} that does not match dataset's rank {}", axes.length, rank); 1080 throw new IllegalArgumentException("axis permutation does not match shape of dataset"); 1081 } 1082 1083 // check all permutation values are within bounds 1084 for (int i = 0; i < rank; i++) { 1085 axes[i] = ShapeUtils.checkAxis(rank, axes[i]); 1086 } 1087 1088 // check for a valid permutation (is this an unnecessary restriction?) 1089 int[] perm = axes.clone(); 1090 Arrays.sort(perm); 1091 1092 for (int i = 0; i < rank; i++) { 1093 if (perm[i] != i) { 1094 logger.error("axis permutation is not valid: it does not contain complete set of axes"); 1095 throw new IllegalArgumentException("axis permutation does not contain complete set of axes"); 1096 } 1097 } 1098 1099 if (Arrays.equals(axes, perm)) 1100 return null; // signal identity or trivial permutation 1101 1102 return axes; 1103 } 1104}