Merge branch 'addingWritingFullImageWithPyramids'

Author: xulman

src/main/ijm/DataStore-To-TIFFs.ijm

@@ -32,7 +32,7 @@ for (t = downloadTimePoint_FROM; t <= downloadTimePoint_TILL; t++) {
 	filename = outputDir+"/"+fileNameBeforeTPNumber+padding(t,zeroPaddingWidth)+fileNameAfterTPNumber;
 	print("Doing TP "+t+", that is a file: "+filename);
 
-	run("Read full image", "url="+host_port+" datasetid="+dataset+" timepoint="+t+" channel="+downloadFromThisChannel+" angle="+downloadThisAngle+" resolutionlevelsasstr=[[1, 1, 1]] versionasstr="+version+" timeout="+(timeout*1000)+" verboselog="+verboseDownload);
+	run("Read full image", "url="+host_port+" datasetid="+dataset+" versionasstr="+version+" timepoint="+t+" channel="+downloadFromThisChannel+" angle="+downloadThisAngle+" resolutionlevelsasstr=[[1, 1, 1]] timeout="+(timeout*1000)+" verboselog="+verboseDownload);
 
 	saveAs("Tiff", filename);
 	close();

src/main/ijm/TIFFs_Downscale_And_To_DataStore.ijm renamed to src/main/ijm/TIFFs-Downscale-And-To-DataStore.ijm

@@ -28,7 +28,7 @@ function padding(number, width) {
 }
 
 function justSaveAsLevel(resLevelX,resLevelY,resLevelZ) {
-	run("Write full image", "url="+host_port+" datasetid="+dataset+" timepoint="+t+" channel="+uploadIntoThisChannel+" angle="+uploadIntoThisAngle+" resolutionlevelsasstr=[["+resLevelX+", "+resLevelY+", "+resLevelZ+"]] versionasstr="+version+" timeout="+(timeout*1000)+" verboselog="+verboseUpload);
+	run("Write full image", "url="+host_port+" datasetid="+dataset+" versionasstr="+version+" timepoint="+t+" channel="+uploadIntoThisChannel+" angle="+uploadIntoThisAngle+" resolutionlevelsasstr=[["+resLevelX+", "+resLevelY+", "+resLevelZ+"]] timeout="+(timeout*1000)+" verboselog="+verboseUpload+" uploadrespyramids=false");
 }
 
 function scaleAndSave(refMainImg, resLevelX,resLevelY,resLevelZ) {

src/main/ijm/TIFFs-To-DataStore.ijm

@@ -0,0 +1,41 @@
+#@ String(persistkey="datasetserverurl") host_port
+#@ String(persistkey="datasetdatasetid") dataset
+#@ String(description="Use single number like 0,1,2 or keywords latest or new") version = "latest"
+
+#@ int(min="0") uploadIntoThisChannel
+#@ int(min="0") uploadIntoThisAngle
+#@ int(min="0") uploadTimePoint_FROM
+#@ int(min="0") uploadTimePoint_TILL
+
+#@ String(description="The part of file name before the time-point running numbers", default="img") fileNameBeforeTPNumber
+#@ int(description="Pad running numbers with zeros up to the given width, set to 0 for no padding.", min="0", default="0") zeroPaddingWidth
+#@ String(description="The part of file name right after the time-point running numbers", default=".tif") fileNameAfterTPNumber = ".tif"
+#@ File(style="directory", description="Folder with the above defined files") inputDir
+
+#@ boolean verboseUpload = false
+#@ int(min="0", label="Timeout in seconds, e.g. 60") timeout
+
+//fixup boolean: 1 -> true
+if (verboseUpload == 1) { verboseUpload = "true"; }
+
+function padding(number, width) {
+	if (width > 0) {
+		numStr = "00000000000000000000"+number;
+		return substring(numStr, numStr.length-width);
+	} else {
+		return number;
+	}
+}
+
+print("Uploading started....");
+for (t = uploadTimePoint_FROM; t <= uploadTimePoint_TILL; t++) {
+	filename = inputDir+"/"+fileNameBeforeTPNumber+padding(t,zeroPaddingWidth)+fileNameAfterTPNumber;
+	print("Doing TP "+t+", that is a file: "+filename);
+
+	open(filename);
+
+	run("Write full image", "url="+host_port+" datasetid="+dataset+" versionasstr="+version+" timepoint="+t+" channel="+uploadIntoThisChannel+" angle="+uploadIntoThisAngle+" resolutionlevelsasstr=[[1, 1, 1]] timeout="+(timeout*1000)+" verboselog="+verboseUpload+" uploadrespyramids=true");
+
+	close();
+}
+print("Uploading finished....");

src/main/java/cz/it4i/fiji/legacy/AddVersionDataset.java

@@ -25,7 +25,7 @@ public class AddVersionDataset implements Command {
 	public int newlyAddedVersion = 0;
 
 	@Parameter(type = ItemIO.OUTPUT)
-	public String universalReferenceHint = "Try the 'last' keyword instead of a particular version in scripts...";
+	public String universalReferenceHint = "Try the 'latest' keyword instead of a particular version in scripts...";
 
 	@Parameter
 	public DataStoreService dataStoreService;

src/main/java/cz/it4i/fiji/legacy/ReadFullImage.java

@@ -72,6 +72,7 @@ public void run() {
 					timepoint,channel,angle,
 					resolutionLevelsAsStr,versionAsStr,
 					timeout,verboseLog);
+			log.info("transfer is finished");
 		} catch (IOException e) {
 			log.error("Problem reading full image: "+e.getMessage());
 			//e.printStackTrace();
@@ -160,6 +161,10 @@ Dataset readNow(final String url, final String datasetID,
 			rangeSpatialY();
 			rangeSpatialZ();
 
+			if (currentResLevel == null)
+				throw new IOException("Cannot read at res level "+resolutionLevelsAsStr
+						+" because the dataset is not having this one.");
+
 			return this.readWithAType();
 		}
 	}

src/main/java/cz/it4i/fiji/legacy/ReadIntoImagePlus.java

@@ -19,6 +19,7 @@ public void run() {
 		adjustReportingVerbosity();
 
 		outDatasetImg = readWithAType();
+		mainLogger.info("transfer is finished");
 		if (showRunCmd) {
 			final String howToRun = reportAsMacroCommand("Read full image");
 			mainLogger.info("Corresponding IJM command: "+howToRun);

src/main/java/cz/it4i/fiji/legacy/WriteFromImagePlus.java

@@ -19,6 +19,7 @@ public void run() {
 		adjustReportingVerbosity();
 
 		writeWithAType((Img)inDatasetImg);
+		mainLogger.info("transfer is finished");
 		if (showRunCmd) {
 			final String howToRun = reportAsMacroCommand("Write full image");
 			mainLogger.info("Corresponding IJM command: "+howToRun);

src/main/java/cz/it4i/fiji/legacy/WriteFullImage.java

@@ -10,6 +10,7 @@
 import cz.it4i.fiji.legacy.common.ImagePlusTransferrer;
 import cz.it4i.fiji.rest.util.DatasetInfo;
 import net.imglib2.img.Img;
+import net.imglib2.view.Views;
 import net.imglib2.type.NativeType;
 import net.imglib2.type.numeric.RealType;
 import org.scijava.Context;
@@ -44,6 +45,9 @@ public class WriteFullImage implements Command {
 			persistKey="datasetreslevel")
 	public String resolutionLevelsAsStr = "[1, 1, 1]";
 
+	@Parameter(label = "Write also lower resolutions:", required = false)
+	public boolean uploadResPyramids = true;
+
 	@Parameter(label = "Selected version:",
 			description = "provide number, or keyword: latest, new",
 			persistKey="datasetversion")
@@ -69,14 +73,14 @@ public class WriteFullImage implements Command {
 	@Override
 	public void run() {
 		try {
-			new LocalWriter(log.getContext()).writeNow(inDatasetImg.getImgPlus().getImg(),
+			new LocalWriter(log.getContext()).writeNow((Img)inDatasetImg.getImgPlus().getImg(),
 					URL,datasetID,
 					timepoint,channel,angle,
-					resolutionLevelsAsStr,versionAsStr,
+					resolutionLevelsAsStr,uploadResPyramids,versionAsStr,
 					timeout,verboseLog);
-		} catch (IOException e) {
+			log.info("transfer is finished");
+		} catch (IOException | IllegalArgumentException e) {
 			log.error("Problem writing full image: "+e.getMessage());
-			//e.printStackTrace();
 		}
 	}
 
@@ -86,7 +90,7 @@ void to(final Img<? extends RealType<?>> image, final String url, final String d
 	        final int timepoint, final int channel, final int angle,
 	        final int downscaleX, final int downscaleY, final int downscaleZ,
 	        final String versionAsStr)
-	throws IOException {
+	throws IOException,IllegalArgumentException {
 		to(image, url,datasetID,timepoint,channel,angle,
 				createResStr(downscaleX,downscaleY,downscaleZ),
 				versionAsStr,120000,false);
@@ -96,7 +100,7 @@ void to(final Img<? extends RealType<?>> image, final String url, final String d
 	void to(final Img<? extends RealType<?>> image, final String url, final String datasetID,
 	        final int timepoint, final int channel, final int angle,
 	        final String resolutionLevelsAsStr, final String versionAsStr)
-	throws IOException {
+	throws IOException,IllegalArgumentException {
 		to(image, url,datasetID,timepoint,channel,angle,
 				resolutionLevelsAsStr,
 				versionAsStr,120000,false);
@@ -107,8 +111,40 @@ void to(final Img<? extends RealType<?>> image, final String url, final String d
 	        final int timepoint, final int channel, final int angle,
 	        final String resolutionLevelsAsStr, final String versionAsStr,
 	        final int serverTimeout, final boolean verboseLog)
-	throws IOException {
-		new LocalWriter().writeNow(image, url, datasetID, timepoint, channel, angle,
+	throws IOException,IllegalArgumentException {
+		new LocalWriter().writeNow((Img)image, url, datasetID, timepoint, channel, angle,
+				resolutionLevelsAsStr, versionAsStr, serverTimeout, verboseLog);
+	}
+
+
+	public static
+	void toWithoutPyramids(final Img<? extends RealType<?>> image, final String url, final String datasetID,
+	        final int timepoint, final int channel, final int angle,
+	        final int downscaleX, final int downscaleY, final int downscaleZ,
+	        final String versionAsStr)
+	throws IOException,IllegalArgumentException {
+		toWithoutPyramids(image, url,datasetID,timepoint,channel,angle,
+				createResStr(downscaleX,downscaleY,downscaleZ),
+				versionAsStr,120000,false);
+	}
+
+	public static
+	void toWithoutPyramids(final Img<? extends RealType<?>> image, final String url, final String datasetID,
+	        final int timepoint, final int channel, final int angle,
+	        final String resolutionLevelsAsStr, final String versionAsStr)
+	throws IOException,IllegalArgumentException {
+		toWithoutPyramids(image, url,datasetID,timepoint,channel,angle,
+				resolutionLevelsAsStr,
+				versionAsStr,120000,false);
+	}
+
+	public static
+	void toWithoutPyramids(final Img<? extends RealType<?>> image, final String url, final String datasetID,
+	        final int timepoint, final int channel, final int angle,
+	        final String resolutionLevelsAsStr, final String versionAsStr,
+	        final int serverTimeout, final boolean verboseLog)
+	throws IOException,IllegalArgumentException {
+		new LocalWriter().writeNowWithoutPyramids((Img)image, url, datasetID, timepoint, channel, angle,
 				resolutionLevelsAsStr, versionAsStr, serverTimeout, verboseLog);
 	}
 
@@ -125,17 +161,38 @@ static class LocalWriter extends ImagePlusTransferrer {
 			this.setContext(useThisCtx);
 		}
 
-		void writeNow(final Img<? extends RealType<?>> image, final String url, final String datasetID,
+		<T extends RealType<T>>
+		void writeNow(final Img<T> img, final String url, final String datasetID,
 		              final int timepoint, final int channel, final int angle,
 		              final String resolutionLevelsAsStr, final String versionAsStr,
 		              final int serverTimeout, final boolean verboseLog)
-		throws IOException {
+		throws IOException,IllegalArgumentException
+		{
+			writeNow(img,url,datasetID,timepoint,channel,angle,
+					resolutionLevelsAsStr,true,versionAsStr,serverTimeout,verboseLog);
+		}
+
+		<T extends RealType<T>>
+		void writeNowWithoutPyramids(final Img<T> img, final String url, final String datasetID,
+		              final int timepoint, final int channel, final int angle,
+		              final String resolutionLevelsAsStr, final String versionAsStr,
+		              final int serverTimeout, final boolean verboseLog)
+		throws IOException,IllegalArgumentException
+		{
+			writeNow(img,url,datasetID,timepoint,channel,angle,
+					resolutionLevelsAsStr,false,versionAsStr,serverTimeout,verboseLog);
+		}
 
-			if (!(image.firstElement() instanceof NativeType)) {
-				throw new IllegalArgumentException("Provided type ("
-						+image.firstElement().getClass().getSimpleName()
-						+") is not derived from NativeType.");
-			}
+
+		<TR extends RealType<TR>, TNR extends NativeType<TNR> & RealType<TNR>>
+		void writeNow(final Img<TR> img, final String url, final String datasetID,
+		              final int timepoint, final int channel, final int angle,
+		              final String resolutionLevelsAsStr, final boolean uploadResPyramids,
+		              final String versionAsStr,
+		              final int serverTimeout, final boolean verboseLog)
+		throws IOException,IllegalArgumentException
+		{
+			final Img<TNR> image = checkForAndExtendWithNativeType(img);
 
 			this.URL = url;
 			this.datasetID = datasetID;
@@ -169,7 +226,59 @@ void writeNow(final Img<? extends RealType<?>> image, final String url, final St
 			rangeSpatialY();
 			rangeSpatialZ();
 
-			this.writeWithAType((Img)image);
+			if (currentResLevel == null)
+				throw new IOException("Cannot write to res level "+resolutionLevelsAsStr
+						+" because the dataset is not having this one.");
+
+			this.writeWithAType(image);
+
+			if (uploadResPyramids)
+			{
+				//prevent from creating new version with every next resolution
+				if (versionAsStr.startsWith("new")) this.versionAsStr = "latest";
+
+				//plan: find upper res levels, for each create downscaled image and upload it
+				int resLevelIdx = 0;
+				while (resLevelIdx < di.resolutionLevels.size()
+						&& di.resolutionLevels.get(resLevelIdx) != currentResLevel) ++resLevelIdx;
+				if (resLevelIdx == di.resolutionLevels.size())
+					throw new RuntimeException("Failed re-matching the resolution level, that's odd...");
+				myLogger.info("Starting to upload down-scaled versions from level "+ resLevelIdx +".");
+
+				//base downscale factors:
+				final int rx = currentResLevel.resolutions.get(0);
+				final int ry = currentResLevel.resolutions.get(1);
+				final int rz = currentResLevel.resolutions.get(2);
+
+				//these remaining res levels we're gonna write too
+				++resLevelIdx;
+				while (resLevelIdx < di.resolutionLevels.size())
+				{
+					currentResLevel = di.resolutionLevels.get(resLevelIdx);
+					this.resolutionLevelsAsStr = currentResLevel.resolutions.toString();
+					myLogger.info("==> Writing also: "+this.resolutionLevelsAsStr);
+
+					rangeSpatialX();
+					rangeSpatialY();
+					rangeSpatialZ();
+
+					//downscale factor w.r.t to the base factor
+					final int dx = currentResLevel.resolutions.get(0) / rx;
+					final int dy = currentResLevel.resolutions.get(1) / ry;
+					final int dz = currentResLevel.resolutions.get(2) / rz;
+
+					//check the scaling is integer, or skip this scaling
+					if (dx*rx != currentResLevel.resolutions.get(0)
+						|| dy*ry != currentResLevel.resolutions.get(1)
+						|| dz*rz != currentResLevel.resolutions.get(2))
+					{
+						myLogger.info("Cannot reach res level "+resLevelIdx+" from "+ resLevelIdx +" with integer-scaling.");
+					} else {
+						writeWithAType( Views.subsample(image, dx,dy,dz), image.firstElement() );
+					}
+					++resLevelIdx;
+				}
+			}
 		}
 	}
 }

src/main/java/cz/it4i/fiji/legacy/common/ImagePlusDialogHandler.java

@@ -40,19 +40,19 @@ abstract class ImagePlusDialogHandler extends DynamicCommand {
 	public String datasetLabel = "";
 
 	@Parameter(label="min X [px]:", min="0", callback = "rangeSpatialX")
-	public int minX = -1;
+	public int minX = -5;
 	@Parameter(label="max X [px]:", min="0", callback = "rangeSpatialX")
-	public int maxX = -1;
+	public int maxX = -5;
 
 	@Parameter(label="min Y [px]:", min="0", callback = "rangeSpatialY")
-	public int minY = -1;
+	public int minY = -5;
 	@Parameter(label="max Y [px]:", min="0", callback = "rangeSpatialY")
-	public int maxY = -1;
+	public int maxY = -5;
 
 	@Parameter(label="min Z [px]:", min="0", callback = "rangeSpatialZ")
-	public int minZ = -1;
+	public int minZ = -5;
 	@Parameter(label="max Z [px]:", min="0", callback = "rangeSpatialZ")
-	public int maxZ = -1;
+	public int maxZ = -5;
 
 	@Parameter(label="time point:", min="0", callback = "rangeTPs",
 			description="In units of the respective dataset.",
@@ -276,7 +276,8 @@ private void rangeSpatial(int cn,int cx, int nI,int xI, int blockSize, int maxx)
 		//upper bound: align to block size
 		if (max < 1) max = 1;
 		max = (int)Math.ceil((double)max / (double)blockSize) * blockSize -1;
-		if (max > maxx) max = maxx-1;
+		if (max > maxx || max < 0) max = maxx-1;
+		//NB: multiplication after ceil() can make 'max' overflow (and end up negative as a consequence)
 
 		//lower bound: align to block size
 		if (min < 0) min = 0;

src/main/java/cz/it4i/fiji/legacy/common/ImagePlusTransferrer.java

@@ -3,6 +3,7 @@
 import net.imagej.Dataset;
 import net.imagej.DefaultDataset;
 import net.imagej.ImgPlus;
+import net.imglib2.RandomAccessibleInterval;
 import net.imglib2.img.Img;
 import net.imglib2.view.Views;
 import net.imglib2.type.NativeType;
@@ -135,6 +136,21 @@ void printTransferPlan() {
 	}
 
 
+	// ----------------------------------------------
+	public static
+	<TR extends RealType<TR>, TNR extends NativeType<TNR> & RealType<TNR>>
+	Img<TNR> checkForAndExtendWithNativeType(final Img<TR> image)
+	throws IllegalArgumentException
+	{
+		if (!(image.firstElement() instanceof NativeType)) {
+			throw new IllegalArgumentException("Provided type ("
+					+ image.firstElement().getClass().getSimpleName()
+					+ ") is not derived from NativeType.");
+		}
+		return (Img)image;
+	}
+
+
 	// ----------------------------------------------
 	public <T extends NativeType<T> & RealType<T>>
 	Dataset readWithAType() {
@@ -267,9 +283,14 @@ Dataset readWithAType() {
 
 	public <T extends NativeType<T> & RealType<T>>
 	void writeWithAType(final Img<T> img) {
+		writeWithAType(img, img.firstElement());
+	}
+
+	public <T extends NativeType<T> & RealType<T>>
+	void writeWithAType(final RandomAccessibleInterval<T> rai, final T ofThisType) {
 		try {
 			@SuppressWarnings("unchecked")
-			final Imglib2Types.TypeHandler<T> th = Imglib2Types.getTypeHandler(img.firstElement());
+			final Imglib2Types.TypeHandler<T> th = Imglib2Types.getTypeHandler(ofThisType);
 
 			//sanity check already at the client
 			@SuppressWarnings("unchecked")
@@ -357,7 +378,7 @@ void writeWithAType(final Img<T> img) {
 							pxData = new byte[blockLength];
 
 						//copy the current image block into the buffer
-						th.imgIntervalIntoBlock( Views.interval(img,
+						th.imgIntervalIntoBlock( Views.interval(rai,
 								new long[]{x-minX,      y-minY,      z-minZ},
 								new long[]{x-minX+ex-1, y-minY+ey-1, z-minZ+ez-1}),
 								ex*ey*ez, pxData);