Skip to content

Commit e52256e

Browse files
ThechopseeThechopsee
committed
feat: Added support for ZARR format
1 parent 1dab42c commit e52256e

File tree

5 files changed

+193
-6
lines changed

5 files changed

+193
-6
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,3 +45,4 @@ cookies.txt
4545

4646
# Vim #
4747
.*.sw*
48+
*.tif

‎src/main/java/cz/it4i/fiji/datastore/rest_client/WriteSequenceToN5.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,7 @@ public static void writeN5File(final AbstractSequenceDescription<?, ?, ?> seq,
181181
int numCompletedTasks = 0;
182182

183183
final ExecutorService executorService = Executors.newFixedThreadPool( numCellCreatorThreads );
184+
184185
try (N5Writer n5 = writerSupplier.get()) {
185186
// write image data for all views
186187
final int numTimepoints = timepointIds.size();

‎src/main/java/cz/it4i/fiji/legacy/CloneEmptyDatasetFromN5toZarr.java‎

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
package cz.it4i.fiji.legacy;
2+
3+
import com.fasterxml.jackson.databind.ObjectMapper;
4+
import cz.it4i.fiji.legacy.util.GuiResolutionLevelParams;
5+
import cz.it4i.fiji.rest.util.DatasetInfo;
6+
import net.imagej.Dataset;
7+
import org.scijava.ItemIO;
8+
import org.scijava.ItemVisibility;
9+
import org.scijava.command.Command;
10+
import org.scijava.command.CommandInfo;
11+
import org.scijava.command.CommandModule;
12+
import org.scijava.command.CommandService;
13+
import org.scijava.log.LogLevel;
14+
import org.scijava.log.LogService;
15+
import org.scijava.log.Logger;
16+
import org.scijava.module.Module;
17+
import org.scijava.module.ModuleException;
18+
import org.scijava.module.process.PostprocessorPlugin;
19+
import org.scijava.module.process.PreprocessorPlugin;
20+
import org.scijava.plugin.Parameter;
21+
import org.scijava.plugin.Plugin;
22+
import org.scijava.plugin.PluginService;
23+
import org.scijava.prefs.PrefService;
24+
25+
import java.io.IOException;
26+
import java.util.List;
27+
import java.util.concurrent.ExecutionException;
28+
import java.util.concurrent.Future;
29+
30+
@Plugin(type = Command.class, headless = true, menuPath = "Plugins>HPC DataStore>ZARR Convert>Create new dataset from existing N5")
31+
32+
public class CloneEmptyDatasetFromN5toZarr implements Command {
33+
@Parameter(label = "URL of a reference DatasetsRegisterService:", persistKey = "datasetserverurl")
34+
public String src_url = "someHostname:9080";
35+
36+
@Parameter(label = "UUID of a reference N5 dataset on that service:", persistKey = "datasetdatasetid")
37+
public String src_uuid = "someDatasetUUID";
38+
39+
@Parameter(label = "URL of a target DatasetsRegisterService:", persistKey = "datasetserverurl")
40+
public String tgt_url = "someHostname:9080";
41+
42+
@Parameter(label = "Modify params before creating the new dataset:")
43+
public boolean shallUpdateParams = false;
44+
45+
@Parameter
46+
public PrefService prefs;
47+
48+
@Parameter
49+
public CommandService cs;
50+
@Parameter
51+
public PluginService ps;
52+
53+
@Parameter
54+
public LogService mainLogger;
55+
protected Logger myLogger;
56+
57+
@Parameter(label = "Report corresponding macro command:", required = false)
58+
public boolean showRunCmd = false;
59+
60+
@Parameter(type = ItemIO.OUTPUT, label="UUID of the created Zarr dataset:")
61+
public String newDatasetUUID;
62+
@Parameter(type = ItemIO.OUTPUT, label="Label of the created Zarr dataset:")
63+
public String newDatasetLabel;
64+
@Override
65+
public void run() {
66+
//logging facility
67+
myLogger = mainLogger.subLogger("HPC CloneDataset", LogLevel.INFO);
68+
69+
try {
70+
//retrieve reference data
71+
myLogger.info("Reading "+src_uuid+" from "+src_url);
72+
DatasetInfo di = DatasetInfo.createFrom(src_url, src_uuid);
73+
myLogger.info(di.toString());
74+
75+
if (!shallUpdateParams) {
76+
final String json = new ObjectMapper().writeValueAsString(di);
77+
myLogger.info("CREATED JSON:\n"+json);
78+
79+
final Future<CommandModule> subcall = cs.run(CreateNewDatasetFromJSON.class, true,
80+
"url",tgt_url, "json",json, "showRunCmd",showRunCmd);
81+
newDatasetUUID = (String)subcall.get().getOutput("newDatasetUUID");
82+
newDatasetLabel = (String)subcall.get().getOutput("newDatasetLabel");
83+
} else {
84+
//we gonna preset the prefs store so that the "params of resolution level" dialog(s)
85+
//will get preloaded with the reference values
86+
final GuiResolutionLevelParams prefsHandler = new GuiResolutionLevelParams();
87+
prefsHandler.prefs = prefs;
88+
for (int level = 0; level < di.resolutionLevels.size(); ++level) {
89+
final DatasetInfo.ResolutionLevel l = di.resolutionLevels.get(level);
90+
prefsHandler.resLevelNumber=level+1;
91+
prefsHandler.down_x = l.resolutions.get(0);
92+
prefsHandler.down_y = l.resolutions.get(1);
93+
prefsHandler.down_z = l.resolutions.get(2);
94+
prefsHandler.block_x = l.blockDimensions.get(0);
95+
prefsHandler.block_y = l.blockDimensions.get(1);
96+
prefsHandler.block_z = l.blockDimensions.get(2);
97+
prefsHandler.storeIntoPrefs();
98+
}
99+
100+
final CommandModule cm = new CommandModule( new CommandInfo(CreateNewDataset.class) );
101+
cm.setInput("datasetType","Zarr");
102+
cm.setInput("url", tgt_url);
103+
cm.setInput("label", di.label);
104+
cm.setInput("voxelType", di.voxelType);
105+
cm.setInput("fullResSizeX", di.dimensions.get(0));
106+
cm.setInput("fullResSizeY", di.dimensions.get(1));
107+
cm.setInput("fullResSizeZ", di.dimensions.get(2));
108+
cm.setInput("timepoints", di.timepoints);
109+
cm.setInput("channels", di.channels);
110+
cm.setInput("angles", di.angles);
111+
cm.setInput("res_x", di.voxelResolution.get(0));
112+
cm.setInput("res_y", di.voxelResolution.get(1));
113+
cm.setInput("res_z", di.voxelResolution.get(2));
114+
cm.setInput("res_unit", di.voxelUnit);
115+
116+
if (di.timepointResolution != null) {
117+
cm.setInput("time_res", di.timepointResolution.value);
118+
cm.setInput("time_unit", di.timepointResolution.unit);
119+
}
120+
if (di.channelResolution != null) {
121+
cm.setInput("channel_res", di.channelResolution.value);
122+
cm.setInput("channel_unit", di.channelResolution.unit);
123+
}
124+
if (di.angleResolution != null) {
125+
cm.setInput("angle_res", di.angleResolution.value);
126+
cm.setInput("angle_unit", di.angleResolution.unit);
127+
}
128+
cm.setInput("numberOfAllResLevels", di.resolutionLevels.size());
129+
cm.setInput("compression", di.compression);
130+
cm.setInput("showRunCmd", showRunCmd);
131+
132+
// 1) this is a standard list of pre- and post-processors,
133+
// this is how ModuleService.java creates them for every call of run()
134+
final List<PreprocessorPlugin> pre = ps.createInstancesOfType(PreprocessorPlugin.class);
135+
final List<PostprocessorPlugin> post = ps.createInstancesOfType(PostprocessorPlugin.class);
136+
137+
// 2) but we find and remove one particular service...
138+
for (int i = 0; i < pre.size(); ++i)
139+
if (pre.get(i).getClass().getSimpleName().startsWith("LoadInputsPreprocessor")) {
140+
pre.remove(i);
141+
break;
142+
}
143+
144+
// 3) finally, the command (module actually) is started
145+
final Future<Module> subcall = cs.moduleService().run(cm,pre,post);
146+
147+
newDatasetUUID = (String)subcall.get().getOutput("newDatasetUUID");
148+
newDatasetLabel = (String)subcall.get().getOutput("newDatasetLabel");
149+
}
150+
} catch (IOException | ModuleException e) {
151+
myLogger.error("Some error accessing the reference service and dataset: "+e.getMessage());
152+
} catch (ExecutionException e) {
153+
myLogger.error(e.getMessage());
154+
155+
} catch (InterruptedException e) {
156+
//dialog interrupted, do nothing special...
157+
myLogger.error("eee!? "+e.getMessage());
158+
}
159+
}
160+
}

‎src/main/java/cz/it4i/fiji/legacy/CreateNewDataset.java‎

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,13 @@ public class CreateNewDataset implements Command {
9797
@Parameter(type = ItemIO.OUTPUT, label="Label of the created dataset:")
9898
public String newDatasetLabel;
9999

100+
@Parameter(label = "DatasetType:", choices = { "N5", "Zarr" })
101+
public String datasetType;
102+
protected String getDatasetType()
103+
{
104+
return this.datasetType;
105+
}
106+
100107
@Override
101108
public void run() {
102109
//logging facility
@@ -115,9 +122,18 @@ public void run() {
115122
di.timepointResolution = new DatasetInfo.ResolutionWithOwnUnit(time_res, time_unit);
116123
di.channelResolution = new DatasetInfo.ResolutionWithOwnUnit(channel_res, channel_unit);
117124
di.angleResolution = new DatasetInfo.ResolutionWithOwnUnit(angle_res, angle_unit);
118-
119-
di.compression = this.compression.equals("none") ? "raw" : this.compression;
120-
125+
//This solution was chosen so that there was no need to change the DatasetDTO object,
126+
// so the information about the dataset type is placed after the compression with the delimiter '/',
127+
// according to which it is then divided in the registerServiceEndpoint and only the actual value of the compression is stored in the compression.
128+
// datasetType is then normally stored in the Dataset class and is used to select a suitable handler.
129+
// The most likely solution would be to add the datasetType attribute to the DatasetDTO class, but unfortunately this was not entirely within my power.
130+
if(this.datasetType.equals("Zarr")) {
131+
di.compression = this.compression.equals("none") ? "raw" : this.compression + "/Zarr";
132+
}
133+
else
134+
{
135+
di.compression = this.compression.equals("none") ? "raw" : this.compression + "/N5";
136+
}
121137
di.versions = Collections.emptyList();
122138
di.resolutionLevels = new ArrayList<>(numberOfAllResLevels);
123139
di.label = label;
@@ -162,7 +178,7 @@ public void run() {
162178
myLogger.info("CREATED JSON:\n"+json);
163179

164180
final Future<CommandModule> subcall = cs.run(CreateNewDatasetFromJSON.class, true,
165-
"url",url, "json",json, "showRunCmd",showRunCmd);
181+
"url",url, "json",json, "showRunCmd",showRunCmd,"datasetType",getDatasetType());
166182
newDatasetUUID = (String)subcall.get().getOutput("newDatasetUUID");
167183
newDatasetLabel = di.getLabel();
168184
} catch (ExecutionException e) {

‎src/main/java/cz/it4i/fiji/legacy/CreateNewDatasetFromImage.java‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,9 @@ void readParams() {
115115
@Parameter(type = ItemIO.OUTPUT, label="Label of the created dataset:")
116116
public String newDatasetLabel;
117117

118+
@Parameter(label = "DatasetType:", choices = { "N5", "Zarr" })
119+
public String datasetType;
120+
118121
@Override
119122
public void run() {
120123
//logging facility
@@ -134,7 +137,13 @@ public void run() {
134137
di.channelResolution = new DatasetInfo.ResolutionWithOwnUnit(channel_res, channel_unit);
135138
di.angleResolution = new DatasetInfo.ResolutionWithOwnUnit(angle_res, angle_unit);
136139

137-
di.compression = this.compression.equals("none") ? "raw" : this.compression;
140+
if(this.datasetType.equals("Zarr")) {
141+
di.compression = this.compression.equals("none") ? "raw" : this.compression + "/Zarr";
142+
}
143+
else
144+
{
145+
di.compression = this.compression.equals("none") ? "raw" : this.compression + "/N5";
146+
}
138147

139148
di.versions = Collections.emptyList();
140149
di.resolutionLevels = new ArrayList<>(numberOfAllResLevels);
@@ -179,7 +188,7 @@ public void run() {
179188
myLogger.info("CREATED JSON:\n"+json);
180189

181190
final Future<CommandModule> subcall = cs.run(CreateNewDatasetFromJSON.class, true,
182-
"url",url, "json",json, "showRunCmd",showRunCmd);
191+
"url",url, "json",json, "showRunCmd",showRunCmd,"datasetType");
183192
newDatasetUUID = (String)subcall.get().getOutput("newDatasetUUID");
184193
newDatasetLabel = di.getLabel();
185194
} catch (ExecutionException e) {

0 commit comments

Comments
 (0)