add batch id

Author: wei

tests/README.md

@@ -17,7 +17,8 @@ Inside `dir/test_code.py` pytest will run any code in the main file and then cal
 
 # gets evaluated first
 my_two = 2.0
-my_three = add_one(my_two)
+my_three = add_one(my_two) 
+
 
 def add_one(x):
     # calls this function

tidy3d/plugins/adjoint/web.py

@@ -281,7 +281,7 @@ def start(self) -> None:
         To monitor progress of the :class:`Job`, call :meth:`Job.monitor` after started.
         """
         if self.jax_info is not None:
-            upload_jax_info(task_id=self.task_id, jax_info=self.jax_info, verbose=self.verbose)
+            upload_jax_info(task_id=self.task_id(), jax_info=self.jax_info, verbose=self.verbose)
         super().start()
 
 

tidy3d/plugins/invdes/test.py

@@ -0,0 +1,167 @@
+import numpy as np
+
+import tidy3d as td
+import tidy3d.plugins.invdes as tdi
+
+from .web import Job
+
+# source info
+wavelength = 1.0
+
+# waveguide parameters
+num_output_waveguides = 3
+ly_wg = 0.5 * wavelength
+buffer_wg = 0.9 * wavelength
+
+# buffer between design region, pml, and sources
+buffer = 1 * wavelength
+
+# relative permittivity of material
+eps_mat = 4.0
+
+# resolution (for both the FDTD simulation and for the design region)
+min_steps_per_wvl = 30
+pixel_size = wavelength / min_steps_per_wvl / np.sqrt(eps_mat)
+
+# spectral information
+freq0 = td.C_0 / wavelength
+fwidth = freq0 / 10
+run_time = 50 / fwidth
+
+# design region size in y
+ly_des = num_output_waveguides * (ly_wg + buffer_wg)
+lx_des = 4 * wavelength
+
+# simulation size
+Lx = 2 * buffer + lx_des + 2 * buffer
+Ly = buffer + ly_des + buffer
+
+# source and monitor locations
+x_src = -lx_des / 2 - buffer
+x_mnt = -x_src
+
+# material Medium
+medium = td.Medium(permittivity=eps_mat)
+
+# grid spec
+grid_spec = td.GridSpec.auto(wavelength=wavelength, min_steps_per_wvl=min_steps_per_wvl)
+
+
+# monitor names
+def output_monitor_name(i: int) -> str:
+    return f"MNT_{i}"
+
+
+field_mnt_name = "field"
+
+# mode spec
+mode_spec = td.ModeSpec(num_modes=1)
+
+
+if __name__ == "__main__":
+    # radius (um) of the conic filter that is convolved with th parameter array.
+    # Larger values tend to help create larger feature sizes in the final device.
+    projection_radius = 0.120
+
+    # projection strength, larger values lead to more binarization and tend to push intermediate parameters towards (0,1) density
+    beta = 10.0
+
+    # transformations on the parameters that lead to the material density array (0,1)
+    filter_project = tdi.FilterProject(radius=projection_radius, beta=beta)
+
+    # length scale (um) of the erosion dilation penalty.
+    # features smaller than this scale will be penalized
+    length_scale = 0.120
+
+    # penalty weight, the penalty contributes its raw value (max of 1) times this weight to the objective function
+    weight = 0.8
+
+    # penalties applied to the state of the material density, after these transformations are applied
+    penalty = tdi.ErosionDilationPenalty(weight=weight, length_scale=length_scale)
+
+    design_region = tdi.TopologyDesignRegion(
+        size=(lx_des, ly_des, td.inf),
+        center=(0, 0, 0),
+        eps_bounds=(1.0, eps_mat),  # the minimum and maximum permittivity values in the final grid
+        transformations=[filter_project],
+        penalties=[penalty],
+        pixel_size=pixel_size,
+    )
+
+    waveguide_in = td.Structure(
+        geometry=td.Box(
+            size=(Lx, ly_wg, td.inf),
+            center=(-Lx + 2 * buffer, 0, 0),
+        ),
+        medium=medium,
+    )
+
+    y_max_wg_centers = ly_des / 2 - buffer_wg / 2 - ly_wg / 2
+    wg_y_centers_out = np.linspace(-y_max_wg_centers, y_max_wg_centers, num_output_waveguides)
+
+    # put a waveguide and mode monitor at each of the outputs
+    waveguides_out = []
+    monitors_out = []
+    for i, wg_y_center in enumerate(wg_y_centers_out):
+        wg_out = td.Structure(
+            geometry=td.Box(
+                size=(Lx, ly_wg, td.inf),
+                center=(Lx - 2 * buffer, wg_y_center, 0),
+            ),
+            medium=medium,
+        )
+
+        waveguides_out.append(wg_out)
+
+        mnt_out = td.ModeMonitor(
+            size=(0, ly_wg + 1.8 * buffer_wg, td.inf),
+            center=(x_mnt, wg_y_center, 0),
+            freqs=[freq0],
+            name=output_monitor_name(i),
+            mode_spec=mode_spec,
+        )
+
+        monitors_out.append(mnt_out)
+
+    source = td.ModeSource(
+        size=(0, ly_wg + 1.8 * buffer_wg, td.inf),
+        center=(x_src, 0, 0),
+        source_time=td.GaussianPulse(freq0=freq0, fwidth=fwidth),
+        mode_index=0,
+        direction="+",
+    )
+
+    # used to visualize fields in the plane, not for optimization
+    fld_mnt = td.FieldMonitor(
+        center=(0, 0, 0),
+        size=(td.inf, td.inf, 0),
+        freqs=[freq0],
+        name=field_mnt_name,
+    )
+
+    simulation = td.Simulation(
+        size=(Lx, Ly, 0),
+        grid_spec=grid_spec,
+        boundary_spec=td.BoundarySpec.pml(x=True, y=True, z=False),
+        run_time=run_time,
+        structures=[waveguide_in] + waveguides_out,
+        sources=[source],
+        monitors=[fld_mnt] + monitors_out,
+    )
+
+    design = tdi.InverseDesign(
+        simulation=simulation,
+        design_region=design_region,
+        task_name="invdes",
+        output_monitor_names=[mnt.name for mnt in monitors_out],
+    )
+
+    optimizer = tdi.AdamOptimizer(
+        design=design,
+        num_steps=10,
+        learning_rate=0.1,
+        results_cache_fname="data/invdes_history.hdf5",
+    )
+
+    optimizer.to_hdf5("data/invdes_history.hdf5")
+    job = Job(optimizer=optimizer)

tidy3d/web/api/autograd/autograd.py

@@ -1167,7 +1167,6 @@ def _run_async_tidy3d_bwd(
     _ = run_kwargs.pop("path_dir", None)
     batch = Batch(simulations=simulations, **batch_init_kwargs)
     td.log.info(f"running {batch.simulation_type} batch with '_run_async_tidy3d_bwd()'")
-
     batch.start()
     batch.monitor()
 

tidy3d/web/api/container.py

@@ -44,9 +44,9 @@ def _check_path_dir(path: str) -> None:
         """Make sure local output directory exists and create it if not."""
 
     @staticmethod
-    def _check_folder(folder_name: str) -> None:
+    def _check_folder(folder_name: str) -> Folder:
         """Make sure ``folder_name`` exists on the web UI and create it if not."""
-        Folder.get(folder_name, create=True)
+        return Folder.get(folder_name, create=True)
 
 
 class Job(WebContainer):
@@ -235,24 +235,24 @@ def run(self, path: str = DEFAULT_DATA_PATH) -> SimulationDataType:
         self.monitor()
         return self.load(path=path)
 
-    @cached_property
-    def task_id(self) -> TaskId:
+    def task_id(self, batch_id: str = None) -> TaskId:
         """The task ID for this ``Job``. Uploads the ``Job`` if it hasn't already been uploaded."""
         if self.task_id_cached:
             return self.task_id_cached
         self._check_folder(self.folder_name)
-        return self._upload()
+        return self._upload(batch_id)
 
-    def _upload(self) -> TaskId:
+    def _upload(self, batch_id: str) -> TaskId:
         """Upload this job and return the task ID for handling."""
         # upload kwargs with all fields except task_id
         upload_kwargs = {key: getattr(self, key) for key in self._upload_fields}
+        upload_kwargs["batch_id"] = batch_id
         task_id = web.upload(**upload_kwargs)
         return task_id
 
-    def upload(self) -> None:
+    def upload(self, batch_id: str = None) -> None:
         """Upload this ``Job``."""
-        _ = self.task_id
+        _ = self.task_id(batch_id)
 
     def get_info(self) -> TaskInfo:
         """Return information about a :class:`Job`.
@@ -370,7 +370,9 @@ def estimate_cost(self, verbose: bool = True) -> float:
         Cost is calculated assuming the simulation runs for
         the full ``run_time``. If early shut-off is triggered, the cost is adjusted proportionately.
         """
-        return web.estimate_cost(self.task_id, verbose=verbose, solver_version=self.solver_version)
+        return web.estimate_cost(
+            self.task_id(), verbose=verbose, solver_version=self.solver_version
+        )
 
     @staticmethod
     def _check_path_dir(path: str) -> None:
@@ -511,6 +513,18 @@ class Batch(WebContainer):
         description="Name of folder to store member of each batch on web UI.",
     )
 
+    batch_id: str = pd.Field(
+        None,
+        title="Batch Id",
+        description="ID of batch to store member of each batch on web UI.",
+    )
+
+    batch_name: str = pd.Field(
+        "batch",
+        title="Batch Name",
+        description="Name of batch to store member of each batch on web UI.",
+    )
+
     verbose: bool = pd.Field(
         True, title="Verbose", description="Whether to print info messages and progressbars."
     )
@@ -674,8 +688,15 @@ def num_jobs(self) -> int:
     def upload(self) -> None:
         """Upload a series of tasks associated with this ``Batch`` using multi-threading."""
         self._check_folder(self.folder_name)
+        batch_id = self.create_batch()
         with ThreadPoolExecutor(max_workers=self.num_workers) as executor:
-            futures = [executor.submit(job.upload) for _, job in self.jobs.items()]
+            futures = [
+                executor.submit(
+                    job.upload,
+                    batch_id,
+                )
+                for _, job in self.jobs.items()
+            ]
 
             # progressbar (number of tasks uploaded)
             if self.verbose:
@@ -711,6 +732,10 @@ def get_info(self) -> dict[TaskName, TaskInfo]:
     def start(self) -> None:
         """Start running all tasks in the :class:`Batch`.
 
+        Parameters:
+            batch_id: str
+                Batch ID to start the tasks in.
+
         Note
         ----
         To monitor the running simulations, can call :meth:`Batch.monitor`.
@@ -1068,3 +1093,8 @@ def _check_path_dir(path_dir: str) -> None:
         """
         if len(path_dir) > 0 and not os.path.exists(path_dir):
             os.makedirs(path_dir, exist_ok=True)
+
+    def create_batch(self) -> str:
+        """Create batch."""
+        folder = self._check_folder(self.folder_name)
+        return OptimizationBatch.create(self.batch_name, folder.folder_id)

tidy3d/web/api/webapi.py

@@ -201,6 +201,7 @@ def upload(
     source_required: bool = True,
     solver_version: Optional[str] = None,
     reduce_simulation: Literal["auto", True, False] = "auto",
+    batch_id: str = None,
 ) -> TaskId:
     """
     Upload simulation to server, but do not start running :class:`.Simulation`.
@@ -230,7 +231,8 @@ def upload(
         target solver version.
     reduce_simulation: Literal["auto", True, False] = "auto"
         Whether to reduce structures in the simulation to the simulation domain only. Note: currently only implemented for the mode solver.
-
+    batch_id: str = None
+        Batch id to upload tasks to.
     Returns
     -------
     str
@@ -260,7 +262,14 @@ def upload(
     task_type = stub.get_type()
 
     task = SimulationTask.create(
-        task_type, task_name, folder_name, callback_url, simulation_type, parent_tasks, "Gz"
+        task_type,
+        task_name,
+        folder_name,
+        callback_url,
+        simulation_type,
+        parent_tasks,
+        "Gz",
+        batch_id,
     )
     if verbose:
         console = get_logging_console()
@@ -385,6 +394,8 @@ def start(
         worker group
     pay_type: Union[PayType, str] = PayType.AUTO
         Which method to pay the simulation
+    batch_id: str
+        batch id
     Note
     ----
     To monitor progress, can call :meth:`monitor` after starting simulation.