ThreadRunner

yaml
my_data:
   type: MyCustomImageClass
   data_location: path/to/files
   metadata_location: path/to/sepc.xml

metadata_location

data_location

python
class DaskAlpha3TifsDataset(AbstractDataSet):
    def __init__(self, filepath: str, params: Dict[str, Any] = None):
    
        # parse the path and protocol (e.g. file, http, s3, etc.)
        protocol, path = get_protocol_and_path(filepath)
        self._protocol = protocol
        self._filepath = PurePosixPath(path)
        self._fs = fsspec.filesystem(self._protocol)
        load_path = get_filepath_str(self._filepath, self._protocol)
        self.xml = get_meta_alpha3(load_path)
        self.xml['filters'] = params
        self.xml['ch_names'] = [params[i] for i in self.xml['filter_order']]
     

    def _load(self) -> da.Array:
        """Loads data from the image file.

        Returns:
            Data from the image file as a numpy array
        """
        # using get_filepath_str ensures that the protocol and path are appended correctly for different filesystems
        file_shapes = self.xml['file_shapes']
        base_dir = self.xml['base_dir'] 
        files = glob.glob(os.path.join(base_dir, 'Raw', '*.tiff'))
        logger.info(f'Found {len(files)} files in {base_dir} Raw folder')
        sizes = [(file_shapes[3], file_shapes[4])] * len(files)
        delay = [dask.delayed(load_tiff_stack)(fn) for fn in files]
        both = list(zip(delay, sizes))
        slices = [slice(i, i+file_shapes[2]) for i in range(0, len(both), file_shapes[2])]
        lazy_arrays = [da.from_delayed(x, shape=y, dtype=np.uint16) for x, y in both]
        lazy_arrays_conZ = [ da.stack(lazy_arrays[s], axis=0) for s in slices]
        lazy_arrays_conTileCh = da.stack(lazy_arrays_conZ, axis=0).reshape(file_shapes[[5,0,1,2,3,4]])
        return lazy_arrays_conTileCh

get_meta_alpha3

node

params

yml
gel3_round1:
  type: alpha3_expand.extra.datasets.dask_alpha3_tifs_dataset.DaskAlpha3TifsDataset
  filepath: /nrs/svoboda/moharb/ExM/Alpha3/20220310_YFP_ANM1_Gel3_R1_v2/Basal/
  params:
    2: YFP
    3: PSD95
    4: GluA1

yaml
my_data:
   type: DaskAlpha3TifsDataset
   filepath: path/to/files
   params: 
     xml: path/to/sepc.xml

DaskArray

self.xml

DaskAlpha3TifsDataset._load

python
 def _load(self) -> Dict[str, Any]:
        """Loads data from the image file.

        Returns:
            Data from the image file as a numpy array
        """
        # using get_filepath_str ensures that the protocol and path are appended correctly for different filesystems
        file_shapes = self.xml['file_shapes']
        base_dir = self.xml['base_dir'] 
        files = glob.glob(os.path.join(base_dir, 'Raw', '*.tiff'))
        logger.info(f'Found {len(files)} files in {base_dir} Raw folder')
        sizes = [(file_shapes[3], file_shapes[4])] * len(files)
        delay = [dask.delayed(load_tiff_stack)(fn) for fn in files]
        both = list(zip(delay, sizes))
        slices = [slice(i, i+file_shapes[2]) for i in range(0, len(both), file_shapes[2])]
        lazy_arrays = [da.from_delayed(x, shape=y, dtype=np.uint16) for x, y in both]
        lazy_arrays_conZ = [ da.stack(lazy_arrays[s], axis=0) for s in slices]
        lazy_arrays_conTileCh = da.stack(lazy_arrays_conZ, axis=0).reshape(file_shapes[[5,0,1,2,3,4]])
        return {'data': lazy_arrays_conTileCh, 'meta':self.xml}