https://kedro.org/ logo
#advanced-need-help
Title
# advanced-need-help
b

boazmohar

03/11/2022, 4:28 PM
No it is fixed in regards to
data_location
so these is not even a need to a xml path... Here is part of the class
Copy code
python
class DaskAlpha3TifsDataset(AbstractDataSet):
    def __init__(self, filepath: str, params: Dict[str, Any] = None):
    
        # parse the path and protocol (e.g. file, http, s3, etc.)
        protocol, path = get_protocol_and_path(filepath)
        self._protocol = protocol
        self._filepath = PurePosixPath(path)
        self._fs = fsspec.filesystem(self._protocol)
        load_path = get_filepath_str(self._filepath, self._protocol)
        self.xml = get_meta_alpha3(load_path)
        self.xml['filters'] = params
        self.xml['ch_names'] = [params[i] for i in self.xml['filter_order']]
     

    def _load(self) -> da.Array:
        """Loads data from the image file.

        Returns:
            Data from the image file as a numpy array
        """
        # using get_filepath_str ensures that the protocol and path are appended correctly for different filesystems
        file_shapes = self.xml['file_shapes']
        base_dir = self.xml['base_dir'] 
        files = glob.glob(os.path.join(base_dir, 'Raw', '*.tiff'))
        logger.info(f'Found {len(files)} files in {base_dir} Raw folder')
        sizes = [(file_shapes[3], file_shapes[4])] * len(files)
        delay = [dask.delayed(load_tiff_stack)(fn) for fn in files]
        both = list(zip(delay, sizes))
        slices = [slice(i, i+file_shapes[2]) for i in range(0, len(both), file_shapes[2])]
        lazy_arrays = [da.from_delayed(x, shape=y, dtype=np.uint16) for x, y in both]
        lazy_arrays_conZ = [ da.stack(lazy_arrays[s], axis=0) for s in slices]
        lazy_arrays_conTileCh = da.stack(lazy_arrays_conZ, axis=0).reshape(file_shapes[[5,0,1,2,3,4]])
        return lazy_arrays_conTileCh