Isaac89
12/09/2021, 3:51 PM# requires paramiko -> pip install paramiko
# in conf/local/catalog.yml
test_fsspec:
type: pandas.CSVDataSet
filepath: "sftp:///path/to/remote_cluster/cool_data.csv"
credentials: cluster_credentials
load_args:
sep: ","
index_col: 0
save_args:
index: True
encoding: "utf-8"
# in conf/local/credentials.yml
cluster_credentials:
username: my_username
host: host_address
port: 22
password: password
# in jupyter lab
catalog.load("test_fsspec")