[{"_aux":null,"name":null,"id":85,"reference":null,"started_at":"2024-12-04T16:10:55.060182+00:00","finished_at":"2024-12-04T16:12:04.418894+00:00","_status_code":0,"uid":"wW6GOHDq44lbrdKi08cr","created_at":"2024-12-04T16:10:55.060220+00:00","_is_consecutive":false,"_branch_code":1,"reference_type":null,"created_by":{"id":2,"uid":"vbppW10z","name":null,"handle":"mkeller7","created_at":"2024-08-30T13:39:52.768943+00:00","updated_at":"2024-08-30T13:39:52.768949+00:00"},"transform":{"id":18,"key":"vitessce.ipynb","uid":"BZhZQ6uIbkWv0000","_aux":null,"hash":"BNmgpsOXkmoLOMVuguzWaQ","type":"notebook","version":null,"is_latest":true,"reference":null,"created_at":"2024-12-04T16:10:54.987564+00:00","updated_at":"2024-12-04T16:12:05.865185+00:00","description":"Vitessce","source_code":"# %% [markdown]\n# [![](https://img.shields.io/badge/Source%20on%20GitHub-orange)](https://github.com/laminlabs/lamin-spatial/blob/main/docs/vitessce.ipynb) [![hub](https://img.shields.io/badge/View%20on%20LaminHub-mediumseagreen)](https://lamin.ai/laminlabs/lamindata/transform/hqtT4OTr5Tiq5zKv/URbaThYhljgXbZEzgj5x)\n\n# %% [markdown] nbsphinx=\"hidden\"\n# # transform.name\n\n# %% [markdown]\n# This tutorial has been adopted from the data preparation examples in [the Vitessce documention](https://vitessce.github.io/vitessce-python).\n#\n# We demonstrate how to use Vitessce to create interactive visualizations for data stored in LaminDB Artifacts, using multiple data formats.\n\n# %% editable=true slideshow={\"slide_type\": \"\"} tags=[\"hide-output\"]\n# # !pip install \"vitessce[all]>=3.5.0\"\n# # !pip install \"generate-tiff-offsets>=0.1.9\"\n# # !pip install \"lamindb[jupyter,aws,bionty]\"\n# !lamin connect laminlabs/lamin-dev # <-- replace with your instance\n\n# %% editable=true slideshow={\"slide_type\": \"\"} tags=[\"hide-output\"]\nfrom urllib.request import urlretrieve\nfrom pathlib import Path\nfrom anndata import read_h5ad\nimport vitessce as vit\nfrom vitessce import data_utils as vitdu\nfrom generate_tiff_offsets import get_offsets\nimport lamindb as ln\nimport zipfile\nimport json\n\n# [optional] track the current notebook or script\nln.track(\"BZhZQ6uIbkWv0000\")\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# ## Visualize an AnnData object (H5AD format)\n\n# %% [markdown]\n# Here, we download an example H5AD file (approximately 22 MB). This is a [dataset](https://www.covid19cellatlas.org/index.healthy.html#habib17) from the COVID-19 Cell Atlas.\n\n# %% editable=true slideshow={\"slide_type\": \"\"}\nh5ad_raw_filepath = \"./habib17.raw.h5ad\"\nh5ad_processed_filepath = \"./habib17.processed.h5ad\"\nif not Path(h5ad_raw_filepath).exists():\n urlretrieve(\n \"https://covid19.cog.sanger.ac.uk/habib17.processed.h5ad\", h5ad_raw_filepath\n )\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# This AnnData file was not saved recently and `read_h5ad` will warn us about this. By reading and subsequently re-writing the file using an up-to-date `anndata` package version, we ensure that the dataset is saved using the latest AnnData H5AD format.\n\n# %% editable=true slideshow={\"slide_type\": \"\"}\nadata = read_h5ad(h5ad_raw_filepath)\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# Optionally, we can modify the AnnData object, for example to subset it to the highly variable genes (which will affect the amount of data that is displayed in the heatmap visualization).\n\n# %% editable=true slideshow={\"slide_type\": \"\"}\nadata = adata[:, adata.var[\"highly_variable\"]].copy()\n\n# %% editable=true slideshow={\"slide_type\": \"\"}\nadata.write_h5ad(h5ad_processed_filepath)\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# ### Save the dataset\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# Save the `.h5ad` file as a LaminDB Artifact.\n\n# %% editable=true slideshow={\"slide_type\": \"\"}\nh5ad_artifact = ln.Artifact(\n h5ad_processed_filepath,\n key=\"vitessce_examples/habib17.h5ad\",\n description=\"Habib et al., 2017 Nature Methods, h5ad\",\n type=\"dataset\",\n)\nh5ad_artifact.save()\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# When using `.h5ad` files, we construct a [Reference Specification](https://fsspec.github.io/kerchunk/spec.html) which enables interoperability with the [Zarr](https://zarrita.dev/packages/storage.html#referencestore) interface.\n\n# %% editable=true slideshow={\"slide_type\": \"\"}\nref_spec = vitdu.generate_h5ad_ref_spec(h5ad_processed_filepath)\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# We next need to save the corresponding Reference Specification to a JSON file and upload it to LaminDB as an Artifact.\n\n# %% editable=true slideshow={\"slide_type\": \"\"}\nref_path = \"./habib17.processed.reference.json\"\nwith open(ref_path, \"w\") as file:\n json.dump(ref_spec, file)\nref_artifact = ln.Artifact(\n ref_path,\n key=\"vitessce_examples/habib17.reference.json\",\n description=\"Reference JSON for H5AD file, Habib et al., 2017 Nature Methods\"\n).save()\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# ### Save a VitessceConfig object\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# You can pass LaminDB Artifacts to the AnnDataWrapper class using the `adata_artifact` and `ref_artifact` [parameters](https://python-docs.vitessce.io/api_data.html#vitessce.wrappers.AnnDataWrapper).\n\n# %% editable=true slideshow={\"slide_type\": \"\"}\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=h5ad_artifact.description,\n)\ndataset = vc.add_dataset(name=\"Habib 2017\").add_object(\n vit.AnnDataWrapper(\n adata_artifact=h5ad_artifact,\n ref_artifact=ref_artifact,\n obs_feature_matrix_path=\"X\",\n obs_embedding_paths=[\"obsm/X_umap\"],\n obs_embedding_names=[\"UMAP\"],\n obs_set_paths=[\"obs/CellType\"],\n obs_set_names=[\"Cell Type\"],\n )\n)\nobs_sets = vc.add_view(vit.ViewType.OBS_SETS, dataset=dataset)\nobs_sets_sizes = vc.add_view(vit.ViewType.OBS_SET_SIZES, dataset=dataset)\nscatterplot = vc.add_view(vit.ViewType.SCATTERPLOT, dataset=dataset, mapping=\"UMAP\")\nheatmap = vc.add_view(vit.ViewType.HEATMAP, dataset=dataset)\ngenes = vc.add_view(vit.ViewType.FEATURE_LIST, dataset=dataset)\nvc.link_views([scatterplot, heatmap], [\"featureValueColormapRange\"], [[0.0, 0.1]])\nvc.layout(((scatterplot | obs_sets) / heatmap) | (obs_sets_sizes / genes));\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# Save the `VitessceConfig` object.\n\n# %% editable=true slideshow={\"slide_type\": \"\"} tags=[\"hide-output\"]\nh5ad_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View Habib17 (h5ad) in Vitessce\",\n)\n\n# %% editable=true slideshow={\"slide_type\": \"\"}\nh5ad_vc_artifact.view_lineage()\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# If your `VitessceConfig` object references data from multiple artifacts, the Vitessce button will appear next to a `Collection` that groups these artifacts (on the [Collections](https://lamin.ai/vitessce/examples/collections) tab of the Artifacts page).\n# Note that when using an `.h5ad`-based Artifact, the presence of the corresponding `.reference.json` file will result in the creation of a Collection.\n#\n# :::\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# ## Visualize an AnnData object (Zarr format)\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# AnnData objects can be saved on-disk to not only `.h5ad` files, but also to [Zarr stores](https://zarr.readthedocs.io/en/stable/tutorial.html#storage-alternatives) using AnnData's [write_zarr](https://anndata.readthedocs.io/en/latest/generated/anndata.AnnData.write_zarr.html) method.\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# ### Save the dataset\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# Convert the dataset to `.zarr` format. Often, Zarr data is stored on-disk using a [DirectoryStore](https://zarr.readthedocs.io/en/stable/api/storage.html#zarr.storage.DirectoryStore). By convention, these directories are named using a `.zarr` suffix.\n\n# %% editable=true slideshow={\"slide_type\": \"\"}\nzarr_processed_filepath = \"./habib17.anndata.zarr\"\n\n# %% editable=true slideshow={\"slide_type\": \"\"}\nadata = read_h5ad(h5ad_raw_filepath)\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# Just like in the above section, we may want to modify the AnnData object prior to saving it as a LaminDB Artifact that we will use for visualization.\n\n# %% editable=true slideshow={\"slide_type\": \"\"}\nadata = adata[:, adata.var[\"highly_variable\"]].copy()\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# Note the usage of `.write_zarr` below, as opposed to `.write_h5ad`.\n\n# %% editable=true slideshow={\"slide_type\": \"\"}\nadata.write_zarr(zarr_processed_filepath)\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# Save the `.zarr` directory as a LaminDB Artifact.\n\n# %% editable=true slideshow={\"slide_type\": \"\"}\nadata_zarr_artifact = ln.Artifact(\n zarr_processed_filepath,\n key=\"vitessce_examples/habib17.adata.zarr\",\n description=\"Habib et al., 2017 Nature Methods, zarr\",\n type=\"dataset\",\n)\nadata_zarr_artifact.save()\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# ### Save a VitessceConfig object\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n# Here, we configure the visualization the same way as above in the `.h5ad`-based example, with the exception of the `ref_artifact` parameter, as `.zarr`-based AnnData objects do not require a Reference Specification for Zarr interoperability.\n\n# %% editable=true slideshow={\"slide_type\": \"\"} tags=[\"hide-output\"]\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=adata_zarr_artifact.description,\n)\ndataset = vc.add_dataset(name=\"Habib 2017\").add_object(\n vit.AnnDataWrapper(\n adata_artifact=adata_zarr_artifact,\n obs_feature_matrix_path=\"X\",\n obs_embedding_paths=[\"obsm/X_umap\"],\n obs_embedding_names=[\"UMAP\"],\n obs_set_paths=[\"obs/CellType\"],\n obs_set_names=[\"Cell Type\"],\n )\n)\nobs_sets = vc.add_view(vit.Component.OBS_SETS, dataset=dataset)\nobs_sets_sizes = vc.add_view(vit.Component.OBS_SET_SIZES, dataset=dataset)\nscatterplot = vc.add_view(vit.Component.SCATTERPLOT, dataset=dataset, mapping=\"UMAP\")\nheatmap = vc.add_view(vit.Component.HEATMAP, dataset=dataset)\ngenes = vc.add_view(vit.Component.FEATURE_LIST, dataset=dataset)\nvc.link_views([scatterplot, heatmap], [\"featureValueColormapRange\"], [[0.0, 0.1]])\nvc.layout(((scatterplot | obs_sets) / heatmap) | (obs_sets_sizes / genes));\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# Save the `VitessceConfig` object.\n\n# %% editable=true slideshow={\"slide_type\": \"\"} tags=[\"hide-output\"]\nadata_zarr_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View Habib17 in Vitessce\",\n)\n\n# %% editable=true slideshow={\"slide_type\": \"\"}\nadata_zarr_vc_artifact.view_lineage()\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# :::\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# ## Visualize a SpatialData object (Zarr format)\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# Download and unzip the example SpatialData dataset (approximately 1.47 GB unzipped). Once unzipped, the data will be available via a local Zarr DirectoryStore.\n\n# %% editable=true slideshow={\"slide_type\": \"\"}\nsdata_zip_filepath = \"./visium.spatialdata.zarr.zip\"\nsdata_zarr_filepath = \"./visium.spatialdata.zarr\"\nif not Path(sdata_zarr_filepath).exists():\n if not Path(sdata_zip_filepath).exists():\n urlretrieve(\n \"https://s3.embl.de/spatialdata/spatialdata-sandbox/visium_associated_xenium_io.zip\",\n sdata_zip_filepath,\n )\n with zipfile.ZipFile(sdata_zip_filepath, \"r\") as zip_ref:\n zip_ref.extractall(\".\")\n Path(\"data.zarr\").rename(sdata_zarr_filepath)\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# ### Save the dataset\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# Save the `.zarr` DirectoryStore as a LaminDB Artifact.\n\n# %% editable=true slideshow={\"slide_type\": \"\"}\nsdata_zarr_artifact = ln.Artifact(\n sdata_zarr_filepath,\n key=\"vitessce_examples/visium.sdata.zarr\",\n description=\"Visium SpatialData Example\",\n type=\"dataset\",\n)\nsdata_zarr_artifact.save()\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# ### Save a VitessceConfig object\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n# Here, we use the [SpatialDataWrapper](https://python-docs.vitessce.io/api_data.html#vitessce.wrappers.SpatialDataWrapper) class to specify which parts of the SpatialData object will be loaded for visualization.\n\n# %% editable=true slideshow={\"slide_type\": \"\"}\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=sdata_zarr_artifact.description,\n)\n# Add data to the configuration:\ndataset_uid = \"sdata_visium\"\ndataset = vc.add_dataset(name='Breast Cancer Visium', uid=dataset_uid).add_object(\n vit.SpatialDataWrapper(\n sdata_artifact=sdata_zarr_artifact,\n # The following paths are relative to the root of the SpatialData zarr store on-disk.\n image_path=\"images/CytAssist_FFPE_Human_Breast_Cancer_full_image\",\n table_path=\"tables/table\",\n obs_feature_matrix_path=\"tables/table/X\",\n obs_spots_path=\"shapes/CytAssist_FFPE_Human_Breast_Cancer\",\n region=\"CytAssist_FFPE_Human_Breast_Cancer\",\n coordinate_system=\"global\",\n coordination_values={\n # The following tells Vitessce to consider each observation as a \"spot\"\n \"obsType\": \"spot\",\n },\n )\n)\n\n# Add views (visualizations) to the configuration:\nspatial = vc.add_view(\"spatialBeta\", dataset=dataset)\nfeature_list = vc.add_view(\"featureList\", dataset=dataset)\nlayer_controller = vc.add_view(\"layerControllerBeta\", dataset=dataset)\n\n# Initialize visual properties for multiple linked views:\nvc.link_views_by_dict([spatial, layer_controller], {\n \"imageLayer\": vit.CoordinationLevel([{\n \"photometricInterpretation\": \"RGB\",\n }]),\n}, scope_prefix=vit.get_initial_coordination_scope_prefix(dataset_uid, \"image\"))\nvc.link_views([spatial, layer_controller, feature_list, obs_sets], [\"obsType\"], [\"spot\"])\n\n# Layout the views\nvc.layout(spatial | (feature_list / layer_controller));\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# Save the `VitessceConfig` object.\n\n# %% editable=true slideshow={\"slide_type\": \"\"} tags=[\"hide-output\"]\nsdata_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View Visium SpatialData Example in Vitessce\",\n)\n\n# %% editable=true slideshow={\"slide_type\": \"\"}\nsdata_vc_artifact.view_lineage()\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# :::\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# ## Visualize an image (OME-TIFF format)\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# Vitesse can visualize data from multiple bioimaging file formats, including [OME-TIFF](https://ome-model.readthedocs.io/en/stable/ome-tiff/).\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# Here, we download an example OME-TIFF file (approximately 1.02 GB).\n\n# %% editable=true slideshow={\"slide_type\": \"\"}\nome_tiff_filepath = \"./VAN0006-LK-2-85-PAS_registered.ome.tif\"\nif not Path(ome_tiff_filepath).exists():\n urlretrieve(\n \"https://assets.hubmapconsortium.org/f4188a148e4c759092d19369d310883b/ometiff-pyramids/processedMicroscopy/VAN0006-LK-2-85-PAS_images/VAN0006-LK-2-85-PAS_registered.ome.tif\",\n ome_tiff_filepath,\n )\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# ### Save the dataset\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# Save the `.ome.tif` file as a LaminDB Artifact.\n\n# %% editable=true slideshow={\"slide_type\": \"\"}\nome_tiff_artifact = ln.Artifact(\n ome_tiff_filepath,\n key=\"vitessce_examples/VAN0006-LK-2-85-PAS_registered.ome.tif\",\n description=\"PAS OME-TIFF file, Neumann et al., 2020\",\n type=\"dataset\",\n)\nome_tiff_artifact.save()\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# When using OME-TIFF files, we can use [generate-tiff-offsets](https://github.com/hms-dbmi/generate-tiff-offsets) to create an index for the bytes within the OME-TIFF file. We store these to a companion `.offsets.json` file which makes loading subsets of the image more efficient.\n\n# %% editable=true slideshow={\"slide_type\": \"\"}\noffsets = get_offsets(ome_tiff_filepath)\n\n# %% editable=true slideshow={\"slide_type\": \"\"}\noffsets_path = \"./VAN0006-LK-2-85-PAS_registered.offsets.json\"\nwith open(offsets_path, \"w\") as file:\n json.dump(offsets, file)\noffsets_artifact = ln.Artifact(\n offsets_path,\n key=\"vitessce_examples/VAN0006-LK-2-85-PAS_registered.offsets.json\",\n description=\"PAS offsets file, Neumann et al., 2020\"\n).save()\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# ### Save a VitessceConfig object\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n# Here, we use the [ImageOmeTiffWrapper](https://python-docs.vitessce.io/api_data.html#vitessce.wrappers.ImageOmeTiffWrapper) class to specify which pair of OME-TIFF file and offsets JSON file to load.\n\n# %% editable=true slideshow={\"slide_type\": \"\"}\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=ome_tiff_artifact.description\n)\ndataset = vc.add_dataset(\"Image\").add_object(\n vit.ImageOmeTiffWrapper(\n img_artifact=ome_tiff_artifact,\n offsets_artifact=offsets_artifact,\n )\n)\nspatial = vc.add_view(\"spatialBeta\", dataset=dataset)\nlayer_controller = vc.add_view(\"layerControllerBeta\", dataset=dataset)\nvc.layout(spatial | layer_controller);\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# Save the `VitessceConfig` object.\n\n# %% editable=true slideshow={\"slide_type\": \"\"} tags=[\"hide-output\"]\nome_tiff_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View PAS OME-TIFF, Neumann et al., 2020 in Vitessce\",\n)\n\n# %% editable=true slideshow={\"slide_type\": \"\"}\nome_tiff_vc_artifact.view_lineage()\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# If your `VitessceConfig` object references data from multiple artifacts, the Vitessce button will appear next to a `Collection` that groups these artifacts (on the [Collections](https://lamin.ai/vitessce/examples/collections) tab of the Artifacts page).\n# In the case of OME-TIFF, the presence of the corresponding offsets JSON file will result in the creation of a Collection.\n#\n# :::\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# ## Visualize an image (OME-Zarr format)\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# Retrieve an OME-Zarr (also known as OME-NGFF) formatted image. We could download an unzip such an image, but in downloading the SpatialData object above, we already have access to a valid OME-Zarr image locally.\n\n# %% editable=true slideshow={\"slide_type\": \"\"}\nome_zarr_filepath = Path(sdata_zarr_filepath) / \"images\" / \"CytAssist_FFPE_Human_Breast_Cancer_full_image\"\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# ### Save the dataset\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# Save the `.ome.zarr` DirectoryStore as a LaminDB Artifact.\n\n# %% editable=true slideshow={\"slide_type\": \"\"}\nome_zarr_artifact = ln.Artifact(\n ome_zarr_filepath,\n key=\"vitessce_examples/visium.ome.zarr\",\n description=\"Visium OME-Zarr Example\",\n type=\"dataset\",\n)\nome_zarr_artifact.save()\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# ### Save a VitessceConfig object\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n# Here, we use the [ImageOmeZarrWrapper](https://python-docs.vitessce.io/api_data.html#vitessce.wrappers.ImageOmeZarrWrapper) class to specify an OME-Zarr file to load for visualization.\n\n# %% editable=true slideshow={\"slide_type\": \"\"}\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=ome_zarr_artifact.description\n)\ndataset_uid = \"ome_zarr_image\"\ndataset = vc.add_dataset(\"Image\", uid=dataset_uid).add_object(\n vit.ImageOmeZarrWrapper(\n img_artifact=ome_zarr_artifact,\n )\n)\nspatial = vc.add_view(\"spatialBeta\", dataset=dataset)\nlayer_controller = vc.add_view(\"layerControllerBeta\", dataset=dataset)\nvc.link_views_by_dict([spatial, layer_controller], {\n \"imageLayer\": vit.CoordinationLevel([{\n \"photometricInterpretation\": \"RGB\",\n }]),\n}, scope_prefix=vit.get_initial_coordination_scope_prefix(dataset_uid, \"image\"))\nvc.layout(spatial | layer_controller);\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# Save the `VitessceConfig` object.\n\n# %% editable=true slideshow={\"slide_type\": \"\"} tags=[\"hide-output\"]\nome_zarr_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View Visium OME-Zarr Example in Vitessce\",\n)\n\n# %% editable=true slideshow={\"slide_type\": \"\"}\nome_zarr_vc_artifact.view_lineage()\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# :::\n\n# %% editable=true slideshow={\"slide_type\": \"\"}\n# [optional] finish run context and auto-save the notebook\nln.finish()\n\n# %% [markdown] editable=true slideshow={\"slide_type\": \"\"}\n# :::{dropdown} Upload speed\n#\n# Here is [a note](https://lamin.ai/laminlabs/lamindata/transform/WDjio16cQsdW5zKv) on folder upload speed and why `lamindb` does not use the `.export(to=\"s3\")` functionality of Vitessce.\n#\n# :::\n\n# %% editable=true slideshow={\"slide_type\": \"\"} tags=[\"hide-cell\"]\n# clean up artifacts in CI run\nh5ad_artifact.delete(permanent=True)\nref_artifact.delete(permanent=True)\n\nadata_zarr_artifact.delete(permanent=True)\nsdata_zarr_artifact.delete(permanent=True)\n\nome_tiff_artifact.delete(permanent=True)\noffsets_artifact.delete(permanent=True)\n\nome_zarr_artifact.delete(permanent=True)\n\nh5ad_vc_artifact.delete(permanent=True)\nadata_zarr_vc_artifact.delete(permanent=True)\nsdata_vc_artifact.delete(permanent=True)\nome_tiff_vc_artifact.delete(permanent=True)\nome_zarr_vc_artifact.delete(permanent=True)\n","_branch_code":1,"reference_type":null},"report":{"id":89,"key":null,"uid":"jmAYXpcb2kBkR8030000","_aux":null,"hash":"nDAbwD3D3jprCpsFQPPvMQ","kind":null,"size":464000,"otype":null,"suffix":".html","n_files":null,"version":null,"is_latest":true,"_hash_type":"md5","created_at":"2024-12-04T16:12:04.530256+00:00","updated_at":"2024-12-04T16:12:04.530296+00:00","description":"Report of run wW6GOHDq44lbrdKi08cr","_branch_code":0,"n_observations":null,"_key_is_virtual":true,"_overwrite_versions":false,"storage":{"id":1,"uid":"TwPtYAyQwlyO","_aux":null,"root":"s3://lamin-us-east-1/TwPtYAyQwlyO","type":"s3","region":"us-east-1","created_at":"2024-08-30T13:34:02.166149+00:00","updated_at":"2024-08-30T13:34:02.166182+00:00","description":null,"_branch_code":1,"instance_uid":"3ytNpiOQry0S"}},"environment":{"id":81,"key":null,"uid":"9vGtQN6bRSFPebY10000","_aux":null,"hash":"VMfe0zYYHef6uFbE4_IWTw","kind":null,"size":5037,"otype":null,"suffix":".txt","n_files":null,"version":null,"is_latest":true,"_hash_type":"md5","created_at":"2024-12-04T15:43:13.871194+00:00","updated_at":"2024-12-04T15:43:13.871224+00:00","description":"requirements.txt","_branch_code":0,"n_observations":null,"_key_is_virtual":true,"_overwrite_versions":false,"storage":{"id":1,"uid":"TwPtYAyQwlyO","_aux":null,"root":"s3://lamin-us-east-1/TwPtYAyQwlyO","type":"s3","region":"us-east-1","created_at":"2024-08-30T13:34:02.166149+00:00","updated_at":"2024-08-30T13:34:02.166182+00:00","description":null,"_branch_code":1,"instance_uid":"3ytNpiOQry0S"}},"_param_values":[],"input_collections":[],"output_collections":[],"input_artifacts":[],"output_artifacts":[],"_recreated_artifacts":[{"id":87,"key":"vitessce_examples/habib17.adata.zarr","uid":"kWwG45Hr26KZUW3g0004","_aux":null,"hash":"Ap_EL5yPdfyD07Xjpe1SlA","kind":"dataset","size":10601062,"otype":"AnnData","suffix":".anndata.zarr","n_files":208,"version":null,"is_latest":true,"_hash_type":"md5-d","created_at":"2024-12-04T16:01:11.125029+00:00","updated_at":"2024-12-04T16:11:20.334727+00:00","description":"Habib et al., 2017 Nature Methods, zarr","_branch_code":1,"n_observations":null,"_key_is_virtual":true,"_overwrite_versions":true,"run":{"id":79,"uid":"xt2ErUgo7KKqUOOAnVad","_aux":null,"name":null,"reference":null,"created_at":"2024-12-04T16:00:56.039291+00:00","started_at":"2024-12-04T16:00:56.039234+00:00","finished_at":"2024-12-04T16:01:54.937113+00:00","_branch_code":1,"_status_code":0,"reference_type":null,"_is_consecutive":false,"transform":{"id":17,"key":"vitessce.ipynb","uid":"XmcumVqvKZd30002","_aux":null,"hash":"F2yYH-hVHL0gmje1L98nhQ","type":"notebook","version":null,"is_latest":true,"reference":null,"created_at":"2024-12-04T16:00:55.942775+00:00","updated_at":"2024-12-04T16:01:55.890913+00:00","description":"Vitessce","source_code":"# %% [markdown]\n# [![](https://img.shields.io/badge/Source%20on%20GitHub-orange)](https://github.com/laminlabs/lamin-spatial/blob/main/docs/vitessce.ipynb) [![hub](https://img.shields.io/badge/View%20on%20LaminHub-mediumseagreen)](https://lamin.ai/laminlabs/lamindata/transform/hqtT4OTr5Tiq5zKv/URbaThYhljgXbZEzgj5x)\n\n# %% [markdown] nbsphinx=\"hidden\"\n# # transform.name\n\n# %% [markdown]\n# This tutorial has been adopted from the data preparation examples in [the Vitessce documention](https://vitessce.github.io/vitessce-python).\n#\n# We demonstrate how to use Vitessce to create interactive visualizations for data stored in LaminDB Artifacts, using multiple data formats.\n\n# %% tags=[\"hide-output\"]\n# # !pip install \"vitessce[all]\">=3.5.0\n# # !pip install \"generate-tiff-offsets>=0.1.9\"\n# # !pip install 'lamindb[jupyter,aws,bionty]'\n# !lamin connect laminlabs/lamin-dev # <-- replace with your instance\n\n# %% tags=[\"hide-output\"]\nfrom urllib.request import urlretrieve\nfrom pathlib import Path\nfrom anndata import read_h5ad\nimport vitessce as vit\nfrom vitessce import data_utils as vitdu\nfrom generate_tiff_offsets import get_offsets\nimport numpy as np\nimport lamindb as ln\nimport zipfile\nimport json\n\n# [optional] track the current notebook or script\nln.track(\"XmcumVqvKZd30002\")\n\n# %% [markdown]\n# ## Visualize an AnnData object (H5AD format)\n\n# %% [markdown]\n# Here, we download an example H5AD file (approximately 22 MB). This is a [dataset](https://www.covid19cellatlas.org/index.healthy.html#habib17) from the COVID-19 Cell Atlas.\n\n# %%\nh5ad_raw_filepath = \"./habib17.raw.h5ad\"\nh5ad_processed_filepath = \"./habib17.processed.h5ad\"\nif not Path(h5ad_raw_filepath).exists():\n urlretrieve(\n \"https://covid19.cog.sanger.ac.uk/habib17.processed.h5ad\", h5ad_raw_filepath\n )\n\n# %% [markdown]\n# This AnnData file was not saved recently and `read_h5ad` will warn us about this. By reading and subsequently re-writing the file using an up-to-date `anndata` package version, we ensure that the dataset is saved using the latest AnnData H5AD format.\n\n# %%\nadata = read_h5ad(h5ad_raw_filepath)\n\n# %% [markdown]\n# Optionally, we can modify the AnnData object, for example to subset it to the highly variable genes (which will affect the amount of data that is displayed in the heatmap visualization).\n\n# %%\nadata = adata[:, adata.var[\"highly_variable\"]].copy()\n\n# %%\nadata.write_h5ad(h5ad_processed_filepath)\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Save the `.h5ad` file as a LaminDB Artifact.\n\n# %%\nh5ad_artifact = ln.Artifact(\n h5ad_processed_filepath,\n key=\"vitessce_examples/habib17.h5ad\",\n description=\"Habib et al., 2017 Nature Methods, h5ad\",\n type=\"dataset\",\n)\nh5ad_artifact.save()\n\n# %% [markdown]\n# When using `.h5ad` files, we construct a [Reference Specification](https://fsspec.github.io/kerchunk/spec.html) which enables interoperability with the [Zarr](https://zarrita.dev/packages/storage.html#referencestore) interface.\n\n# %%\nref_spec = vitdu.generate_h5ad_ref_spec(h5ad_processed_filepath)\n\n# %% [markdown]\n# We next need to save the corresponding Reference Specification to a JSON file and upload it to LaminDB as an Artifact.\n\n# %%\nref_path = \"./habib17.processed.reference.json\"\nwith open(ref_path, \"w\") as file:\n json.dump(ref_spec, file)\nref_artifact = ln.Artifact(\n ref_path,\n key=\"vitessce_examples/habib17.reference.json\",\n description=\"Reference JSON for H5AD file, Habib et al., 2017 Nature Methods\"\n).save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n\n# %% [markdown]\n# You can pass LaminDB Artifacts to the AnnDataWrapper class using the `adata_artifact` and `ref_artifact` [parameters](https://python-docs.vitessce.io/api_data.html#vitessce.wrappers.AnnDataWrapper).\n\n# %%\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=h5ad_artifact.description,\n)\ndataset = vc.add_dataset(name=\"Habib 2017\").add_object(\n vit.AnnDataWrapper(\n adata_artifact=h5ad_artifact,\n ref_artifact=ref_artifact,\n obs_feature_matrix_path=\"X\",\n obs_embedding_paths=[\"obsm/X_umap\"],\n obs_embedding_names=[\"UMAP\"],\n obs_set_paths=[\"obs/CellType\"],\n obs_set_names=[\"Cell Type\"],\n )\n)\nobs_sets = vc.add_view(vit.ViewType.OBS_SETS, dataset=dataset)\nobs_sets_sizes = vc.add_view(vit.ViewType.OBS_SET_SIZES, dataset=dataset)\nscatterplot = vc.add_view(vit.ViewType.SCATTERPLOT, dataset=dataset, mapping=\"UMAP\")\nheatmap = vc.add_view(vit.ViewType.HEATMAP, dataset=dataset)\ngenes = vc.add_view(vit.ViewType.FEATURE_LIST, dataset=dataset)\nvc.link_views([scatterplot, heatmap], [\"featureValueColormapRange\"], [[0.0, 0.1]])\nvc.layout(((scatterplot | obs_sets) / heatmap) | (obs_sets_sizes / genes));\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nh5ad_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View Habib17 (h5ad) in Vitessce\",\n)\n\n# %%\nh5ad_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# If your `VitessceConfig` object references data from multiple artifacts, the Vitessce button will appear next to a `Collection` that groups these artifacts (on the [Collections](https://lamin.ai/vitessce/examples/collections) tab of the Artifacts page).\n# Note that when using an `.h5ad`-based Artifact, the presence of the corresponding `.reference.json` file will result in the creation of a Collection.\n#\n# :::\n\n# %% [markdown]\n# ## Visualize an AnnData object (Zarr format)\n\n# %% [markdown]\n# AnnData objects can be saved on-disk to not only `.h5ad` files, but also to [Zarr stores](https://zarr.readthedocs.io/en/stable/tutorial.html#storage-alternatives) using AnnData's [write_zarr](https://anndata.readthedocs.io/en/latest/generated/anndata.AnnData.write_zarr.html) method.\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Convert the dataset to `.zarr` format. Often, Zarr data is stored on-disk using a [DirectoryStore](https://zarr.readthedocs.io/en/stable/api/storage.html#zarr.storage.DirectoryStore). By convention, these directories are named using a `.zarr` suffix.\n\n# %%\nzarr_processed_filepath = \"./habib17.anndata.zarr\"\n\n# %%\nadata = read_h5ad(h5ad_raw_filepath)\n\n# %% [markdown]\n# Just like in the above section, we may want to modify the AnnData object prior to saving it as a LaminDB Artifact that we will use for visualization.\n\n# %%\nadata = adata[:, adata.var[\"highly_variable\"]].copy()\n\n# %% [markdown]\n# Note the usage of `.write_zarr` below, as opposed to `.write_h5ad`.\n\n# %%\nadata.write_zarr(zarr_processed_filepath)\n\n# %% [markdown]\n# Save the `.zarr` directory as a LaminDB Artifact.\n\n# %%\nadata_zarr_artifact = ln.Artifact(\n zarr_processed_filepath,\n key=\"vitessce_examples/habib17.adata.zarr\",\n description=\"Habib et al., 2017 Nature Methods, zarr\",\n type=\"dataset\",\n)\nadata_zarr_artifact.save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n# Here, we configure the visualization the same way as above in the `.h5ad`-based example, with the exception of the `ref_artifact` parameter, as `.zarr`-based AnnData objects do not require a Reference Specification for Zarr interoperability.\n\n# %% tags=[\"hide-output\"]\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=adata_zarr_artifact.description,\n)\ndataset = vc.add_dataset(name=\"Habib 2017\").add_object(\n vit.AnnDataWrapper(\n adata_artifact=adata_zarr_artifact,\n obs_feature_matrix_path=\"X\",\n obs_embedding_paths=[\"obsm/X_umap\"],\n obs_embedding_names=[\"UMAP\"],\n obs_set_paths=[\"obs/CellType\"],\n obs_set_names=[\"Cell Type\"],\n )\n)\nobs_sets = vc.add_view(vit.Component.OBS_SETS, dataset=dataset)\nobs_sets_sizes = vc.add_view(vit.Component.OBS_SET_SIZES, dataset=dataset)\nscatterplot = vc.add_view(vit.Component.SCATTERPLOT, dataset=dataset, mapping=\"UMAP\")\nheatmap = vc.add_view(vit.Component.HEATMAP, dataset=dataset)\ngenes = vc.add_view(vit.Component.FEATURE_LIST, dataset=dataset)\nvc.link_views([scatterplot, heatmap], [\"featureValueColormapRange\"], [[0.0, 0.1]])\nvc.layout(((scatterplot | obs_sets) / heatmap) | (obs_sets_sizes / genes));\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nadata_zarr_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View Habib17 in Vitessce\",\n)\n\n# %%\nadata_zarr_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# :::\n\n# %% [markdown]\n# ## Visualize a SpatialData object (Zarr format)\n\n# %% [markdown]\n# Download and unzip the example SpatialData dataset (approximately 1.47 GB unzipped). Once unzipped, the data will be available via a local Zarr DirectoryStore.\n\n# %%\nsdata_zip_filepath = \"./visium.spatialdata.zarr.zip\"\nsdata_zarr_filepath = \"./visium.spatialdata.zarr\"\nif not Path(sdata_zarr_filepath).exists():\n if not Path(sdata_zip_filepath).exists():\n urlretrieve(\n \"https://s3.embl.de/spatialdata/spatialdata-sandbox/visium_associated_xenium_io.zip\",\n sdata_zip_filepath,\n )\n with zipfile.ZipFile(sdata_zip_filepath, \"r\") as zip_ref:\n zip_ref.extractall(\".\")\n Path(\"data.zarr\").rename(sdata_zarr_filepath)\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Save the `.zarr` DirectoryStore as a LaminDB Artifact.\n\n# %%\nsdata_zarr_artifact = ln.Artifact(\n sdata_zarr_filepath,\n key=\"vitessce_examples/visium.sdata.zarr\",\n description=\"Visium SpatialData Example\",\n type=\"dataset\",\n)\nsdata_zarr_artifact.save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n# Here, we use the [SpatialDataWrapper](https://python-docs.vitessce.io/api_data.html#vitessce.wrappers.SpatialDataWrapper) class to specify which parts of the SpatialData object will be loaded for visualization.\n\n# %%\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=sdata_zarr_artifact.description,\n)\n# Add data to the configuration:\ndataset_uid = \"sdata_visium\"\ndataset = vc.add_dataset(name='Breast Cancer Visium', uid=dataset_uid).add_object(\n vit.SpatialDataWrapper(\n sdata_artifact=sdata_zarr_artifact,\n # The following paths are relative to the root of the SpatialData zarr store on-disk.\n image_path=\"images/CytAssist_FFPE_Human_Breast_Cancer_full_image\",\n table_path=\"tables/table\",\n obs_feature_matrix_path=\"tables/table/X\",\n obs_spots_path=\"shapes/CytAssist_FFPE_Human_Breast_Cancer\",\n region=\"CytAssist_FFPE_Human_Breast_Cancer\",\n coordinate_system=\"global\",\n coordination_values={\n # The following tells Vitessce to consider each observation as a \"spot\"\n \"obsType\": \"spot\",\n },\n )\n)\n\n# Add views (visualizations) to the configuration:\nspatial = vc.add_view(\"spatialBeta\", dataset=dataset)\nfeature_list = vc.add_view(\"featureList\", dataset=dataset)\nlayer_controller = vc.add_view(\"layerControllerBeta\", dataset=dataset)\n\n# Initialize visual properties for multiple linked views:\nvc.link_views_by_dict([spatial, layer_controller], {\n \"imageLayer\": vit.CoordinationLevel([{\n \"photometricInterpretation\": \"RGB\",\n }]),\n}, scope_prefix=vit.get_initial_coordination_scope_prefix(dataset_uid, \"image\"))\nvc.link_views([spatial, layer_controller, feature_list, obs_sets], [\"obsType\"], [\"spot\"])\n\n# Layout the views\nvc.layout(spatial | (feature_list / layer_controller));\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nsdata_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View Visium SpatialData Example in Vitessce\",\n)\n\n# %%\nsdata_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# :::\n\n# %% [markdown]\n# ## Visualize an image (OME-TIFF format)\n\n# %% [markdown]\n# Vitesse can visualize data from multiple bioimaging file formats, including [OME-TIFF](https://ome-model.readthedocs.io/en/stable/ome-tiff/).\n\n# %% [markdown]\n# Here, we download an example OME-TIFF file (approximately 1.02 GB).\n\n# %%\nome_tiff_filepath = \"./VAN0006-LK-2-85-PAS_registered.ome.tif\"\nif not Path(ome_tiff_filepath).exists():\n urlretrieve(\n \"https://assets.hubmapconsortium.org/f4188a148e4c759092d19369d310883b/ometiff-pyramids/processedMicroscopy/VAN0006-LK-2-85-PAS_images/VAN0006-LK-2-85-PAS_registered.ome.tif\",\n ome_tiff_filepath,\n )\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Save the `.ome.tif` file as a LaminDB Artifact.\n\n# %%\nome_tiff_artifact = ln.Artifact(\n ome_tiff_filepath,\n key=\"vitessce_examples/VAN0006-LK-2-85-PAS_registered.ome.tif\",\n description=\"PAS OME-TIFF file, Neumann et al., 2020\",\n type=\"dataset\",\n)\nome_tiff_artifact.save()\n\n# %% [markdown]\n# When using OME-TIFF files, we can use [generate-tiff-offsets](https://github.com/hms-dbmi/generate-tiff-offsets) to create an index for the bytes within the OME-TIFF file. We store these to a companion `.offsets.json` file which makes loading subsets of the image more efficient.\n\n# %%\noffsets = get_offsets(ome_tiff_filepath)\n\n# %%\noffsets_path = \"./VAN0006-LK-2-85-PAS_registered.offsets.json\"\nwith open(offsets_path, \"w\") as file:\n json.dump(offsets, file)\noffsets_artifact = ln.Artifact(\n offsets_path,\n key=\"vitessce_examples/VAN0006-LK-2-85-PAS_registered.offsets.json\",\n description=\"PAS offsets file, Neumann et al., 2020\"\n).save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n# Here, we use the [ImageOmeTiffWrapper](https://python-docs.vitessce.io/api_data.html#vitessce.wrappers.ImageOmeTiffWrapper) class to specify which pair of OME-TIFF file and offsets JSON file to load.\n\n# %%\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=ome_tiff_artifact.description\n)\ndataset = vc.add_dataset(\"Image\").add_object(\n vit.ImageOmeTiffWrapper(\n img_artifact=ome_tiff_artifact,\n offsets_artifact=offsets_artifact,\n )\n)\nspatial = vc.add_view(\"spatialBeta\", dataset=dataset)\nlayer_controller = vc.add_view(\"layerControllerBeta\", dataset=dataset)\nvc.layout(spatial | layer_controller);\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nome_tiff_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View PAS OME-TIFF, Neumann et al., 2020 in Vitessce\",\n)\n\n# %%\nome_tiff_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# If your `VitessceConfig` object references data from multiple artifacts, the Vitessce button will appear next to a `Collection` that groups these artifacts (on the [Collections](https://lamin.ai/vitessce/examples/collections) tab of the Artifacts page).\n# In the case of OME-TIFF, the presence of the corresponding offsets JSON file will result in the creation of a Collection.\n#\n# :::\n\n# %% [markdown]\n# ## Visualize an image (OME-Zarr format)\n\n# %% [markdown]\n# Retrieve an OME-Zarr (also known as OME-NGFF) formatted image. We could download an unzip such an image, but in downloading the SpatialData object above, we already have access to a valid OME-Zarr image locally.\n\n# %%\nome_zarr_filepath = Path(sdata_zarr_filepath) / \"images\" / \"CytAssist_FFPE_Human_Breast_Cancer_full_image\"\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Save the `.ome.zarr` DirectoryStore as a LaminDB Artifact.\n\n# %%\nome_zarr_artifact = ln.Artifact(\n ome_zarr_filepath,\n key=\"vitessce_examples/visium.ome.zarr\",\n description=\"Visium OME-Zarr Example\",\n type=\"dataset\",\n)\nome_zarr_artifact.save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n# Here, we use the [ImageOmeZarrWrapper](https://python-docs.vitessce.io/api_data.html#vitessce.wrappers.ImageOmeZarrWrapper) class to specify an OME-Zarr file to load for visualization.\n\n# %%\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=ome_zarr_artifact.description\n)\ndataset_uid = \"ome_zarr_image\"\ndataset = vc.add_dataset(\"Image\", uid=dataset_uid).add_object(\n vit.ImageOmeZarrWrapper(\n img_artifact=ome_zarr_artifact,\n )\n)\nspatial = vc.add_view(\"spatialBeta\", dataset=dataset)\nlayer_controller = vc.add_view(\"layerControllerBeta\", dataset=dataset)\nvc.link_views_by_dict([spatial, layer_controller], {\n \"imageLayer\": vit.CoordinationLevel([{\n \"photometricInterpretation\": \"RGB\",\n }]),\n}, scope_prefix=vit.get_initial_coordination_scope_prefix(dataset_uid, \"image\"))\nvc.layout(spatial | layer_controller);\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nome_zarr_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View Visium OME-Zarr Example in Vitessce\",\n)\n\n# %%\nome_zarr_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# :::\n\n# %%\n# [optional] finish run context and auto-save the notebook\nln.finish()\n\n# %% [markdown]\n# :::{dropdown} Upload speed\n#\n# Here is [a note](https://lamin.ai/laminlabs/lamindata/transform/WDjio16cQsdW5zKv) on folder upload speed and why `lamindb` does not use the `.export(to=\"s3\")` functionality of Vitessce.\n#\n# :::\n\n# %% tags=[\"hide-cell\"]\n# clean up artifacts in CI run\nh5ad_artifact.delete(permanent=True)\nref_artifact.delete(permanent=True)\n\nadata_zarr_artifact.delete(permanent=True)\nsdata_zarr_artifact.delete(permanent=True)\n\nome_tiff_artifact.delete(permanent=True)\noffsets_artifact.delete(permanent=True)\n\nome_zarr_artifact.delete(permanent=True)\n\nh5ad_vc_artifact.delete(permanent=True)\nadata_zarr_vc_artifact.delete(permanent=True)\nsdata_vc_artifact.delete(permanent=True)\nome_tiff_vc_artifact.delete(permanent=True)\nome_zarr_vc_artifact.delete(permanent=True)\n","_branch_code":1,"reference_type":null}}},{"id":65,"key":"vitessce_examples/habib17.h5ad","uid":"1g3a41z0hggWNBrk0000","_aux":null,"hash":"-O1gvNOMqXm96TNFJ4yZXQ","kind":"dataset","size":22586523,"otype":"AnnData","suffix":".h5ad","n_files":null,"version":null,"is_latest":true,"_hash_type":"md5","created_at":"2024-12-03T21:31:03.222544+00:00","updated_at":"2024-12-04T16:11:01.559593+00:00","description":"Habib et al., 2017 Nature Methods, h5ad","_branch_code":1,"n_observations":null,"_key_is_virtual":true,"_overwrite_versions":false,"run":{"id":43,"uid":"MfsMSwTjRQpibcLciOyO","_aux":null,"name":null,"reference":null,"created_at":"2024-12-03T20:43:15.292036+00:00","started_at":"2024-12-04T15:42:15.45694+00:00","finished_at":"2024-12-04T15:43:14.152546+00:00","_branch_code":1,"_status_code":0,"reference_type":null,"_is_consecutive":false,"transform":{"id":14,"key":"vitessce.ipynb","uid":"XmcumVqvKZd30000","_aux":null,"hash":"sqz7fEl7R7YwU-R5fz7LnQ","type":"notebook","version":null,"is_latest":false,"reference":null,"created_at":"2024-12-03T20:43:15.109256+00:00","updated_at":"2024-12-04T15:52:35.669773+00:00","description":"Vitessce","source_code":"# %% [markdown]\n# [![](https://img.shields.io/badge/Source%20on%20GitHub-orange)](https://github.com/laminlabs/lamin-spatial/blob/main/docs/vitessce.ipynb) [![hub](https://img.shields.io/badge/View%20on%20LaminHub-mediumseagreen)](https://lamin.ai/laminlabs/lamindata/transform/hqtT4OTr5Tiq5zKv/URbaThYhljgXbZEzgj5x)\n\n# %% [markdown] nbsphinx=\"hidden\"\n# # transform.name\n\n# %% [markdown]\n# This tutorial has been adopted from the data preparation examples in [the Vitessce documention](https://vitessce.github.io/vitessce-python).\n#\n\n# %% tags=[\"hide-output\"]\n# # !pip install \"vitessce[all]\">=3.5.0\n# # !pip install \"generate-tiff-offsets==0.1.7\"\n# # !pip install 'lamindb[jupyter,aws,bionty]'\n# !lamin connect laminlabs/lamin-dev # <-- replace with your instance\n\n# %% tags=[\"hide-output\"]\nfrom urllib.request import urlretrieve\nfrom pathlib import Path\nfrom anndata import read_h5ad\nimport vitessce as vit\nfrom vitessce import data_utils as vitdu\nfrom generate_tiff_offsets import get_offsets\nimport numpy as np\nimport lamindb as ln\nimport zipfile\nimport json\n\n# [optional] track the current notebook or script\nln.context.uid = \"XmcumVqvKZd30000\"\nln.track()\n\n# %% [markdown]\n# ## Visualize an AnnData object (H5AD format)\n\n# %% [markdown]\n# Here, we download an example H5AD file (approximately 22 MB). This is a [dataset](https://www.covid19cellatlas.org/index.healthy.html#habib17) from the COVID-19 Cell Atlas.\n\n# %%\nh5ad_raw_filepath = \"./habib17.raw.h5ad\"\nh5ad_processed_filepath = \"./habib17.processed.h5ad\"\nif not Path(h5ad_raw_filepath).exists():\n urlretrieve(\n \"https://covid19.cog.sanger.ac.uk/habib17.processed.h5ad\", h5ad_raw_filepath\n )\n\n# %% [markdown]\n# This AnnData file was not saved recently and `read_h5ad` will warn us about this. By reading and subsequently re-writing the file using an up-to-date `anndata` package version, we ensure that the dataset is saved using the latest AnnData H5AD format.\n\n# %%\nadata = read_h5ad(h5ad_raw_filepath)\n\n# %% [markdown]\n# Optionally, we can modify the AnnData object, for example to subset it to the highly variable genes (which will affect the amount of data that is displayed in the heatmap visualization).\n\n# %%\nadata = adata[:, adata.var[\"highly_variable\"]].copy()\n\n# %%\nadata.write_h5ad(h5ad_processed_filepath)\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Save the `.h5ad` file as a LaminDB Artifact.\n\n# %%\nh5ad_artifact = ln.Artifact(\n h5ad_processed_filepath,\n key=\"vitessce_examples/habib17.h5ad\",\n description=\"Habib et al., 2017 Nature Methods, h5ad\",\n type=\"dataset\",\n)\nh5ad_artifact.save()\n\n# %% [markdown]\n# When using `.h5ad` files, we construct a [Reference Specification](https://fsspec.github.io/kerchunk/spec.html) which enables interoperability with the [Zarr](https://zarrita.dev/packages/storage.html#referencestore) interface.\n\n# %%\nref_spec = vitdu.generate_h5ad_ref_spec(h5ad_processed_filepath)\n\n# %% [markdown]\n# We next need to save the corresponding Reference Specification to a JSON file and upload it to LaminDB as an Artifact.\n\n# %%\nref_path = \"./habib17.processed.reference.json\"\nwith open(ref_path, \"w\") as file:\n json.dump(ref_spec, file)\nref_artifact = ln.Artifact(\n ref_path,\n key=\"vitessce_examples/habib17.reference.json\",\n description=\"Reference JSON for H5AD file, Habib et al., 2017 Nature Methods\"\n).save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n\n# %% [markdown]\n# You can pass LaminDB Artifacts to the AnnDataWrapper class using the `adata_artifact` and `ref_artifact` [parameters](https://python-docs.vitessce.io/api_data.html#vitessce.wrappers.AnnDataWrapper).\n\n# %%\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=h5ad_artifact.description,\n)\ndataset = vc.add_dataset(name=\"Habib 2017\").add_object(\n vit.AnnDataWrapper(\n adata_artifact=h5ad_artifact,\n ref_artifact=ref_artifact,\n obs_feature_matrix_path=\"X\",\n obs_embedding_paths=[\"obsm/X_umap\"],\n obs_embedding_names=[\"UMAP\"],\n obs_set_paths=[\"obs/CellType\"],\n obs_set_names=[\"Cell Type\"],\n )\n)\nobs_sets = vc.add_view(vit.ViewType.OBS_SETS, dataset=dataset)\nobs_sets_sizes = vc.add_view(vit.ViewType.OBS_SET_SIZES, dataset=dataset)\nscatterplot = vc.add_view(vit.ViewType.SCATTERPLOT, dataset=dataset, mapping=\"UMAP\")\nheatmap = vc.add_view(vit.ViewType.HEATMAP, dataset=dataset)\ngenes = vc.add_view(vit.ViewType.FEATURE_LIST, dataset=dataset)\nvc.link_views([scatterplot, heatmap], [\"featureValueColormapRange\"], [[0.0, 0.1]])\nvc.layout(((scatterplot | obs_sets) / heatmap) | (obs_sets_sizes / genes));\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nh5ad_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View Habib17 (h5ad) in Vitessce\"\n)\n\n# %%\nh5ad_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# If your `VitessceConfig` object references data from multiple artifacts, the Vitessce button will appear next to a `Collection` that groups these artifacts (on the [Collections](https://lamin.ai/vitessce/examples/collections) tab of the Artifacts page).\n# Note that when using an `.h5ad`-based Artifact, the presence of the corresponding `.reference.json` file will result in the creation of a Collection.\n#\n# :::\n\n# %% [markdown]\n# ## Visualize an AnnData object (Zarr format)\n\n# %% [markdown]\n# AnnData objects can be saved on-disk to not only `.h5ad` files, but also to [Zarr stores](https://zarr.readthedocs.io/en/stable/tutorial.html#storage-alternatives) using AnnData's [write_zarr](https://anndata.readthedocs.io/en/latest/generated/anndata.AnnData.write_zarr.html) method.\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Convert the dataset to `.zarr` format. Often, Zarr data is stored on-disk using a [DirectoryStore](https://zarr.readthedocs.io/en/stable/api/storage.html#zarr.storage.DirectoryStore). By convention, these directories are named using a `.zarr` suffix.\n\n# %%\nzarr_processed_filepath = \"./habib17.anndata.zarr\"\n\n# %%\nadata = read_h5ad(h5ad_raw_filepath)\n\n# %% [markdown]\n# Just like in the above section, we may want to modify the AnnData object prior to saving it as a LaminDB Artifact that we will use for visualization.\n\n# %%\nadata = adata[:, adata.var[\"highly_variable\"]].copy()\n\n# %% [markdown]\n# Note the usage of `.write_zarr` below, as opposed to `.write_h5ad`.\n\n# %%\nadata.write_zarr(zarr_processed_filepath)\n\n# %% [markdown]\n# Save the `.zarr` directory as a LaminDB Artifact.\n\n# %%\nadata_zarr_artifact = ln.Artifact(\n zarr_processed_filepath,\n key=\"vitessce_examples/habib17.adata.zarr\",\n description=\"Habib et al., 2017 Nature Methods, zarr\",\n type=\"dataset\",\n)\nadata_zarr_artifact.save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n# Here, we configure the visualization the same way as above in the `.h5ad`-based example, with the exception of the `ref_artifact` parameter, as `.zarr`-based AnnData objects do not require a Reference Specification for Zarr interoperability.\n\n# %% tags=[\"hide-output\"]\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=adata_zarr_artifact.description,\n)\ndataset = vc.add_dataset(name=\"Habib 2017\").add_object(\n vit.AnnDataWrapper(\n adata_artifact=adata_zarr_artifact,\n obs_feature_matrix_path=\"X\",\n obs_embedding_paths=[\"obsm/X_umap\"],\n obs_embedding_names=[\"UMAP\"],\n obs_set_paths=[\"obs/CellType\"],\n obs_set_names=[\"Cell Type\"],\n )\n)\nobs_sets = vc.add_view(vit.Component.OBS_SETS, dataset=dataset)\nobs_sets_sizes = vc.add_view(vit.Component.OBS_SET_SIZES, dataset=dataset)\nscatterplot = vc.add_view(vit.Component.SCATTERPLOT, dataset=dataset, mapping=\"UMAP\")\nheatmap = vc.add_view(vit.Component.HEATMAP, dataset=dataset)\ngenes = vc.add_view(vit.Component.FEATURE_LIST, dataset=dataset)\nvc.link_views([scatterplot, heatmap], [\"featureValueColormapRange\"], [[0.0, 0.1]])\nvc.layout(((scatterplot | obs_sets) / heatmap) | (obs_sets_sizes / genes));\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nadata_zarr_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View Habib17 in Vitessce\"\n)\n\n# %%\nadata_zarr_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# :::\n\n# %% [markdown]\n# ## Visualize a SpatialData object (Zarr format)\n\n# %% [markdown]\n# Download and unzip the example SpatialData dataset (approximately 1.47 GB unzipped). Once unzipped, the data will be available via a local Zarr DirectoryStore.\n\n# %%\nsdata_zip_filepath = \"./visium.spatialdata.zarr.zip\"\nsdata_zarr_filepath = \"./visium.spatialdata.zarr\"\nif not Path(sdata_zarr_filepath).exists():\n if not Path(sdata_zip_filepath).exists():\n urlretrieve(\n \"https://s3.embl.de/spatialdata/spatialdata-sandbox/visium_associated_xenium_io.zip\", sdata_zip_filepath\n )\n with zipfile.ZipFile(sdata_zip_filepath, \"r\") as zip_ref:\n zip_ref.extractall(\".\")\n Path(\"data.zarr\").rename(sdata_zarr_filepath)\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Save the `.zarr` DirectoryStore as a LaminDB Artifact.\n\n# %%\nsdata_zarr_artifact = ln.Artifact(\n sdata_zarr_filepath,\n key=\"vitessce_examples/visium.sdata.zarr\",\n description=\"Visium SpatialData Example\",\n type=\"dataset\",\n)\nsdata_zarr_artifact.save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n# Here, we use the [SpatialDataWrapper](https://python-docs.vitessce.io/api_data.html#vitessce.wrappers.SpatialDataWrapper) class to specify which parts of the SpatialData object will be loaded for visualization.\n\n# %%\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=sdata_zarr_artifact.description,\n)\n# Add data to the configuration:\ndataset = vc.add_dataset(name='Breast Cancer Visium').add_object(\n vit.SpatialDataWrapper(\n sdata_artifact=sdata_zarr_artifact,\n # The following paths are relative to the root of the SpatialData zarr store on-disk.\n image_path=\"images/CytAssist_FFPE_Human_Breast_Cancer_full_image\",\n table_path=\"tables/table\",\n obs_feature_matrix_path=\"tables/table/X\",\n obs_spots_path=\"shapes/CytAssist_FFPE_Human_Breast_Cancer\",\n region=\"CytAssist_FFPE_Human_Breast_Cancer\",\n coordinate_system=\"global\",\n coordination_values={\n # The following tells Vitessce to consider each observation as a \"spot\"\n \"obsType\": \"spot\",\n }\n )\n)\n\n# Add views (visualizations) to the configuration:\nspatial = vc.add_view(\"spatialBeta\", dataset=dataset)\nfeature_list = vc.add_view(\"featureList\", dataset=dataset)\nlayer_controller = vc.add_view(\"layerControllerBeta\", dataset=dataset)\nvc.link_views_by_dict([spatial, layer_controller], {\n 'imageLayer': vit.CoordinationLevel([{\n 'photometricInterpretation': 'RGB',\n }]),\n}, scope_prefix=vit.get_initial_coordination_scope_prefix(\"A\", \"image\"))\nvc.link_views([spatial, layer_controller, feature_list, obs_sets], ['obsType'], [\"spot\"])\n\n# Layout the views\nvc.layout(spatial | (feature_list / layer_controller));\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nsdata_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View Visium SpatialData Example in Vitessce\"\n)\n\n# %%\nsdata_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# :::\n\n# %% [markdown]\n# ## Visualize an image (OME-TIFF format)\n\n# %% [markdown]\n# Vitesse can visualize data from multiple bioimaging file formats, including [OME-TIFF](https://ome-model.readthedocs.io/en/stable/ome-tiff/).\n\n# %% [markdown]\n# Here, we download an example OME-TIFF file (approximately 1.02 GB).\n\n# %%\nome_tiff_filepath = \"./VAN0006-LK-2-85-PAS_registered.ome.tif\"\nif not Path(ome_tiff_filepath).exists():\n urlretrieve(\n \"https://assets.hubmapconsortium.org/f4188a148e4c759092d19369d310883b/ometiff-pyramids/processedMicroscopy/VAN0006-LK-2-85-PAS_images/VAN0006-LK-2-85-PAS_registered.ome.tif\", ome_tiff_filepath\n )\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Save the `.ome.tif` file as a LaminDB Artifact.\n\n# %%\nome_tiff_artifact = ln.Artifact(\n ome_tiff_filepath,\n key=\"vitessce_examples/VAN0006-LK-2-85-PAS_registered.ome.tif\",\n description=\"PAS OME-TIFF file, Neumann et al., 2020\",\n type=\"dataset\",\n)\nome_tiff_artifact.save()\n\n# %% [markdown]\n# When using OME-TIFF files, we can use [generate-tiff-offsets](https://github.com/hms-dbmi/generate-tiff-offsets) to create an index for the bytes within the OME-TIFF file. We store these to a companion `.offsets.json` file which makes loading subsets of the image more efficient.\n\n# %%\noffsets = get_offsets(ome_tiff_filepath)\n\n# %%\noffsets_path = \"./VAN0006-LK-2-85-PAS_registered.offsets.json\"\nwith open(offsets_path, \"w\") as file:\n json.dump(offsets, file)\noffsets_artifact = ln.Artifact(\n offsets_path,\n key=\"vitessce_examples/VAN0006-LK-2-85-PAS_registered.offsets.json\",\n description=\"PAS offsets file, Neumann et al., 2020\"\n).save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n# Here, we use the [ImageOmeTiffWrapper](https://python-docs.vitessce.io/api_data.html#vitessce.wrappers.ImageOmeTiffWrapper) class to specify which pair of OME-TIFF file and offsets JSON file to load.\n\n# %%\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=ome_tiff_artifact.description\n)\ndataset = vc.add_dataset(\"Image\").add_object(\n vit.ImageOmeTiffWrapper(\n img_artifact=ome_tiff_artifact,\n offsets_artifact=offsets_artifact,\n )\n)\nspatial = vc.add_view(\"spatialBeta\", dataset=dataset)\nlayer_controller = vc.add_view(\"layerControllerBeta\", dataset=dataset)\nvc.layout(spatial | layer_controller);\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nome_tiff_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View PAS OME-TIFF, Neumann et al., 2020 in Vitessce\"\n)\n\n# %%\nome_tiff_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# If your `VitessceConfig` object references data from multiple artifacts, the Vitessce button will appear next to a `Collection` that groups these artifacts (on the [Collections](https://lamin.ai/vitessce/examples/collections) tab of the Artifacts page).\n# In the case of OME-TIFF, the presence of the corresponding offsets JSON file will result in the creation of a Collection.\n#\n# :::\n\n# %% [markdown]\n# ## Visualize an image (OME-Zarr format)\n\n# %% [markdown]\n# Retrieve an OME-Zarr (also known as OME-NGFF) formatted image. We could download an unzip such an image, but in downloading the SpatialData object above, we already have access to a valid OME-Zarr image locally.\n\n# %%\nome_zarr_filepath = Path(sdata_zarr_filepath) / \"images\" / \"CytAssist_FFPE_Human_Breast_Cancer_full_image\"\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Save the `.ome.zarr` DirectoryStore as a LaminDB Artifact.\n\n# %%\nome_zarr_artifact = ln.Artifact(\n ome_zarr_filepath,\n key=\"vitessce_examples/visium.ome.zarr\",\n description=\"Visium OME-Zarr Example\",\n type=\"dataset\",\n)\nome_zarr_artifact.save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n# Here, we use the [ImageOmeZarrWrapper](https://python-docs.vitessce.io/api_data.html#vitessce.wrappers.ImageOmeZarrWrapper) class to specify an OME-Zarr file to load for visualization.\n\n# %%\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=ome_zarr_artifact.description\n)\ndataset = vc.add_dataset(\"Image\").add_object(\n vit.ImageOmeZarrWrapper(\n img_artifact=ome_zarr_artifact,\n )\n)\nspatial = vc.add_view(\"spatialBeta\", dataset=dataset)\nlayer_controller = vc.add_view(\"layerControllerBeta\", dataset=dataset)\nvc.link_views_by_dict([spatial, layer_controller], {\n 'imageLayer': vit.CoordinationLevel([{\n 'photometricInterpretation': 'RGB',\n }]),\n}, scope_prefix=vit.get_initial_coordination_scope_prefix(\"A\", \"image\"))\nvc.layout(spatial | layer_controller);\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nome_zarr_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View Visium OME-Zarr Example in Vitessce\"\n)\n\n# %%\nome_zarr_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# :::\n\n# %%\n# [optional] finish run context and auto-save the notebook\nln.finish()\n\n# %% [markdown]\n# :::{dropdown} Upload speed\n#\n# Here is [a note](https://lamin.ai/laminlabs/lamindata/transform/WDjio16cQsdW5zKv) on folder upload speed and why `lamindb` does not use the `.export(to=\"s3\")` functionality of Vitessce.\n#\n# :::\n\n# %% tags=[\"hide-cell\"]\n# clean up artifacts in CI run\nh5ad_artifact.delete(permanent=True)\nref_artifact.delete(permanent=True)\n\nadata_zarr_artifact.delete(permanent=True)\nsdata_zarr_artifact.delete(permanent=True)\n\nome_tiff_artifact.delete(permanent=True)\noffsets_artifact.delete(permanent=True)\n\nome_zarr_artifact.delete(permanent=True)\n\nh5ad_vc_artifact.delete(permanent=True)\nadata_zarr_vc_artifact.delete(permanent=True)\nsdata_vc_artifact.delete(permanent=True)\nome_tiff_vc_artifact.delete(permanent=True)\nome_zarr_vc_artifact.delete(permanent=True)\n","_branch_code":1,"reference_type":null}}},{"id":66,"key":"vitessce_examples/habib17.reference.json","uid":"O7CTVi6XjnfsceQ20000","_aux":null,"hash":"XFOV2X6Bw6_zTugTOEBehg","kind":null,"size":543575,"otype":null,"suffix":".json","n_files":null,"version":null,"is_latest":true,"_hash_type":"md5","created_at":"2024-12-03T21:31:14.820835+00:00","updated_at":"2024-12-04T16:11:03.162122+00:00","description":"Reference JSON for H5AD file, Habib et al., 2017 Nature Methods","_branch_code":1,"n_observations":null,"_key_is_virtual":true,"_overwrite_versions":false,"run":{"id":43,"uid":"MfsMSwTjRQpibcLciOyO","_aux":null,"name":null,"reference":null,"created_at":"2024-12-03T20:43:15.292036+00:00","started_at":"2024-12-04T15:42:15.45694+00:00","finished_at":"2024-12-04T15:43:14.152546+00:00","_branch_code":1,"_status_code":0,"reference_type":null,"_is_consecutive":false,"transform":{"id":14,"key":"vitessce.ipynb","uid":"XmcumVqvKZd30000","_aux":null,"hash":"sqz7fEl7R7YwU-R5fz7LnQ","type":"notebook","version":null,"is_latest":false,"reference":null,"created_at":"2024-12-03T20:43:15.109256+00:00","updated_at":"2024-12-04T15:52:35.669773+00:00","description":"Vitessce","source_code":"# %% [markdown]\n# [![](https://img.shields.io/badge/Source%20on%20GitHub-orange)](https://github.com/laminlabs/lamin-spatial/blob/main/docs/vitessce.ipynb) [![hub](https://img.shields.io/badge/View%20on%20LaminHub-mediumseagreen)](https://lamin.ai/laminlabs/lamindata/transform/hqtT4OTr5Tiq5zKv/URbaThYhljgXbZEzgj5x)\n\n# %% [markdown] nbsphinx=\"hidden\"\n# # transform.name\n\n# %% [markdown]\n# This tutorial has been adopted from the data preparation examples in [the Vitessce documention](https://vitessce.github.io/vitessce-python).\n#\n\n# %% tags=[\"hide-output\"]\n# # !pip install \"vitessce[all]\">=3.5.0\n# # !pip install \"generate-tiff-offsets==0.1.7\"\n# # !pip install 'lamindb[jupyter,aws,bionty]'\n# !lamin connect laminlabs/lamin-dev # <-- replace with your instance\n\n# %% tags=[\"hide-output\"]\nfrom urllib.request import urlretrieve\nfrom pathlib import Path\nfrom anndata import read_h5ad\nimport vitessce as vit\nfrom vitessce import data_utils as vitdu\nfrom generate_tiff_offsets import get_offsets\nimport numpy as np\nimport lamindb as ln\nimport zipfile\nimport json\n\n# [optional] track the current notebook or script\nln.context.uid = \"XmcumVqvKZd30000\"\nln.track()\n\n# %% [markdown]\n# ## Visualize an AnnData object (H5AD format)\n\n# %% [markdown]\n# Here, we download an example H5AD file (approximately 22 MB). This is a [dataset](https://www.covid19cellatlas.org/index.healthy.html#habib17) from the COVID-19 Cell Atlas.\n\n# %%\nh5ad_raw_filepath = \"./habib17.raw.h5ad\"\nh5ad_processed_filepath = \"./habib17.processed.h5ad\"\nif not Path(h5ad_raw_filepath).exists():\n urlretrieve(\n \"https://covid19.cog.sanger.ac.uk/habib17.processed.h5ad\", h5ad_raw_filepath\n )\n\n# %% [markdown]\n# This AnnData file was not saved recently and `read_h5ad` will warn us about this. By reading and subsequently re-writing the file using an up-to-date `anndata` package version, we ensure that the dataset is saved using the latest AnnData H5AD format.\n\n# %%\nadata = read_h5ad(h5ad_raw_filepath)\n\n# %% [markdown]\n# Optionally, we can modify the AnnData object, for example to subset it to the highly variable genes (which will affect the amount of data that is displayed in the heatmap visualization).\n\n# %%\nadata = adata[:, adata.var[\"highly_variable\"]].copy()\n\n# %%\nadata.write_h5ad(h5ad_processed_filepath)\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Save the `.h5ad` file as a LaminDB Artifact.\n\n# %%\nh5ad_artifact = ln.Artifact(\n h5ad_processed_filepath,\n key=\"vitessce_examples/habib17.h5ad\",\n description=\"Habib et al., 2017 Nature Methods, h5ad\",\n type=\"dataset\",\n)\nh5ad_artifact.save()\n\n# %% [markdown]\n# When using `.h5ad` files, we construct a [Reference Specification](https://fsspec.github.io/kerchunk/spec.html) which enables interoperability with the [Zarr](https://zarrita.dev/packages/storage.html#referencestore) interface.\n\n# %%\nref_spec = vitdu.generate_h5ad_ref_spec(h5ad_processed_filepath)\n\n# %% [markdown]\n# We next need to save the corresponding Reference Specification to a JSON file and upload it to LaminDB as an Artifact.\n\n# %%\nref_path = \"./habib17.processed.reference.json\"\nwith open(ref_path, \"w\") as file:\n json.dump(ref_spec, file)\nref_artifact = ln.Artifact(\n ref_path,\n key=\"vitessce_examples/habib17.reference.json\",\n description=\"Reference JSON for H5AD file, Habib et al., 2017 Nature Methods\"\n).save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n\n# %% [markdown]\n# You can pass LaminDB Artifacts to the AnnDataWrapper class using the `adata_artifact` and `ref_artifact` [parameters](https://python-docs.vitessce.io/api_data.html#vitessce.wrappers.AnnDataWrapper).\n\n# %%\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=h5ad_artifact.description,\n)\ndataset = vc.add_dataset(name=\"Habib 2017\").add_object(\n vit.AnnDataWrapper(\n adata_artifact=h5ad_artifact,\n ref_artifact=ref_artifact,\n obs_feature_matrix_path=\"X\",\n obs_embedding_paths=[\"obsm/X_umap\"],\n obs_embedding_names=[\"UMAP\"],\n obs_set_paths=[\"obs/CellType\"],\n obs_set_names=[\"Cell Type\"],\n )\n)\nobs_sets = vc.add_view(vit.ViewType.OBS_SETS, dataset=dataset)\nobs_sets_sizes = vc.add_view(vit.ViewType.OBS_SET_SIZES, dataset=dataset)\nscatterplot = vc.add_view(vit.ViewType.SCATTERPLOT, dataset=dataset, mapping=\"UMAP\")\nheatmap = vc.add_view(vit.ViewType.HEATMAP, dataset=dataset)\ngenes = vc.add_view(vit.ViewType.FEATURE_LIST, dataset=dataset)\nvc.link_views([scatterplot, heatmap], [\"featureValueColormapRange\"], [[0.0, 0.1]])\nvc.layout(((scatterplot | obs_sets) / heatmap) | (obs_sets_sizes / genes));\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nh5ad_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View Habib17 (h5ad) in Vitessce\"\n)\n\n# %%\nh5ad_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# If your `VitessceConfig` object references data from multiple artifacts, the Vitessce button will appear next to a `Collection` that groups these artifacts (on the [Collections](https://lamin.ai/vitessce/examples/collections) tab of the Artifacts page).\n# Note that when using an `.h5ad`-based Artifact, the presence of the corresponding `.reference.json` file will result in the creation of a Collection.\n#\n# :::\n\n# %% [markdown]\n# ## Visualize an AnnData object (Zarr format)\n\n# %% [markdown]\n# AnnData objects can be saved on-disk to not only `.h5ad` files, but also to [Zarr stores](https://zarr.readthedocs.io/en/stable/tutorial.html#storage-alternatives) using AnnData's [write_zarr](https://anndata.readthedocs.io/en/latest/generated/anndata.AnnData.write_zarr.html) method.\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Convert the dataset to `.zarr` format. Often, Zarr data is stored on-disk using a [DirectoryStore](https://zarr.readthedocs.io/en/stable/api/storage.html#zarr.storage.DirectoryStore). By convention, these directories are named using a `.zarr` suffix.\n\n# %%\nzarr_processed_filepath = \"./habib17.anndata.zarr\"\n\n# %%\nadata = read_h5ad(h5ad_raw_filepath)\n\n# %% [markdown]\n# Just like in the above section, we may want to modify the AnnData object prior to saving it as a LaminDB Artifact that we will use for visualization.\n\n# %%\nadata = adata[:, adata.var[\"highly_variable\"]].copy()\n\n# %% [markdown]\n# Note the usage of `.write_zarr` below, as opposed to `.write_h5ad`.\n\n# %%\nadata.write_zarr(zarr_processed_filepath)\n\n# %% [markdown]\n# Save the `.zarr` directory as a LaminDB Artifact.\n\n# %%\nadata_zarr_artifact = ln.Artifact(\n zarr_processed_filepath,\n key=\"vitessce_examples/habib17.adata.zarr\",\n description=\"Habib et al., 2017 Nature Methods, zarr\",\n type=\"dataset\",\n)\nadata_zarr_artifact.save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n# Here, we configure the visualization the same way as above in the `.h5ad`-based example, with the exception of the `ref_artifact` parameter, as `.zarr`-based AnnData objects do not require a Reference Specification for Zarr interoperability.\n\n# %% tags=[\"hide-output\"]\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=adata_zarr_artifact.description,\n)\ndataset = vc.add_dataset(name=\"Habib 2017\").add_object(\n vit.AnnDataWrapper(\n adata_artifact=adata_zarr_artifact,\n obs_feature_matrix_path=\"X\",\n obs_embedding_paths=[\"obsm/X_umap\"],\n obs_embedding_names=[\"UMAP\"],\n obs_set_paths=[\"obs/CellType\"],\n obs_set_names=[\"Cell Type\"],\n )\n)\nobs_sets = vc.add_view(vit.Component.OBS_SETS, dataset=dataset)\nobs_sets_sizes = vc.add_view(vit.Component.OBS_SET_SIZES, dataset=dataset)\nscatterplot = vc.add_view(vit.Component.SCATTERPLOT, dataset=dataset, mapping=\"UMAP\")\nheatmap = vc.add_view(vit.Component.HEATMAP, dataset=dataset)\ngenes = vc.add_view(vit.Component.FEATURE_LIST, dataset=dataset)\nvc.link_views([scatterplot, heatmap], [\"featureValueColormapRange\"], [[0.0, 0.1]])\nvc.layout(((scatterplot | obs_sets) / heatmap) | (obs_sets_sizes / genes));\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nadata_zarr_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View Habib17 in Vitessce\"\n)\n\n# %%\nadata_zarr_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# :::\n\n# %% [markdown]\n# ## Visualize a SpatialData object (Zarr format)\n\n# %% [markdown]\n# Download and unzip the example SpatialData dataset (approximately 1.47 GB unzipped). Once unzipped, the data will be available via a local Zarr DirectoryStore.\n\n# %%\nsdata_zip_filepath = \"./visium.spatialdata.zarr.zip\"\nsdata_zarr_filepath = \"./visium.spatialdata.zarr\"\nif not Path(sdata_zarr_filepath).exists():\n if not Path(sdata_zip_filepath).exists():\n urlretrieve(\n \"https://s3.embl.de/spatialdata/spatialdata-sandbox/visium_associated_xenium_io.zip\", sdata_zip_filepath\n )\n with zipfile.ZipFile(sdata_zip_filepath, \"r\") as zip_ref:\n zip_ref.extractall(\".\")\n Path(\"data.zarr\").rename(sdata_zarr_filepath)\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Save the `.zarr` DirectoryStore as a LaminDB Artifact.\n\n# %%\nsdata_zarr_artifact = ln.Artifact(\n sdata_zarr_filepath,\n key=\"vitessce_examples/visium.sdata.zarr\",\n description=\"Visium SpatialData Example\",\n type=\"dataset\",\n)\nsdata_zarr_artifact.save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n# Here, we use the [SpatialDataWrapper](https://python-docs.vitessce.io/api_data.html#vitessce.wrappers.SpatialDataWrapper) class to specify which parts of the SpatialData object will be loaded for visualization.\n\n# %%\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=sdata_zarr_artifact.description,\n)\n# Add data to the configuration:\ndataset = vc.add_dataset(name='Breast Cancer Visium').add_object(\n vit.SpatialDataWrapper(\n sdata_artifact=sdata_zarr_artifact,\n # The following paths are relative to the root of the SpatialData zarr store on-disk.\n image_path=\"images/CytAssist_FFPE_Human_Breast_Cancer_full_image\",\n table_path=\"tables/table\",\n obs_feature_matrix_path=\"tables/table/X\",\n obs_spots_path=\"shapes/CytAssist_FFPE_Human_Breast_Cancer\",\n region=\"CytAssist_FFPE_Human_Breast_Cancer\",\n coordinate_system=\"global\",\n coordination_values={\n # The following tells Vitessce to consider each observation as a \"spot\"\n \"obsType\": \"spot\",\n }\n )\n)\n\n# Add views (visualizations) to the configuration:\nspatial = vc.add_view(\"spatialBeta\", dataset=dataset)\nfeature_list = vc.add_view(\"featureList\", dataset=dataset)\nlayer_controller = vc.add_view(\"layerControllerBeta\", dataset=dataset)\nvc.link_views_by_dict([spatial, layer_controller], {\n 'imageLayer': vit.CoordinationLevel([{\n 'photometricInterpretation': 'RGB',\n }]),\n}, scope_prefix=vit.get_initial_coordination_scope_prefix(\"A\", \"image\"))\nvc.link_views([spatial, layer_controller, feature_list, obs_sets], ['obsType'], [\"spot\"])\n\n# Layout the views\nvc.layout(spatial | (feature_list / layer_controller));\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nsdata_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View Visium SpatialData Example in Vitessce\"\n)\n\n# %%\nsdata_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# :::\n\n# %% [markdown]\n# ## Visualize an image (OME-TIFF format)\n\n# %% [markdown]\n# Vitesse can visualize data from multiple bioimaging file formats, including [OME-TIFF](https://ome-model.readthedocs.io/en/stable/ome-tiff/).\n\n# %% [markdown]\n# Here, we download an example OME-TIFF file (approximately 1.02 GB).\n\n# %%\nome_tiff_filepath = \"./VAN0006-LK-2-85-PAS_registered.ome.tif\"\nif not Path(ome_tiff_filepath).exists():\n urlretrieve(\n \"https://assets.hubmapconsortium.org/f4188a148e4c759092d19369d310883b/ometiff-pyramids/processedMicroscopy/VAN0006-LK-2-85-PAS_images/VAN0006-LK-2-85-PAS_registered.ome.tif\", ome_tiff_filepath\n )\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Save the `.ome.tif` file as a LaminDB Artifact.\n\n# %%\nome_tiff_artifact = ln.Artifact(\n ome_tiff_filepath,\n key=\"vitessce_examples/VAN0006-LK-2-85-PAS_registered.ome.tif\",\n description=\"PAS OME-TIFF file, Neumann et al., 2020\",\n type=\"dataset\",\n)\nome_tiff_artifact.save()\n\n# %% [markdown]\n# When using OME-TIFF files, we can use [generate-tiff-offsets](https://github.com/hms-dbmi/generate-tiff-offsets) to create an index for the bytes within the OME-TIFF file. We store these to a companion `.offsets.json` file which makes loading subsets of the image more efficient.\n\n# %%\noffsets = get_offsets(ome_tiff_filepath)\n\n# %%\noffsets_path = \"./VAN0006-LK-2-85-PAS_registered.offsets.json\"\nwith open(offsets_path, \"w\") as file:\n json.dump(offsets, file)\noffsets_artifact = ln.Artifact(\n offsets_path,\n key=\"vitessce_examples/VAN0006-LK-2-85-PAS_registered.offsets.json\",\n description=\"PAS offsets file, Neumann et al., 2020\"\n).save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n# Here, we use the [ImageOmeTiffWrapper](https://python-docs.vitessce.io/api_data.html#vitessce.wrappers.ImageOmeTiffWrapper) class to specify which pair of OME-TIFF file and offsets JSON file to load.\n\n# %%\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=ome_tiff_artifact.description\n)\ndataset = vc.add_dataset(\"Image\").add_object(\n vit.ImageOmeTiffWrapper(\n img_artifact=ome_tiff_artifact,\n offsets_artifact=offsets_artifact,\n )\n)\nspatial = vc.add_view(\"spatialBeta\", dataset=dataset)\nlayer_controller = vc.add_view(\"layerControllerBeta\", dataset=dataset)\nvc.layout(spatial | layer_controller);\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nome_tiff_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View PAS OME-TIFF, Neumann et al., 2020 in Vitessce\"\n)\n\n# %%\nome_tiff_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# If your `VitessceConfig` object references data from multiple artifacts, the Vitessce button will appear next to a `Collection` that groups these artifacts (on the [Collections](https://lamin.ai/vitessce/examples/collections) tab of the Artifacts page).\n# In the case of OME-TIFF, the presence of the corresponding offsets JSON file will result in the creation of a Collection.\n#\n# :::\n\n# %% [markdown]\n# ## Visualize an image (OME-Zarr format)\n\n# %% [markdown]\n# Retrieve an OME-Zarr (also known as OME-NGFF) formatted image. We could download an unzip such an image, but in downloading the SpatialData object above, we already have access to a valid OME-Zarr image locally.\n\n# %%\nome_zarr_filepath = Path(sdata_zarr_filepath) / \"images\" / \"CytAssist_FFPE_Human_Breast_Cancer_full_image\"\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Save the `.ome.zarr` DirectoryStore as a LaminDB Artifact.\n\n# %%\nome_zarr_artifact = ln.Artifact(\n ome_zarr_filepath,\n key=\"vitessce_examples/visium.ome.zarr\",\n description=\"Visium OME-Zarr Example\",\n type=\"dataset\",\n)\nome_zarr_artifact.save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n# Here, we use the [ImageOmeZarrWrapper](https://python-docs.vitessce.io/api_data.html#vitessce.wrappers.ImageOmeZarrWrapper) class to specify an OME-Zarr file to load for visualization.\n\n# %%\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=ome_zarr_artifact.description\n)\ndataset = vc.add_dataset(\"Image\").add_object(\n vit.ImageOmeZarrWrapper(\n img_artifact=ome_zarr_artifact,\n )\n)\nspatial = vc.add_view(\"spatialBeta\", dataset=dataset)\nlayer_controller = vc.add_view(\"layerControllerBeta\", dataset=dataset)\nvc.link_views_by_dict([spatial, layer_controller], {\n 'imageLayer': vit.CoordinationLevel([{\n 'photometricInterpretation': 'RGB',\n }]),\n}, scope_prefix=vit.get_initial_coordination_scope_prefix(\"A\", \"image\"))\nvc.layout(spatial | layer_controller);\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nome_zarr_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View Visium OME-Zarr Example in Vitessce\"\n)\n\n# %%\nome_zarr_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# :::\n\n# %%\n# [optional] finish run context and auto-save the notebook\nln.finish()\n\n# %% [markdown]\n# :::{dropdown} Upload speed\n#\n# Here is [a note](https://lamin.ai/laminlabs/lamindata/transform/WDjio16cQsdW5zKv) on folder upload speed and why `lamindb` does not use the `.export(to=\"s3\")` functionality of Vitessce.\n#\n# :::\n\n# %% tags=[\"hide-cell\"]\n# clean up artifacts in CI run\nh5ad_artifact.delete(permanent=True)\nref_artifact.delete(permanent=True)\n\nadata_zarr_artifact.delete(permanent=True)\nsdata_zarr_artifact.delete(permanent=True)\n\nome_tiff_artifact.delete(permanent=True)\noffsets_artifact.delete(permanent=True)\n\nome_zarr_artifact.delete(permanent=True)\n\nh5ad_vc_artifact.delete(permanent=True)\nadata_zarr_vc_artifact.delete(permanent=True)\nsdata_vc_artifact.delete(permanent=True)\nome_tiff_vc_artifact.delete(permanent=True)\nome_zarr_vc_artifact.delete(permanent=True)\n","_branch_code":1,"reference_type":null}}},{"id":71,"key":"vitessce_examples/visium.sdata.zarr","uid":"nelDyBfwvEIIAKcC0000","_aux":null,"hash":"BkflAMTiin88wE3Wc84rDQ","kind":"dataset","size":1473180201,"otype":null,"suffix":".zarr","n_files":9167,"version":null,"is_latest":true,"_hash_type":"md5-d","created_at":"2024-12-04T13:25:22.397942+00:00","updated_at":"2024-12-04T16:11:31.168705+00:00","description":"Visium SpatialData Example","_branch_code":1,"n_observations":null,"_key_is_virtual":true,"_overwrite_versions":true,"run":{"id":43,"uid":"MfsMSwTjRQpibcLciOyO","_aux":null,"name":null,"reference":null,"created_at":"2024-12-03T20:43:15.292036+00:00","started_at":"2024-12-04T15:42:15.45694+00:00","finished_at":"2024-12-04T15:43:14.152546+00:00","_branch_code":1,"_status_code":0,"reference_type":null,"_is_consecutive":false,"transform":{"id":14,"key":"vitessce.ipynb","uid":"XmcumVqvKZd30000","_aux":null,"hash":"sqz7fEl7R7YwU-R5fz7LnQ","type":"notebook","version":null,"is_latest":false,"reference":null,"created_at":"2024-12-03T20:43:15.109256+00:00","updated_at":"2024-12-04T15:52:35.669773+00:00","description":"Vitessce","source_code":"# %% [markdown]\n# [![](https://img.shields.io/badge/Source%20on%20GitHub-orange)](https://github.com/laminlabs/lamin-spatial/blob/main/docs/vitessce.ipynb) [![hub](https://img.shields.io/badge/View%20on%20LaminHub-mediumseagreen)](https://lamin.ai/laminlabs/lamindata/transform/hqtT4OTr5Tiq5zKv/URbaThYhljgXbZEzgj5x)\n\n# %% [markdown] nbsphinx=\"hidden\"\n# # transform.name\n\n# %% [markdown]\n# This tutorial has been adopted from the data preparation examples in [the Vitessce documention](https://vitessce.github.io/vitessce-python).\n#\n\n# %% tags=[\"hide-output\"]\n# # !pip install \"vitessce[all]\">=3.5.0\n# # !pip install \"generate-tiff-offsets==0.1.7\"\n# # !pip install 'lamindb[jupyter,aws,bionty]'\n# !lamin connect laminlabs/lamin-dev # <-- replace with your instance\n\n# %% tags=[\"hide-output\"]\nfrom urllib.request import urlretrieve\nfrom pathlib import Path\nfrom anndata import read_h5ad\nimport vitessce as vit\nfrom vitessce import data_utils as vitdu\nfrom generate_tiff_offsets import get_offsets\nimport numpy as np\nimport lamindb as ln\nimport zipfile\nimport json\n\n# [optional] track the current notebook or script\nln.context.uid = \"XmcumVqvKZd30000\"\nln.track()\n\n# %% [markdown]\n# ## Visualize an AnnData object (H5AD format)\n\n# %% [markdown]\n# Here, we download an example H5AD file (approximately 22 MB). This is a [dataset](https://www.covid19cellatlas.org/index.healthy.html#habib17) from the COVID-19 Cell Atlas.\n\n# %%\nh5ad_raw_filepath = \"./habib17.raw.h5ad\"\nh5ad_processed_filepath = \"./habib17.processed.h5ad\"\nif not Path(h5ad_raw_filepath).exists():\n urlretrieve(\n \"https://covid19.cog.sanger.ac.uk/habib17.processed.h5ad\", h5ad_raw_filepath\n )\n\n# %% [markdown]\n# This AnnData file was not saved recently and `read_h5ad` will warn us about this. By reading and subsequently re-writing the file using an up-to-date `anndata` package version, we ensure that the dataset is saved using the latest AnnData H5AD format.\n\n# %%\nadata = read_h5ad(h5ad_raw_filepath)\n\n# %% [markdown]\n# Optionally, we can modify the AnnData object, for example to subset it to the highly variable genes (which will affect the amount of data that is displayed in the heatmap visualization).\n\n# %%\nadata = adata[:, adata.var[\"highly_variable\"]].copy()\n\n# %%\nadata.write_h5ad(h5ad_processed_filepath)\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Save the `.h5ad` file as a LaminDB Artifact.\n\n# %%\nh5ad_artifact = ln.Artifact(\n h5ad_processed_filepath,\n key=\"vitessce_examples/habib17.h5ad\",\n description=\"Habib et al., 2017 Nature Methods, h5ad\",\n type=\"dataset\",\n)\nh5ad_artifact.save()\n\n# %% [markdown]\n# When using `.h5ad` files, we construct a [Reference Specification](https://fsspec.github.io/kerchunk/spec.html) which enables interoperability with the [Zarr](https://zarrita.dev/packages/storage.html#referencestore) interface.\n\n# %%\nref_spec = vitdu.generate_h5ad_ref_spec(h5ad_processed_filepath)\n\n# %% [markdown]\n# We next need to save the corresponding Reference Specification to a JSON file and upload it to LaminDB as an Artifact.\n\n# %%\nref_path = \"./habib17.processed.reference.json\"\nwith open(ref_path, \"w\") as file:\n json.dump(ref_spec, file)\nref_artifact = ln.Artifact(\n ref_path,\n key=\"vitessce_examples/habib17.reference.json\",\n description=\"Reference JSON for H5AD file, Habib et al., 2017 Nature Methods\"\n).save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n\n# %% [markdown]\n# You can pass LaminDB Artifacts to the AnnDataWrapper class using the `adata_artifact` and `ref_artifact` [parameters](https://python-docs.vitessce.io/api_data.html#vitessce.wrappers.AnnDataWrapper).\n\n# %%\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=h5ad_artifact.description,\n)\ndataset = vc.add_dataset(name=\"Habib 2017\").add_object(\n vit.AnnDataWrapper(\n adata_artifact=h5ad_artifact,\n ref_artifact=ref_artifact,\n obs_feature_matrix_path=\"X\",\n obs_embedding_paths=[\"obsm/X_umap\"],\n obs_embedding_names=[\"UMAP\"],\n obs_set_paths=[\"obs/CellType\"],\n obs_set_names=[\"Cell Type\"],\n )\n)\nobs_sets = vc.add_view(vit.ViewType.OBS_SETS, dataset=dataset)\nobs_sets_sizes = vc.add_view(vit.ViewType.OBS_SET_SIZES, dataset=dataset)\nscatterplot = vc.add_view(vit.ViewType.SCATTERPLOT, dataset=dataset, mapping=\"UMAP\")\nheatmap = vc.add_view(vit.ViewType.HEATMAP, dataset=dataset)\ngenes = vc.add_view(vit.ViewType.FEATURE_LIST, dataset=dataset)\nvc.link_views([scatterplot, heatmap], [\"featureValueColormapRange\"], [[0.0, 0.1]])\nvc.layout(((scatterplot | obs_sets) / heatmap) | (obs_sets_sizes / genes));\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nh5ad_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View Habib17 (h5ad) in Vitessce\"\n)\n\n# %%\nh5ad_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# If your `VitessceConfig` object references data from multiple artifacts, the Vitessce button will appear next to a `Collection` that groups these artifacts (on the [Collections](https://lamin.ai/vitessce/examples/collections) tab of the Artifacts page).\n# Note that when using an `.h5ad`-based Artifact, the presence of the corresponding `.reference.json` file will result in the creation of a Collection.\n#\n# :::\n\n# %% [markdown]\n# ## Visualize an AnnData object (Zarr format)\n\n# %% [markdown]\n# AnnData objects can be saved on-disk to not only `.h5ad` files, but also to [Zarr stores](https://zarr.readthedocs.io/en/stable/tutorial.html#storage-alternatives) using AnnData's [write_zarr](https://anndata.readthedocs.io/en/latest/generated/anndata.AnnData.write_zarr.html) method.\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Convert the dataset to `.zarr` format. Often, Zarr data is stored on-disk using a [DirectoryStore](https://zarr.readthedocs.io/en/stable/api/storage.html#zarr.storage.DirectoryStore). By convention, these directories are named using a `.zarr` suffix.\n\n# %%\nzarr_processed_filepath = \"./habib17.anndata.zarr\"\n\n# %%\nadata = read_h5ad(h5ad_raw_filepath)\n\n# %% [markdown]\n# Just like in the above section, we may want to modify the AnnData object prior to saving it as a LaminDB Artifact that we will use for visualization.\n\n# %%\nadata = adata[:, adata.var[\"highly_variable\"]].copy()\n\n# %% [markdown]\n# Note the usage of `.write_zarr` below, as opposed to `.write_h5ad`.\n\n# %%\nadata.write_zarr(zarr_processed_filepath)\n\n# %% [markdown]\n# Save the `.zarr` directory as a LaminDB Artifact.\n\n# %%\nadata_zarr_artifact = ln.Artifact(\n zarr_processed_filepath,\n key=\"vitessce_examples/habib17.adata.zarr\",\n description=\"Habib et al., 2017 Nature Methods, zarr\",\n type=\"dataset\",\n)\nadata_zarr_artifact.save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n# Here, we configure the visualization the same way as above in the `.h5ad`-based example, with the exception of the `ref_artifact` parameter, as `.zarr`-based AnnData objects do not require a Reference Specification for Zarr interoperability.\n\n# %% tags=[\"hide-output\"]\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=adata_zarr_artifact.description,\n)\ndataset = vc.add_dataset(name=\"Habib 2017\").add_object(\n vit.AnnDataWrapper(\n adata_artifact=adata_zarr_artifact,\n obs_feature_matrix_path=\"X\",\n obs_embedding_paths=[\"obsm/X_umap\"],\n obs_embedding_names=[\"UMAP\"],\n obs_set_paths=[\"obs/CellType\"],\n obs_set_names=[\"Cell Type\"],\n )\n)\nobs_sets = vc.add_view(vit.Component.OBS_SETS, dataset=dataset)\nobs_sets_sizes = vc.add_view(vit.Component.OBS_SET_SIZES, dataset=dataset)\nscatterplot = vc.add_view(vit.Component.SCATTERPLOT, dataset=dataset, mapping=\"UMAP\")\nheatmap = vc.add_view(vit.Component.HEATMAP, dataset=dataset)\ngenes = vc.add_view(vit.Component.FEATURE_LIST, dataset=dataset)\nvc.link_views([scatterplot, heatmap], [\"featureValueColormapRange\"], [[0.0, 0.1]])\nvc.layout(((scatterplot | obs_sets) / heatmap) | (obs_sets_sizes / genes));\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nadata_zarr_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View Habib17 in Vitessce\"\n)\n\n# %%\nadata_zarr_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# :::\n\n# %% [markdown]\n# ## Visualize a SpatialData object (Zarr format)\n\n# %% [markdown]\n# Download and unzip the example SpatialData dataset (approximately 1.47 GB unzipped). Once unzipped, the data will be available via a local Zarr DirectoryStore.\n\n# %%\nsdata_zip_filepath = \"./visium.spatialdata.zarr.zip\"\nsdata_zarr_filepath = \"./visium.spatialdata.zarr\"\nif not Path(sdata_zarr_filepath).exists():\n if not Path(sdata_zip_filepath).exists():\n urlretrieve(\n \"https://s3.embl.de/spatialdata/spatialdata-sandbox/visium_associated_xenium_io.zip\", sdata_zip_filepath\n )\n with zipfile.ZipFile(sdata_zip_filepath, \"r\") as zip_ref:\n zip_ref.extractall(\".\")\n Path(\"data.zarr\").rename(sdata_zarr_filepath)\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Save the `.zarr` DirectoryStore as a LaminDB Artifact.\n\n# %%\nsdata_zarr_artifact = ln.Artifact(\n sdata_zarr_filepath,\n key=\"vitessce_examples/visium.sdata.zarr\",\n description=\"Visium SpatialData Example\",\n type=\"dataset\",\n)\nsdata_zarr_artifact.save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n# Here, we use the [SpatialDataWrapper](https://python-docs.vitessce.io/api_data.html#vitessce.wrappers.SpatialDataWrapper) class to specify which parts of the SpatialData object will be loaded for visualization.\n\n# %%\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=sdata_zarr_artifact.description,\n)\n# Add data to the configuration:\ndataset = vc.add_dataset(name='Breast Cancer Visium').add_object(\n vit.SpatialDataWrapper(\n sdata_artifact=sdata_zarr_artifact,\n # The following paths are relative to the root of the SpatialData zarr store on-disk.\n image_path=\"images/CytAssist_FFPE_Human_Breast_Cancer_full_image\",\n table_path=\"tables/table\",\n obs_feature_matrix_path=\"tables/table/X\",\n obs_spots_path=\"shapes/CytAssist_FFPE_Human_Breast_Cancer\",\n region=\"CytAssist_FFPE_Human_Breast_Cancer\",\n coordinate_system=\"global\",\n coordination_values={\n # The following tells Vitessce to consider each observation as a \"spot\"\n \"obsType\": \"spot\",\n }\n )\n)\n\n# Add views (visualizations) to the configuration:\nspatial = vc.add_view(\"spatialBeta\", dataset=dataset)\nfeature_list = vc.add_view(\"featureList\", dataset=dataset)\nlayer_controller = vc.add_view(\"layerControllerBeta\", dataset=dataset)\nvc.link_views_by_dict([spatial, layer_controller], {\n 'imageLayer': vit.CoordinationLevel([{\n 'photometricInterpretation': 'RGB',\n }]),\n}, scope_prefix=vit.get_initial_coordination_scope_prefix(\"A\", \"image\"))\nvc.link_views([spatial, layer_controller, feature_list, obs_sets], ['obsType'], [\"spot\"])\n\n# Layout the views\nvc.layout(spatial | (feature_list / layer_controller));\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nsdata_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View Visium SpatialData Example in Vitessce\"\n)\n\n# %%\nsdata_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# :::\n\n# %% [markdown]\n# ## Visualize an image (OME-TIFF format)\n\n# %% [markdown]\n# Vitesse can visualize data from multiple bioimaging file formats, including [OME-TIFF](https://ome-model.readthedocs.io/en/stable/ome-tiff/).\n\n# %% [markdown]\n# Here, we download an example OME-TIFF file (approximately 1.02 GB).\n\n# %%\nome_tiff_filepath = \"./VAN0006-LK-2-85-PAS_registered.ome.tif\"\nif not Path(ome_tiff_filepath).exists():\n urlretrieve(\n \"https://assets.hubmapconsortium.org/f4188a148e4c759092d19369d310883b/ometiff-pyramids/processedMicroscopy/VAN0006-LK-2-85-PAS_images/VAN0006-LK-2-85-PAS_registered.ome.tif\", ome_tiff_filepath\n )\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Save the `.ome.tif` file as a LaminDB Artifact.\n\n# %%\nome_tiff_artifact = ln.Artifact(\n ome_tiff_filepath,\n key=\"vitessce_examples/VAN0006-LK-2-85-PAS_registered.ome.tif\",\n description=\"PAS OME-TIFF file, Neumann et al., 2020\",\n type=\"dataset\",\n)\nome_tiff_artifact.save()\n\n# %% [markdown]\n# When using OME-TIFF files, we can use [generate-tiff-offsets](https://github.com/hms-dbmi/generate-tiff-offsets) to create an index for the bytes within the OME-TIFF file. We store these to a companion `.offsets.json` file which makes loading subsets of the image more efficient.\n\n# %%\noffsets = get_offsets(ome_tiff_filepath)\n\n# %%\noffsets_path = \"./VAN0006-LK-2-85-PAS_registered.offsets.json\"\nwith open(offsets_path, \"w\") as file:\n json.dump(offsets, file)\noffsets_artifact = ln.Artifact(\n offsets_path,\n key=\"vitessce_examples/VAN0006-LK-2-85-PAS_registered.offsets.json\",\n description=\"PAS offsets file, Neumann et al., 2020\"\n).save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n# Here, we use the [ImageOmeTiffWrapper](https://python-docs.vitessce.io/api_data.html#vitessce.wrappers.ImageOmeTiffWrapper) class to specify which pair of OME-TIFF file and offsets JSON file to load.\n\n# %%\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=ome_tiff_artifact.description\n)\ndataset = vc.add_dataset(\"Image\").add_object(\n vit.ImageOmeTiffWrapper(\n img_artifact=ome_tiff_artifact,\n offsets_artifact=offsets_artifact,\n )\n)\nspatial = vc.add_view(\"spatialBeta\", dataset=dataset)\nlayer_controller = vc.add_view(\"layerControllerBeta\", dataset=dataset)\nvc.layout(spatial | layer_controller);\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nome_tiff_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View PAS OME-TIFF, Neumann et al., 2020 in Vitessce\"\n)\n\n# %%\nome_tiff_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# If your `VitessceConfig` object references data from multiple artifacts, the Vitessce button will appear next to a `Collection` that groups these artifacts (on the [Collections](https://lamin.ai/vitessce/examples/collections) tab of the Artifacts page).\n# In the case of OME-TIFF, the presence of the corresponding offsets JSON file will result in the creation of a Collection.\n#\n# :::\n\n# %% [markdown]\n# ## Visualize an image (OME-Zarr format)\n\n# %% [markdown]\n# Retrieve an OME-Zarr (also known as OME-NGFF) formatted image. We could download an unzip such an image, but in downloading the SpatialData object above, we already have access to a valid OME-Zarr image locally.\n\n# %%\nome_zarr_filepath = Path(sdata_zarr_filepath) / \"images\" / \"CytAssist_FFPE_Human_Breast_Cancer_full_image\"\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Save the `.ome.zarr` DirectoryStore as a LaminDB Artifact.\n\n# %%\nome_zarr_artifact = ln.Artifact(\n ome_zarr_filepath,\n key=\"vitessce_examples/visium.ome.zarr\",\n description=\"Visium OME-Zarr Example\",\n type=\"dataset\",\n)\nome_zarr_artifact.save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n# Here, we use the [ImageOmeZarrWrapper](https://python-docs.vitessce.io/api_data.html#vitessce.wrappers.ImageOmeZarrWrapper) class to specify an OME-Zarr file to load for visualization.\n\n# %%\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=ome_zarr_artifact.description\n)\ndataset = vc.add_dataset(\"Image\").add_object(\n vit.ImageOmeZarrWrapper(\n img_artifact=ome_zarr_artifact,\n )\n)\nspatial = vc.add_view(\"spatialBeta\", dataset=dataset)\nlayer_controller = vc.add_view(\"layerControllerBeta\", dataset=dataset)\nvc.link_views_by_dict([spatial, layer_controller], {\n 'imageLayer': vit.CoordinationLevel([{\n 'photometricInterpretation': 'RGB',\n }]),\n}, scope_prefix=vit.get_initial_coordination_scope_prefix(\"A\", \"image\"))\nvc.layout(spatial | layer_controller);\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nome_zarr_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View Visium OME-Zarr Example in Vitessce\"\n)\n\n# %%\nome_zarr_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# :::\n\n# %%\n# [optional] finish run context and auto-save the notebook\nln.finish()\n\n# %% [markdown]\n# :::{dropdown} Upload speed\n#\n# Here is [a note](https://lamin.ai/laminlabs/lamindata/transform/WDjio16cQsdW5zKv) on folder upload speed and why `lamindb` does not use the `.export(to=\"s3\")` functionality of Vitessce.\n#\n# :::\n\n# %% tags=[\"hide-cell\"]\n# clean up artifacts in CI run\nh5ad_artifact.delete(permanent=True)\nref_artifact.delete(permanent=True)\n\nadata_zarr_artifact.delete(permanent=True)\nsdata_zarr_artifact.delete(permanent=True)\n\nome_tiff_artifact.delete(permanent=True)\noffsets_artifact.delete(permanent=True)\n\nome_zarr_artifact.delete(permanent=True)\n\nh5ad_vc_artifact.delete(permanent=True)\nadata_zarr_vc_artifact.delete(permanent=True)\nsdata_vc_artifact.delete(permanent=True)\nome_tiff_vc_artifact.delete(permanent=True)\nome_zarr_vc_artifact.delete(permanent=True)\n","_branch_code":1,"reference_type":null}}},{"id":42,"key":"vitessce_examples/VAN0006-LK-2-85-PAS_registered.ome.tif","uid":"0I5aMNdh2UrECtFX0000","_aux":null,"hash":"pa3FYqKL6afG8PduprhRJk","kind":"dataset","size":1021561156,"otype":null,"suffix":".tif","n_files":null,"version":null,"is_latest":true,"_hash_type":"sha1-fl","created_at":"2024-10-05T20:00:49.509347+00:00","updated_at":"2024-12-04T16:11:39.785667+00:00","description":"PAS OME-TIFF file, Neumann et al., 2020","_branch_code":1,"n_observations":null,"_key_is_virtual":true,"_overwrite_versions":false,"run":{"id":43,"uid":"MfsMSwTjRQpibcLciOyO","_aux":null,"name":null,"reference":null,"created_at":"2024-12-03T20:43:15.292036+00:00","started_at":"2024-12-04T15:42:15.45694+00:00","finished_at":"2024-12-04T15:43:14.152546+00:00","_branch_code":1,"_status_code":0,"reference_type":null,"_is_consecutive":false,"transform":{"id":14,"key":"vitessce.ipynb","uid":"XmcumVqvKZd30000","_aux":null,"hash":"sqz7fEl7R7YwU-R5fz7LnQ","type":"notebook","version":null,"is_latest":false,"reference":null,"created_at":"2024-12-03T20:43:15.109256+00:00","updated_at":"2024-12-04T15:52:35.669773+00:00","description":"Vitessce","source_code":"# %% [markdown]\n# [![](https://img.shields.io/badge/Source%20on%20GitHub-orange)](https://github.com/laminlabs/lamin-spatial/blob/main/docs/vitessce.ipynb) [![hub](https://img.shields.io/badge/View%20on%20LaminHub-mediumseagreen)](https://lamin.ai/laminlabs/lamindata/transform/hqtT4OTr5Tiq5zKv/URbaThYhljgXbZEzgj5x)\n\n# %% [markdown] nbsphinx=\"hidden\"\n# # transform.name\n\n# %% [markdown]\n# This tutorial has been adopted from the data preparation examples in [the Vitessce documention](https://vitessce.github.io/vitessce-python).\n#\n\n# %% tags=[\"hide-output\"]\n# # !pip install \"vitessce[all]\">=3.5.0\n# # !pip install \"generate-tiff-offsets==0.1.7\"\n# # !pip install 'lamindb[jupyter,aws,bionty]'\n# !lamin connect laminlabs/lamin-dev # <-- replace with your instance\n\n# %% tags=[\"hide-output\"]\nfrom urllib.request import urlretrieve\nfrom pathlib import Path\nfrom anndata import read_h5ad\nimport vitessce as vit\nfrom vitessce import data_utils as vitdu\nfrom generate_tiff_offsets import get_offsets\nimport numpy as np\nimport lamindb as ln\nimport zipfile\nimport json\n\n# [optional] track the current notebook or script\nln.context.uid = \"XmcumVqvKZd30000\"\nln.track()\n\n# %% [markdown]\n# ## Visualize an AnnData object (H5AD format)\n\n# %% [markdown]\n# Here, we download an example H5AD file (approximately 22 MB). This is a [dataset](https://www.covid19cellatlas.org/index.healthy.html#habib17) from the COVID-19 Cell Atlas.\n\n# %%\nh5ad_raw_filepath = \"./habib17.raw.h5ad\"\nh5ad_processed_filepath = \"./habib17.processed.h5ad\"\nif not Path(h5ad_raw_filepath).exists():\n urlretrieve(\n \"https://covid19.cog.sanger.ac.uk/habib17.processed.h5ad\", h5ad_raw_filepath\n )\n\n# %% [markdown]\n# This AnnData file was not saved recently and `read_h5ad` will warn us about this. By reading and subsequently re-writing the file using an up-to-date `anndata` package version, we ensure that the dataset is saved using the latest AnnData H5AD format.\n\n# %%\nadata = read_h5ad(h5ad_raw_filepath)\n\n# %% [markdown]\n# Optionally, we can modify the AnnData object, for example to subset it to the highly variable genes (which will affect the amount of data that is displayed in the heatmap visualization).\n\n# %%\nadata = adata[:, adata.var[\"highly_variable\"]].copy()\n\n# %%\nadata.write_h5ad(h5ad_processed_filepath)\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Save the `.h5ad` file as a LaminDB Artifact.\n\n# %%\nh5ad_artifact = ln.Artifact(\n h5ad_processed_filepath,\n key=\"vitessce_examples/habib17.h5ad\",\n description=\"Habib et al., 2017 Nature Methods, h5ad\",\n type=\"dataset\",\n)\nh5ad_artifact.save()\n\n# %% [markdown]\n# When using `.h5ad` files, we construct a [Reference Specification](https://fsspec.github.io/kerchunk/spec.html) which enables interoperability with the [Zarr](https://zarrita.dev/packages/storage.html#referencestore) interface.\n\n# %%\nref_spec = vitdu.generate_h5ad_ref_spec(h5ad_processed_filepath)\n\n# %% [markdown]\n# We next need to save the corresponding Reference Specification to a JSON file and upload it to LaminDB as an Artifact.\n\n# %%\nref_path = \"./habib17.processed.reference.json\"\nwith open(ref_path, \"w\") as file:\n json.dump(ref_spec, file)\nref_artifact = ln.Artifact(\n ref_path,\n key=\"vitessce_examples/habib17.reference.json\",\n description=\"Reference JSON for H5AD file, Habib et al., 2017 Nature Methods\"\n).save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n\n# %% [markdown]\n# You can pass LaminDB Artifacts to the AnnDataWrapper class using the `adata_artifact` and `ref_artifact` [parameters](https://python-docs.vitessce.io/api_data.html#vitessce.wrappers.AnnDataWrapper).\n\n# %%\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=h5ad_artifact.description,\n)\ndataset = vc.add_dataset(name=\"Habib 2017\").add_object(\n vit.AnnDataWrapper(\n adata_artifact=h5ad_artifact,\n ref_artifact=ref_artifact,\n obs_feature_matrix_path=\"X\",\n obs_embedding_paths=[\"obsm/X_umap\"],\n obs_embedding_names=[\"UMAP\"],\n obs_set_paths=[\"obs/CellType\"],\n obs_set_names=[\"Cell Type\"],\n )\n)\nobs_sets = vc.add_view(vit.ViewType.OBS_SETS, dataset=dataset)\nobs_sets_sizes = vc.add_view(vit.ViewType.OBS_SET_SIZES, dataset=dataset)\nscatterplot = vc.add_view(vit.ViewType.SCATTERPLOT, dataset=dataset, mapping=\"UMAP\")\nheatmap = vc.add_view(vit.ViewType.HEATMAP, dataset=dataset)\ngenes = vc.add_view(vit.ViewType.FEATURE_LIST, dataset=dataset)\nvc.link_views([scatterplot, heatmap], [\"featureValueColormapRange\"], [[0.0, 0.1]])\nvc.layout(((scatterplot | obs_sets) / heatmap) | (obs_sets_sizes / genes));\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nh5ad_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View Habib17 (h5ad) in Vitessce\"\n)\n\n# %%\nh5ad_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# If your `VitessceConfig` object references data from multiple artifacts, the Vitessce button will appear next to a `Collection` that groups these artifacts (on the [Collections](https://lamin.ai/vitessce/examples/collections) tab of the Artifacts page).\n# Note that when using an `.h5ad`-based Artifact, the presence of the corresponding `.reference.json` file will result in the creation of a Collection.\n#\n# :::\n\n# %% [markdown]\n# ## Visualize an AnnData object (Zarr format)\n\n# %% [markdown]\n# AnnData objects can be saved on-disk to not only `.h5ad` files, but also to [Zarr stores](https://zarr.readthedocs.io/en/stable/tutorial.html#storage-alternatives) using AnnData's [write_zarr](https://anndata.readthedocs.io/en/latest/generated/anndata.AnnData.write_zarr.html) method.\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Convert the dataset to `.zarr` format. Often, Zarr data is stored on-disk using a [DirectoryStore](https://zarr.readthedocs.io/en/stable/api/storage.html#zarr.storage.DirectoryStore). By convention, these directories are named using a `.zarr` suffix.\n\n# %%\nzarr_processed_filepath = \"./habib17.anndata.zarr\"\n\n# %%\nadata = read_h5ad(h5ad_raw_filepath)\n\n# %% [markdown]\n# Just like in the above section, we may want to modify the AnnData object prior to saving it as a LaminDB Artifact that we will use for visualization.\n\n# %%\nadata = adata[:, adata.var[\"highly_variable\"]].copy()\n\n# %% [markdown]\n# Note the usage of `.write_zarr` below, as opposed to `.write_h5ad`.\n\n# %%\nadata.write_zarr(zarr_processed_filepath)\n\n# %% [markdown]\n# Save the `.zarr` directory as a LaminDB Artifact.\n\n# %%\nadata_zarr_artifact = ln.Artifact(\n zarr_processed_filepath,\n key=\"vitessce_examples/habib17.adata.zarr\",\n description=\"Habib et al., 2017 Nature Methods, zarr\",\n type=\"dataset\",\n)\nadata_zarr_artifact.save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n# Here, we configure the visualization the same way as above in the `.h5ad`-based example, with the exception of the `ref_artifact` parameter, as `.zarr`-based AnnData objects do not require a Reference Specification for Zarr interoperability.\n\n# %% tags=[\"hide-output\"]\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=adata_zarr_artifact.description,\n)\ndataset = vc.add_dataset(name=\"Habib 2017\").add_object(\n vit.AnnDataWrapper(\n adata_artifact=adata_zarr_artifact,\n obs_feature_matrix_path=\"X\",\n obs_embedding_paths=[\"obsm/X_umap\"],\n obs_embedding_names=[\"UMAP\"],\n obs_set_paths=[\"obs/CellType\"],\n obs_set_names=[\"Cell Type\"],\n )\n)\nobs_sets = vc.add_view(vit.Component.OBS_SETS, dataset=dataset)\nobs_sets_sizes = vc.add_view(vit.Component.OBS_SET_SIZES, dataset=dataset)\nscatterplot = vc.add_view(vit.Component.SCATTERPLOT, dataset=dataset, mapping=\"UMAP\")\nheatmap = vc.add_view(vit.Component.HEATMAP, dataset=dataset)\ngenes = vc.add_view(vit.Component.FEATURE_LIST, dataset=dataset)\nvc.link_views([scatterplot, heatmap], [\"featureValueColormapRange\"], [[0.0, 0.1]])\nvc.layout(((scatterplot | obs_sets) / heatmap) | (obs_sets_sizes / genes));\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nadata_zarr_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View Habib17 in Vitessce\"\n)\n\n# %%\nadata_zarr_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# :::\n\n# %% [markdown]\n# ## Visualize a SpatialData object (Zarr format)\n\n# %% [markdown]\n# Download and unzip the example SpatialData dataset (approximately 1.47 GB unzipped). Once unzipped, the data will be available via a local Zarr DirectoryStore.\n\n# %%\nsdata_zip_filepath = \"./visium.spatialdata.zarr.zip\"\nsdata_zarr_filepath = \"./visium.spatialdata.zarr\"\nif not Path(sdata_zarr_filepath).exists():\n if not Path(sdata_zip_filepath).exists():\n urlretrieve(\n \"https://s3.embl.de/spatialdata/spatialdata-sandbox/visium_associated_xenium_io.zip\", sdata_zip_filepath\n )\n with zipfile.ZipFile(sdata_zip_filepath, \"r\") as zip_ref:\n zip_ref.extractall(\".\")\n Path(\"data.zarr\").rename(sdata_zarr_filepath)\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Save the `.zarr` DirectoryStore as a LaminDB Artifact.\n\n# %%\nsdata_zarr_artifact = ln.Artifact(\n sdata_zarr_filepath,\n key=\"vitessce_examples/visium.sdata.zarr\",\n description=\"Visium SpatialData Example\",\n type=\"dataset\",\n)\nsdata_zarr_artifact.save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n# Here, we use the [SpatialDataWrapper](https://python-docs.vitessce.io/api_data.html#vitessce.wrappers.SpatialDataWrapper) class to specify which parts of the SpatialData object will be loaded for visualization.\n\n# %%\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=sdata_zarr_artifact.description,\n)\n# Add data to the configuration:\ndataset = vc.add_dataset(name='Breast Cancer Visium').add_object(\n vit.SpatialDataWrapper(\n sdata_artifact=sdata_zarr_artifact,\n # The following paths are relative to the root of the SpatialData zarr store on-disk.\n image_path=\"images/CytAssist_FFPE_Human_Breast_Cancer_full_image\",\n table_path=\"tables/table\",\n obs_feature_matrix_path=\"tables/table/X\",\n obs_spots_path=\"shapes/CytAssist_FFPE_Human_Breast_Cancer\",\n region=\"CytAssist_FFPE_Human_Breast_Cancer\",\n coordinate_system=\"global\",\n coordination_values={\n # The following tells Vitessce to consider each observation as a \"spot\"\n \"obsType\": \"spot\",\n }\n )\n)\n\n# Add views (visualizations) to the configuration:\nspatial = vc.add_view(\"spatialBeta\", dataset=dataset)\nfeature_list = vc.add_view(\"featureList\", dataset=dataset)\nlayer_controller = vc.add_view(\"layerControllerBeta\", dataset=dataset)\nvc.link_views_by_dict([spatial, layer_controller], {\n 'imageLayer': vit.CoordinationLevel([{\n 'photometricInterpretation': 'RGB',\n }]),\n}, scope_prefix=vit.get_initial_coordination_scope_prefix(\"A\", \"image\"))\nvc.link_views([spatial, layer_controller, feature_list, obs_sets], ['obsType'], [\"spot\"])\n\n# Layout the views\nvc.layout(spatial | (feature_list / layer_controller));\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nsdata_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View Visium SpatialData Example in Vitessce\"\n)\n\n# %%\nsdata_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# :::\n\n# %% [markdown]\n# ## Visualize an image (OME-TIFF format)\n\n# %% [markdown]\n# Vitesse can visualize data from multiple bioimaging file formats, including [OME-TIFF](https://ome-model.readthedocs.io/en/stable/ome-tiff/).\n\n# %% [markdown]\n# Here, we download an example OME-TIFF file (approximately 1.02 GB).\n\n# %%\nome_tiff_filepath = \"./VAN0006-LK-2-85-PAS_registered.ome.tif\"\nif not Path(ome_tiff_filepath).exists():\n urlretrieve(\n \"https://assets.hubmapconsortium.org/f4188a148e4c759092d19369d310883b/ometiff-pyramids/processedMicroscopy/VAN0006-LK-2-85-PAS_images/VAN0006-LK-2-85-PAS_registered.ome.tif\", ome_tiff_filepath\n )\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Save the `.ome.tif` file as a LaminDB Artifact.\n\n# %%\nome_tiff_artifact = ln.Artifact(\n ome_tiff_filepath,\n key=\"vitessce_examples/VAN0006-LK-2-85-PAS_registered.ome.tif\",\n description=\"PAS OME-TIFF file, Neumann et al., 2020\",\n type=\"dataset\",\n)\nome_tiff_artifact.save()\n\n# %% [markdown]\n# When using OME-TIFF files, we can use [generate-tiff-offsets](https://github.com/hms-dbmi/generate-tiff-offsets) to create an index for the bytes within the OME-TIFF file. We store these to a companion `.offsets.json` file which makes loading subsets of the image more efficient.\n\n# %%\noffsets = get_offsets(ome_tiff_filepath)\n\n# %%\noffsets_path = \"./VAN0006-LK-2-85-PAS_registered.offsets.json\"\nwith open(offsets_path, \"w\") as file:\n json.dump(offsets, file)\noffsets_artifact = ln.Artifact(\n offsets_path,\n key=\"vitessce_examples/VAN0006-LK-2-85-PAS_registered.offsets.json\",\n description=\"PAS offsets file, Neumann et al., 2020\"\n).save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n# Here, we use the [ImageOmeTiffWrapper](https://python-docs.vitessce.io/api_data.html#vitessce.wrappers.ImageOmeTiffWrapper) class to specify which pair of OME-TIFF file and offsets JSON file to load.\n\n# %%\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=ome_tiff_artifact.description\n)\ndataset = vc.add_dataset(\"Image\").add_object(\n vit.ImageOmeTiffWrapper(\n img_artifact=ome_tiff_artifact,\n offsets_artifact=offsets_artifact,\n )\n)\nspatial = vc.add_view(\"spatialBeta\", dataset=dataset)\nlayer_controller = vc.add_view(\"layerControllerBeta\", dataset=dataset)\nvc.layout(spatial | layer_controller);\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nome_tiff_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View PAS OME-TIFF, Neumann et al., 2020 in Vitessce\"\n)\n\n# %%\nome_tiff_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# If your `VitessceConfig` object references data from multiple artifacts, the Vitessce button will appear next to a `Collection` that groups these artifacts (on the [Collections](https://lamin.ai/vitessce/examples/collections) tab of the Artifacts page).\n# In the case of OME-TIFF, the presence of the corresponding offsets JSON file will result in the creation of a Collection.\n#\n# :::\n\n# %% [markdown]\n# ## Visualize an image (OME-Zarr format)\n\n# %% [markdown]\n# Retrieve an OME-Zarr (also known as OME-NGFF) formatted image. We could download an unzip such an image, but in downloading the SpatialData object above, we already have access to a valid OME-Zarr image locally.\n\n# %%\nome_zarr_filepath = Path(sdata_zarr_filepath) / \"images\" / \"CytAssist_FFPE_Human_Breast_Cancer_full_image\"\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Save the `.ome.zarr` DirectoryStore as a LaminDB Artifact.\n\n# %%\nome_zarr_artifact = ln.Artifact(\n ome_zarr_filepath,\n key=\"vitessce_examples/visium.ome.zarr\",\n description=\"Visium OME-Zarr Example\",\n type=\"dataset\",\n)\nome_zarr_artifact.save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n# Here, we use the [ImageOmeZarrWrapper](https://python-docs.vitessce.io/api_data.html#vitessce.wrappers.ImageOmeZarrWrapper) class to specify an OME-Zarr file to load for visualization.\n\n# %%\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=ome_zarr_artifact.description\n)\ndataset = vc.add_dataset(\"Image\").add_object(\n vit.ImageOmeZarrWrapper(\n img_artifact=ome_zarr_artifact,\n )\n)\nspatial = vc.add_view(\"spatialBeta\", dataset=dataset)\nlayer_controller = vc.add_view(\"layerControllerBeta\", dataset=dataset)\nvc.link_views_by_dict([spatial, layer_controller], {\n 'imageLayer': vit.CoordinationLevel([{\n 'photometricInterpretation': 'RGB',\n }]),\n}, scope_prefix=vit.get_initial_coordination_scope_prefix(\"A\", \"image\"))\nvc.layout(spatial | layer_controller);\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nome_zarr_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View Visium OME-Zarr Example in Vitessce\"\n)\n\n# %%\nome_zarr_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# :::\n\n# %%\n# [optional] finish run context and auto-save the notebook\nln.finish()\n\n# %% [markdown]\n# :::{dropdown} Upload speed\n#\n# Here is [a note](https://lamin.ai/laminlabs/lamindata/transform/WDjio16cQsdW5zKv) on folder upload speed and why `lamindb` does not use the `.export(to=\"s3\")` functionality of Vitessce.\n#\n# :::\n\n# %% tags=[\"hide-cell\"]\n# clean up artifacts in CI run\nh5ad_artifact.delete(permanent=True)\nref_artifact.delete(permanent=True)\n\nadata_zarr_artifact.delete(permanent=True)\nsdata_zarr_artifact.delete(permanent=True)\n\nome_tiff_artifact.delete(permanent=True)\noffsets_artifact.delete(permanent=True)\n\nome_zarr_artifact.delete(permanent=True)\n\nh5ad_vc_artifact.delete(permanent=True)\nadata_zarr_vc_artifact.delete(permanent=True)\nsdata_vc_artifact.delete(permanent=True)\nome_tiff_vc_artifact.delete(permanent=True)\nome_zarr_vc_artifact.delete(permanent=True)\n","_branch_code":1,"reference_type":null}}},{"id":43,"key":"vitessce_examples/VAN0006-LK-2-85-PAS_registered.offsets.json","uid":"CMBUzbNNCEDGCKcQ0000","_aux":null,"hash":"5ADv5d7DI5oYVaMOPnhqiw","kind":"dataset","size":36,"otype":null,"suffix":".json","n_files":null,"version":null,"is_latest":true,"_hash_type":"md5","created_at":"2024-10-05T20:18:38.092796+00:00","updated_at":"2024-12-04T16:11:40.945337+00:00","description":"PAS offsets file, Neumann et al., 2020","_branch_code":1,"n_observations":null,"_key_is_virtual":true,"_overwrite_versions":false,"run":{"id":43,"uid":"MfsMSwTjRQpibcLciOyO","_aux":null,"name":null,"reference":null,"created_at":"2024-12-03T20:43:15.292036+00:00","started_at":"2024-12-04T15:42:15.45694+00:00","finished_at":"2024-12-04T15:43:14.152546+00:00","_branch_code":1,"_status_code":0,"reference_type":null,"_is_consecutive":false,"transform":{"id":14,"key":"vitessce.ipynb","uid":"XmcumVqvKZd30000","_aux":null,"hash":"sqz7fEl7R7YwU-R5fz7LnQ","type":"notebook","version":null,"is_latest":false,"reference":null,"created_at":"2024-12-03T20:43:15.109256+00:00","updated_at":"2024-12-04T15:52:35.669773+00:00","description":"Vitessce","source_code":"# %% [markdown]\n# [![](https://img.shields.io/badge/Source%20on%20GitHub-orange)](https://github.com/laminlabs/lamin-spatial/blob/main/docs/vitessce.ipynb) [![hub](https://img.shields.io/badge/View%20on%20LaminHub-mediumseagreen)](https://lamin.ai/laminlabs/lamindata/transform/hqtT4OTr5Tiq5zKv/URbaThYhljgXbZEzgj5x)\n\n# %% [markdown] nbsphinx=\"hidden\"\n# # transform.name\n\n# %% [markdown]\n# This tutorial has been adopted from the data preparation examples in [the Vitessce documention](https://vitessce.github.io/vitessce-python).\n#\n\n# %% tags=[\"hide-output\"]\n# # !pip install \"vitessce[all]\">=3.5.0\n# # !pip install \"generate-tiff-offsets==0.1.7\"\n# # !pip install 'lamindb[jupyter,aws,bionty]'\n# !lamin connect laminlabs/lamin-dev # <-- replace with your instance\n\n# %% tags=[\"hide-output\"]\nfrom urllib.request import urlretrieve\nfrom pathlib import Path\nfrom anndata import read_h5ad\nimport vitessce as vit\nfrom vitessce import data_utils as vitdu\nfrom generate_tiff_offsets import get_offsets\nimport numpy as np\nimport lamindb as ln\nimport zipfile\nimport json\n\n# [optional] track the current notebook or script\nln.context.uid = \"XmcumVqvKZd30000\"\nln.track()\n\n# %% [markdown]\n# ## Visualize an AnnData object (H5AD format)\n\n# %% [markdown]\n# Here, we download an example H5AD file (approximately 22 MB). This is a [dataset](https://www.covid19cellatlas.org/index.healthy.html#habib17) from the COVID-19 Cell Atlas.\n\n# %%\nh5ad_raw_filepath = \"./habib17.raw.h5ad\"\nh5ad_processed_filepath = \"./habib17.processed.h5ad\"\nif not Path(h5ad_raw_filepath).exists():\n urlretrieve(\n \"https://covid19.cog.sanger.ac.uk/habib17.processed.h5ad\", h5ad_raw_filepath\n )\n\n# %% [markdown]\n# This AnnData file was not saved recently and `read_h5ad` will warn us about this. By reading and subsequently re-writing the file using an up-to-date `anndata` package version, we ensure that the dataset is saved using the latest AnnData H5AD format.\n\n# %%\nadata = read_h5ad(h5ad_raw_filepath)\n\n# %% [markdown]\n# Optionally, we can modify the AnnData object, for example to subset it to the highly variable genes (which will affect the amount of data that is displayed in the heatmap visualization).\n\n# %%\nadata = adata[:, adata.var[\"highly_variable\"]].copy()\n\n# %%\nadata.write_h5ad(h5ad_processed_filepath)\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Save the `.h5ad` file as a LaminDB Artifact.\n\n# %%\nh5ad_artifact = ln.Artifact(\n h5ad_processed_filepath,\n key=\"vitessce_examples/habib17.h5ad\",\n description=\"Habib et al., 2017 Nature Methods, h5ad\",\n type=\"dataset\",\n)\nh5ad_artifact.save()\n\n# %% [markdown]\n# When using `.h5ad` files, we construct a [Reference Specification](https://fsspec.github.io/kerchunk/spec.html) which enables interoperability with the [Zarr](https://zarrita.dev/packages/storage.html#referencestore) interface.\n\n# %%\nref_spec = vitdu.generate_h5ad_ref_spec(h5ad_processed_filepath)\n\n# %% [markdown]\n# We next need to save the corresponding Reference Specification to a JSON file and upload it to LaminDB as an Artifact.\n\n# %%\nref_path = \"./habib17.processed.reference.json\"\nwith open(ref_path, \"w\") as file:\n json.dump(ref_spec, file)\nref_artifact = ln.Artifact(\n ref_path,\n key=\"vitessce_examples/habib17.reference.json\",\n description=\"Reference JSON for H5AD file, Habib et al., 2017 Nature Methods\"\n).save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n\n# %% [markdown]\n# You can pass LaminDB Artifacts to the AnnDataWrapper class using the `adata_artifact` and `ref_artifact` [parameters](https://python-docs.vitessce.io/api_data.html#vitessce.wrappers.AnnDataWrapper).\n\n# %%\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=h5ad_artifact.description,\n)\ndataset = vc.add_dataset(name=\"Habib 2017\").add_object(\n vit.AnnDataWrapper(\n adata_artifact=h5ad_artifact,\n ref_artifact=ref_artifact,\n obs_feature_matrix_path=\"X\",\n obs_embedding_paths=[\"obsm/X_umap\"],\n obs_embedding_names=[\"UMAP\"],\n obs_set_paths=[\"obs/CellType\"],\n obs_set_names=[\"Cell Type\"],\n )\n)\nobs_sets = vc.add_view(vit.ViewType.OBS_SETS, dataset=dataset)\nobs_sets_sizes = vc.add_view(vit.ViewType.OBS_SET_SIZES, dataset=dataset)\nscatterplot = vc.add_view(vit.ViewType.SCATTERPLOT, dataset=dataset, mapping=\"UMAP\")\nheatmap = vc.add_view(vit.ViewType.HEATMAP, dataset=dataset)\ngenes = vc.add_view(vit.ViewType.FEATURE_LIST, dataset=dataset)\nvc.link_views([scatterplot, heatmap], [\"featureValueColormapRange\"], [[0.0, 0.1]])\nvc.layout(((scatterplot | obs_sets) / heatmap) | (obs_sets_sizes / genes));\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nh5ad_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View Habib17 (h5ad) in Vitessce\"\n)\n\n# %%\nh5ad_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# If your `VitessceConfig` object references data from multiple artifacts, the Vitessce button will appear next to a `Collection` that groups these artifacts (on the [Collections](https://lamin.ai/vitessce/examples/collections) tab of the Artifacts page).\n# Note that when using an `.h5ad`-based Artifact, the presence of the corresponding `.reference.json` file will result in the creation of a Collection.\n#\n# :::\n\n# %% [markdown]\n# ## Visualize an AnnData object (Zarr format)\n\n# %% [markdown]\n# AnnData objects can be saved on-disk to not only `.h5ad` files, but also to [Zarr stores](https://zarr.readthedocs.io/en/stable/tutorial.html#storage-alternatives) using AnnData's [write_zarr](https://anndata.readthedocs.io/en/latest/generated/anndata.AnnData.write_zarr.html) method.\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Convert the dataset to `.zarr` format. Often, Zarr data is stored on-disk using a [DirectoryStore](https://zarr.readthedocs.io/en/stable/api/storage.html#zarr.storage.DirectoryStore). By convention, these directories are named using a `.zarr` suffix.\n\n# %%\nzarr_processed_filepath = \"./habib17.anndata.zarr\"\n\n# %%\nadata = read_h5ad(h5ad_raw_filepath)\n\n# %% [markdown]\n# Just like in the above section, we may want to modify the AnnData object prior to saving it as a LaminDB Artifact that we will use for visualization.\n\n# %%\nadata = adata[:, adata.var[\"highly_variable\"]].copy()\n\n# %% [markdown]\n# Note the usage of `.write_zarr` below, as opposed to `.write_h5ad`.\n\n# %%\nadata.write_zarr(zarr_processed_filepath)\n\n# %% [markdown]\n# Save the `.zarr` directory as a LaminDB Artifact.\n\n# %%\nadata_zarr_artifact = ln.Artifact(\n zarr_processed_filepath,\n key=\"vitessce_examples/habib17.adata.zarr\",\n description=\"Habib et al., 2017 Nature Methods, zarr\",\n type=\"dataset\",\n)\nadata_zarr_artifact.save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n# Here, we configure the visualization the same way as above in the `.h5ad`-based example, with the exception of the `ref_artifact` parameter, as `.zarr`-based AnnData objects do not require a Reference Specification for Zarr interoperability.\n\n# %% tags=[\"hide-output\"]\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=adata_zarr_artifact.description,\n)\ndataset = vc.add_dataset(name=\"Habib 2017\").add_object(\n vit.AnnDataWrapper(\n adata_artifact=adata_zarr_artifact,\n obs_feature_matrix_path=\"X\",\n obs_embedding_paths=[\"obsm/X_umap\"],\n obs_embedding_names=[\"UMAP\"],\n obs_set_paths=[\"obs/CellType\"],\n obs_set_names=[\"Cell Type\"],\n )\n)\nobs_sets = vc.add_view(vit.Component.OBS_SETS, dataset=dataset)\nobs_sets_sizes = vc.add_view(vit.Component.OBS_SET_SIZES, dataset=dataset)\nscatterplot = vc.add_view(vit.Component.SCATTERPLOT, dataset=dataset, mapping=\"UMAP\")\nheatmap = vc.add_view(vit.Component.HEATMAP, dataset=dataset)\ngenes = vc.add_view(vit.Component.FEATURE_LIST, dataset=dataset)\nvc.link_views([scatterplot, heatmap], [\"featureValueColormapRange\"], [[0.0, 0.1]])\nvc.layout(((scatterplot | obs_sets) / heatmap) | (obs_sets_sizes / genes));\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nadata_zarr_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View Habib17 in Vitessce\"\n)\n\n# %%\nadata_zarr_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# :::\n\n# %% [markdown]\n# ## Visualize a SpatialData object (Zarr format)\n\n# %% [markdown]\n# Download and unzip the example SpatialData dataset (approximately 1.47 GB unzipped). Once unzipped, the data will be available via a local Zarr DirectoryStore.\n\n# %%\nsdata_zip_filepath = \"./visium.spatialdata.zarr.zip\"\nsdata_zarr_filepath = \"./visium.spatialdata.zarr\"\nif not Path(sdata_zarr_filepath).exists():\n if not Path(sdata_zip_filepath).exists():\n urlretrieve(\n \"https://s3.embl.de/spatialdata/spatialdata-sandbox/visium_associated_xenium_io.zip\", sdata_zip_filepath\n )\n with zipfile.ZipFile(sdata_zip_filepath, \"r\") as zip_ref:\n zip_ref.extractall(\".\")\n Path(\"data.zarr\").rename(sdata_zarr_filepath)\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Save the `.zarr` DirectoryStore as a LaminDB Artifact.\n\n# %%\nsdata_zarr_artifact = ln.Artifact(\n sdata_zarr_filepath,\n key=\"vitessce_examples/visium.sdata.zarr\",\n description=\"Visium SpatialData Example\",\n type=\"dataset\",\n)\nsdata_zarr_artifact.save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n# Here, we use the [SpatialDataWrapper](https://python-docs.vitessce.io/api_data.html#vitessce.wrappers.SpatialDataWrapper) class to specify which parts of the SpatialData object will be loaded for visualization.\n\n# %%\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=sdata_zarr_artifact.description,\n)\n# Add data to the configuration:\ndataset = vc.add_dataset(name='Breast Cancer Visium').add_object(\n vit.SpatialDataWrapper(\n sdata_artifact=sdata_zarr_artifact,\n # The following paths are relative to the root of the SpatialData zarr store on-disk.\n image_path=\"images/CytAssist_FFPE_Human_Breast_Cancer_full_image\",\n table_path=\"tables/table\",\n obs_feature_matrix_path=\"tables/table/X\",\n obs_spots_path=\"shapes/CytAssist_FFPE_Human_Breast_Cancer\",\n region=\"CytAssist_FFPE_Human_Breast_Cancer\",\n coordinate_system=\"global\",\n coordination_values={\n # The following tells Vitessce to consider each observation as a \"spot\"\n \"obsType\": \"spot\",\n }\n )\n)\n\n# Add views (visualizations) to the configuration:\nspatial = vc.add_view(\"spatialBeta\", dataset=dataset)\nfeature_list = vc.add_view(\"featureList\", dataset=dataset)\nlayer_controller = vc.add_view(\"layerControllerBeta\", dataset=dataset)\nvc.link_views_by_dict([spatial, layer_controller], {\n 'imageLayer': vit.CoordinationLevel([{\n 'photometricInterpretation': 'RGB',\n }]),\n}, scope_prefix=vit.get_initial_coordination_scope_prefix(\"A\", \"image\"))\nvc.link_views([spatial, layer_controller, feature_list, obs_sets], ['obsType'], [\"spot\"])\n\n# Layout the views\nvc.layout(spatial | (feature_list / layer_controller));\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nsdata_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View Visium SpatialData Example in Vitessce\"\n)\n\n# %%\nsdata_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# :::\n\n# %% [markdown]\n# ## Visualize an image (OME-TIFF format)\n\n# %% [markdown]\n# Vitesse can visualize data from multiple bioimaging file formats, including [OME-TIFF](https://ome-model.readthedocs.io/en/stable/ome-tiff/).\n\n# %% [markdown]\n# Here, we download an example OME-TIFF file (approximately 1.02 GB).\n\n# %%\nome_tiff_filepath = \"./VAN0006-LK-2-85-PAS_registered.ome.tif\"\nif not Path(ome_tiff_filepath).exists():\n urlretrieve(\n \"https://assets.hubmapconsortium.org/f4188a148e4c759092d19369d310883b/ometiff-pyramids/processedMicroscopy/VAN0006-LK-2-85-PAS_images/VAN0006-LK-2-85-PAS_registered.ome.tif\", ome_tiff_filepath\n )\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Save the `.ome.tif` file as a LaminDB Artifact.\n\n# %%\nome_tiff_artifact = ln.Artifact(\n ome_tiff_filepath,\n key=\"vitessce_examples/VAN0006-LK-2-85-PAS_registered.ome.tif\",\n description=\"PAS OME-TIFF file, Neumann et al., 2020\",\n type=\"dataset\",\n)\nome_tiff_artifact.save()\n\n# %% [markdown]\n# When using OME-TIFF files, we can use [generate-tiff-offsets](https://github.com/hms-dbmi/generate-tiff-offsets) to create an index for the bytes within the OME-TIFF file. We store these to a companion `.offsets.json` file which makes loading subsets of the image more efficient.\n\n# %%\noffsets = get_offsets(ome_tiff_filepath)\n\n# %%\noffsets_path = \"./VAN0006-LK-2-85-PAS_registered.offsets.json\"\nwith open(offsets_path, \"w\") as file:\n json.dump(offsets, file)\noffsets_artifact = ln.Artifact(\n offsets_path,\n key=\"vitessce_examples/VAN0006-LK-2-85-PAS_registered.offsets.json\",\n description=\"PAS offsets file, Neumann et al., 2020\"\n).save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n# Here, we use the [ImageOmeTiffWrapper](https://python-docs.vitessce.io/api_data.html#vitessce.wrappers.ImageOmeTiffWrapper) class to specify which pair of OME-TIFF file and offsets JSON file to load.\n\n# %%\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=ome_tiff_artifact.description\n)\ndataset = vc.add_dataset(\"Image\").add_object(\n vit.ImageOmeTiffWrapper(\n img_artifact=ome_tiff_artifact,\n offsets_artifact=offsets_artifact,\n )\n)\nspatial = vc.add_view(\"spatialBeta\", dataset=dataset)\nlayer_controller = vc.add_view(\"layerControllerBeta\", dataset=dataset)\nvc.layout(spatial | layer_controller);\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nome_tiff_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View PAS OME-TIFF, Neumann et al., 2020 in Vitessce\"\n)\n\n# %%\nome_tiff_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# If your `VitessceConfig` object references data from multiple artifacts, the Vitessce button will appear next to a `Collection` that groups these artifacts (on the [Collections](https://lamin.ai/vitessce/examples/collections) tab of the Artifacts page).\n# In the case of OME-TIFF, the presence of the corresponding offsets JSON file will result in the creation of a Collection.\n#\n# :::\n\n# %% [markdown]\n# ## Visualize an image (OME-Zarr format)\n\n# %% [markdown]\n# Retrieve an OME-Zarr (also known as OME-NGFF) formatted image. We could download an unzip such an image, but in downloading the SpatialData object above, we already have access to a valid OME-Zarr image locally.\n\n# %%\nome_zarr_filepath = Path(sdata_zarr_filepath) / \"images\" / \"CytAssist_FFPE_Human_Breast_Cancer_full_image\"\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Save the `.ome.zarr` DirectoryStore as a LaminDB Artifact.\n\n# %%\nome_zarr_artifact = ln.Artifact(\n ome_zarr_filepath,\n key=\"vitessce_examples/visium.ome.zarr\",\n description=\"Visium OME-Zarr Example\",\n type=\"dataset\",\n)\nome_zarr_artifact.save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n# Here, we use the [ImageOmeZarrWrapper](https://python-docs.vitessce.io/api_data.html#vitessce.wrappers.ImageOmeZarrWrapper) class to specify an OME-Zarr file to load for visualization.\n\n# %%\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=ome_zarr_artifact.description\n)\ndataset = vc.add_dataset(\"Image\").add_object(\n vit.ImageOmeZarrWrapper(\n img_artifact=ome_zarr_artifact,\n )\n)\nspatial = vc.add_view(\"spatialBeta\", dataset=dataset)\nlayer_controller = vc.add_view(\"layerControllerBeta\", dataset=dataset)\nvc.link_views_by_dict([spatial, layer_controller], {\n 'imageLayer': vit.CoordinationLevel([{\n 'photometricInterpretation': 'RGB',\n }]),\n}, scope_prefix=vit.get_initial_coordination_scope_prefix(\"A\", \"image\"))\nvc.layout(spatial | layer_controller);\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nome_zarr_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View Visium OME-Zarr Example in Vitessce\"\n)\n\n# %%\nome_zarr_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# :::\n\n# %%\n# [optional] finish run context and auto-save the notebook\nln.finish()\n\n# %% [markdown]\n# :::{dropdown} Upload speed\n#\n# Here is [a note](https://lamin.ai/laminlabs/lamindata/transform/WDjio16cQsdW5zKv) on folder upload speed and why `lamindb` does not use the `.export(to=\"s3\")` functionality of Vitessce.\n#\n# :::\n\n# %% tags=[\"hide-cell\"]\n# clean up artifacts in CI run\nh5ad_artifact.delete(permanent=True)\nref_artifact.delete(permanent=True)\n\nadata_zarr_artifact.delete(permanent=True)\nsdata_zarr_artifact.delete(permanent=True)\n\nome_tiff_artifact.delete(permanent=True)\noffsets_artifact.delete(permanent=True)\n\nome_zarr_artifact.delete(permanent=True)\n\nh5ad_vc_artifact.delete(permanent=True)\nadata_zarr_vc_artifact.delete(permanent=True)\nsdata_vc_artifact.delete(permanent=True)\nome_tiff_vc_artifact.delete(permanent=True)\nome_zarr_vc_artifact.delete(permanent=True)\n","_branch_code":1,"reference_type":null}}},{"id":77,"key":"vitessce_examples/visium.ome.zarr","uid":"toCvpsqAiH3XcAvT0000","_aux":null,"hash":"UtsNOkk05EFL34hgUabEwg","kind":"dataset","size":1400251840,"otype":null,"suffix":"","n_files":8809,"version":null,"is_latest":true,"_hash_type":"md5-d","created_at":"2024-12-04T14:38:29.474709+00:00","updated_at":"2024-12-04T16:11:51.300094+00:00","description":"Visium OME-Zarr Example","_branch_code":1,"n_observations":null,"_key_is_virtual":true,"_overwrite_versions":true,"run":{"id":43,"uid":"MfsMSwTjRQpibcLciOyO","_aux":null,"name":null,"reference":null,"created_at":"2024-12-03T20:43:15.292036+00:00","started_at":"2024-12-04T15:42:15.45694+00:00","finished_at":"2024-12-04T15:43:14.152546+00:00","_branch_code":1,"_status_code":0,"reference_type":null,"_is_consecutive":false,"transform":{"id":14,"key":"vitessce.ipynb","uid":"XmcumVqvKZd30000","_aux":null,"hash":"sqz7fEl7R7YwU-R5fz7LnQ","type":"notebook","version":null,"is_latest":false,"reference":null,"created_at":"2024-12-03T20:43:15.109256+00:00","updated_at":"2024-12-04T15:52:35.669773+00:00","description":"Vitessce","source_code":"# %% [markdown]\n# [![](https://img.shields.io/badge/Source%20on%20GitHub-orange)](https://github.com/laminlabs/lamin-spatial/blob/main/docs/vitessce.ipynb) [![hub](https://img.shields.io/badge/View%20on%20LaminHub-mediumseagreen)](https://lamin.ai/laminlabs/lamindata/transform/hqtT4OTr5Tiq5zKv/URbaThYhljgXbZEzgj5x)\n\n# %% [markdown] nbsphinx=\"hidden\"\n# # transform.name\n\n# %% [markdown]\n# This tutorial has been adopted from the data preparation examples in [the Vitessce documention](https://vitessce.github.io/vitessce-python).\n#\n\n# %% tags=[\"hide-output\"]\n# # !pip install \"vitessce[all]\">=3.5.0\n# # !pip install \"generate-tiff-offsets==0.1.7\"\n# # !pip install 'lamindb[jupyter,aws,bionty]'\n# !lamin connect laminlabs/lamin-dev # <-- replace with your instance\n\n# %% tags=[\"hide-output\"]\nfrom urllib.request import urlretrieve\nfrom pathlib import Path\nfrom anndata import read_h5ad\nimport vitessce as vit\nfrom vitessce import data_utils as vitdu\nfrom generate_tiff_offsets import get_offsets\nimport numpy as np\nimport lamindb as ln\nimport zipfile\nimport json\n\n# [optional] track the current notebook or script\nln.context.uid = \"XmcumVqvKZd30000\"\nln.track()\n\n# %% [markdown]\n# ## Visualize an AnnData object (H5AD format)\n\n# %% [markdown]\n# Here, we download an example H5AD file (approximately 22 MB). This is a [dataset](https://www.covid19cellatlas.org/index.healthy.html#habib17) from the COVID-19 Cell Atlas.\n\n# %%\nh5ad_raw_filepath = \"./habib17.raw.h5ad\"\nh5ad_processed_filepath = \"./habib17.processed.h5ad\"\nif not Path(h5ad_raw_filepath).exists():\n urlretrieve(\n \"https://covid19.cog.sanger.ac.uk/habib17.processed.h5ad\", h5ad_raw_filepath\n )\n\n# %% [markdown]\n# This AnnData file was not saved recently and `read_h5ad` will warn us about this. By reading and subsequently re-writing the file using an up-to-date `anndata` package version, we ensure that the dataset is saved using the latest AnnData H5AD format.\n\n# %%\nadata = read_h5ad(h5ad_raw_filepath)\n\n# %% [markdown]\n# Optionally, we can modify the AnnData object, for example to subset it to the highly variable genes (which will affect the amount of data that is displayed in the heatmap visualization).\n\n# %%\nadata = adata[:, adata.var[\"highly_variable\"]].copy()\n\n# %%\nadata.write_h5ad(h5ad_processed_filepath)\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Save the `.h5ad` file as a LaminDB Artifact.\n\n# %%\nh5ad_artifact = ln.Artifact(\n h5ad_processed_filepath,\n key=\"vitessce_examples/habib17.h5ad\",\n description=\"Habib et al., 2017 Nature Methods, h5ad\",\n type=\"dataset\",\n)\nh5ad_artifact.save()\n\n# %% [markdown]\n# When using `.h5ad` files, we construct a [Reference Specification](https://fsspec.github.io/kerchunk/spec.html) which enables interoperability with the [Zarr](https://zarrita.dev/packages/storage.html#referencestore) interface.\n\n# %%\nref_spec = vitdu.generate_h5ad_ref_spec(h5ad_processed_filepath)\n\n# %% [markdown]\n# We next need to save the corresponding Reference Specification to a JSON file and upload it to LaminDB as an Artifact.\n\n# %%\nref_path = \"./habib17.processed.reference.json\"\nwith open(ref_path, \"w\") as file:\n json.dump(ref_spec, file)\nref_artifact = ln.Artifact(\n ref_path,\n key=\"vitessce_examples/habib17.reference.json\",\n description=\"Reference JSON for H5AD file, Habib et al., 2017 Nature Methods\"\n).save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n\n# %% [markdown]\n# You can pass LaminDB Artifacts to the AnnDataWrapper class using the `adata_artifact` and `ref_artifact` [parameters](https://python-docs.vitessce.io/api_data.html#vitessce.wrappers.AnnDataWrapper).\n\n# %%\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=h5ad_artifact.description,\n)\ndataset = vc.add_dataset(name=\"Habib 2017\").add_object(\n vit.AnnDataWrapper(\n adata_artifact=h5ad_artifact,\n ref_artifact=ref_artifact,\n obs_feature_matrix_path=\"X\",\n obs_embedding_paths=[\"obsm/X_umap\"],\n obs_embedding_names=[\"UMAP\"],\n obs_set_paths=[\"obs/CellType\"],\n obs_set_names=[\"Cell Type\"],\n )\n)\nobs_sets = vc.add_view(vit.ViewType.OBS_SETS, dataset=dataset)\nobs_sets_sizes = vc.add_view(vit.ViewType.OBS_SET_SIZES, dataset=dataset)\nscatterplot = vc.add_view(vit.ViewType.SCATTERPLOT, dataset=dataset, mapping=\"UMAP\")\nheatmap = vc.add_view(vit.ViewType.HEATMAP, dataset=dataset)\ngenes = vc.add_view(vit.ViewType.FEATURE_LIST, dataset=dataset)\nvc.link_views([scatterplot, heatmap], [\"featureValueColormapRange\"], [[0.0, 0.1]])\nvc.layout(((scatterplot | obs_sets) / heatmap) | (obs_sets_sizes / genes));\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nh5ad_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View Habib17 (h5ad) in Vitessce\"\n)\n\n# %%\nh5ad_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# If your `VitessceConfig` object references data from multiple artifacts, the Vitessce button will appear next to a `Collection` that groups these artifacts (on the [Collections](https://lamin.ai/vitessce/examples/collections) tab of the Artifacts page).\n# Note that when using an `.h5ad`-based Artifact, the presence of the corresponding `.reference.json` file will result in the creation of a Collection.\n#\n# :::\n\n# %% [markdown]\n# ## Visualize an AnnData object (Zarr format)\n\n# %% [markdown]\n# AnnData objects can be saved on-disk to not only `.h5ad` files, but also to [Zarr stores](https://zarr.readthedocs.io/en/stable/tutorial.html#storage-alternatives) using AnnData's [write_zarr](https://anndata.readthedocs.io/en/latest/generated/anndata.AnnData.write_zarr.html) method.\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Convert the dataset to `.zarr` format. Often, Zarr data is stored on-disk using a [DirectoryStore](https://zarr.readthedocs.io/en/stable/api/storage.html#zarr.storage.DirectoryStore). By convention, these directories are named using a `.zarr` suffix.\n\n# %%\nzarr_processed_filepath = \"./habib17.anndata.zarr\"\n\n# %%\nadata = read_h5ad(h5ad_raw_filepath)\n\n# %% [markdown]\n# Just like in the above section, we may want to modify the AnnData object prior to saving it as a LaminDB Artifact that we will use for visualization.\n\n# %%\nadata = adata[:, adata.var[\"highly_variable\"]].copy()\n\n# %% [markdown]\n# Note the usage of `.write_zarr` below, as opposed to `.write_h5ad`.\n\n# %%\nadata.write_zarr(zarr_processed_filepath)\n\n# %% [markdown]\n# Save the `.zarr` directory as a LaminDB Artifact.\n\n# %%\nadata_zarr_artifact = ln.Artifact(\n zarr_processed_filepath,\n key=\"vitessce_examples/habib17.adata.zarr\",\n description=\"Habib et al., 2017 Nature Methods, zarr\",\n type=\"dataset\",\n)\nadata_zarr_artifact.save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n# Here, we configure the visualization the same way as above in the `.h5ad`-based example, with the exception of the `ref_artifact` parameter, as `.zarr`-based AnnData objects do not require a Reference Specification for Zarr interoperability.\n\n# %% tags=[\"hide-output\"]\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=adata_zarr_artifact.description,\n)\ndataset = vc.add_dataset(name=\"Habib 2017\").add_object(\n vit.AnnDataWrapper(\n adata_artifact=adata_zarr_artifact,\n obs_feature_matrix_path=\"X\",\n obs_embedding_paths=[\"obsm/X_umap\"],\n obs_embedding_names=[\"UMAP\"],\n obs_set_paths=[\"obs/CellType\"],\n obs_set_names=[\"Cell Type\"],\n )\n)\nobs_sets = vc.add_view(vit.Component.OBS_SETS, dataset=dataset)\nobs_sets_sizes = vc.add_view(vit.Component.OBS_SET_SIZES, dataset=dataset)\nscatterplot = vc.add_view(vit.Component.SCATTERPLOT, dataset=dataset, mapping=\"UMAP\")\nheatmap = vc.add_view(vit.Component.HEATMAP, dataset=dataset)\ngenes = vc.add_view(vit.Component.FEATURE_LIST, dataset=dataset)\nvc.link_views([scatterplot, heatmap], [\"featureValueColormapRange\"], [[0.0, 0.1]])\nvc.layout(((scatterplot | obs_sets) / heatmap) | (obs_sets_sizes / genes));\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nadata_zarr_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View Habib17 in Vitessce\"\n)\n\n# %%\nadata_zarr_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# :::\n\n# %% [markdown]\n# ## Visualize a SpatialData object (Zarr format)\n\n# %% [markdown]\n# Download and unzip the example SpatialData dataset (approximately 1.47 GB unzipped). Once unzipped, the data will be available via a local Zarr DirectoryStore.\n\n# %%\nsdata_zip_filepath = \"./visium.spatialdata.zarr.zip\"\nsdata_zarr_filepath = \"./visium.spatialdata.zarr\"\nif not Path(sdata_zarr_filepath).exists():\n if not Path(sdata_zip_filepath).exists():\n urlretrieve(\n \"https://s3.embl.de/spatialdata/spatialdata-sandbox/visium_associated_xenium_io.zip\", sdata_zip_filepath\n )\n with zipfile.ZipFile(sdata_zip_filepath, \"r\") as zip_ref:\n zip_ref.extractall(\".\")\n Path(\"data.zarr\").rename(sdata_zarr_filepath)\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Save the `.zarr` DirectoryStore as a LaminDB Artifact.\n\n# %%\nsdata_zarr_artifact = ln.Artifact(\n sdata_zarr_filepath,\n key=\"vitessce_examples/visium.sdata.zarr\",\n description=\"Visium SpatialData Example\",\n type=\"dataset\",\n)\nsdata_zarr_artifact.save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n# Here, we use the [SpatialDataWrapper](https://python-docs.vitessce.io/api_data.html#vitessce.wrappers.SpatialDataWrapper) class to specify which parts of the SpatialData object will be loaded for visualization.\n\n# %%\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=sdata_zarr_artifact.description,\n)\n# Add data to the configuration:\ndataset = vc.add_dataset(name='Breast Cancer Visium').add_object(\n vit.SpatialDataWrapper(\n sdata_artifact=sdata_zarr_artifact,\n # The following paths are relative to the root of the SpatialData zarr store on-disk.\n image_path=\"images/CytAssist_FFPE_Human_Breast_Cancer_full_image\",\n table_path=\"tables/table\",\n obs_feature_matrix_path=\"tables/table/X\",\n obs_spots_path=\"shapes/CytAssist_FFPE_Human_Breast_Cancer\",\n region=\"CytAssist_FFPE_Human_Breast_Cancer\",\n coordinate_system=\"global\",\n coordination_values={\n # The following tells Vitessce to consider each observation as a \"spot\"\n \"obsType\": \"spot\",\n }\n )\n)\n\n# Add views (visualizations) to the configuration:\nspatial = vc.add_view(\"spatialBeta\", dataset=dataset)\nfeature_list = vc.add_view(\"featureList\", dataset=dataset)\nlayer_controller = vc.add_view(\"layerControllerBeta\", dataset=dataset)\nvc.link_views_by_dict([spatial, layer_controller], {\n 'imageLayer': vit.CoordinationLevel([{\n 'photometricInterpretation': 'RGB',\n }]),\n}, scope_prefix=vit.get_initial_coordination_scope_prefix(\"A\", \"image\"))\nvc.link_views([spatial, layer_controller, feature_list, obs_sets], ['obsType'], [\"spot\"])\n\n# Layout the views\nvc.layout(spatial | (feature_list / layer_controller));\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nsdata_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View Visium SpatialData Example in Vitessce\"\n)\n\n# %%\nsdata_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# :::\n\n# %% [markdown]\n# ## Visualize an image (OME-TIFF format)\n\n# %% [markdown]\n# Vitesse can visualize data from multiple bioimaging file formats, including [OME-TIFF](https://ome-model.readthedocs.io/en/stable/ome-tiff/).\n\n# %% [markdown]\n# Here, we download an example OME-TIFF file (approximately 1.02 GB).\n\n# %%\nome_tiff_filepath = \"./VAN0006-LK-2-85-PAS_registered.ome.tif\"\nif not Path(ome_tiff_filepath).exists():\n urlretrieve(\n \"https://assets.hubmapconsortium.org/f4188a148e4c759092d19369d310883b/ometiff-pyramids/processedMicroscopy/VAN0006-LK-2-85-PAS_images/VAN0006-LK-2-85-PAS_registered.ome.tif\", ome_tiff_filepath\n )\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Save the `.ome.tif` file as a LaminDB Artifact.\n\n# %%\nome_tiff_artifact = ln.Artifact(\n ome_tiff_filepath,\n key=\"vitessce_examples/VAN0006-LK-2-85-PAS_registered.ome.tif\",\n description=\"PAS OME-TIFF file, Neumann et al., 2020\",\n type=\"dataset\",\n)\nome_tiff_artifact.save()\n\n# %% [markdown]\n# When using OME-TIFF files, we can use [generate-tiff-offsets](https://github.com/hms-dbmi/generate-tiff-offsets) to create an index for the bytes within the OME-TIFF file. We store these to a companion `.offsets.json` file which makes loading subsets of the image more efficient.\n\n# %%\noffsets = get_offsets(ome_tiff_filepath)\n\n# %%\noffsets_path = \"./VAN0006-LK-2-85-PAS_registered.offsets.json\"\nwith open(offsets_path, \"w\") as file:\n json.dump(offsets, file)\noffsets_artifact = ln.Artifact(\n offsets_path,\n key=\"vitessce_examples/VAN0006-LK-2-85-PAS_registered.offsets.json\",\n description=\"PAS offsets file, Neumann et al., 2020\"\n).save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n# Here, we use the [ImageOmeTiffWrapper](https://python-docs.vitessce.io/api_data.html#vitessce.wrappers.ImageOmeTiffWrapper) class to specify which pair of OME-TIFF file and offsets JSON file to load.\n\n# %%\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=ome_tiff_artifact.description\n)\ndataset = vc.add_dataset(\"Image\").add_object(\n vit.ImageOmeTiffWrapper(\n img_artifact=ome_tiff_artifact,\n offsets_artifact=offsets_artifact,\n )\n)\nspatial = vc.add_view(\"spatialBeta\", dataset=dataset)\nlayer_controller = vc.add_view(\"layerControllerBeta\", dataset=dataset)\nvc.layout(spatial | layer_controller);\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nome_tiff_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View PAS OME-TIFF, Neumann et al., 2020 in Vitessce\"\n)\n\n# %%\nome_tiff_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# If your `VitessceConfig` object references data from multiple artifacts, the Vitessce button will appear next to a `Collection` that groups these artifacts (on the [Collections](https://lamin.ai/vitessce/examples/collections) tab of the Artifacts page).\n# In the case of OME-TIFF, the presence of the corresponding offsets JSON file will result in the creation of a Collection.\n#\n# :::\n\n# %% [markdown]\n# ## Visualize an image (OME-Zarr format)\n\n# %% [markdown]\n# Retrieve an OME-Zarr (also known as OME-NGFF) formatted image. We could download an unzip such an image, but in downloading the SpatialData object above, we already have access to a valid OME-Zarr image locally.\n\n# %%\nome_zarr_filepath = Path(sdata_zarr_filepath) / \"images\" / \"CytAssist_FFPE_Human_Breast_Cancer_full_image\"\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Save the `.ome.zarr` DirectoryStore as a LaminDB Artifact.\n\n# %%\nome_zarr_artifact = ln.Artifact(\n ome_zarr_filepath,\n key=\"vitessce_examples/visium.ome.zarr\",\n description=\"Visium OME-Zarr Example\",\n type=\"dataset\",\n)\nome_zarr_artifact.save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n# Here, we use the [ImageOmeZarrWrapper](https://python-docs.vitessce.io/api_data.html#vitessce.wrappers.ImageOmeZarrWrapper) class to specify an OME-Zarr file to load for visualization.\n\n# %%\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=ome_zarr_artifact.description\n)\ndataset = vc.add_dataset(\"Image\").add_object(\n vit.ImageOmeZarrWrapper(\n img_artifact=ome_zarr_artifact,\n )\n)\nspatial = vc.add_view(\"spatialBeta\", dataset=dataset)\nlayer_controller = vc.add_view(\"layerControllerBeta\", dataset=dataset)\nvc.link_views_by_dict([spatial, layer_controller], {\n 'imageLayer': vit.CoordinationLevel([{\n 'photometricInterpretation': 'RGB',\n }]),\n}, scope_prefix=vit.get_initial_coordination_scope_prefix(\"A\", \"image\"))\nvc.layout(spatial | layer_controller);\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nome_zarr_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View Visium OME-Zarr Example in Vitessce\"\n)\n\n# %%\nome_zarr_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# :::\n\n# %%\n# [optional] finish run context and auto-save the notebook\nln.finish()\n\n# %% [markdown]\n# :::{dropdown} Upload speed\n#\n# Here is [a note](https://lamin.ai/laminlabs/lamindata/transform/WDjio16cQsdW5zKv) on folder upload speed and why `lamindb` does not use the `.export(to=\"s3\")` functionality of Vitessce.\n#\n# :::\n\n# %% tags=[\"hide-cell\"]\n# clean up artifacts in CI run\nh5ad_artifact.delete(permanent=True)\nref_artifact.delete(permanent=True)\n\nadata_zarr_artifact.delete(permanent=True)\nsdata_zarr_artifact.delete(permanent=True)\n\nome_tiff_artifact.delete(permanent=True)\noffsets_artifact.delete(permanent=True)\n\nome_zarr_artifact.delete(permanent=True)\n\nh5ad_vc_artifact.delete(permanent=True)\nadata_zarr_vc_artifact.delete(permanent=True)\nsdata_vc_artifact.delete(permanent=True)\nome_tiff_vc_artifact.delete(permanent=True)\nome_zarr_vc_artifact.delete(permanent=True)\n","_branch_code":1,"reference_type":null}}}],"_recreated_collections":[{"id":7,"key":"View Habib17 (h5ad) in Vitessce","uid":"obXokdzOnSH2tytJ0000","_aux":null,"hash":"U9WUAN_l9mLUWYqO2pJ9-g","version":null,"is_latest":true,"reference":null,"created_at":"2024-12-03T21:33:48.870791+00:00","updated_at":"2024-12-04T16:11:09.095459+00:00","description":null,"_branch_code":1,"reference_type":null,"artifacts":[{"uid":"1g3a41z0hggWNBrk0000"},{"uid":"O7CTVi6XjnfsceQ20000"}],"run":{"id":43,"uid":"MfsMSwTjRQpibcLciOyO","_aux":null,"name":null,"reference":null,"created_at":"2024-12-03T20:43:15.292036+00:00","started_at":"2024-12-04T15:42:15.45694+00:00","finished_at":"2024-12-04T15:43:14.152546+00:00","_branch_code":1,"_status_code":0,"reference_type":null,"_is_consecutive":false,"transform":{"id":14,"key":"vitessce.ipynb","uid":"XmcumVqvKZd30000","_aux":null,"hash":"sqz7fEl7R7YwU-R5fz7LnQ","type":"notebook","version":null,"is_latest":false,"reference":null,"created_at":"2024-12-03T20:43:15.109256+00:00","updated_at":"2024-12-04T15:52:35.669773+00:00","description":"Vitessce","source_code":"# %% [markdown]\n# [![](https://img.shields.io/badge/Source%20on%20GitHub-orange)](https://github.com/laminlabs/lamin-spatial/blob/main/docs/vitessce.ipynb) [![hub](https://img.shields.io/badge/View%20on%20LaminHub-mediumseagreen)](https://lamin.ai/laminlabs/lamindata/transform/hqtT4OTr5Tiq5zKv/URbaThYhljgXbZEzgj5x)\n\n# %% [markdown] nbsphinx=\"hidden\"\n# # transform.name\n\n# %% [markdown]\n# This tutorial has been adopted from the data preparation examples in [the Vitessce documention](https://vitessce.github.io/vitessce-python).\n#\n\n# %% tags=[\"hide-output\"]\n# # !pip install \"vitessce[all]\">=3.5.0\n# # !pip install \"generate-tiff-offsets==0.1.7\"\n# # !pip install 'lamindb[jupyter,aws,bionty]'\n# !lamin connect laminlabs/lamin-dev # <-- replace with your instance\n\n# %% tags=[\"hide-output\"]\nfrom urllib.request import urlretrieve\nfrom pathlib import Path\nfrom anndata import read_h5ad\nimport vitessce as vit\nfrom vitessce import data_utils as vitdu\nfrom generate_tiff_offsets import get_offsets\nimport numpy as np\nimport lamindb as ln\nimport zipfile\nimport json\n\n# [optional] track the current notebook or script\nln.context.uid = \"XmcumVqvKZd30000\"\nln.track()\n\n# %% [markdown]\n# ## Visualize an AnnData object (H5AD format)\n\n# %% [markdown]\n# Here, we download an example H5AD file (approximately 22 MB). This is a [dataset](https://www.covid19cellatlas.org/index.healthy.html#habib17) from the COVID-19 Cell Atlas.\n\n# %%\nh5ad_raw_filepath = \"./habib17.raw.h5ad\"\nh5ad_processed_filepath = \"./habib17.processed.h5ad\"\nif not Path(h5ad_raw_filepath).exists():\n urlretrieve(\n \"https://covid19.cog.sanger.ac.uk/habib17.processed.h5ad\", h5ad_raw_filepath\n )\n\n# %% [markdown]\n# This AnnData file was not saved recently and `read_h5ad` will warn us about this. By reading and subsequently re-writing the file using an up-to-date `anndata` package version, we ensure that the dataset is saved using the latest AnnData H5AD format.\n\n# %%\nadata = read_h5ad(h5ad_raw_filepath)\n\n# %% [markdown]\n# Optionally, we can modify the AnnData object, for example to subset it to the highly variable genes (which will affect the amount of data that is displayed in the heatmap visualization).\n\n# %%\nadata = adata[:, adata.var[\"highly_variable\"]].copy()\n\n# %%\nadata.write_h5ad(h5ad_processed_filepath)\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Save the `.h5ad` file as a LaminDB Artifact.\n\n# %%\nh5ad_artifact = ln.Artifact(\n h5ad_processed_filepath,\n key=\"vitessce_examples/habib17.h5ad\",\n description=\"Habib et al., 2017 Nature Methods, h5ad\",\n type=\"dataset\",\n)\nh5ad_artifact.save()\n\n# %% [markdown]\n# When using `.h5ad` files, we construct a [Reference Specification](https://fsspec.github.io/kerchunk/spec.html) which enables interoperability with the [Zarr](https://zarrita.dev/packages/storage.html#referencestore) interface.\n\n# %%\nref_spec = vitdu.generate_h5ad_ref_spec(h5ad_processed_filepath)\n\n# %% [markdown]\n# We next need to save the corresponding Reference Specification to a JSON file and upload it to LaminDB as an Artifact.\n\n# %%\nref_path = \"./habib17.processed.reference.json\"\nwith open(ref_path, \"w\") as file:\n json.dump(ref_spec, file)\nref_artifact = ln.Artifact(\n ref_path,\n key=\"vitessce_examples/habib17.reference.json\",\n description=\"Reference JSON for H5AD file, Habib et al., 2017 Nature Methods\"\n).save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n\n# %% [markdown]\n# You can pass LaminDB Artifacts to the AnnDataWrapper class using the `adata_artifact` and `ref_artifact` [parameters](https://python-docs.vitessce.io/api_data.html#vitessce.wrappers.AnnDataWrapper).\n\n# %%\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=h5ad_artifact.description,\n)\ndataset = vc.add_dataset(name=\"Habib 2017\").add_object(\n vit.AnnDataWrapper(\n adata_artifact=h5ad_artifact,\n ref_artifact=ref_artifact,\n obs_feature_matrix_path=\"X\",\n obs_embedding_paths=[\"obsm/X_umap\"],\n obs_embedding_names=[\"UMAP\"],\n obs_set_paths=[\"obs/CellType\"],\n obs_set_names=[\"Cell Type\"],\n )\n)\nobs_sets = vc.add_view(vit.ViewType.OBS_SETS, dataset=dataset)\nobs_sets_sizes = vc.add_view(vit.ViewType.OBS_SET_SIZES, dataset=dataset)\nscatterplot = vc.add_view(vit.ViewType.SCATTERPLOT, dataset=dataset, mapping=\"UMAP\")\nheatmap = vc.add_view(vit.ViewType.HEATMAP, dataset=dataset)\ngenes = vc.add_view(vit.ViewType.FEATURE_LIST, dataset=dataset)\nvc.link_views([scatterplot, heatmap], [\"featureValueColormapRange\"], [[0.0, 0.1]])\nvc.layout(((scatterplot | obs_sets) / heatmap) | (obs_sets_sizes / genes));\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nh5ad_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View Habib17 (h5ad) in Vitessce\"\n)\n\n# %%\nh5ad_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# If your `VitessceConfig` object references data from multiple artifacts, the Vitessce button will appear next to a `Collection` that groups these artifacts (on the [Collections](https://lamin.ai/vitessce/examples/collections) tab of the Artifacts page).\n# Note that when using an `.h5ad`-based Artifact, the presence of the corresponding `.reference.json` file will result in the creation of a Collection.\n#\n# :::\n\n# %% [markdown]\n# ## Visualize an AnnData object (Zarr format)\n\n# %% [markdown]\n# AnnData objects can be saved on-disk to not only `.h5ad` files, but also to [Zarr stores](https://zarr.readthedocs.io/en/stable/tutorial.html#storage-alternatives) using AnnData's [write_zarr](https://anndata.readthedocs.io/en/latest/generated/anndata.AnnData.write_zarr.html) method.\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Convert the dataset to `.zarr` format. Often, Zarr data is stored on-disk using a [DirectoryStore](https://zarr.readthedocs.io/en/stable/api/storage.html#zarr.storage.DirectoryStore). By convention, these directories are named using a `.zarr` suffix.\n\n# %%\nzarr_processed_filepath = \"./habib17.anndata.zarr\"\n\n# %%\nadata = read_h5ad(h5ad_raw_filepath)\n\n# %% [markdown]\n# Just like in the above section, we may want to modify the AnnData object prior to saving it as a LaminDB Artifact that we will use for visualization.\n\n# %%\nadata = adata[:, adata.var[\"highly_variable\"]].copy()\n\n# %% [markdown]\n# Note the usage of `.write_zarr` below, as opposed to `.write_h5ad`.\n\n# %%\nadata.write_zarr(zarr_processed_filepath)\n\n# %% [markdown]\n# Save the `.zarr` directory as a LaminDB Artifact.\n\n# %%\nadata_zarr_artifact = ln.Artifact(\n zarr_processed_filepath,\n key=\"vitessce_examples/habib17.adata.zarr\",\n description=\"Habib et al., 2017 Nature Methods, zarr\",\n type=\"dataset\",\n)\nadata_zarr_artifact.save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n# Here, we configure the visualization the same way as above in the `.h5ad`-based example, with the exception of the `ref_artifact` parameter, as `.zarr`-based AnnData objects do not require a Reference Specification for Zarr interoperability.\n\n# %% tags=[\"hide-output\"]\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=adata_zarr_artifact.description,\n)\ndataset = vc.add_dataset(name=\"Habib 2017\").add_object(\n vit.AnnDataWrapper(\n adata_artifact=adata_zarr_artifact,\n obs_feature_matrix_path=\"X\",\n obs_embedding_paths=[\"obsm/X_umap\"],\n obs_embedding_names=[\"UMAP\"],\n obs_set_paths=[\"obs/CellType\"],\n obs_set_names=[\"Cell Type\"],\n )\n)\nobs_sets = vc.add_view(vit.Component.OBS_SETS, dataset=dataset)\nobs_sets_sizes = vc.add_view(vit.Component.OBS_SET_SIZES, dataset=dataset)\nscatterplot = vc.add_view(vit.Component.SCATTERPLOT, dataset=dataset, mapping=\"UMAP\")\nheatmap = vc.add_view(vit.Component.HEATMAP, dataset=dataset)\ngenes = vc.add_view(vit.Component.FEATURE_LIST, dataset=dataset)\nvc.link_views([scatterplot, heatmap], [\"featureValueColormapRange\"], [[0.0, 0.1]])\nvc.layout(((scatterplot | obs_sets) / heatmap) | (obs_sets_sizes / genes));\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nadata_zarr_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View Habib17 in Vitessce\"\n)\n\n# %%\nadata_zarr_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# :::\n\n# %% [markdown]\n# ## Visualize a SpatialData object (Zarr format)\n\n# %% [markdown]\n# Download and unzip the example SpatialData dataset (approximately 1.47 GB unzipped). Once unzipped, the data will be available via a local Zarr DirectoryStore.\n\n# %%\nsdata_zip_filepath = \"./visium.spatialdata.zarr.zip\"\nsdata_zarr_filepath = \"./visium.spatialdata.zarr\"\nif not Path(sdata_zarr_filepath).exists():\n if not Path(sdata_zip_filepath).exists():\n urlretrieve(\n \"https://s3.embl.de/spatialdata/spatialdata-sandbox/visium_associated_xenium_io.zip\", sdata_zip_filepath\n )\n with zipfile.ZipFile(sdata_zip_filepath, \"r\") as zip_ref:\n zip_ref.extractall(\".\")\n Path(\"data.zarr\").rename(sdata_zarr_filepath)\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Save the `.zarr` DirectoryStore as a LaminDB Artifact.\n\n# %%\nsdata_zarr_artifact = ln.Artifact(\n sdata_zarr_filepath,\n key=\"vitessce_examples/visium.sdata.zarr\",\n description=\"Visium SpatialData Example\",\n type=\"dataset\",\n)\nsdata_zarr_artifact.save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n# Here, we use the [SpatialDataWrapper](https://python-docs.vitessce.io/api_data.html#vitessce.wrappers.SpatialDataWrapper) class to specify which parts of the SpatialData object will be loaded for visualization.\n\n# %%\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=sdata_zarr_artifact.description,\n)\n# Add data to the configuration:\ndataset = vc.add_dataset(name='Breast Cancer Visium').add_object(\n vit.SpatialDataWrapper(\n sdata_artifact=sdata_zarr_artifact,\n # The following paths are relative to the root of the SpatialData zarr store on-disk.\n image_path=\"images/CytAssist_FFPE_Human_Breast_Cancer_full_image\",\n table_path=\"tables/table\",\n obs_feature_matrix_path=\"tables/table/X\",\n obs_spots_path=\"shapes/CytAssist_FFPE_Human_Breast_Cancer\",\n region=\"CytAssist_FFPE_Human_Breast_Cancer\",\n coordinate_system=\"global\",\n coordination_values={\n # The following tells Vitessce to consider each observation as a \"spot\"\n \"obsType\": \"spot\",\n }\n )\n)\n\n# Add views (visualizations) to the configuration:\nspatial = vc.add_view(\"spatialBeta\", dataset=dataset)\nfeature_list = vc.add_view(\"featureList\", dataset=dataset)\nlayer_controller = vc.add_view(\"layerControllerBeta\", dataset=dataset)\nvc.link_views_by_dict([spatial, layer_controller], {\n 'imageLayer': vit.CoordinationLevel([{\n 'photometricInterpretation': 'RGB',\n }]),\n}, scope_prefix=vit.get_initial_coordination_scope_prefix(\"A\", \"image\"))\nvc.link_views([spatial, layer_controller, feature_list, obs_sets], ['obsType'], [\"spot\"])\n\n# Layout the views\nvc.layout(spatial | (feature_list / layer_controller));\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nsdata_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View Visium SpatialData Example in Vitessce\"\n)\n\n# %%\nsdata_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# :::\n\n# %% [markdown]\n# ## Visualize an image (OME-TIFF format)\n\n# %% [markdown]\n# Vitesse can visualize data from multiple bioimaging file formats, including [OME-TIFF](https://ome-model.readthedocs.io/en/stable/ome-tiff/).\n\n# %% [markdown]\n# Here, we download an example OME-TIFF file (approximately 1.02 GB).\n\n# %%\nome_tiff_filepath = \"./VAN0006-LK-2-85-PAS_registered.ome.tif\"\nif not Path(ome_tiff_filepath).exists():\n urlretrieve(\n \"https://assets.hubmapconsortium.org/f4188a148e4c759092d19369d310883b/ometiff-pyramids/processedMicroscopy/VAN0006-LK-2-85-PAS_images/VAN0006-LK-2-85-PAS_registered.ome.tif\", ome_tiff_filepath\n )\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Save the `.ome.tif` file as a LaminDB Artifact.\n\n# %%\nome_tiff_artifact = ln.Artifact(\n ome_tiff_filepath,\n key=\"vitessce_examples/VAN0006-LK-2-85-PAS_registered.ome.tif\",\n description=\"PAS OME-TIFF file, Neumann et al., 2020\",\n type=\"dataset\",\n)\nome_tiff_artifact.save()\n\n# %% [markdown]\n# When using OME-TIFF files, we can use [generate-tiff-offsets](https://github.com/hms-dbmi/generate-tiff-offsets) to create an index for the bytes within the OME-TIFF file. We store these to a companion `.offsets.json` file which makes loading subsets of the image more efficient.\n\n# %%\noffsets = get_offsets(ome_tiff_filepath)\n\n# %%\noffsets_path = \"./VAN0006-LK-2-85-PAS_registered.offsets.json\"\nwith open(offsets_path, \"w\") as file:\n json.dump(offsets, file)\noffsets_artifact = ln.Artifact(\n offsets_path,\n key=\"vitessce_examples/VAN0006-LK-2-85-PAS_registered.offsets.json\",\n description=\"PAS offsets file, Neumann et al., 2020\"\n).save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n# Here, we use the [ImageOmeTiffWrapper](https://python-docs.vitessce.io/api_data.html#vitessce.wrappers.ImageOmeTiffWrapper) class to specify which pair of OME-TIFF file and offsets JSON file to load.\n\n# %%\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=ome_tiff_artifact.description\n)\ndataset = vc.add_dataset(\"Image\").add_object(\n vit.ImageOmeTiffWrapper(\n img_artifact=ome_tiff_artifact,\n offsets_artifact=offsets_artifact,\n )\n)\nspatial = vc.add_view(\"spatialBeta\", dataset=dataset)\nlayer_controller = vc.add_view(\"layerControllerBeta\", dataset=dataset)\nvc.layout(spatial | layer_controller);\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nome_tiff_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View PAS OME-TIFF, Neumann et al., 2020 in Vitessce\"\n)\n\n# %%\nome_tiff_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# If your `VitessceConfig` object references data from multiple artifacts, the Vitessce button will appear next to a `Collection` that groups these artifacts (on the [Collections](https://lamin.ai/vitessce/examples/collections) tab of the Artifacts page).\n# In the case of OME-TIFF, the presence of the corresponding offsets JSON file will result in the creation of a Collection.\n#\n# :::\n\n# %% [markdown]\n# ## Visualize an image (OME-Zarr format)\n\n# %% [markdown]\n# Retrieve an OME-Zarr (also known as OME-NGFF) formatted image. We could download an unzip such an image, but in downloading the SpatialData object above, we already have access to a valid OME-Zarr image locally.\n\n# %%\nome_zarr_filepath = Path(sdata_zarr_filepath) / \"images\" / \"CytAssist_FFPE_Human_Breast_Cancer_full_image\"\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Save the `.ome.zarr` DirectoryStore as a LaminDB Artifact.\n\n# %%\nome_zarr_artifact = ln.Artifact(\n ome_zarr_filepath,\n key=\"vitessce_examples/visium.ome.zarr\",\n description=\"Visium OME-Zarr Example\",\n type=\"dataset\",\n)\nome_zarr_artifact.save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n# Here, we use the [ImageOmeZarrWrapper](https://python-docs.vitessce.io/api_data.html#vitessce.wrappers.ImageOmeZarrWrapper) class to specify an OME-Zarr file to load for visualization.\n\n# %%\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=ome_zarr_artifact.description\n)\ndataset = vc.add_dataset(\"Image\").add_object(\n vit.ImageOmeZarrWrapper(\n img_artifact=ome_zarr_artifact,\n )\n)\nspatial = vc.add_view(\"spatialBeta\", dataset=dataset)\nlayer_controller = vc.add_view(\"layerControllerBeta\", dataset=dataset)\nvc.link_views_by_dict([spatial, layer_controller], {\n 'imageLayer': vit.CoordinationLevel([{\n 'photometricInterpretation': 'RGB',\n }]),\n}, scope_prefix=vit.get_initial_coordination_scope_prefix(\"A\", \"image\"))\nvc.layout(spatial | layer_controller);\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nome_zarr_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View Visium OME-Zarr Example in Vitessce\"\n)\n\n# %%\nome_zarr_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# :::\n\n# %%\n# [optional] finish run context and auto-save the notebook\nln.finish()\n\n# %% [markdown]\n# :::{dropdown} Upload speed\n#\n# Here is [a note](https://lamin.ai/laminlabs/lamindata/transform/WDjio16cQsdW5zKv) on folder upload speed and why `lamindb` does not use the `.export(to=\"s3\")` functionality of Vitessce.\n#\n# :::\n\n# %% tags=[\"hide-cell\"]\n# clean up artifacts in CI run\nh5ad_artifact.delete(permanent=True)\nref_artifact.delete(permanent=True)\n\nadata_zarr_artifact.delete(permanent=True)\nsdata_zarr_artifact.delete(permanent=True)\n\nome_tiff_artifact.delete(permanent=True)\noffsets_artifact.delete(permanent=True)\n\nome_zarr_artifact.delete(permanent=True)\n\nh5ad_vc_artifact.delete(permanent=True)\nadata_zarr_vc_artifact.delete(permanent=True)\nsdata_vc_artifact.delete(permanent=True)\nome_tiff_vc_artifact.delete(permanent=True)\nome_zarr_vc_artifact.delete(permanent=True)\n","_branch_code":1,"reference_type":null}}},{"id":1,"key":"View PAS OME-TIFF, Neumann et al., 2020 in Vitessce","uid":"HqfzZmkfYOy20ZNN0000","_aux":null,"hash":"vpleElXxZiaZblQ09q9K-g","version":null,"is_latest":true,"reference":null,"created_at":"2024-10-05T20:49:25.47867+00:00","updated_at":"2024-12-04T16:11:43.549879+00:00","description":null,"_branch_code":1,"reference_type":null,"artifacts":[{"uid":"0I5aMNdh2UrECtFX0000"},{"uid":"CMBUzbNNCEDGCKcQ0000"}],"run":{"id":43,"uid":"MfsMSwTjRQpibcLciOyO","_aux":null,"name":null,"reference":null,"created_at":"2024-12-03T20:43:15.292036+00:00","started_at":"2024-12-04T15:42:15.45694+00:00","finished_at":"2024-12-04T15:43:14.152546+00:00","_branch_code":1,"_status_code":0,"reference_type":null,"_is_consecutive":false,"transform":{"id":14,"key":"vitessce.ipynb","uid":"XmcumVqvKZd30000","_aux":null,"hash":"sqz7fEl7R7YwU-R5fz7LnQ","type":"notebook","version":null,"is_latest":false,"reference":null,"created_at":"2024-12-03T20:43:15.109256+00:00","updated_at":"2024-12-04T15:52:35.669773+00:00","description":"Vitessce","source_code":"# %% [markdown]\n# [![](https://img.shields.io/badge/Source%20on%20GitHub-orange)](https://github.com/laminlabs/lamin-spatial/blob/main/docs/vitessce.ipynb) [![hub](https://img.shields.io/badge/View%20on%20LaminHub-mediumseagreen)](https://lamin.ai/laminlabs/lamindata/transform/hqtT4OTr5Tiq5zKv/URbaThYhljgXbZEzgj5x)\n\n# %% [markdown] nbsphinx=\"hidden\"\n# # transform.name\n\n# %% [markdown]\n# This tutorial has been adopted from the data preparation examples in [the Vitessce documention](https://vitessce.github.io/vitessce-python).\n#\n\n# %% tags=[\"hide-output\"]\n# # !pip install \"vitessce[all]\">=3.5.0\n# # !pip install \"generate-tiff-offsets==0.1.7\"\n# # !pip install 'lamindb[jupyter,aws,bionty]'\n# !lamin connect laminlabs/lamin-dev # <-- replace with your instance\n\n# %% tags=[\"hide-output\"]\nfrom urllib.request import urlretrieve\nfrom pathlib import Path\nfrom anndata import read_h5ad\nimport vitessce as vit\nfrom vitessce import data_utils as vitdu\nfrom generate_tiff_offsets import get_offsets\nimport numpy as np\nimport lamindb as ln\nimport zipfile\nimport json\n\n# [optional] track the current notebook or script\nln.context.uid = \"XmcumVqvKZd30000\"\nln.track()\n\n# %% [markdown]\n# ## Visualize an AnnData object (H5AD format)\n\n# %% [markdown]\n# Here, we download an example H5AD file (approximately 22 MB). This is a [dataset](https://www.covid19cellatlas.org/index.healthy.html#habib17) from the COVID-19 Cell Atlas.\n\n# %%\nh5ad_raw_filepath = \"./habib17.raw.h5ad\"\nh5ad_processed_filepath = \"./habib17.processed.h5ad\"\nif not Path(h5ad_raw_filepath).exists():\n urlretrieve(\n \"https://covid19.cog.sanger.ac.uk/habib17.processed.h5ad\", h5ad_raw_filepath\n )\n\n# %% [markdown]\n# This AnnData file was not saved recently and `read_h5ad` will warn us about this. By reading and subsequently re-writing the file using an up-to-date `anndata` package version, we ensure that the dataset is saved using the latest AnnData H5AD format.\n\n# %%\nadata = read_h5ad(h5ad_raw_filepath)\n\n# %% [markdown]\n# Optionally, we can modify the AnnData object, for example to subset it to the highly variable genes (which will affect the amount of data that is displayed in the heatmap visualization).\n\n# %%\nadata = adata[:, adata.var[\"highly_variable\"]].copy()\n\n# %%\nadata.write_h5ad(h5ad_processed_filepath)\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Save the `.h5ad` file as a LaminDB Artifact.\n\n# %%\nh5ad_artifact = ln.Artifact(\n h5ad_processed_filepath,\n key=\"vitessce_examples/habib17.h5ad\",\n description=\"Habib et al., 2017 Nature Methods, h5ad\",\n type=\"dataset\",\n)\nh5ad_artifact.save()\n\n# %% [markdown]\n# When using `.h5ad` files, we construct a [Reference Specification](https://fsspec.github.io/kerchunk/spec.html) which enables interoperability with the [Zarr](https://zarrita.dev/packages/storage.html#referencestore) interface.\n\n# %%\nref_spec = vitdu.generate_h5ad_ref_spec(h5ad_processed_filepath)\n\n# %% [markdown]\n# We next need to save the corresponding Reference Specification to a JSON file and upload it to LaminDB as an Artifact.\n\n# %%\nref_path = \"./habib17.processed.reference.json\"\nwith open(ref_path, \"w\") as file:\n json.dump(ref_spec, file)\nref_artifact = ln.Artifact(\n ref_path,\n key=\"vitessce_examples/habib17.reference.json\",\n description=\"Reference JSON for H5AD file, Habib et al., 2017 Nature Methods\"\n).save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n\n# %% [markdown]\n# You can pass LaminDB Artifacts to the AnnDataWrapper class using the `adata_artifact` and `ref_artifact` [parameters](https://python-docs.vitessce.io/api_data.html#vitessce.wrappers.AnnDataWrapper).\n\n# %%\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=h5ad_artifact.description,\n)\ndataset = vc.add_dataset(name=\"Habib 2017\").add_object(\n vit.AnnDataWrapper(\n adata_artifact=h5ad_artifact,\n ref_artifact=ref_artifact,\n obs_feature_matrix_path=\"X\",\n obs_embedding_paths=[\"obsm/X_umap\"],\n obs_embedding_names=[\"UMAP\"],\n obs_set_paths=[\"obs/CellType\"],\n obs_set_names=[\"Cell Type\"],\n )\n)\nobs_sets = vc.add_view(vit.ViewType.OBS_SETS, dataset=dataset)\nobs_sets_sizes = vc.add_view(vit.ViewType.OBS_SET_SIZES, dataset=dataset)\nscatterplot = vc.add_view(vit.ViewType.SCATTERPLOT, dataset=dataset, mapping=\"UMAP\")\nheatmap = vc.add_view(vit.ViewType.HEATMAP, dataset=dataset)\ngenes = vc.add_view(vit.ViewType.FEATURE_LIST, dataset=dataset)\nvc.link_views([scatterplot, heatmap], [\"featureValueColormapRange\"], [[0.0, 0.1]])\nvc.layout(((scatterplot | obs_sets) / heatmap) | (obs_sets_sizes / genes));\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nh5ad_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View Habib17 (h5ad) in Vitessce\"\n)\n\n# %%\nh5ad_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# If your `VitessceConfig` object references data from multiple artifacts, the Vitessce button will appear next to a `Collection` that groups these artifacts (on the [Collections](https://lamin.ai/vitessce/examples/collections) tab of the Artifacts page).\n# Note that when using an `.h5ad`-based Artifact, the presence of the corresponding `.reference.json` file will result in the creation of a Collection.\n#\n# :::\n\n# %% [markdown]\n# ## Visualize an AnnData object (Zarr format)\n\n# %% [markdown]\n# AnnData objects can be saved on-disk to not only `.h5ad` files, but also to [Zarr stores](https://zarr.readthedocs.io/en/stable/tutorial.html#storage-alternatives) using AnnData's [write_zarr](https://anndata.readthedocs.io/en/latest/generated/anndata.AnnData.write_zarr.html) method.\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Convert the dataset to `.zarr` format. Often, Zarr data is stored on-disk using a [DirectoryStore](https://zarr.readthedocs.io/en/stable/api/storage.html#zarr.storage.DirectoryStore). By convention, these directories are named using a `.zarr` suffix.\n\n# %%\nzarr_processed_filepath = \"./habib17.anndata.zarr\"\n\n# %%\nadata = read_h5ad(h5ad_raw_filepath)\n\n# %% [markdown]\n# Just like in the above section, we may want to modify the AnnData object prior to saving it as a LaminDB Artifact that we will use for visualization.\n\n# %%\nadata = adata[:, adata.var[\"highly_variable\"]].copy()\n\n# %% [markdown]\n# Note the usage of `.write_zarr` below, as opposed to `.write_h5ad`.\n\n# %%\nadata.write_zarr(zarr_processed_filepath)\n\n# %% [markdown]\n# Save the `.zarr` directory as a LaminDB Artifact.\n\n# %%\nadata_zarr_artifact = ln.Artifact(\n zarr_processed_filepath,\n key=\"vitessce_examples/habib17.adata.zarr\",\n description=\"Habib et al., 2017 Nature Methods, zarr\",\n type=\"dataset\",\n)\nadata_zarr_artifact.save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n# Here, we configure the visualization the same way as above in the `.h5ad`-based example, with the exception of the `ref_artifact` parameter, as `.zarr`-based AnnData objects do not require a Reference Specification for Zarr interoperability.\n\n# %% tags=[\"hide-output\"]\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=adata_zarr_artifact.description,\n)\ndataset = vc.add_dataset(name=\"Habib 2017\").add_object(\n vit.AnnDataWrapper(\n adata_artifact=adata_zarr_artifact,\n obs_feature_matrix_path=\"X\",\n obs_embedding_paths=[\"obsm/X_umap\"],\n obs_embedding_names=[\"UMAP\"],\n obs_set_paths=[\"obs/CellType\"],\n obs_set_names=[\"Cell Type\"],\n )\n)\nobs_sets = vc.add_view(vit.Component.OBS_SETS, dataset=dataset)\nobs_sets_sizes = vc.add_view(vit.Component.OBS_SET_SIZES, dataset=dataset)\nscatterplot = vc.add_view(vit.Component.SCATTERPLOT, dataset=dataset, mapping=\"UMAP\")\nheatmap = vc.add_view(vit.Component.HEATMAP, dataset=dataset)\ngenes = vc.add_view(vit.Component.FEATURE_LIST, dataset=dataset)\nvc.link_views([scatterplot, heatmap], [\"featureValueColormapRange\"], [[0.0, 0.1]])\nvc.layout(((scatterplot | obs_sets) / heatmap) | (obs_sets_sizes / genes));\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nadata_zarr_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View Habib17 in Vitessce\"\n)\n\n# %%\nadata_zarr_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# :::\n\n# %% [markdown]\n# ## Visualize a SpatialData object (Zarr format)\n\n# %% [markdown]\n# Download and unzip the example SpatialData dataset (approximately 1.47 GB unzipped). Once unzipped, the data will be available via a local Zarr DirectoryStore.\n\n# %%\nsdata_zip_filepath = \"./visium.spatialdata.zarr.zip\"\nsdata_zarr_filepath = \"./visium.spatialdata.zarr\"\nif not Path(sdata_zarr_filepath).exists():\n if not Path(sdata_zip_filepath).exists():\n urlretrieve(\n \"https://s3.embl.de/spatialdata/spatialdata-sandbox/visium_associated_xenium_io.zip\", sdata_zip_filepath\n )\n with zipfile.ZipFile(sdata_zip_filepath, \"r\") as zip_ref:\n zip_ref.extractall(\".\")\n Path(\"data.zarr\").rename(sdata_zarr_filepath)\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Save the `.zarr` DirectoryStore as a LaminDB Artifact.\n\n# %%\nsdata_zarr_artifact = ln.Artifact(\n sdata_zarr_filepath,\n key=\"vitessce_examples/visium.sdata.zarr\",\n description=\"Visium SpatialData Example\",\n type=\"dataset\",\n)\nsdata_zarr_artifact.save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n# Here, we use the [SpatialDataWrapper](https://python-docs.vitessce.io/api_data.html#vitessce.wrappers.SpatialDataWrapper) class to specify which parts of the SpatialData object will be loaded for visualization.\n\n# %%\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=sdata_zarr_artifact.description,\n)\n# Add data to the configuration:\ndataset = vc.add_dataset(name='Breast Cancer Visium').add_object(\n vit.SpatialDataWrapper(\n sdata_artifact=sdata_zarr_artifact,\n # The following paths are relative to the root of the SpatialData zarr store on-disk.\n image_path=\"images/CytAssist_FFPE_Human_Breast_Cancer_full_image\",\n table_path=\"tables/table\",\n obs_feature_matrix_path=\"tables/table/X\",\n obs_spots_path=\"shapes/CytAssist_FFPE_Human_Breast_Cancer\",\n region=\"CytAssist_FFPE_Human_Breast_Cancer\",\n coordinate_system=\"global\",\n coordination_values={\n # The following tells Vitessce to consider each observation as a \"spot\"\n \"obsType\": \"spot\",\n }\n )\n)\n\n# Add views (visualizations) to the configuration:\nspatial = vc.add_view(\"spatialBeta\", dataset=dataset)\nfeature_list = vc.add_view(\"featureList\", dataset=dataset)\nlayer_controller = vc.add_view(\"layerControllerBeta\", dataset=dataset)\nvc.link_views_by_dict([spatial, layer_controller], {\n 'imageLayer': vit.CoordinationLevel([{\n 'photometricInterpretation': 'RGB',\n }]),\n}, scope_prefix=vit.get_initial_coordination_scope_prefix(\"A\", \"image\"))\nvc.link_views([spatial, layer_controller, feature_list, obs_sets], ['obsType'], [\"spot\"])\n\n# Layout the views\nvc.layout(spatial | (feature_list / layer_controller));\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nsdata_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View Visium SpatialData Example in Vitessce\"\n)\n\n# %%\nsdata_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# :::\n\n# %% [markdown]\n# ## Visualize an image (OME-TIFF format)\n\n# %% [markdown]\n# Vitesse can visualize data from multiple bioimaging file formats, including [OME-TIFF](https://ome-model.readthedocs.io/en/stable/ome-tiff/).\n\n# %% [markdown]\n# Here, we download an example OME-TIFF file (approximately 1.02 GB).\n\n# %%\nome_tiff_filepath = \"./VAN0006-LK-2-85-PAS_registered.ome.tif\"\nif not Path(ome_tiff_filepath).exists():\n urlretrieve(\n \"https://assets.hubmapconsortium.org/f4188a148e4c759092d19369d310883b/ometiff-pyramids/processedMicroscopy/VAN0006-LK-2-85-PAS_images/VAN0006-LK-2-85-PAS_registered.ome.tif\", ome_tiff_filepath\n )\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Save the `.ome.tif` file as a LaminDB Artifact.\n\n# %%\nome_tiff_artifact = ln.Artifact(\n ome_tiff_filepath,\n key=\"vitessce_examples/VAN0006-LK-2-85-PAS_registered.ome.tif\",\n description=\"PAS OME-TIFF file, Neumann et al., 2020\",\n type=\"dataset\",\n)\nome_tiff_artifact.save()\n\n# %% [markdown]\n# When using OME-TIFF files, we can use [generate-tiff-offsets](https://github.com/hms-dbmi/generate-tiff-offsets) to create an index for the bytes within the OME-TIFF file. We store these to a companion `.offsets.json` file which makes loading subsets of the image more efficient.\n\n# %%\noffsets = get_offsets(ome_tiff_filepath)\n\n# %%\noffsets_path = \"./VAN0006-LK-2-85-PAS_registered.offsets.json\"\nwith open(offsets_path, \"w\") as file:\n json.dump(offsets, file)\noffsets_artifact = ln.Artifact(\n offsets_path,\n key=\"vitessce_examples/VAN0006-LK-2-85-PAS_registered.offsets.json\",\n description=\"PAS offsets file, Neumann et al., 2020\"\n).save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n# Here, we use the [ImageOmeTiffWrapper](https://python-docs.vitessce.io/api_data.html#vitessce.wrappers.ImageOmeTiffWrapper) class to specify which pair of OME-TIFF file and offsets JSON file to load.\n\n# %%\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=ome_tiff_artifact.description\n)\ndataset = vc.add_dataset(\"Image\").add_object(\n vit.ImageOmeTiffWrapper(\n img_artifact=ome_tiff_artifact,\n offsets_artifact=offsets_artifact,\n )\n)\nspatial = vc.add_view(\"spatialBeta\", dataset=dataset)\nlayer_controller = vc.add_view(\"layerControllerBeta\", dataset=dataset)\nvc.layout(spatial | layer_controller);\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nome_tiff_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View PAS OME-TIFF, Neumann et al., 2020 in Vitessce\"\n)\n\n# %%\nome_tiff_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# If your `VitessceConfig` object references data from multiple artifacts, the Vitessce button will appear next to a `Collection` that groups these artifacts (on the [Collections](https://lamin.ai/vitessce/examples/collections) tab of the Artifacts page).\n# In the case of OME-TIFF, the presence of the corresponding offsets JSON file will result in the creation of a Collection.\n#\n# :::\n\n# %% [markdown]\n# ## Visualize an image (OME-Zarr format)\n\n# %% [markdown]\n# Retrieve an OME-Zarr (also known as OME-NGFF) formatted image. We could download an unzip such an image, but in downloading the SpatialData object above, we already have access to a valid OME-Zarr image locally.\n\n# %%\nome_zarr_filepath = Path(sdata_zarr_filepath) / \"images\" / \"CytAssist_FFPE_Human_Breast_Cancer_full_image\"\n\n# %% [markdown]\n# ### Save the dataset\n\n# %% [markdown]\n# Save the `.ome.zarr` DirectoryStore as a LaminDB Artifact.\n\n# %%\nome_zarr_artifact = ln.Artifact(\n ome_zarr_filepath,\n key=\"vitessce_examples/visium.ome.zarr\",\n description=\"Visium OME-Zarr Example\",\n type=\"dataset\",\n)\nome_zarr_artifact.save()\n\n# %% [markdown]\n# ### Save a VitessceConfig object\n\n# %% [markdown]\n# You can create a dashboard for one or several datasets by using Vitessce's component API.\n# Here, we use the [ImageOmeZarrWrapper](https://python-docs.vitessce.io/api_data.html#vitessce.wrappers.ImageOmeZarrWrapper) class to specify an OME-Zarr file to load for visualization.\n\n# %%\nvc = vit.VitessceConfig(\n schema_version=\"1.0.17\",\n description=ome_zarr_artifact.description\n)\ndataset = vc.add_dataset(\"Image\").add_object(\n vit.ImageOmeZarrWrapper(\n img_artifact=ome_zarr_artifact,\n )\n)\nspatial = vc.add_view(\"spatialBeta\", dataset=dataset)\nlayer_controller = vc.add_view(\"layerControllerBeta\", dataset=dataset)\nvc.link_views_by_dict([spatial, layer_controller], {\n 'imageLayer': vit.CoordinationLevel([{\n 'photometricInterpretation': 'RGB',\n }]),\n}, scope_prefix=vit.get_initial_coordination_scope_prefix(\"A\", \"image\"))\nvc.layout(spatial | layer_controller);\n\n# %% [markdown]\n# Save the `VitessceConfig` object.\n\n# %% tags=[\"hide-output\"]\nome_zarr_vc_artifact = ln.integrations.save_vitessce_config(\n vc, description=\"View Visium OME-Zarr Example in Vitessce\"\n)\n\n# %%\nome_zarr_vc_artifact.view_lineage()\n\n# %% [markdown]\n# :::{note}\n#\n# After running `save_vitessce_config`, a Vitessce button will appear next to the dataset in the [Artifacts](https://lamin.ai/vitessce/examples/artifacts) page of the web interface.\n#\n# :::\n\n# %%\n# [optional] finish run context and auto-save the notebook\nln.finish()\n\n# %% [markdown]\n# :::{dropdown} Upload speed\n#\n# Here is [a note](https://lamin.ai/laminlabs/lamindata/transform/WDjio16cQsdW5zKv) on folder upload speed and why `lamindb` does not use the `.export(to=\"s3\")` functionality of Vitessce.\n#\n# :::\n\n# %% tags=[\"hide-cell\"]\n# clean up artifacts in CI run\nh5ad_artifact.delete(permanent=True)\nref_artifact.delete(permanent=True)\n\nadata_zarr_artifact.delete(permanent=True)\nsdata_zarr_artifact.delete(permanent=True)\n\nome_tiff_artifact.delete(permanent=True)\noffsets_artifact.delete(permanent=True)\n\nome_zarr_artifact.delete(permanent=True)\n\nh5ad_vc_artifact.delete(permanent=True)\nadata_zarr_vc_artifact.delete(permanent=True)\nsdata_vc_artifact.delete(permanent=True)\nome_tiff_vc_artifact.delete(permanent=True)\nome_zarr_vc_artifact.delete(permanent=True)\n","_branch_code":1,"reference_type":null}}}],"initiated_by_run":null}]

Runs