Quickstart

Reading Data

To get the fun part out of the way, you can read ur data:

from pathlib import Path
from rich.pretty import pprint, pretty_repr
from rich.console import Console
from nwb_linkml.io import HDF5IO

# set up for pprinting in notebooks
console = Console(width=100)
print = console.print

# find sample data file and read
nwb_file = Path('../../nwb_linkml/tests/data/aibs.nwb')
data = HDF5IO(nwb_file).read()
print(data) 

Show code cell output Hide code cell output

---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
Cell In[1], line 12
     10 # find sample data file and read
     11 nwb_file = Path('../../nwb_linkml/tests/data/aibs.nwb')
---> 12 data = HDF5IO(nwb_file).read()
     13 print(data) 

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/linkml-arrays/lib/python3.11/site-packages/nwb_linkml/io/hdf5.py:106, in HDF5IO.read(self, path)
     66 def read(self, path: Optional[str] = None) -> Union["NWBFile", BaseModel, Dict[str, BaseModel]]:
     67     """
     68     Read data into models from an NWB File.
     69 
   (...)
    103         otherwise whatever Model or dictionary of models applies to the requested ``path``
    104     """
--> 106     provider = self.make_provider()
    108     h5f = h5py.File(str(self.path))
    109     src = h5f.get(path) if path else h5f

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/linkml-arrays/lib/python3.11/site-packages/nwb_linkml/io/hdf5.py:185, in HDF5IO.make_provider(self)
    182 provider = SchemaProvider(versions=versions)
    184 # build schema so we have them cached
--> 185 provider.build_from_dicts(schema)
    186 h5f.close()
    187 return provider

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/linkml-arrays/lib/python3.11/site-packages/nwb_linkml/providers/linkml.py:132, in LinkMLProvider.build_from_dicts(self, schemas, **kwargs)
    130 res = {}
    131 for adapter in ns_adapters.values():
--> 132     res.update(self.build(adapter, **kwargs))
    134 return res

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/linkml-arrays/lib/python3.11/site-packages/nwb_linkml/providers/schema.py:128, in SchemaProvider.build(self, ns_adapter, verbose, linkml_kwargs, pydantic_kwargs, **kwargs)
    125 linkml_provider = LinkMLProvider(path=self.path, verbose=verbose)
    126 pydantic_provider = PydanticProvider(path=self.path, verbose=verbose)
--> 128 linkml_res = linkml_provider.build(
    129     ns_adapter=ns_adapter, versions=self.versions, **linkml_kwargs
    130 )
    131 results = {}
    132 for ns, ns_result in linkml_res.items():

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/linkml-arrays/lib/python3.11/site-packages/nwb_linkml/providers/linkml.py:181, in LinkMLProvider.build(self, ns_adapter, versions, dump, force)
    179     progress = AdapterProgress(ns_adapter)
    180     with progress:
--> 181         built = ns_adapter.build(progress=progress)
    182 else:
    183     built = ns_adapter.build()

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/linkml-arrays/lib/python3.11/site-packages/nwb_linkml/adapters/namespaces.py:82, in NamespacesAdapter.build(self, skip_imports, progress)
     79     with contextlib.suppress(KeyError):
     80         # happens when we skip builds due to caching
     81         progress.update(sch.namespace, action=sch.name)
---> 82 sch_result += sch.build()
     83 if progress is not None:
     84     with contextlib.suppress(KeyError):
     85         # happens when we skip builds due to caching

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/linkml-arrays/lib/python3.11/site-packages/nwb_linkml/adapters/schema.py:83, in SchemaAdapter.build(self)
     81     res += new_res
     82 for group in self.groups:
---> 83     new_res = GroupAdapter(cls=group).build()
     84     if len(new_res.slots) > 0:
     85         pdb.set_trace()

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/linkml-arrays/lib/python3.11/site-packages/nwb_linkml/adapters/group.py:60, in GroupAdapter.build(self)
     52 if (
     53     len(self.cls.groups) == 0
     54     and len(self.cls.datasets) == 0
     55     and self.cls.neurodata_type_inc is not None
     56     and self.parent is not None
     57 ):
     58     return self.handle_container_slot(self.cls)
---> 60 nested_res = self.build_subclasses()
     61 # we don't propagate slots up to the next level since they are meant for this
     62 # level (ie. a way to refer to our children)
     63 res = self.build_base(extra_attrs=nested_res.slots)

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/linkml-arrays/lib/python3.11/site-packages/nwb_linkml/adapters/group.py:162, in GroupAdapter.build_subclasses(self)
    158 for dset in self.cls.datasets:
    159     # if dset.name == 'timestamps':
    160     #     pdb.set_trace()
    161     dset_adapter = DatasetAdapter(cls=dset, parent=self)
--> 162     dataset_res += dset_adapter.build()
    164 # Actually i'm not sure we have to special case this, we could handle it in
    165 # i/o instead
    166 
   (...)
    170 # and quantity of *,
    171 # the group can then contain any number of groups of those included types as direct children
    173 group_res = BuildResult()

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/linkml-arrays/lib/python3.11/site-packages/nwb_linkml/adapters/dataset.py:802, in DatasetAdapter.build(self)
    800 # apply matching maps
    801 if map is not None:
--> 802     res = map.apply(self.cls, res, self._get_full_name())
    804 return res

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/linkml-arrays/lib/python3.11/site-packages/nwb_linkml/adapters/dataset.py:533, in MapArrayLikeAttributes.apply(c, cls, res, name)
    529 """
    530 Map to an arraylike class
    531 """
    532 array_adapter = ArrayAdapter(cls.dims, cls.shape)
--> 533 expressions = array_adapter.make_slot()
    534 # make a slot for the arraylike class
    535 array_slot = SlotDefinition(
    536     name="array", range=ClassAdapter.handle_dtype(cls.dtype), **expressions
    537 )

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/linkml-arrays/lib/python3.11/site-packages/nwb_linkml/adapters/array.py:113, in ArrayAdapter.make_slot(self)
    103 def make_slot(
    104     self,
    105 ) -> Union[
    106     Dict[Literal["array"], ArrayExpression],
    107     Dict[Literal["any_of"], Dict[Literal["array"], List[ArrayExpression]]],
    108 ]:
    109     """
    110     Make the array expressions in a dict form that can be ``**kwarg``'d into a SlotDefinition,
    111     taking into account needing to use ``any_of`` for multiple array range specifications.
    112     """
--> 113     expressions = self.make()
    114     if len(expressions) == 1:
    115         return {"array": expressions[0]}

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/linkml-arrays/lib/python3.11/site-packages/nwb_linkml/adapters/array.py:100, in ArrayAdapter.make(self)
     98 """Create an array specification from self.dims and self.shape"""
     99 shapes = self.pivot_dims()
--> 100 expressions = [self.make_expression(shape) for shape in shapes]
    101 return expressions

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/linkml-arrays/lib/python3.11/site-packages/nwb_linkml/adapters/array.py:100, in <listcomp>(.0)
     98 """Create an array specification from self.dims and self.shape"""
     99 shapes = self.pivot_dims()
--> 100 expressions = [self.make_expression(shape) for shape in shapes]
    101 return expressions

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/linkml-arrays/lib/python3.11/site-packages/nwb_linkml/adapters/array.py:91, in ArrayAdapter.make_expression(self, shape)
     87 def make_expression(self, shape: Shape) -> ArrayExpression:
     88     """
     89     Create the corresponding array specification from a shape
     90     """
---> 91     dims = [
     92         DimensionExpression(alias=snake_case(dim.dims), exact_cardinality=dim.shape)
     93         for dim in shape
     94     ]
     95     return ArrayExpression(dimensions=dims)

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/linkml-arrays/lib/python3.11/site-packages/nwb_linkml/adapters/array.py:92, in <listcomp>(.0)
     87 def make_expression(self, shape: Shape) -> ArrayExpression:
     88     """
     89     Create the corresponding array specification from a shape
     90     """
     91     dims = [
---> 92         DimensionExpression(alias=snake_case(dim.dims), exact_cardinality=dim.shape)
     93         for dim in shape
     94     ]
     95     return ArrayExpression(dimensions=dims)

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/linkml-arrays/lib/python3.11/site-packages/nwb_linkml/maps/naming.py:27, in snake_case(name)
     24 if name is None:
     25     return None
---> 27 name = name.strip()
     28 name = re.sub(r"\W+", "_", name)
     29 name = name.lower()

AttributeError: 'int' object has no attribute 'strip'

Load and manipulate NWB schemas

A git provider module can manage a repository to provide a given NWB namespace at a given version, and cast the schema into Pydantic models from nwb_schema_language.

For the nwb-core schema, loading first just the namespaces file (without adjoining schema):

from nwb_linkml.providers.git import NWB_CORE_REPO
from nwb_linkml.io.schema import load_namespaces

namespace_file: 'Path' = NWB_CORE_REPO.provide_from_git('2.6.0')
core_namespaces = load_namespaces(namespace_file)
print(core_namespaces)

Or for a schema file…

from nwb_linkml.io.schema import load_schema_file

base_schema_file =  namespace_file.parent / 'nwb.base.yaml'
nwb_core_base = load_schema_file(base_schema_file)
print(nwb_core_base)

And additional adapters are used to handle some of the implicit behavior in nwb schema files, like importing other namespaces at a specific version, and inter-schema class imports. Eg. the NamespacesAdapter finds the implicitly imported hdmf-common namespace (again provided by the git schema provider).

from nwb_linkml.adapters import NamespacesAdapter

core_ns = NamespacesAdapter.from_yaml(namespace_file)
print(core_ns.imported) 

The classes in nwb_schema_language are just pydantic models, so they can be used like any other to create new, validated schemas.

Translating to LinkML

adapters handle the conversion from NWB schema language to LinkML.

core_linkml = core_ns.build()
print(core_linkml)

The BuildResult class holds the LinkML representation of each of the schemas and their classes, which are now in linkml_runtime.linkml_model.SchemaDefinition and ClassDefinition classes:

print(core_linkml.schemas[0])

Generating Pydantic Models

Todo

Document Pydantic model generation

Caching Output with Providers

Todo

Document provider usage