Quickstart

Reading Data

To get the fun part out of the way, you can read ur data:

from pathlib import Path
from rich.pretty import pprint, pretty_repr
from rich.console import Console
from nwb_linkml.io import HDF5IO

# set up for pprinting in notebooks
console = Console(width=100)
print = console.print

# find sample data file and read
nwb_file = Path('../../nwb_linkml/tests/data/aibs.nwb')
data = HDF5IO(nwb_file).read()
print(data) 

Show code cell output Hide code cell output

---------------------------------------------------------------------------
NotImplementedError                       Traceback (most recent call last)
Cell In[1], line 12
     10 # find sample data file and read
     11 nwb_file = Path('../../nwb_linkml/tests/data/aibs.nwb')
---> 12 data = HDF5IO(nwb_file).read()
     13 print(data) 

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/pdm/lib/python3.11/site-packages/nwb_linkml/io/hdf5.py:115, in HDF5IO.read(self, path)
    113 queue.apply_phase(ReadPhases.plan)
    114 # Read operations gather the data before casting into models
--> 115 queue.apply_phase(ReadPhases.read)
    116 # Construction operations actually cast the models
    117 # this often needs to run several times as models with dependencies wait for their
    118 # dependents to be cast
    119 queue.apply_phase(ReadPhases.construct)

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/pdm/lib/python3.11/site-packages/nwb_linkml/maps/hdf5.py:668, in ReadQueue.apply_phase(self, phase, max_passes)
    661     for op in phase_maps:
    662         if op.check(item, self.provider, self.completed):
    663             # Formerly there was an "exclusive" property in the maps which let potentially multiple
    664             # operations be applied per stage, except if an operation was `exclusive` which would break
    665             # iteration over the operations. This was removed because it was badly implemented, but
    666             # if there is ever a need to do that, then we would need to decide what to do with the
    667             # multiple results.
--> 668             results.append(op.apply(item, self.provider, self.completed))
    669             break # out of inner iteration
    671 # remake the source queue and save results

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/pdm/lib/python3.11/site-packages/nwb_linkml/maps/hdf5.py:237, in ResolveDynamicTable.apply(cls, src, provider, completed)
    235     # make a populated model :)
    236     base_model = provider.get_class(src.namespace, src.neurodata_type)
--> 237     model = dynamictable_to_model(obj, base=base_model)
    239     completes = [HDF5_Path(child.name) for child in obj.values()]
    241 return H5ReadResult(
    242     path=src.path,
    243     source=src,
   (...)
    247     applied=['ResolveDynamicTable']
    248 )

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/pdm/lib/python3.11/site-packages/nwb_linkml/maps/hdmf.py:52, in dynamictable_to_model(group, model, base)
     46 """
     47 Instantiate a dynamictable model
     48 
     49 Calls :func:`.model_from_dynamictable` if ``model`` is not provided.
     50 """
     51 if model is None:
---> 52     model = model_from_dynamictable(group, base)
     54 items = {}
     55 for col, col_type in model.model_fields.items():

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/pdm/lib/python3.11/site-packages/nwb_linkml/maps/hdmf.py:38, in model_from_dynamictable(group, base)
     34     # FIXME: handling nested column types that appear only in some versions?
     35     #types[col] = (List[type_ | None], ...)
     36     types[col] = (type_, None)
---> 38 model = create_model(group.name.split('/')[-1], **types, __base__=base)
     39 return model

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/pdm/lib/python3.11/site-packages/pydantic/main.py:1569, in create_model(model_name, __config__, __doc__, __base__, __module__, __validators__, __cls_kwargs__, __slots__, **field_definitions)
   1566     ns['__orig_bases__'] = __base__
   1567 namespace.update(ns)
-> 1569 return meta(
   1570     model_name,
   1571     resolved_bases,
   1572     namespace,
   1573     __pydantic_reset_parent_namespace__=False,
   1574     _create_model_module=__module__,
   1575     **kwds,
   1576 )

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/pdm/lib/python3.11/site-packages/pydantic/_internal/_model_construction.py:205, in ModelMetaclass.__new__(mcs, cls_name, bases, namespace, __pydantic_generic_metadata__, __pydantic_reset_parent_namespace__, _create_model_module, **kwargs)
    202 if config_wrapper.frozen and '__hash__' not in namespace:
    203     set_default_hash_func(cls, bases)
--> 205 complete_model_class(
    206     cls,
    207     cls_name,
    208     config_wrapper,
    209     raise_errors=False,
    210     types_namespace=types_namespace,
    211     create_model_module=_create_model_module,
    212 )
    214 # If this is placed before the complete_model_class call above,
    215 # the generic computed fields return type is set to PydanticUndefined
    216 cls.model_computed_fields = {k: v.info for k, v in cls.__pydantic_decorators__.computed_fields.items()}

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/pdm/lib/python3.11/site-packages/pydantic/_internal/_model_construction.py:534, in complete_model_class(cls, cls_name, config_wrapper, raise_errors, types_namespace, create_model_module)
    531     return False
    533 try:
--> 534     schema = cls.__get_pydantic_core_schema__(cls, handler)
    535 except PydanticUndefinedAnnotation as e:
    536     if raise_errors:

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/pdm/lib/python3.11/site-packages/pydantic/main.py:642, in BaseModel.__get_pydantic_core_schema__(cls, source, handler)
    639     if not cls.__pydantic_generic_metadata__['origin']:
    640         return cls.__pydantic_core_schema__
--> 642 return handler(source)

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/pdm/lib/python3.11/site-packages/pydantic/_internal/_schema_generation_shared.py:83, in CallbackGetCoreSchemaHandler.__call__(self, source_type)
     82 def __call__(self, source_type: Any, /) -> core_schema.CoreSchema:
---> 83     schema = self._handler(source_type)
     84     ref = schema.get('ref')
     85     if self._ref_mode == 'to-def':

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/pdm/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:512, in GenerateSchema.generate_schema(self, obj, from_dunder_get_core_schema)
    509         schema = from_property
    511 if schema is None:
--> 512     schema = self._generate_schema_inner(obj)
    514 metadata_js_function = _extract_get_pydantic_json_schema(obj, schema)
    515 if metadata_js_function is not None:

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/pdm/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:784, in GenerateSchema._generate_schema_inner(self, obj)
    782 if lenient_issubclass(obj, BaseModel):
    783     with self.model_type_stack.push(obj):
--> 784         return self._model_schema(obj)
    786 if isinstance(obj, PydanticRecursiveRef):
    787     return core_schema.definition_reference_schema(schema_ref=obj.type_ref)

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/pdm/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:591, in GenerateSchema._model_schema(self, cls)
    579     model_schema = core_schema.model_schema(
    580         cls,
    581         inner_schema,
   (...)
    587         metadata=metadata,
    588     )
    589 else:
    590     fields_schema: core_schema.CoreSchema = core_schema.model_fields_schema(
--> 591         {k: self._generate_md_field_schema(k, v, decorators) for k, v in fields.items()},
    592         computed_fields=[
    593             self._computed_field_schema(d, decorators.field_serializers)
    594             for d in computed_fields.values()
    595         ],
    596         extras_schema=extras_schema,
    597         model_name=cls.__name__,
    598     )
    599     inner_schema = apply_validators(fields_schema, decorators.root_validators.values(), None)
    600     new_inner_schema = define_expected_missing_refs(inner_schema, recursively_defined_type_refs())

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/pdm/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:591, in <dictcomp>(.0)
    579     model_schema = core_schema.model_schema(
    580         cls,
    581         inner_schema,
   (...)
    587         metadata=metadata,
    588     )
    589 else:
    590     fields_schema: core_schema.CoreSchema = core_schema.model_fields_schema(
--> 591         {k: self._generate_md_field_schema(k, v, decorators) for k, v in fields.items()},
    592         computed_fields=[
    593             self._computed_field_schema(d, decorators.field_serializers)
    594             for d in computed_fields.values()
    595         ],
    596         extras_schema=extras_schema,
    597         model_name=cls.__name__,
    598     )
    599     inner_schema = apply_validators(fields_schema, decorators.root_validators.values(), None)
    600     new_inner_schema = define_expected_missing_refs(inner_schema, recursively_defined_type_refs())

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/pdm/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:947, in GenerateSchema._generate_md_field_schema(self, name, field_info, decorators)
    940 def _generate_md_field_schema(
    941     self,
    942     name: str,
    943     field_info: FieldInfo,
    944     decorators: DecoratorInfos,
    945 ) -> core_schema.ModelField:
    946     """Prepare a ModelField to represent a model field."""
--> 947     common_field = self._common_field_schema(name, field_info, decorators)
    948     return core_schema.model_field(
    949         common_field['schema'],
    950         serialization_exclude=common_field['serialization_exclude'],
   (...)
    954         metadata=common_field['metadata'],
    955     )

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/pdm/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:1134, in GenerateSchema._common_field_schema(self, name, field_info, decorators)
   1132         schema = self._apply_annotations(source_type, annotations, transform_inner_schema=set_discriminator)
   1133     else:
-> 1134         schema = self._apply_annotations(
   1135             source_type,
   1136             annotations,
   1137         )
   1139 # This V1 compatibility shim should eventually be removed
   1140 # push down any `each_item=True` validators
   1141 # note that this won't work for any Annotated types that get wrapped by a function validator
   1142 # but that's okay because that didn't exist in V1
   1143 this_field_validators = filter_field_decorator_info_by_field(decorators.validators.values(), name)

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/pdm/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:1890, in GenerateSchema._apply_annotations(self, source_type, annotations, transform_inner_schema)
   1885         continue
   1886     get_inner_schema = self._get_wrapped_inner_schema(
   1887         get_inner_schema, annotation, pydantic_js_annotation_functions
   1888     )
-> 1890 schema = get_inner_schema(source_type)
   1891 if pydantic_js_annotation_functions:
   1892     metadata = CoreMetadataHandler(schema).metadata

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/pdm/lib/python3.11/site-packages/pydantic/_internal/_schema_generation_shared.py:83, in CallbackGetCoreSchemaHandler.__call__(self, source_type)
     82 def __call__(self, source_type: Any, /) -> core_schema.CoreSchema:
---> 83     schema = self._handler(source_type)
     84     ref = schema.get('ref')
     85     if self._ref_mode == 'to-def':

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/pdm/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:1871, in GenerateSchema._apply_annotations.<locals>.inner_handler(obj)
   1869 from_property = self._generate_schema_from_property(obj, source_type)
   1870 if from_property is None:
-> 1871     schema = self._generate_schema_inner(obj)
   1872 else:
   1873     schema = from_property

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/pdm/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:789, in GenerateSchema._generate_schema_inner(self, obj)
    786 if isinstance(obj, PydanticRecursiveRef):
    787     return core_schema.definition_reference_schema(schema_ref=obj.type_ref)
--> 789 return self.match_type(obj)

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/pdm/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:871, in GenerateSchema.match_type(self, obj)
    869 origin = get_origin(obj)
    870 if origin is not None:
--> 871     return self._match_generic_type(obj, origin)
    873 if self._arbitrary_types:
    874     return self._arbitrary_type_schema(obj)

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/pdm/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:895, in GenerateSchema._match_generic_type(self, obj, origin)
    892     return from_property
    894 if _typing_extra.origin_is_union(origin):
--> 895     return self._union_schema(obj)
    896 elif origin in TUPLE_TYPES:
    897     return self._tuple_schema(obj)

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/pdm/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:1207, in GenerateSchema._union_schema(self, union_type)
   1205         nullable = True
   1206     else:
-> 1207         choices.append(self.generate_schema(arg))
   1209 if len(choices) == 1:
   1210     s = choices[0]

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/pdm/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:507, in GenerateSchema.generate_schema(self, obj, from_dunder_get_core_schema)
    504 schema: CoreSchema | None = None
    506 if from_dunder_get_core_schema:
--> 507     from_property = self._generate_schema_from_property(obj, obj)
    508     if from_property is not None:
    509         schema = from_property

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/pdm/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:679, in GenerateSchema._generate_schema_from_property(self, obj, source)
    677         schema = get_schema(source)
    678     else:
--> 679         schema = get_schema(
    680             source, CallbackGetCoreSchemaHandler(self._generate_schema_inner, self, ref_mode=ref_mode)
    681         )
    682 # fmt: off
    683 elif (
    684     (existing_schema := getattr(obj, '__pydantic_core_schema__', None)) is not None
    685     and not isinstance(existing_schema, MockCoreSchema)
    686     and existing_schema.get('cls', None) == obj
    687 ):

File ~/checkouts/readthedocs.org/user_builds/nwb-linkml/envs/pdm/lib/python3.11/site-packages/nwb_linkml/types/ndarray.py:106, in NDArray.__get_pydantic_core_schema__(cls, _source_type, _handler)
    104 # get pydantic core schema for the given specified type
    105 if isinstance(dtype, nptyping.structure.StructureMeta):
--> 106     raise NotImplementedError('Jonny finish this')
    107     # functools.reduce(operator.or_, [int, float, str])
    108 else:
    109     array_type_handler = _handler.generate_schema(
    110         np_to_python[dtype])

NotImplementedError: Jonny finish this

Load and manipulate NWB schemas

A git provider module can manage a repository to provide a given NWB namespace at a given version, and cast the schema into Pydantic models from nwb_schema_language.

For the nwb-core schema, loading first just the namespaces file (without adjoining schema):

from nwb_linkml.providers.git import NWB_CORE_REPO
from nwb_linkml.io.schema import load_namespaces

namespace_file: 'Path' = NWB_CORE_REPO.provide_from_git('2.6.0')
core_namespaces = load_namespaces(namespace_file)
print(core_namespaces)

Or for a schema file…

from nwb_linkml.io.schema import load_schema_file

base_schema_file =  namespace_file.parent / 'nwb.base.yaml'
nwb_core_base = load_schema_file(base_schema_file)
print(nwb_core_base)

And additional adapters are used to handle some of the implicit behavior in nwb schema files, like importing other namespaces at a specific version, and inter-schema class imports. Eg. the NamespacesAdapter finds the implicitly imported hdmf-common namespace (again provided by the git schema provider).

from nwb_linkml.adapters import NamespacesAdapter

core_ns = NamespacesAdapter.from_yaml(namespace_file)
print(core_ns.imported) 

The classes in nwb_schema_language are just pydantic models, so they can be used like any other to create new, validated schemas.

Translating to LinkML

adapters handle the conversion from NWB schema language to LinkML.

core_linkml = core_ns.build()
print(core_linkml)

The BuildResult class holds the LinkML representation of each of the schemas and their classes, which are now in linkml_runtime.linkml_model.SchemaDefinition and ClassDefinition classes:

print(core_linkml.schemas[0])

Generating Pydantic Models

Todo

Document Pydantic model generation

Caching Output with Providers

Todo

Document provider usage