Address performance issues with DataSelector when thousands of files are selected (23954df2) · Commits · NDIP / Nova Packages / nova-trame

CHANGELOG.md

+8 −0

Original line number	Diff line number	Diff line
		### nova-trame, 0.20.3

		* Performance of the DataSelector for large numbers of files should be improved (thanks to John Duggan).

		### nova-trame, 0.20.2

		* Matplotlib figure will no longer raise a TypeError when running on Python >= 3.11 (thanks to John Duggan).

		### nova-trame, 0.20.1

		* DataSelector now supports a `show_user_directories` flag that will allow users to choose datafiles from user directories (thanks to John Duggan).

poetry.lock

+32 −1

Original line number	Diff line number	Diff line
		@@ -1558,6 +1558,22 @@ pyspark = ["pyspark (>=3.5.0)"]
		pyspark-connect = ["pyspark[connect] (>=3.5.0)"]
		sqlframe = ["sqlframe (>=3.22.0)"]

		[[package]]
		name = "natsort"
		version = "8.4.0"
		description = "Simple yet flexible natural sorting in Python."
		optional = false
		python-versions = ">=3.7"
		groups = ["main"]
		files = [
		{file = "natsort-8.4.0-py3-none-any.whl", hash = "sha256:4732914fb471f56b5cce04d7bae6f164a592c7712e1c85f9ef585e197299521c"},
		{file = "natsort-8.4.0.tar.gz", hash = "sha256:45312c4a0e5507593da193dedd04abb1469253b601ecaf63445ad80f0a1ea581"},
		]

		[package.extras]
		fast = ["fastnumbers (>=2.0.0)"]
		icu = ["PyICU (>=1.0.0)"]

		[[package]]
		name = "nodeenv"
		version = "1.9.1"
		@@ -2990,6 +3006,21 @@ files = [
		dev = ["nox", "pre-commit", "pytest (>=6)", "pytest-cov (>=3)", "ruff"]
		test = ["nox", "pytest (>=6)", "pytest-cov (>=3)"]

		[[package]]
		name = "trame-datagrid"
		version = "0.2.1"
		description = "Trame widget for RevoGrid"
		optional = false
		python-versions = "*"
		groups = ["main"]
		files = [
		{file = "trame-datagrid-0.2.1.tar.gz", hash = "sha256:284cbf294dc4c10b32a9fdfcc15391aee5eeb4fd726733ac83a289b959b344ee"},
		{file = "trame_datagrid-0.2.1-py3-none-any.whl", hash = "sha256:93ac6aa6213eab9d28ca37762a1ef8cdbf8e61d85a7a25ec48734b007816325f"},
		]

		[package.dependencies]
		trame-client = "*"

		[[package]]
		name = "trame-matplotlib"
		version = "2.0.3"
		@@ -3382,4 +3413,4 @@ propcache = ">=0.2.1"
		[metadata]
		lock-version = "2.1"
		python-versions = ">=3.10,<4.0"
		content-hash = "928d2f9b4435bebdc59a8b590ec2afb95379253c8b28d9ba5f5bad75b4fb0f3d"
		content-hash = "6f04613f54e5481ade404734f3e36d1c7b0e4c198b5a01d3394fabcde51d9a94"

pyproject.toml

+3 −1

Original line number	Diff line number	Diff line
		@@ -6,7 +6,7 @@ Changelog = "https://code.ornl.gov/ndip/public-packages/nova-trame/blob/main/CHA

		[tool.poetry]
		name = "nova-trame"
		version = "0.20.2"
		version = "0.20.3"
		description = "A Python Package for injecting curated themes and custom components into Trame applications"
		authors = ["Duggan, John <dugganjw@ornl.gov>"]
		readme = "README.md"
		@@ -22,6 +22,7 @@ python = ">=3.10,<4.0"
		tomli = "*"
		tornado = "*"
		trame = "*"
		trame-datagrid = "*"
		trame-matplotlib = "*"
		trame-plotly = "*"
		trame-vega = "*"
		@@ -30,6 +31,7 @@ nova-mvvm = "*"
		pydantic = "*"
		nova-common = ">=0.2.0"
		blinker = "^1.9.0"
		natsort = "^8.4.0"

		[build-system]
		requires = ["poetry-core"]

src/nova/trame/model/data_selector.py

+50 −39

Original line number	Diff line number	Diff line
		@@ -5,9 +5,12 @@ from pathlib import Path
		from typing import Any, List, Optional
		from warnings import warn

		from natsort import natsorted
		from pydantic import BaseModel, Field, field_validator, model_validator
		from typing_extensions import Self

		CUSTOM_DIRECTORIES_LABEL = "Custom Directory"

		INSTRUMENTS = {
		"HFIR": {
		"CG-1A": "CG1A",
		@@ -57,14 +60,14 @@ INSTRUMENTS = {
		class DataSelectorState(BaseModel, validate_assignment=True):
		"""Selection state for identifying datafiles."""

		allow_custom_directories: bool = Field(default=False)
		facility: str = Field(default="", title="Facility")
		instrument: str = Field(default="", title="Instrument")
		experiment: str = Field(default="", title="Experiment")
		user_directory: str = Field(default="", title="User Directory")
		custom_directory: str = Field(default="", title="Custom Directory")
		directory: str = Field(default="")
		extensions: List[str] = Field(default=[])
		prefix: str = Field(default="")
		show_user_directories: bool = Field(default=False)

		@field_validator("experiment", mode="after")
		@classmethod
		@@ -80,7 +83,7 @@ class DataSelectorState(BaseModel, validate_assignment=True):
		warn(f"Facility '{self.facility}' could not be found. Valid options: {valid_facilities}", stacklevel=1)

		valid_instruments = self.get_instruments()
		if self.instrument and self.instrument not in valid_instruments:
		if self.instrument and self.facility != CUSTOM_DIRECTORIES_LABEL and self.instrument not in valid_instruments:
		warn(
		(
		f"Instrument '{self.instrument}' could not be found in '{self.facility}'. "
		@@ -94,8 +97,8 @@ class DataSelectorState(BaseModel, validate_assignment=True):

		def get_facilities(self) -> List[str]:
		facilities = list(INSTRUMENTS.keys())
		if self.show_user_directories:
		facilities.append("User Directory")
		if self.allow_custom_directories:
		facilities.append(CUSTOM_DIRECTORIES_LABEL)
		return facilities

		def get_instruments(self) -> List[str]:
		@@ -106,23 +109,23 @@ class DataSelectorModel:
		"""Manages file system interactions for the DataSelector widget."""

		def __init__(
		self, facility: str, instrument: str, extensions: List[str], prefix: str, show_user_directories: bool
		self, facility: str, instrument: str, extensions: List[str], prefix: str, allow_custom_directories: bool
		) -> None:
		self.state = DataSelectorState()
		self.state.facility = facility
		self.state.instrument = instrument
		self.state.extensions = extensions
		self.state.prefix = prefix
		self.state.show_user_directories = show_user_directories
		self.state.allow_custom_directories = allow_custom_directories

		def get_facilities(self) -> List[str]:
		return sorted(self.state.get_facilities())
		return natsorted(self.state.get_facilities())

		def get_instrument_dir(self) -> str:
		return INSTRUMENTS.get(self.state.facility, {}).get(self.state.instrument, "")

		def get_instruments(self) -> List[str]:
		return sorted(self.state.get_instruments())
		return natsorted(self.state.get_instruments())

		def get_experiments(self) -> List[str]:
		experiments = []
		@@ -135,11 +138,11 @@ class DataSelectorModel:
		except OSError:
		pass

		return sorted(experiments)
		return natsorted(experiments)

		def sort_directories(self, directories: List[Any]) -> List[Any]:
		# Sort the current level of dictionaries
		sorted_dirs = sorted(directories, key=lambda x: x["title"])
		sorted_dirs = natsorted(directories, key=lambda x: x["title"])

		# Process each sorted item to sort their children
		for item in sorted_dirs:
		@@ -154,15 +157,17 @@ class DataSelectorModel:

		return Path("/") / self.state.facility / self.get_instrument_dir() / self.state.experiment

		def get_user_directory_path(self) -> Optional[Path]:
		if not self.state.user_directory:
		def get_custom_directory_path(self) -> Optional[Path]:
		# Don't expose the full file system
		if not self.state.custom_directory:
		return None

		return Path("/SNS/users") / self.state.user_directory
		return Path(self.state.custom_directory)

		def get_directories(self) -> List[str]:
		if self.state.facility == "User Directory":
		base_path = self.get_user_directory_path()
		using_custom_directory = self.state.facility == CUSTOM_DIRECTORIES_LABEL
		if using_custom_directory:
		base_path = self.get_custom_directory_path()
		else:
		base_path = self.get_experiment_directory_path()

		@@ -171,6 +176,12 @@ class DataSelectorModel:

		directories = []
		try:
		if using_custom_directory:
		for entry in os.listdir(base_path):
		path = base_path / entry
		if os.path.isdir(path):
		directories.append({"path": str(path), "title": entry})
		else:
		for dirpath, _, _ in os.walk(base_path):
		# Get the relative path from the start path
		path_parts = os.path.relpath(dirpath, base_path).split(os.sep)
		@@ -224,7 +235,7 @@ class DataSelectorModel:
		except OSError:
		pass

		return sorted(datafiles)
		return natsorted(datafiles)

		def set_directory(self, directory_path: str) -> None:
		self.state.directory = directory_path

src/nova/trame/view/components/data_selector.py

+50 −22

Original line number	Diff line number	Diff line
		@@ -4,11 +4,11 @@ from typing import Any, List, Optional, cast
		from warnings import warn

		from trame.app import get_server
		from trame.widgets import client, html
		from trame.widgets import client, datagrid, html
		from trame.widgets import vuetify3 as vuetify

		from nova.mvvm.trame_binding import TrameBinding
		from nova.trame.model.data_selector import DataSelectorModel
		from nova.trame.model.data_selector import CUSTOM_DIRECTORIES_LABEL, DataSelectorModel
		from nova.trame.view.layouts import GridLayout, VBoxLayout
		from nova.trame.view_model.data_selector import DataSelectorViewModel

		@@ -17,18 +17,18 @@ from .input_field import InputField
		vuetify.enable_lab()


		class DataSelector(vuetify.VDataTableVirtual):
		class DataSelector(datagrid.VGrid):
		"""Allows the user to select datafiles from an IPTS experiment."""

		def __init__(
		self,
		v_model: str,
		allow_custom_directories: bool = False,
		facility: str = "",
		instrument: str = "",
		extensions: Optional[List[str]] = None,
		prefix: str = "",
		select_strategy: str = "all",
		show_user_directories: bool = False,
		**kwargs: Any,
		) -> None:
		"""Constructor for DataSelector.
		@@ -38,6 +38,9 @@ class DataSelector(vuetify.VDataTableVirtual):
		v_model : str
		The name of the state variable to bind to this widget. The state variable will contain a list of the files
		selected by the user.
		allow_custom_directories : bool, optional
		Whether or not to allow users to provide their own directories to search for datafiles in. Ignored if the
		facility parameter is set.
		facility : str, optional
		The facility to restrict data selection to. Options: HFIR, SNS
		instrument : str, optional
		@@ -50,9 +53,6 @@ class DataSelector(vuetify.VDataTableVirtual):
		select_strategy : str, optional
		The selection strategy to pass to the `VDataTable component <https://trame.readthedocs.io/en/latest/trame.widgets.vuetify3.html#trame.widgets.vuetify3.VDataTable>`__.
		If unset, the `all` strategy will be used.
		show_user_directories : bool, optional
		Whether or not to allow users to select data files from user directories. Ignored if the facility parameter
		is set.
		**kwargs
		All other arguments will be passed to the underlying
		`VDataTable component <https://trame.readthedocs.io/en/latest/trame.widgets.vuetify3.html#trame.widgets.vuetify3.VDataTable>`_.
		@@ -69,16 +69,17 @@ class DataSelector(vuetify.VDataTableVirtual):
		else:
		self._label = None

		if facility and show_user_directories:
		warn("show_user_directories will be ignored since the facility parameter is set.", stacklevel=1)
		if facility and allow_custom_directories:
		warn("allow_custom_directories will be ignored since the facility parameter is set.", stacklevel=1)

		self._v_model = v_model
		self._v_model_name_in_state = v_model.split(".")[0]
		self._allow_custom_directories = allow_custom_directories
		self._extensions = extensions if extensions is not None else []
		self._prefix = prefix
		self._select_strategy = select_strategy
		self._show_user_directories = show_user_directories

		self._revogrid_id = f"nova__dataselector_{self._next_id}_rv"
		self._state_name = f"nova__dataselector_{self._next_id}_state"
		self._facilities_name = f"nova__dataselector_{self._next_id}_facilities"
		self._instruments_name = f"nova__dataselector_{self._next_id}_instruments"
		@@ -87,6 +88,9 @@ class DataSelector(vuetify.VDataTableVirtual):
		self._datafiles_name = f"nova__dataselector_{self._next_id}_datafiles"

		self._flush_state = f"flushState('{self._v_model_name_in_state}');"
		self._reset_rv_grid = client.JSEval(
		exec=f"window.grid_manager.get('{self._revogrid_id}').updateCheckboxes()"
		).exec
		self._reset_state = client.JSEval(exec=f"{self._v_model} = []; {self._flush_state}").exec

		self.create_model(facility, instrument)
		@@ -106,19 +110,19 @@ class DataSelector(vuetify.VDataTableVirtual):
		if instrument == "":
		columns -= 1
		InputField(
		v_if=f"{self._state_name}.facility !== 'User Directory'",
		v_if=f"{self._state_name}.facility !== '{CUSTOM_DIRECTORIES_LABEL}'",
		v_model=f"{self._state_name}.instrument",
		items=(self._instruments_name,),
		type="autocomplete",
		)
		InputField(
		v_if=f"{self._state_name}.facility !== 'User Directory'",
		v_if=f"{self._state_name}.facility !== '{CUSTOM_DIRECTORIES_LABEL}'",
		v_model=f"{self._state_name}.experiment",
		column_span=columns,
		items=(self._experiments_name,),
		type="autocomplete",
		)
		InputField(v_else=True, v_model=f"{self._state_name}.user_directory", column_span=2)
		InputField(v_else=True, v_model=f"{self._state_name}.custom_directory", column_span=2)

		with GridLayout(columns=2, classes="flex-1-0 h-0", valign="start"):
		if not self._prefix:
		@@ -137,21 +141,44 @@ class DataSelector(vuetify.VDataTableVirtual):

		super().__init__(
		v_model=self._v_model,
		classes="h-100 overflow-y-auto",
		fixed_header=True,
		headers=("[{ align: 'left', key: 'title', title: 'Available Datafiles' }]",),
		item_title="title",
		item_value="path",
		select_strategy=self._select_strategy,
		show_select=True,
		can_focus=False,
		columns=(
		"[{"
		" cellTemplate: (createElement, props) =>"
		f" window.grid_manager.get('{self._revogrid_id}').cellTemplate(createElement, props),"
		" columnTemplate: (createElement) =>"
		f" window.grid_manager.get('{self._revogrid_id}').columnTemplate(createElement),"
		" name: 'Available Datafiles',"
		" prop: 'title',"
		"}]",
		),
		frame_size=10,
		hide_attribution=True,
		id=self._revogrid_id,
		readonly=True,
		stretch=True,
		source=(self._datafiles_name,),
		theme="compact",
		**kwargs,
		)
		if self._label:
		self.label = self._label
		self.items = (self._datafiles_name,)
		if "update_modelValue" not in kwargs:
		self.update_modelValue = self._flush_state

		# Sets up some JavaScript event handlers when the component is mounted.
		with self:
		client.ClientTriggers(
		mounted=(
		"window.grid_manager.add("
		f" '{self._revogrid_id}',"
		f" '{self._v_model}',"
		f" '{self._datafiles_name}',"
		f" '{self._v_model_name_in_state}'"
		")"
		)
		)

		with cast(
		vuetify.VSelect,
		InputField(
		@@ -171,7 +198,7 @@ class DataSelector(vuetify.VDataTableVirtual):

		def create_model(self, facility: str, instrument: str) -> None:
		self._model = DataSelectorModel(
		facility, instrument, self._extensions, self._prefix, self._show_user_directories
		facility, instrument, self._extensions, self._prefix, self._allow_custom_directories
		)

		def create_viewmodel(self) -> None:
		@@ -191,6 +218,7 @@ class DataSelector(vuetify.VDataTableVirtual):

		def reset(self, _: Any = None) -> None:
		self._reset_state()
		self._reset_rv_grid()

		def set_state(
		self, facility: Optional[str] = None, instrument: Optional[str] = None, experiment: Optional[str] = None