Lazy Loading PyQt Data Models (for QTreeViews)

Sometimes loading all of the required data into a model at the time it’s created is not a great option.  For example, you wouldn’t want to have a file system model enumerating every file – this could take quite a while, and besides, the user is not likely to want to navigate to every file on the filesystem!  A better option is to lazy load the data as required.

Qt data models allow you to do this by reimplementing the virtual functions rowCount , hasChildren , canFetchMore  and fetchMore  of QAbstractItemModel –

  • rowCount – you would normally implement this for a tree model anyway. Just make sure that it returns 0 if the node has children that have not been loaded yet.
  • hasChildren – override to return True for nodes that have children that haven’t been loaded yet and return whatever the base class returns in all other cases.
  • canFetchMore – return True if the node has children that haven’t been loaded yet, False otherwise.
  • fetchMore – this is where you perform whatever logic you need to decide what nodes to create and insert them into the model.

Here’s the basic idea – for nodes that you know have children that haven’t been loaded, return 0 from rowCount and True from canFetchMore and hasChildren. This tells Qt to show a node with an expander next to it even though it currently has no children. When the expander is clicked, fetchMore is called and you populate the children from the given parent.

One thing to note – you must call beginInsertRows and endInsertRows in the fetchMore method. What’s more, you musn’t change the underlying datastore before calling beginInsertRows or after endInsertRows. Unfortunately, you need to know how many rows you are inserting when you call beginInsertRows – so you are probably going to want to generate a list of nodes to add, then make the call to beginInsertRows. If you do it this way though, you can’t set the new nodes’ parent, as it would change the underlying datastore.

The item model is really the only thing that changes from a normal tree model with a couple of state attributes in the node for tracking whether data has been loaded or not.  Code below with comments on the model.

import sys

from PyQt5.QtWidgets import QApplication, QTreeView

from node import Node
from model import FileSystemTreeModel


app = QApplication(sys.argv)

model = FileSystemTreeModel(Node('Filename'), path='c:/')


tree = QTreeView()
tree.setModel(model)

tree.show()

sys.exit(app.exec_())
import os

from PyQt5.QtCore import Qt, QModelIndex, QAbstractItemModel
from PyQt5.QtGui import QIcon

class FileSystemTreeModel(QAbstractItemModel):

    FLAG_DEFAULT = Qt.ItemIsEnabled | Qt.ItemIsSelectable

    def __init__(self, path='c:/', parent=None):
        super(FileSystemTreeModel, self).__init__()

        self.root = Node('Filename')
        self.parent = parent
        self.path = path

        # generate root node children
        for file in os.listdir(path):
            file_path = os.path.join(path, file)

            node = Node(file, file_path, parent=self.root)
            if os.path.isdir(file_path):
                node.is_dir = True

    # takes a model index and returns the related Python node
    def getNode(self, index):
        if index.isValid():
            return index.internalPointer()
        else:
            return self.root

    # check if the note has data that has not been loaded yet
    def canFetchMore(self, index):
        node = self.getNode(index)

        if node.is_dir and not node.is_traversed:
            return True

        return False

    # called if canFetchMore returns True, then dynamically inserts nodes required for
    # directory contents
    def fetchMore(self, index):
        parent = self.getNode(index)

        nodes = []
        for file in os.listdir(parent.path):
            file_path = os.path.join(parent.path, file)

            node = Node(file, file_path)
            if os.path.isdir(file_path):
                node.is_dir = True

            nodes.append(node)

        self.insertNodes(0, nodes, index)
        parent.is_traversed = True

    # returns True for directory nodes so that Qt knows to check if there is more to load
    def hasChildren(self, index):
        node = self.getNode(index)

        if node.is_dir:
            return True

        return super(FileSystemTreeModel, self).hasChildren(index)

    # should return 0 if there is data to fetch (handled implicitly by check length of child list)
    def rowCount(self, parent):
        node = self.getNode(parent)
        return node.child_count()

    def columnCount(self, parent):
        return 1

    def flags(self, index):
        return FileSystemTreeModel.FLAG_DEFAULT

    def parent(self, index):
        node = self.getNode(index)

        parent = node.parent
        if parent == self.root:
            return QModelIndex()

        return self.createIndex(parent.row(), 0, parent)

    def index(self, row, column, parent):
        node = self.getNode(parent)

        child = node.child(row)

        if not child:
            return QModelIndex()

        return self.createIndex(row, column, child)

    def headerData(self, section, orientation, role):
        return self.root.name

    def data(self, index, role):
        if not index.isValid():
            return None

        node = index.internalPointer()

        if role == Qt.DisplayRole:
            return node.name

        if role == Qt.DecorationRole:
            if node.is_dir:
                return QIcon('d:\\folder.png')
            else:
                return QIcon('d:\\document.png')

        else:
            return None

    def insertNodes(self, position, nodes, parent=QModelIndex()):
        node = self.getNode(parent)

        self.beginInsertRows(parent, position, position + len(nodes) - 1)

        for child in nodes:
            success = node.insert_child(position, child)

        self.endInsertRows()

        return True
class Node(object):
    def __init__(self, name, path=None, parent=None):
        super(Node, self).__init__()

        self.name = name
        self.children = []
        self.parent = parent

        self.is_dir = False
        self.path = path
        self.is_traversed = False

        if parent is not None:
            parent.add_child(self)

    def add_child(self, child):
        self.children.append(child)
        child.parent = self

    def insert_child(self, position, child):
        if position < 0 or position > self.child_count():
            return False

        self.children.insert(position, child)
        child.parent = self

        return True

    def child(self, row):
        return self.children
def child_count(self): return len(self.children) def row(self): if self.parent is not None: return self.parent.children.index(self) return 0

Leave a Reply

Your email address will not be published. Required fields are marked *