Commit b1c4a2d6 authored by Jared Casper's avatar Jared Casper
Browse files

Document indexed_dataset.get() a bit.

parent da0562fc

megatron/data/indexed_dataset.py

+5 −0
Original line number Diff line number Diff line
@@ -487,6 +487,11 @@ class MMapIndexedDataset(torch.utils.data.Dataset): 
            return sents



    def get(self, idx, offset=0, length=None):

        """ Retrieves a single item from the dataset with the option to only

        return a portion of the item.



        get(idx) is the same as [idx] but get() does not support slicing.

        """

        ptr, size = self._index[idx]

        if length is None:

            length = size - offset