Commit 9c377105 authored by Azharuddin Mohammed's avatar Azharuddin Mohammed
Browse files

Revert r366458, r366467 and r366468 

r366458 is causing test failures. r366467 and r366468 had to be reverted as
they were casuing conflict while reverting r366458.

r366468 [clangd] Remove dead code from BackgroundIndex
r366467 [clangd] BackgroundIndex stores shards to the closest project
r366458 [clangd] Refactor background-index shard loading

llvm-svn: 366551
parent e9e59ad7

clang-tools-extra/clangd/CMakeLists.txt

+0 −1
Original line number Diff line number Diff line
@@ -73,7 +73,6 @@ add_clang_library(clangDaemon 
  XRefs.cpp



  index/Background.cpp

  index/BackgroundIndexLoader.cpp

  index/BackgroundIndexStorage.cpp

  index/BackgroundQueue.cpp

  index/BackgroundRebuild.cpp

clang-tools-extra/clangd/ClangdServer.cpp

+1 −2
Original line number Diff line number Diff line
@@ -127,8 +127,7 @@ ClangdServer::ClangdServer(const GlobalCompilationDatabase &CDB, 
  if (Opts.BackgroundIndex) {

    BackgroundIdx = llvm::make_unique<BackgroundIndex>(

        Context::current().clone(), FSProvider, CDB,

        BackgroundIndexStorage::createDiskBackedStorageFactory(

            [&CDB](llvm::StringRef File) { return CDB.getProjectInfo(File); }));

        BackgroundIndexStorage::createDiskBackedStorageFactory());

    AddIndex(BackgroundIdx.get());

  }

  if (DynamicIdx)

clang-tools-extra/clangd/index/Background.cpp

+175 −93
Original line number Diff line number Diff line
@@ -10,7 +10,6 @@ 
#include "ClangdUnit.h"

#include "Compiler.h"

#include "Context.h"

#include "FSProvider.h"

#include "Headers.h"

#include "Logger.h"

#include "Path.h"
@@ -19,7 +18,6 @@ 
#include "Threading.h"

#include "Trace.h"

#include "URI.h"

#include "index/BackgroundIndexLoader.h"

#include "index/FileIndex.h"

#include "index/IndexAction.h"

#include "index/MemIndex.h"
@@ -30,8 +28,6 @@ 
#include "clang/Basic/SourceLocation.h"

#include "clang/Basic/SourceManager.h"

#include "clang/Driver/Types.h"

#include "llvm/ADT/ArrayRef.h"

#include "llvm/ADT/DenseSet.h"

#include "llvm/ADT/Hashing.h"

#include "llvm/ADT/STLExtras.h"

#include "llvm/ADT/ScopeExit.h"
@@ -46,7 +42,6 @@ 
#include <atomic>

#include <chrono>

#include <condition_variable>

#include <cstddef>

#include <memory>

#include <mutex>

#include <numeric>
@@ -54,8 +49,6 @@ 
#include <random>

#include <string>

#include <thread>

#include <utility>

#include <vector>



namespace clang {

namespace clangd {
@@ -126,18 +119,6 @@ llvm::SmallString<128> getAbsolutePath(const tooling::CompileCommand &Cmd) { 
  }

  return AbsolutePath;

}



bool shardIsStale(const LoadedShard &LS, llvm::vfs::FileSystem *FS) {

  auto Buf = FS->getBufferForFile(LS.AbsolutePath);

  if (!Buf) {

    elog("Background-index: Couldn't read {0} to validate stored index: {1}",

         LS.AbsolutePath, Buf.getError().message());

    // There is no point in indexing an unreadable file.

    return false;

  }

  return digest(Buf->get()->getBuffer()) != LS.Digest;

}



} // namespace



BackgroundIndex::BackgroundIndex(
@@ -175,14 +156,14 @@ BackgroundQueue::Task BackgroundIndex::changedFilesTask( 
    log("Enqueueing {0} commands for indexing", ChangedFiles.size());

    SPAN_ATTACH(Tracer, "files", int64_t(ChangedFiles.size()));



    auto NeedsReIndexing = loadProject(std::move(ChangedFiles));

    auto NeedsReIndexing = loadShards(std::move(ChangedFiles));

    // Run indexing for files that need to be updated.

    std::shuffle(NeedsReIndexing.begin(), NeedsReIndexing.end(),

                 std::mt19937(std::random_device{}()));

    std::vector<BackgroundQueue::Task> Tasks;

    Tasks.reserve(NeedsReIndexing.size());

    for (auto &Cmd : NeedsReIndexing)

      Tasks.push_back(indexFileTask(std::move(Cmd)));

    for (auto &Elem : NeedsReIndexing)

      Tasks.push_back(indexFileTask(std::move(Elem.first), Elem.second));

    Queue.append(std::move(Tasks));

  });
@@ -197,12 +178,13 @@ static llvm::StringRef filenameWithoutExtension(llvm::StringRef Path) { 
}



BackgroundQueue::Task

BackgroundIndex::indexFileTask(tooling::CompileCommand Cmd) {

  BackgroundQueue::Task T([this, Cmd] {

BackgroundIndex::indexFileTask(tooling::CompileCommand Cmd,

                               BackgroundIndexStorage *Storage) {

  BackgroundQueue::Task T([this, Storage, Cmd] {

    // We can't use llvm::StringRef here since we are going to

    // move from Cmd during the call below.

    const std::string FileName = Cmd.Filename;

    if (auto Error = index(std::move(Cmd)))

    if (auto Error = index(std::move(Cmd), Storage))

      elog("Indexing {0} failed: {1}", FileName, std::move(Error));

  });

  T.QueuePri = IndexFile;
@@ -225,7 +207,7 @@ void BackgroundIndex::boostRelated(llvm::StringRef Path) { 
void BackgroundIndex::update(

    llvm::StringRef MainFile, IndexFileIn Index,

    const llvm::StringMap<ShardVersion> &ShardVersionsSnapshot,

    bool HadErrors) {

    BackgroundIndexStorage *IndexStorage, bool HadErrors) {

  // Partition symbols/references into files.

  struct File {

    llvm::DenseSet<const Symbol *> Symbols;
@@ -309,7 +291,7 @@ void BackgroundIndex::update( 
    // We need to store shards before updating the index, since the latter

    // consumes slabs.

    // FIXME: Also skip serializing the shard if it is already up-to-date.

    BackgroundIndexStorage *IndexStorage = IndexStorageFactory(Path);

    if (IndexStorage) {

      IndexFileOut Shard;

      Shard.Symbols = SS.get();

      Shard.Refs = RS.get();
@@ -324,6 +306,7 @@ void BackgroundIndex::update( 
      if (auto Error = IndexStorage->storeShard(Path, Shard))

        elog("Failed to write background-index shard for file {0}: {1}", Path,

             std::move(Error));

    }



    {

      std::lock_guard<std::mutex> Lock(ShardVersionsMu);
@@ -346,7 +329,8 @@ void BackgroundIndex::update( 
  }

}



llvm::Error BackgroundIndex::index(tooling::CompileCommand Cmd) {

llvm::Error BackgroundIndex::index(tooling::CompileCommand Cmd,

                                   BackgroundIndexStorage *IndexStorage) {

  trace::Span Tracer("BackgroundIndex");

  SPAN_ATTACH(Tracer, "file", Cmd.Filename);

  auto AbsolutePath = getAbsolutePath(Cmd);
@@ -440,78 +424,176 @@ llvm::Error BackgroundIndex::index(tooling::CompileCommand Cmd) { 
    for (auto &It : *Index.Sources)

      It.second.Flags |= IncludeGraphNode::SourceFlag::HadErrors;

  }

  update(AbsolutePath, std::move(Index), ShardVersionsSnapshot, HadErrors);

  update(AbsolutePath, std::move(Index), ShardVersionsSnapshot, IndexStorage,

         HadErrors);



  Rebuilder.indexedTU();

  return llvm::Error::success();

}



// Restores shards for \p MainFiles from index storage. Then checks staleness of

// those shards and returns a list of TUs that needs to be indexed to update

// staleness.

std::vector<tooling::CompileCommand>

BackgroundIndex::loadProject(std::vector<std::string> MainFiles) {

  std::vector<tooling::CompileCommand> NeedsReIndexing;

std::vector<BackgroundIndex::Source>

BackgroundIndex::loadShard(const tooling::CompileCommand &Cmd,

                           BackgroundIndexStorage *IndexStorage,

                           llvm::StringSet<> &LoadedShards) {

  struct ShardInfo {

    std::string AbsolutePath;

    std::unique_ptr<IndexFileIn> Shard;

    FileDigest Digest = {};

    bool CountReferences = false;

    bool HadErrors = false;

  };

  std::vector<ShardInfo> IntermediateSymbols;

  // Make sure we don't have duplicate elements in the queue. Keys are absolute

  // paths.

  llvm::StringSet<> InQueue;

  auto FS = FSProvider.getFileSystem();

  // Dependencies of this TU, paired with the information about whether they

  // need to be re-indexed or not.

  std::vector<Source> Dependencies;

  std::queue<Source> ToVisit;

  std::string AbsolutePath = getAbsolutePath(Cmd).str();

  // Up until we load the shard related to a dependency it needs to be

  // re-indexed.

  ToVisit.emplace(AbsolutePath, true);

  InQueue.insert(AbsolutePath);

  // Goes over each dependency.

  while (!ToVisit.empty()) {

    Dependencies.push_back(std::move(ToVisit.front()));

    // Dependencies is not modified during the rest of the loop, so it is safe

    // to keep the reference.

    auto &CurDependency = Dependencies.back();

    ToVisit.pop();

    // If we have already seen this shard before(either loaded or failed) don't

    // re-try again. Since the information in the shard won't change from one TU

    // to another.

    if (!LoadedShards.try_emplace(CurDependency.Path).second) {

      // If the dependency needs to be re-indexed, first occurence would already

      // have detected that, so we don't need to issue it again.

      CurDependency.NeedsReIndexing = false;

      continue;

    }



  Rebuilder.startLoading();

  // Load shards for all of the mainfiles.

  const std::vector<LoadedShard> Result =

      loadIndexShards(MainFiles, IndexStorageFactory, CDB);

  size_t LoadedShards = 0;

    auto Shard = IndexStorage->loadShard(CurDependency.Path);

    if (!Shard || !Shard->Sources) {

      // File will be returned as requiring re-indexing to caller.

      vlog("Failed to load shard: {0}", CurDependency.Path);

      continue;

    }

    // These are the edges in the include graph for current dependency.

    for (const auto &I : *Shard->Sources) {

      auto U = URI::parse(I.getKey());

      if (!U)

        continue;

      auto AbsolutePath = URI::resolve(*U, CurDependency.Path);

      if (!AbsolutePath)

        continue;

      // Add file as dependency if haven't seen before.

      if (InQueue.try_emplace(*AbsolutePath).second)

        ToVisit.emplace(*AbsolutePath, true);

      // The node contains symbol information only for current file, the rest is

      // just edges.

      if (*AbsolutePath != CurDependency.Path)

        continue;



      // We found source file info for current dependency.

      assert(I.getValue().Digest != FileDigest{{0}} && "Digest is empty?");

      ShardInfo SI;

      SI.AbsolutePath = CurDependency.Path;

      SI.Shard = std::move(Shard);

      SI.Digest = I.getValue().Digest;

      SI.CountReferences =

          I.getValue().Flags & IncludeGraphNode::SourceFlag::IsTU;

      SI.HadErrors =

          I.getValue().Flags & IncludeGraphNode::SourceFlag::HadErrors;

      IntermediateSymbols.push_back(std::move(SI));

      // Check if the source needs re-indexing.

      // Get the digest, skip it if file doesn't exist.

      auto Buf = FS->getBufferForFile(CurDependency.Path);

      if (!Buf) {

        elog("Couldn't get buffer for file: {0}: {1}", CurDependency.Path,

             Buf.getError().message());

        continue;

      }

      // If digests match then dependency doesn't need re-indexing.

      // FIXME: Also check for dependencies(sources) of this shard and compile

      // commands for cache invalidation.

      CurDependency.NeedsReIndexing =

          digest(Buf->get()->getBuffer()) != I.getValue().Digest;

    }

  }

  // Load shard information into background-index.

  {

    // Update in-memory state.

    std::lock_guard<std::mutex> Lock(ShardVersionsMu);

    for (auto &LS : Result) {

      if (!LS.Shard)

        continue;

    // This can override a newer version that is added in another thread,

    // if this thread sees the older version but finishes later. This

    // should be rare in practice.

    for (const ShardInfo &SI : IntermediateSymbols) {

      auto SS =

          LS.Shard->Symbols

              ? llvm::make_unique<SymbolSlab>(std::move(*LS.Shard->Symbols))

          SI.Shard->Symbols

              ? llvm::make_unique<SymbolSlab>(std::move(*SI.Shard->Symbols))

              : nullptr;

      auto RS = LS.Shard->Refs

                    ? llvm::make_unique<RefSlab>(std::move(*LS.Shard->Refs))

      auto RS = SI.Shard->Refs

                    ? llvm::make_unique<RefSlab>(std::move(*SI.Shard->Refs))

                    : nullptr;

      auto RelS =

          LS.Shard->Relations

              ? llvm::make_unique<RelationSlab>(std::move(*LS.Shard->Relations))

          SI.Shard->Relations

              ? llvm::make_unique<RelationSlab>(std::move(*SI.Shard->Relations))

              : nullptr;

      ShardVersion &SV = ShardVersions[LS.AbsolutePath];

      SV.Digest = LS.Digest;

      SV.HadErrors = LS.HadErrors;

      ++LoadedShards;

      ShardVersion &SV = ShardVersions[SI.AbsolutePath];

      SV.Digest = SI.Digest;

      SV.HadErrors = SI.HadErrors;



      IndexedSymbols.update(LS.AbsolutePath, std::move(SS), std::move(RS),

                            std::move(RelS), LS.CountReferences);

      IndexedSymbols.update(SI.AbsolutePath, std::move(SS), std::move(RS),

                            std::move(RelS), SI.CountReferences);

    }

  }

  Rebuilder.loadedShard(LoadedShards);

  Rebuilder.doneLoading();

  if (!IntermediateSymbols.empty())

    Rebuilder.loadedTU();



  auto FS = FSProvider.getFileSystem();

  llvm::DenseSet<PathRef> TUsToIndex;

  // We'll accept data from stale shards, but ensure the files get reindexed

  // soon.

  for (auto &LS : Result) {

    if (!shardIsStale(LS, FS.get()))

  return Dependencies;

}



// Goes over each changed file and loads them from index. Returns the list of

// TUs that had out-of-date/no shards.

std::vector<std::pair<tooling::CompileCommand, BackgroundIndexStorage *>>

BackgroundIndex::loadShards(std::vector<std::string> ChangedFiles) {

  std::vector<std::pair<tooling::CompileCommand, BackgroundIndexStorage *>>

      NeedsReIndexing;

  // Keeps track of the files that will be reindexed, to make sure we won't

  // re-index same dependencies more than once. Keys are AbsolutePaths.

  llvm::StringSet<> FilesToIndex;

  // Keeps track of the loaded shards to make sure we don't perform redundant

  // disk IO. Keys are absolute paths.

  llvm::StringSet<> LoadedShards;

  Rebuilder.startLoading();

  for (const auto &File : ChangedFiles) {

    auto Cmd = CDB.getCompileCommand(File);

    if (!Cmd)

      continue;

    PathRef TUForFile = LS.DependentTU;

    assert(!TUForFile.empty() && "File without a TU!");



    std::string ProjectRoot;

    if (auto PI = CDB.getProjectInfo(File))

      ProjectRoot = std::move(PI->SourceRoot);



    BackgroundIndexStorage *IndexStorage = IndexStorageFactory(ProjectRoot);

    auto Dependencies = loadShard(*Cmd, IndexStorage, LoadedShards);

    for (const auto &Dependency : Dependencies) {

      if (!Dependency.NeedsReIndexing || FilesToIndex.count(Dependency.Path))

        continue;

      // FIXME: Currently, we simply schedule indexing on a TU whenever any of

      // its dependencies needs re-indexing. We might do it smarter by figuring

      // out a minimal set of TUs that will cover all the stale dependencies.

    // FIXME: Try looking at other TUs if no compile commands are available

    // for this TU, i.e TU was deleted after we performed indexing.

    TUsToIndex.insert(TUForFile);

      vlog("Enqueueing TU {0} because its dependency {1} needs re-indexing.",

           Cmd->Filename, Dependency.Path);

      NeedsReIndexing.push_back({std::move(*Cmd), IndexStorage});

      // Mark all of this TU's dependencies as to-be-indexed so that we won't

      // try to re-index those.

      for (const auto &Dependency : Dependencies)

        FilesToIndex.insert(Dependency.Path);

      break;

    }



  for (PathRef TU : TUsToIndex) {

    auto Cmd = CDB.getCompileCommand(TU);

    if (!Cmd)

      continue;

    NeedsReIndexing.emplace_back(std::move(*Cmd));

  }



  Rebuilder.doneLoading();

  return NeedsReIndexing;

}

clang-tools-extra/clangd/index/Background.h

+24 −14
Original line number Diff line number Diff line
@@ -12,7 +12,6 @@ 
#include "Context.h"

#include "FSProvider.h"

#include "GlobalCompilationDatabase.h"

#include "Path.h"

#include "SourceCode.h"

#include "Threading.h"

#include "index/BackgroundRebuild.h"
@@ -50,17 +49,15 @@ public: 
  virtual std::unique_ptr<IndexFileIn>

  loadShard(llvm::StringRef ShardIdentifier) const = 0;



  // The factory provides storage for each File.

  // The factory provides storage for each CDB.

  // It keeps ownership of the storage instances, and should manage caching

  // itself. Factory must be threadsafe and never returns nullptr.

  using Factory = llvm::unique_function<BackgroundIndexStorage *(PathRef)>;

  using Factory =

      llvm::unique_function<BackgroundIndexStorage *(llvm::StringRef)>;



  // Creates an Index Storage that saves shards into disk. Index storage uses

  // CDBDirectory + ".clangd/index/" as the folder to save shards. CDBDirectory

  // is the first directory containing a CDB in parent directories of a file, or

  // user's home directory if none was found, e.g. standard library headers.

  static Factory createDiskBackedStorageFactory(

      std::function<llvm::Optional<ProjectInfo>(PathRef)> GetProjectInfo);

  // CDBDirectory + ".clangd/index/" as the folder to save shards.

  static Factory createDiskBackedStorageFactory();

};



// A priority queue of tasks which can be run on (external) worker threads.
@@ -160,14 +157,15 @@ private: 
  /// information on IndexStorage.

  void update(llvm::StringRef MainFile, IndexFileIn Index,

              const llvm::StringMap<ShardVersion> &ShardVersionsSnapshot,

              bool HadErrors);

              BackgroundIndexStorage *IndexStorage, bool HadErrors);



  // configuration

  const FileSystemProvider &FSProvider;

  const GlobalCompilationDatabase &CDB;

  Context BackgroundContext;



  llvm::Error index(tooling::CompileCommand);

  llvm::Error index(tooling::CompileCommand,

                    BackgroundIndexStorage *IndexStorage);



  FileSymbols IndexedSymbols;

  BackgroundIndexRebuilder Rebuilder;
@@ -175,13 +173,25 @@ private: 
  std::mutex ShardVersionsMu;



  BackgroundIndexStorage::Factory IndexStorageFactory;

  // Tries to load shards for the MainFiles and their dependencies.

  std::vector<tooling::CompileCommand>

  loadProject(std::vector<std::string> MainFiles);

  struct Source {

    std::string Path;

    bool NeedsReIndexing;

    Source(llvm::StringRef Path, bool NeedsReIndexing)

        : Path(Path), NeedsReIndexing(NeedsReIndexing) {}

  };

  // Loads the shards for a single TU and all of its dependencies. Returns the

  // list of sources and whether they need to be re-indexed.

  std::vector<Source> loadShard(const tooling::CompileCommand &Cmd,

                                BackgroundIndexStorage *IndexStorage,

                                llvm::StringSet<> &LoadedShards);

  // Tries to load shards for the ChangedFiles.

  std::vector<std::pair<tooling::CompileCommand, BackgroundIndexStorage *>>

  loadShards(std::vector<std::string> ChangedFiles);



  BackgroundQueue::Task

  changedFilesTask(const std::vector<std::string> &ChangedFiles);

  BackgroundQueue::Task indexFileTask(tooling::CompileCommand Cmd);

  BackgroundQueue::Task indexFileTask(tooling::CompileCommand Cmd,

                                      BackgroundIndexStorage *Storage);



  // from lowest to highest priority

  enum QueuePriority {

clang-tools-extra/clangd/index/BackgroundIndexLoader.cpp

deleted100644 → 0
+0 −151
Original line number Diff line number Diff line 
//===-- BackgroundIndexLoader.cpp - ---------------------------------------===//

//

// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

// See https://llvm.org/LICENSE.txt for license information.

// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

//

//===----------------------------------------------------------------------===//



#include "index/BackgroundIndexLoader.h"

#include "GlobalCompilationDatabase.h"

#include "Logger.h"

#include "Path.h"

#include "index/Background.h"

#include "llvm/ADT/DenseMap.h"

#include "llvm/ADT/DenseSet.h"

#include "llvm/ADT/SmallString.h"

#include "llvm/ADT/StringMap.h"

#include "llvm/Support/Path.h"

#include <string>

#include <utility>

#include <vector>



namespace clang {

namespace clangd {

namespace {



llvm::Optional<Path> uriToAbsolutePath(llvm::StringRef URI, PathRef HintPath) {

  auto U = URI::parse(URI);

  if (!U)

    return llvm::None;

  auto AbsolutePath = URI::resolve(*U, HintPath);

  if (!AbsolutePath)

    return llvm::None;

  return *AbsolutePath;

}



/// A helper class to cache BackgroundIndexStorage operations and keep the

/// inverse dependency mapping.

class BackgroundIndexLoader {

public:

  BackgroundIndexLoader(BackgroundIndexStorage::Factory &IndexStorageFactory)

      : IndexStorageFactory(IndexStorageFactory) {}

  /// Load the shards for \p MainFile and all of its dependencies.

  void load(PathRef MainFile);



  /// Consumes the loader and returns all shards.

  std::vector<LoadedShard> takeResult() &&;



private:

  /// Returns the Shard for \p StartSourceFile from cache or loads it from \p

  /// Storage. Also returns paths for dependencies of \p StartSourceFile if it

  /// wasn't cached yet.

  std::pair<const LoadedShard &, std::vector<Path>>

  loadShard(PathRef StartSourceFile);



  /// Cache for Storage lookups.

  llvm::StringMap<LoadedShard> LoadedShards;



  /// References are into the AbsolutePaths in LoadedShards.

  llvm::DenseMap<PathRef, PathRef> FileToTU;



  BackgroundIndexStorage::Factory &IndexStorageFactory;

};



std::pair<const LoadedShard &, std::vector<Path>>

BackgroundIndexLoader::loadShard(PathRef StartSourceFile) {

  auto It = LoadedShards.try_emplace(StartSourceFile);

  LoadedShard &LS = It.first->getValue();

  std::vector<Path> Edges = {};

  // Return the cached shard.

  if (!It.second)

    return {LS, Edges};



  LS.AbsolutePath = StartSourceFile.str();

  BackgroundIndexStorage *Storage = IndexStorageFactory(LS.AbsolutePath);

  auto Shard = Storage->loadShard(StartSourceFile);

  if (!Shard || !Shard->Sources) {

    vlog("Failed to load shard: {0}", StartSourceFile);

    return {LS, Edges};

  }



  LS.Shard = std::move(Shard);

  for (const auto &It : *LS.Shard->Sources) {

    auto AbsPath = uriToAbsolutePath(It.getKey(), StartSourceFile);

    if (!AbsPath)

      continue;

    // A shard contains only edges for non main-file sources.

    if (*AbsPath != StartSourceFile) {

      Edges.push_back(*AbsPath);

      continue;

    }



    // Fill in shard metadata.

    const IncludeGraphNode &IGN = It.getValue();

    LS.Digest = IGN.Digest;

    LS.CountReferences = IGN.Flags & IncludeGraphNode::SourceFlag::IsTU;

    LS.HadErrors = IGN.Flags & IncludeGraphNode::SourceFlag::HadErrors;

  }

  assert(LS.Digest != FileDigest{{0}} && "Digest is empty?");

  return {LS, Edges};

}



void BackgroundIndexLoader::load(PathRef MainFile) {

  llvm::StringSet<> InQueue;

  // Following containers points to strings inside InQueue.

  std::queue<PathRef> ToVisit;

  InQueue.insert(MainFile);

  ToVisit.push(MainFile);



  while (!ToVisit.empty()) {

    PathRef SourceFile = ToVisit.front();

    ToVisit.pop();



    auto ShardAndEdges = loadShard(SourceFile);

    FileToTU[ShardAndEdges.first.AbsolutePath] = MainFile;

    for (PathRef Edge : ShardAndEdges.second) {

      auto It = InQueue.insert(Edge);

      if (It.second)

        ToVisit.push(It.first->getKey());

    }

  }

}



std::vector<LoadedShard> BackgroundIndexLoader::takeResult() && {

  std::vector<LoadedShard> Result;

  Result.reserve(LoadedShards.size());

  for (auto &It : LoadedShards) {

    Result.push_back(std::move(It.getValue()));

    LoadedShard &LS = Result.back();

    auto TUsIt = FileToTU.find(LS.AbsolutePath);

    assert(TUsIt != FileToTU.end() && "No TU registered for the shard");

    Result.back().DependentTU = TUsIt->second;

  }

  return Result;

}

} // namespace



std::vector<LoadedShard>

loadIndexShards(llvm::ArrayRef<Path> MainFiles,

                BackgroundIndexStorage::Factory &IndexStorageFactory,

                const GlobalCompilationDatabase &CDB) {

  BackgroundIndexLoader Loader(IndexStorageFactory);

  for (llvm::StringRef MainFile : MainFiles) {

    assert(llvm::sys::path::is_absolute(MainFile));

    Loader.load(MainFile);

  }

  return std::move(Loader).takeResult();

}



} // namespace clangd

} // namespace clang