Commit 3f8da5d0 authored by Sam McCall's avatar Sam McCall
Browse files

[Tooling/Syntax] Helpers to find spelled tokens touching a location. 

Summary: Useful when positions are used to target nodes, with before/after ambiguity.

Reviewers: ilya-biryukov, kbobyrev

Subscribers: cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D71356
parent 471d9f3e

clang/include/clang/Basic/SourceLocation.h

+11 −0
Original line number Diff line number Diff line
@@ -188,9 +188,20 @@ inline bool operator!=(const SourceLocation &LHS, const SourceLocation &RHS) { 
  return !(LHS == RHS);

}



// Ordering is meaningful only if LHS and RHS have the same FileID!

// Otherwise use SourceManager::isBeforeInTranslationUnit().

inline bool operator<(const SourceLocation &LHS, const SourceLocation &RHS) {

  return LHS.getRawEncoding() < RHS.getRawEncoding();

}

inline bool operator>(const SourceLocation &LHS, const SourceLocation &RHS) {

  return LHS.getRawEncoding() > RHS.getRawEncoding();

}

inline bool operator<=(const SourceLocation &LHS, const SourceLocation &RHS) {

  return LHS.getRawEncoding() <= RHS.getRawEncoding();

}

inline bool operator>=(const SourceLocation &LHS, const SourceLocation &RHS) {

  return LHS.getRawEncoding() >= RHS.getRawEncoding();

}



/// A trivial tuple used to represent a source range.

class SourceRange {

clang/include/clang/Tooling/Syntax/Tokens.h

+11 −0
Original line number Diff line number Diff line
@@ -309,6 +309,17 @@ private: 
  const SourceManager *SourceMgr;

};



/// The spelled tokens that overlap or touch a spelling location Loc.

/// This always returns 0-2 tokens.

llvm::ArrayRef<syntax::Token>

spelledTokensTouching(SourceLocation Loc, const syntax::TokenBuffer &Tokens);



/// The identifier token that overlaps or touches a spelling location Loc.

/// If there is none, returns nullptr.

const syntax::Token *

spelledIdentifierTouching(SourceLocation Loc,

                          const syntax::TokenBuffer &Tokens);



/// Lex the text buffer, corresponding to \p FID, in raw mode and record the

/// resulting spelled tokens. Does minimal post-processing on raw identifiers,

/// setting the appropriate token kind (instead of the raw_identifier reported

clang/lib/Tooling/Syntax/Tokens.cpp

+25 −0
Original line number Diff line number Diff line
@@ -248,6 +248,31 @@ TokenBuffer::expansionStartingAt(const syntax::Token *Spelled) const { 
  return E;

}



llvm::ArrayRef<syntax::Token>

syntax::spelledTokensTouching(SourceLocation Loc,

                              const syntax::TokenBuffer &Tokens) {

  assert(Loc.isFileID());

  llvm::ArrayRef<syntax::Token> All =

      Tokens.spelledTokens(Tokens.sourceManager().getFileID(Loc));

  // Comparing SourceLocations is well-defined within a FileID.

  auto *Right = llvm::partition_point(

      All, [&](const syntax::Token &Tok) { return Tok.location() < Loc; });

  bool AcceptRight = Right != All.end() && Right->location() <= Loc;

  bool AcceptLeft = Right != All.begin() && (Right - 1)->endLocation() >= Loc;

  return llvm::makeArrayRef(Right - (AcceptLeft ? 1 : 0),

                            Right + (AcceptRight ? 1 : 0));

}



const syntax::Token *

syntax::spelledIdentifierTouching(SourceLocation Loc,

                                  const syntax::TokenBuffer &Tokens) {

  for (const syntax::Token &Tok : spelledTokensTouching(Loc, Tokens)) {

    if (Tok.kind() == tok::identifier)

      return &Tok;

  }

  return nullptr;

}



std::vector<const syntax::Token *>

TokenBuffer::macroExpansions(FileID FID) const {

  auto FileIt = Files.find(FID);

clang/unittests/Tooling/Syntax/TokensTest.cpp

+41 −0
Original line number Diff line number Diff line
@@ -793,4 +793,45 @@ TEST_F(TokenBufferTest, macroExpansions) { 
    ActualMacroRanges.push_back(Expansion->range(SM));

  EXPECT_EQ(ExpectedMacroRanges, ActualMacroRanges);

}



TEST_F(TokenBufferTest, Touching) {

  llvm::Annotations Code("^i^nt^ ^a^b^=^1;^");

  recordTokens(Code.code());



  auto Touching = [&](int Index) {

    SourceLocation Loc = SourceMgr->getComposedLoc(SourceMgr->getMainFileID(),

                                                   Code.points()[Index]);

    return spelledTokensTouching(Loc, Buffer);

  };

  auto Identifier = [&](int Index) {

    SourceLocation Loc = SourceMgr->getComposedLoc(SourceMgr->getMainFileID(),

                                                   Code.points()[Index]);

    const syntax::Token *Tok = spelledIdentifierTouching(Loc, Buffer);

    return Tok ? Tok->text(*SourceMgr) : "";

  };



  EXPECT_THAT(Touching(0), SameRange(findSpelled("int")));

  EXPECT_EQ(Identifier(0), "");

  EXPECT_THAT(Touching(1), SameRange(findSpelled("int")));

  EXPECT_EQ(Identifier(1), "");

  EXPECT_THAT(Touching(2), SameRange(findSpelled("int")));

  EXPECT_EQ(Identifier(2), "");



  EXPECT_THAT(Touching(3), SameRange(findSpelled("ab")));

  EXPECT_EQ(Identifier(3), "ab");

  EXPECT_THAT(Touching(4), SameRange(findSpelled("ab")));

  EXPECT_EQ(Identifier(4), "ab");



  EXPECT_THAT(Touching(5), SameRange(findSpelled("ab =")));

  EXPECT_EQ(Identifier(5), "ab");



  EXPECT_THAT(Touching(6), SameRange(findSpelled("= 1")));

  EXPECT_EQ(Identifier(6), "");



  EXPECT_THAT(Touching(7), SameRange(findSpelled(";")));

  EXPECT_EQ(Identifier(7), "");



  ASSERT_EQ(Code.points().size(), 8u);

}



} // namespace