Unverified Commit 247fa041 authored by Jun Zhang's avatar Jun Zhang
Browse files

[clang] Add a new annotation token: annot_repl_input_end 

This patch is the first part of the below RFC:
https://discourse.llvm.org/t/rfc-handle-execution-results-in-clang-repl/68493



It adds an annotation token which will replace the original EOF token
when we are in the incremental C++ mode. In addition, when we're
parsing an ExprStmt and there's a missing semicolon after the
expression, we set a marker in the annotation token and continue
parsing.

Eventually, we propogate this info in ParseTopLevelStmtDecl and are able
to mark this Decl as something we want to do value printing. Below is a
example:

clang-repl> int x = 42;
clang-repl> x
// `x` is a TopLevelStmtDecl and without a semicolon, we should set
// it's IsSemiMissing bit so we can do something interesting in
// ASTConsumer::HandleTopLevelDecl.

The idea about annotation toke is proposed by Richard Smith, thanks!

Signed-off-by: default avatarJun Zhang <jun@junz.org>

Differential Revision: https://reviews.llvm.org/D148997
parent 8a5450d3

clang/include/clang/AST/Decl.h

+7 −0
Original line number Diff line number Diff line
@@ -4324,6 +4324,7 @@ class TopLevelStmtDecl : public Decl { 
  friend class ASTDeclWriter;



  Stmt *Statement = nullptr;

  bool IsSemiMissing = false;



  TopLevelStmtDecl(DeclContext *DC, SourceLocation L, Stmt *S)

      : Decl(TopLevelStmt, DC, L), Statement(S) {}
@@ -4337,6 +4338,12 @@ public: 
  SourceRange getSourceRange() const override LLVM_READONLY;

  Stmt *getStmt() { return Statement; }

  const Stmt *getStmt() const { return Statement; }

  void setStmt(Stmt *S) {

    assert(IsSemiMissing && "Operation supported for printing values only!");

    Statement = S;

  }

  bool isSemiMissing() const { return IsSemiMissing; }

  void setSemiMissing(bool Missing = true) { IsSemiMissing = Missing; }



  static bool classof(const Decl *D) { return classofKind(D->getKind()); }

  static bool classofKind(Kind K) { return K == TopLevelStmt; }

clang/include/clang/Basic/TokenKinds.def

+3 −0
Original line number Diff line number Diff line
@@ -942,6 +942,9 @@ ANNOTATION(module_end) 
// into the name of a header unit.

ANNOTATION(header_unit)



// Annotation for end of input in clang-repl.

ANNOTATION(repl_input_end)



#undef PRAGMA_ANNOTATION

#undef ANNOTATION

#undef TESTING_KEYWORD

clang/include/clang/Parse/Parser.h

+3 −1
Original line number Diff line number Diff line
@@ -18,6 +18,7 @@ 
#include "clang/Basic/OpenMPKinds.h"

#include "clang/Basic/OperatorPrecedence.h"

#include "clang/Basic/Specifiers.h"

#include "clang/Basic/TokenKinds.h"

#include "clang/Lex/CodeCompletionHandler.h"

#include "clang/Lex/Preprocessor.h"

#include "clang/Sema/DeclSpec.h"
@@ -692,7 +693,8 @@ private: 
  bool isEofOrEom() {

    tok::TokenKind Kind = Tok.getKind();

    return Kind == tok::eof || Kind == tok::annot_module_begin ||

           Kind == tok::annot_module_end || Kind == tok::annot_module_include;

           Kind == tok::annot_module_end || Kind == tok::annot_module_include ||

           Kind == tok::annot_repl_input_end;

  }



  /// Checks if the \p Level is valid for use in a fold expression.

clang/lib/Frontend/PrintPreprocessedOutput.cpp

+5 −1
Original line number Diff line number Diff line
@@ -663,7 +663,8 @@ void PrintPPOutputPPCallbacks::HandleWhitespaceBeforeTok(const Token &Tok, 
  // them.

  if (Tok.is(tok::eof) ||

      (Tok.isAnnotation() && !Tok.is(tok::annot_header_unit) &&

       !Tok.is(tok::annot_module_begin) && !Tok.is(tok::annot_module_end)))

       !Tok.is(tok::annot_module_begin) && !Tok.is(tok::annot_module_end) &&

       !Tok.is(tok::annot_repl_input_end)))

    return;



  // EmittedDirectiveOnThisLine takes priority over RequireSameLine.
@@ -819,6 +820,9 @@ static void PrintPreprocessedTokens(Preprocessor &PP, Token &Tok, 
      // -traditional-cpp the lexer keeps /all/ whitespace, including comments.

      PP.Lex(Tok);

      continue;

    } else if (Tok.is(tok::annot_repl_input_end)) {

      PP.Lex(Tok);

      continue;

    } else if (Tok.is(tok::eod)) {

      // Don't print end of directive tokens, since they are typically newlines

      // that mess up our line tracking. These come from unknown pre-processor

clang/lib/Interpreter/IncrementalParser.cpp

+8 −8
Original line number Diff line number Diff line
@@ -158,8 +158,8 @@ IncrementalParser::ParseOrWrapTopLevelDecl() { 
  LastPTU.TUPart = C.getTranslationUnitDecl();



  // Skip previous eof due to last incremental input.

  if (P->getCurToken().is(tok::eof)) {

    P->ConsumeToken();

  if (P->getCurToken().is(tok::annot_repl_input_end)) {

    P->ConsumeAnyToken();

    // FIXME: Clang does not call ExitScope on finalizing the regular TU, we

    // might want to do that around HandleEndOfTranslationUnit.

    P->ExitScope();
@@ -259,13 +259,13 @@ IncrementalParser::Parse(llvm::StringRef input) { 
    Token Tok;

    do {

      PP.Lex(Tok);

    } while (Tok.isNot(tok::eof));

  }



    } while (Tok.isNot(tok::annot_repl_input_end));

  } else {

    Token AssertTok;

    PP.Lex(AssertTok);

  assert(AssertTok.is(tok::eof) &&

    assert(AssertTok.is(tok::annot_repl_input_end) &&

           "Lexer must be EOF when starting incremental parse!");

  }



  if (CodeGenerator *CG = getCodeGen(Act.get())) {

    std::unique_ptr<llvm::Module> M(CG->ReleaseModule());