Unverified Commit 5111286f authored by Jun Zhang's avatar Jun Zhang
Browse files

Reland "Reland [clang-repl] Introduce Value to capture expression results" 

This reverts commit 094ab478.

Reland with changing `ParseAndExecute` to `Parse` in
`Interpreter::create`. This avoid creating JIT instance everytime even
if we don't really need them.

This should fixes failures like https://lab.llvm.org/buildbot/#/builders/38/builds/11955

The original reverted patch also causes GN bot fails on M1. (https://lab.llvm.org/buildbot/#/builders/38/builds/11955)
However, we can't reproduce it so let's reland it and see what happens.
See discussions here: https://reviews.llvm.org/rGd71a4e02277a64a9dece591cdf2b34f15c3b19a0
parent 09011c4e

clang/include/clang/Interpreter/Interpreter.h

+31 −11
Original line number Diff line number Diff line
@@ -14,14 +14,15 @@ 
#ifndef LLVM_CLANG_INTERPRETER_INTERPRETER_H

#define LLVM_CLANG_INTERPRETER_INTERPRETER_H



#include "clang/Interpreter/PartialTranslationUnit.h"



#include "clang/AST/Decl.h"

#include "clang/AST/GlobalDecl.h"

#include "clang/Interpreter/PartialTranslationUnit.h"

#include "clang/Interpreter/Value.h"



#include "llvm/ADT/DenseMap.h"

#include "llvm/ExecutionEngine/JITSymbol.h"

#include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h"

#include "llvm/Support/Error.h"



#include <memory>

#include <vector>
@@ -54,24 +55,26 @@ class Interpreter { 
  Interpreter(std::unique_ptr<CompilerInstance> CI, llvm::Error &Err);



  llvm::Error CreateExecutor();

  unsigned InitPTUSize = 0;



  // This member holds the last result of the value printing. It's a class

  // member because we might want to access it after more inputs. If no value

  // printing happens, it's in an invalid state.

  Value LastValue;



public:

  ~Interpreter();

  static llvm::Expected<std::unique_ptr<Interpreter>>

  create(std::unique_ptr<CompilerInstance> CI);

  const ASTContext &getASTContext() const;

  ASTContext &getASTContext();

  const CompilerInstance *getCompilerInstance() const;

  llvm::Expected<llvm::orc::LLJIT &> getExecutionEngine();



  llvm::Expected<PartialTranslationUnit &> Parse(llvm::StringRef Code);

  llvm::Error Execute(PartialTranslationUnit &T);

  llvm::Error ParseAndExecute(llvm::StringRef Code) {

    auto PTU = Parse(Code);

    if (!PTU)

      return PTU.takeError();

    if (PTU->TheModule)

      return Execute(*PTU);

    return llvm::Error::success();

  }

  llvm::Error ParseAndExecute(llvm::StringRef Code, Value *V = nullptr);

  llvm::Expected<llvm::orc::ExecutorAddr> CompileDtorCall(CXXRecordDecl *CXXRD);



  /// Undo N previous incremental inputs.

  llvm::Error Undo(unsigned N = 1);
@@ -92,6 +95,23 @@ public: 
  /// file.

  llvm::Expected<llvm::orc::ExecutorAddr>

  getSymbolAddressFromLinkerName(llvm::StringRef LinkerName) const;



  enum InterfaceKind { NoAlloc, WithAlloc, CopyArray };



  const llvm::SmallVectorImpl<Expr *> &getValuePrintingInfo() const {

    return ValuePrintingInfo;

  }



  Expr *SynthesizeExpr(Expr *E);



private:

  size_t getEffectivePTUSize() const;



  bool FindRuntimeInterface();



  llvm::DenseMap<CXXRecordDecl *, llvm::orc::ExecutorAddr> Dtors;



  llvm::SmallVector<Expr *, 3> ValuePrintingInfo;

};

} // namespace clang

clang/include/clang/Interpreter/Value.h

0 → 100644
+202 −0
Original line number Diff line number Diff line 
//===--- Value.h - Definition of interpreter value --------------*- C++ -*-===//

//

// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

// See https://llvm.org/LICENSE.txt for license information.

// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

//

//===----------------------------------------------------------------------===//

//

// Value is a lightweight struct that is used for carrying execution results in

// clang-repl. It's a special runtime that acts like a messager between compiled

// code and interpreted code. This makes it possible to exchange interesting

// information between the compiled & interpreted world.

//

// A typical usage is like the below:

//

// Value V;

// Interp.ParseAndExecute("int x = 42;");

// Interp.ParseAndExecute("x", &V);

// V.getType(); // <-- Yields a clang::QualType.

// V.getInt(); // <-- Yields 42.

//

// The current design is still highly experimental and nobody should rely on the

// API being stable because we're hopefully going to make significant changes to

// it in the relatively near future. For example, Value also intends to be used

// as an exchange token for JIT support enabling remote execution on the embed

// devices where the JIT infrastructure cannot fit. To support that we will need

// to split the memory storage in a different place and perhaps add a resource

// header is similar to intrinsics headers which have stricter performance

// constraints.

//

//===----------------------------------------------------------------------===//



#ifndef LLVM_CLANG_INTERPRETER_VALUE_H

#define LLVM_CLANG_INTERPRETER_VALUE_H



#include "llvm/Support/Compiler.h"

#include <cstdint>



// NOTE: Since the REPL itself could also include this runtime, extreme caution

// should be taken when MAKING CHANGES to this file, especially when INCLUDE NEW

// HEADERS, like <string>, <memory> and etc. (That pulls a large number of

// tokens and will impact the runtime performance of the REPL)



namespace llvm {

class raw_ostream;



} // namespace llvm



namespace clang {



class ASTContext;

class Interpreter;

class QualType;



#if __has_attribute(visibility) &&                                             \

    (!(defined(_WIN32) || defined(__CYGWIN__)) ||                              \

     (defined(__MINGW32__) && defined(__clang__)))

#if defined(LLVM_BUILD_LLVM_DYLIB) || defined(LLVM_BUILD_SHARED_LIBS)

#define REPL_EXTERNAL_VISIBILITY __attribute__((visibility("default")))

#else

#define REPL_EXTERNAL_VISIBILITY

#endif

#else

#if defined(_WIN32)

#define REPL_EXTERNAL_VISIBILITY __declspec(dllexport)

#endif

#endif



#define REPL_BUILTIN_TYPES                                                     \

  X(bool, Bool)                                                                \

  X(char, Char_S)                                                              \

  X(signed char, SChar)                                                        \

  X(unsigned char, UChar)                                                      \

  X(short, Short)                                                              \

  X(unsigned short, UShort)                                                    \

  X(int, Int)                                                                  \

  X(unsigned int, UInt)                                                        \

  X(long, Long)                                                                \

  X(unsigned long, ULong)                                                      \

  X(long long, LongLong)                                                       \

  X(unsigned long long, ULongLong)                                             \

  X(float, Float)                                                              \

  X(double, Double)                                                            \

  X(long double, LongDouble)



class REPL_EXTERNAL_VISIBILITY Value {

  union Storage {

#define X(type, name) type m_##name;

    REPL_BUILTIN_TYPES

#undef X

    void *m_Ptr;

  };



public:

  enum Kind {

#define X(type, name) K_##name,

    REPL_BUILTIN_TYPES

#undef X



    K_Void,

    K_PtrOrObj,

    K_Unspecified

  };



  Value() = default;

  Value(Interpreter *In, void *Ty);

  Value(const Value &RHS);

  Value(Value &&RHS) noexcept;

  Value &operator=(const Value &RHS);

  Value &operator=(Value &&RHS) noexcept;

  ~Value();



  void printType(llvm::raw_ostream &Out) const;

  void printData(llvm::raw_ostream &Out) const;

  void print(llvm::raw_ostream &Out) const;

  void dump() const;

  void clear();



  ASTContext &getASTContext();

  const ASTContext &getASTContext() const;

  Interpreter &getInterpreter();

  const Interpreter &getInterpreter() const;

  QualType getType() const;



  bool isValid() const { return ValueKind != K_Unspecified; }

  bool isVoid() const { return ValueKind == K_Void; }

  bool hasValue() const { return isValid() && !isVoid(); }

  bool isManuallyAlloc() const { return IsManuallyAlloc; }

  Kind getKind() const { return ValueKind; }

  void setKind(Kind K) { ValueKind = K; }

  void setOpaqueType(void *Ty) { OpaqueType = Ty; }



  void *getPtr() const;

  void setPtr(void *Ptr) { Data.m_Ptr = Ptr; }



#define X(type, name)                                                          \

  void set##name(type Val) { Data.m_##name = Val; }                            \

  type get##name() const { return Data.m_##name; }

  REPL_BUILTIN_TYPES

#undef X



  /// \brief Get the value with cast.

  //

  /// Get the value cast to T. This is similar to reinterpret_cast<T>(value),

  /// casting the value of builtins (except void), enums and pointers.

  /// Values referencing an object are treated as pointers to the object.

  template <typename T> T convertTo() const {

    return convertFwd<T>::cast(*this);

  }



protected:

  bool isPointerOrObjectType() const { return ValueKind == K_PtrOrObj; }



  /// \brief Get to the value with type checking casting the underlying

  /// stored value to T.

  template <typename T> T as() const {

    switch (ValueKind) {

    default:

      return T();

#define X(type, name)                                                          \

  case Value::K_##name:                                                        \

    return (T)Data.m_##name;

      REPL_BUILTIN_TYPES

#undef X

    }

  }



  // Allow convertTo to be partially specialized.

  template <typename T> struct convertFwd {

    static T cast(const Value &V) {

      if (V.isPointerOrObjectType())

        return (T)(uintptr_t)V.as<void *>();

      if (!V.isValid() || V.isVoid()) {

        return T();

      }

      return V.as<T>();

    }

  };



  template <typename T> struct convertFwd<T *> {

    static T *cast(const Value &V) {

      if (V.isPointerOrObjectType())

        return (T *)(uintptr_t)V.as<void *>();

      return nullptr;

    }

  };



  Interpreter *Interp = nullptr;

  void *OpaqueType = nullptr;

  Storage Data;

  Kind ValueKind = K_Unspecified;

  bool IsManuallyAlloc = false;

};



template <> inline void *Value::as() const {

  if (isPointerOrObjectType())

    return Data.m_Ptr;

  return (void *)as<uintptr_t>();

}



} // namespace clang

#endif

clang/lib/Interpreter/CMakeLists.txt

+2 −0
Original line number Diff line number Diff line
@@ -14,6 +14,8 @@ add_clang_library(clangInterpreter 
  IncrementalExecutor.cpp

  IncrementalParser.cpp

  Interpreter.cpp

  InterpreterUtils.cpp

  Value.cpp



  DEPENDS

  intrinsics_gen

clang/lib/Interpreter/IncrementalParser.cpp

+90 −9
Original line number Diff line number Diff line
@@ -11,7 +11,6 @@ 
//===----------------------------------------------------------------------===//



#include "IncrementalParser.h"



#include "clang/AST/DeclContextInternals.h"

#include "clang/CodeGen/BackendUtil.h"

#include "clang/CodeGen/CodeGenAction.h"
@@ -19,9 +18,9 @@ 
#include "clang/Frontend/CompilerInstance.h"

#include "clang/Frontend/FrontendAction.h"

#include "clang/FrontendTool/Utils.h"

#include "clang/Interpreter/Interpreter.h"

#include "clang/Parse/Parser.h"

#include "clang/Sema/Sema.h"



#include "llvm/Option/ArgList.h"

#include "llvm/Support/CrashRecoveryContext.h"

#include "llvm/Support/Error.h"
@@ -31,6 +30,79 @@ 


namespace clang {



class IncrementalASTConsumer final : public ASTConsumer {

  Interpreter &Interp;

  std::unique_ptr<ASTConsumer> Consumer;



public:

  IncrementalASTConsumer(Interpreter &InterpRef, std::unique_ptr<ASTConsumer> C)

      : Interp(InterpRef), Consumer(std::move(C)) {}



  bool HandleTopLevelDecl(DeclGroupRef DGR) override final {

    if (DGR.isNull())

      return true;

    if (!Consumer)

      return true;



    for (Decl *D : DGR)

      if (auto *TSD = llvm::dyn_cast<TopLevelStmtDecl>(D);

          TSD && TSD->isSemiMissing())

        TSD->setStmt(Interp.SynthesizeExpr(cast<Expr>(TSD->getStmt())));



    return Consumer->HandleTopLevelDecl(DGR);

  }

  void HandleTranslationUnit(ASTContext &Ctx) override final {

    Consumer->HandleTranslationUnit(Ctx);

  }

  void HandleInlineFunctionDefinition(FunctionDecl *D) override final {

    Consumer->HandleInlineFunctionDefinition(D);

  }

  void HandleInterestingDecl(DeclGroupRef D) override final {

    Consumer->HandleInterestingDecl(D);

  }

  void HandleTagDeclDefinition(TagDecl *D) override final {

    Consumer->HandleTagDeclDefinition(D);

  }

  void HandleTagDeclRequiredDefinition(const TagDecl *D) override final {

    Consumer->HandleTagDeclRequiredDefinition(D);

  }

  void HandleCXXImplicitFunctionInstantiation(FunctionDecl *D) override final {

    Consumer->HandleCXXImplicitFunctionInstantiation(D);

  }

  void HandleTopLevelDeclInObjCContainer(DeclGroupRef D) override final {

    Consumer->HandleTopLevelDeclInObjCContainer(D);

  }

  void HandleImplicitImportDecl(ImportDecl *D) override final {

    Consumer->HandleImplicitImportDecl(D);

  }

  void CompleteTentativeDefinition(VarDecl *D) override final {

    Consumer->CompleteTentativeDefinition(D);

  }

  void CompleteExternalDeclaration(VarDecl *D) override final {

    Consumer->CompleteExternalDeclaration(D);

  }

  void AssignInheritanceModel(CXXRecordDecl *RD) override final {

    Consumer->AssignInheritanceModel(RD);

  }

  void HandleCXXStaticMemberVarInstantiation(VarDecl *D) override final {

    Consumer->HandleCXXStaticMemberVarInstantiation(D);

  }

  void HandleVTable(CXXRecordDecl *RD) override final {

    Consumer->HandleVTable(RD);

  }

  ASTMutationListener *GetASTMutationListener() override final {

    return Consumer->GetASTMutationListener();

  }

  ASTDeserializationListener *GetASTDeserializationListener() override final {

    return Consumer->GetASTDeserializationListener();

  }

  void PrintStats() override final { Consumer->PrintStats(); }

  bool shouldSkipFunctionBody(Decl *D) override final {

    return Consumer->shouldSkipFunctionBody(D);

  }

  static bool classof(const clang::ASTConsumer *) { return true; }

};



/// A custom action enabling the incremental processing functionality.

///

/// The usual \p FrontendAction expects one call to ExecuteAction and once it
@@ -122,7 +194,8 @@ public: 
  }

};



IncrementalParser::IncrementalParser(std::unique_ptr<CompilerInstance> Instance,

IncrementalParser::IncrementalParser(Interpreter &Interp,

                                     std::unique_ptr<CompilerInstance> Instance,

                                     llvm::LLVMContext &LLVMCtx,

                                     llvm::Error &Err)

    : CI(std::move(Instance)) {
@@ -131,6 +204,9 @@ IncrementalParser::IncrementalParser(std::unique_ptr<CompilerInstance> Instance, 
  if (Err)

    return;

  CI->ExecuteAction(*Act);

  std::unique_ptr<ASTConsumer> IncrConsumer =

      std::make_unique<IncrementalASTConsumer>(Interp, CI->takeASTConsumer());

  CI->setASTConsumer(std::move(IncrConsumer));

  Consumer = &CI->getASTConsumer();

  P.reset(

      new Parser(CI->getPreprocessor(), CI->getSema(), /*SkipBodies=*/false));
@@ -267,17 +343,22 @@ IncrementalParser::Parse(llvm::StringRef input) { 
           "Lexer must be EOF when starting incremental parse!");

  }



  if (CodeGenerator *CG = getCodeGen(Act.get())) {

    std::unique_ptr<llvm::Module> M(CG->ReleaseModule());

    CG->StartModule("incr_module_" + std::to_string(PTUs.size()),

                    M->getContext());



  if (std::unique_ptr<llvm::Module> M = GenModule())

    PTU->TheModule = std::move(M);

  }



  return PTU;

}



std::unique_ptr<llvm::Module> IncrementalParser::GenModule() {

  static unsigned ID = 0;

  if (CodeGenerator *CG = getCodeGen(Act.get())) {

    std::unique_ptr<llvm::Module> M(CG->ReleaseModule());

    CG->StartModule("incr_module_" + std::to_string(ID++), M->getContext());

    return M;

  }

  return nullptr;

}



void IncrementalParser::CleanUpPTU(PartialTranslationUnit &PTU) {

  TranslationUnitDecl *MostRecentTU = PTU.TUPart;

  TranslationUnitDecl *FirstTU = MostRecentTU->getFirstDecl();

clang/lib/Interpreter/IncrementalParser.h

+5 −3
Original line number Diff line number Diff line
@@ -16,7 +16,6 @@ 
#include "clang/Interpreter/PartialTranslationUnit.h"



#include "clang/AST/GlobalDecl.h"



#include "llvm/ADT/ArrayRef.h"

#include "llvm/ADT/StringRef.h"

#include "llvm/Support/Error.h"
@@ -31,8 +30,8 @@ namespace clang { 
class ASTConsumer;

class CompilerInstance;

class IncrementalAction;

class Interpreter;

class Parser;



/// Provides support for incremental compilation. Keeps track of the state

/// changes between the subsequent incremental input.

///
@@ -57,7 +56,8 @@ class IncrementalParser { 
  std::list<PartialTranslationUnit> PTUs;



public:

  IncrementalParser(std::unique_ptr<CompilerInstance> Instance,

  IncrementalParser(Interpreter &Interp,

                    std::unique_ptr<CompilerInstance> Instance,

                    llvm::LLVMContext &LLVMCtx, llvm::Error &Err);

  ~IncrementalParser();
@@ -76,6 +76,8 @@ public: 


  std::list<PartialTranslationUnit> &getPTUs() { return PTUs; }



  std::unique_ptr<llvm::Module> GenModule();



private:

  llvm::Expected<PartialTranslationUnit &> ParseOrWrapTopLevelDecl();

};