Commit 5ca84087 authored by Nguyen, Thien Minh's avatar Nguyen, Thien Minh
Browse files

Implement container blob upload for azure job submission 



Create a blob storage for the job then upload data.

Signed-off-by: default avatarThien Nguyen <nguyentm@ornl.gov>
parent 94b7a6aa

mlir/tools/azure/CMakeLists.txt

+1 −1
Original line number Diff line number Diff line 
add_executable(azure-qir-tool azure-qir-tool.cpp)

add_executable(azure-qir-tool azure-qir-tool.cpp azure-utils.cpp)

target_include_directories(azure-qir-tool PUBLIC ${XACC_ROOT}/include ${XACC_ROOT}/include/xacc)

target_link_libraries(azure-qir-tool cpr curl)

target_link_directories(azure-qir-tool PRIVATE ${XACC_ROOT}/lib)

mlir/tools/azure/azure-qir-tool.cpp

+60 −19
Original line number Diff line number Diff line 
#include "cpr/cpr.h"

#include "azure-utils.hpp"

#include <ctime>

#include <filesystem>

#include <fstream>

#include <iostream>

#include <json.hpp>

#include <regex>

#include "cpr/cpr.h"

#include <ctime>

#include <random>

#include <regex>

namespace {

std::vector<std::string> split(const std::string &s, char delim) {

  std::vector<std::string> elems;
@@ -105,6 +106,47 @@ std::string randomIdStr() { 
  }

  return res;

}



std::pair<std::string, std::string>

createContainerBlob(const qcor::utils::Url &url) {

  auto request = url;

  request.AppendQueryParameter("restype", "container");

  cpr::Header cprHeaders;

  cprHeaders.insert({"Content-Length", "0"});

  cprHeaders.insert({"x-ms-version", "2020-02-10"});

  auto response = cpr::Put(cpr::Url{request.GetAbsoluteUrl()}, cprHeaders,

                           cpr::VerifySsl(false));

  if (response.status_code != 201) {

    throw std::runtime_error("Failed to create container blob");

  }

  return std::make_pair(response.header["ETag"],

                        response.header["x-ms-request-id"]);

}



std::string uploadContainerBlob(const qcor::utils::Url &url,

                                const std::string &blobName,

                                std::stringstream &stream) {

  auto request = url;

  request.AppendPath(blobName);

  cpr::Header cprHeaders;

  const auto requestBody = stream.str();

  cprHeaders.insert({"Content-Length", std::to_string(requestBody.size())});

  cprHeaders.insert({"x-ms-version", "2020-02-10"});

  cprHeaders.insert({"Accept-Encoding", "gzip, deflate"});

  cprHeaders.insert({"Connection", "keep-alive"});

  cprHeaders.insert({"x-ms-blob-type", "BlockBlob"});

  cprHeaders.insert({"Accept", "application/xml"});

  cprHeaders.insert({"x-ms-blob-content-encoding", "gzip"});

  cprHeaders.insert({"x-ms-blob-content-type", "application/json"});

  cprHeaders.insert({"Content-Type", "application/octet-stream"});

  auto response = cpr::Put(cpr::Url{request.GetAbsoluteUrl()}, cprHeaders,

                           cpr::Body{requestBody}, cpr::VerifySsl(false));

  if (response.status_code != 201) {

    throw std::runtime_error("Failed to upload container blob");

  }



  return request.GetUrlWithoutQuery(false);

}

} // namespace



int main(int argc, char **argv) {
@@ -116,8 +158,8 @@ int main(int argc, char **argv) { 
  // "qcor -set-credentials azure"

  if (std::filesystem::exists(azureConfigFilename)) {

    const auto [baseUrl, accessToken] = getConfigInfo(azureConfigFilename);

    std::cout << "baseUrl: " << baseUrl << std::endl;

    std::cout << "token: " << accessToken << std::endl;

    // std::cout << "baseUrl: " << baseUrl << std::endl;

    // std::cout << "token: " << accessToken << std::endl;

    cpr::Header cprHeaders;

    cprHeaders.insert({"Content-type", "application/json"});

    cprHeaders.insert({"Connection", "keep-alive"});
@@ -148,19 +190,18 @@ int main(int argc, char **argv) { 
    // PUT the JOB data to the Azure Storage

    // TODO: This should be a QIR file

    const std::string body = "howdy-qcor";

    cpr::Header blobUploadHeaders;

    blobUploadHeaders.insert({"Accept-Encoding", "gzip, deflate"});

    blobUploadHeaders.insert({"Connection", "keep-alive"});

    blobUploadHeaders.insert({"x-ms-blob-type", "BlockBlob"});

    blobUploadHeaders.insert({"Accept", "application/xml"});

    blobUploadHeaders.insert({"Content-Length", std::to_string(body.length())});

    blobUploadHeaders.insert({"x-ms-blob-content-encoding", "gzip"});

    blobUploadHeaders.insert({"x-ms-blob-content-type", "application/json"});

    blobUploadHeaders.insert({"Content-Type", "application/octet-stream"});

    blobUploadHeaders.insert({"Authorization", "Bearer " + accessToken});

    auto uploadResponse = cpr::Put(cpr::Url{sasUri}, blobUploadHeaders,

                                   cpr::Body{body}, cpr::VerifySsl(false));

    std::cout << "Upload Response:" << uploadResponse.text << "\n";



    const auto [eTag, requestId] =

        createContainerBlob(qcor::utils::Url{sasUri});

    std::cout << "Etag:" << eTag << "\n";

    std::stringstream stream;

    stream << body;

    static constexpr const char *blobName = "inputData";

    // Blob URI to construct Azure Quantum job

    const auto blobUri =

        uploadContainerBlob(qcor::utils::Url{sasUri}, blobName, stream);

    std::cout << "Blob access url:" << blobUri << std::endl;



    // 3. Create the job metadata and submit a job request to Azure.

    // https://docs.microsoft.com/en-us/rest/api/azurequantum/dataplane/jobs

  } else {

mlir/tools/azure/azure-utils.cpp

0 → 100644
+140 −0
Original line number Diff line number Diff line 
#include "azure-utils.hpp"

#include <algorithm>

#include <cctype>

#include <iterator>

#include <limits>

#include <stdexcept>

namespace {

std::string FormatEncodedUrlQueryParameters(

    std::map<std::string, std::string> const &encodedQueryParameters) {

  {

    std::string queryStr;

    if (!encodedQueryParameters.empty()) {

      auto separator = '?';

      for (const auto &q : encodedQueryParameters) {

        queryStr += separator + q.first + '=' + q.second;

        separator = '&';

      }

    }



    return queryStr;

  }

}

} // namespace



namespace qcor {

namespace utils {

std::string Url::GetUrlWithoutQuery(bool relative) const {

  std::string url;



  if (!relative) {

    if (!m_scheme.empty()) {

      url += m_scheme + "://";

    }

    url += m_host;

    if (m_port != 0) {

      url += ":" + std::to_string(m_port);

    }

  }



  if (!m_encodedPath.empty()) {

    if (!relative) {

      url += "/";

    }



    url += m_encodedPath;

  }



  return url;

}



void Url::AppendQueryParameters(const std::string &query) {

  std::string::const_iterator cur = query.begin();

  if (cur != query.end() && *cur == '?') {

    ++cur;

  }



  while (cur != query.end()) {

    auto key_end = std::find(cur, query.end(), '=');

    std::string query_key = std::string(cur, key_end);



    cur = key_end;

    if (cur != query.end()) {

      ++cur;

    }



    auto value_end = std::find(cur, query.end(), '&');

    std::string query_value = std::string(cur, value_end);



    cur = value_end;

    if (cur != query.end()) {

      ++cur;

    }

    m_encodedQueryParameters[std::move(query_key)] = std::move(query_value);

  }

}



Url::Url(const std::string &url) {

  std::string::const_iterator pos = url.begin();

  const std::string schemeEnd = "://";

  auto schemeIter = url.find(schemeEnd);

  if (schemeIter != std::string::npos) {

    std::transform(url.begin(), url.begin() + schemeIter,

                   std::back_inserter(m_scheme), ::tolower);



    pos = url.begin() + schemeIter + schemeEnd.length();

  }



  auto hostIter = std::find_if(

      pos, url.end(), [](char c) { return c == '/' || c == '?' || c == ':'; });

  m_host = std::string(pos, hostIter);

  pos = hostIter;



  if (pos != url.end() && *pos == ':') {

    auto port_ite = std::find_if_not(pos + 1, url.end(), [](char c) {

      return std::isdigit(static_cast<unsigned char>(c));

    });

    auto portNumber = std::stoi(std::string(pos + 1, port_ite));



    // stoi will throw out_of_range when `int` is overflow, but we need to throw

    // if uint16 is overflow

    auto maxPortNumberSupported = std::numeric_limits<uint16_t>::max();

    if (portNumber > maxPortNumberSupported) {

      throw std::out_of_range(

          "The port number is out of range. The max supported number is " +

          std::to_string(maxPortNumberSupported) + ".");

    }

    // cast is safe because the overflow was detected before

    m_port = static_cast<uint16_t>(portNumber);

    pos = port_ite;

  }



  if (pos != url.end() && (*pos != '/') && (*pos != '?')) {

    // only char `\` or `?` is valid after the port (or the end of the URL). Any

    // other char is an invalid input

    throw std::invalid_argument("The port number contains invalid characters.");

  }



  if (pos != url.end() && (*pos == '/')) {

    auto pathIter = std::find(pos + 1, url.end(), '?');

    m_encodedPath = std::string(pos + 1, pathIter);

    pos = pathIter;

  }



  if (pos != url.end() && *pos == '?') {

    auto queryIter = std::find(pos + 1, url.end(), '#');

    AppendQueryParameters(std::string(pos + 1, queryIter));

    pos = queryIter;

  }

}



std::string Url::GetRelativeUrl() const {

  return GetUrlWithoutQuery(true) +

         FormatEncodedUrlQueryParameters(m_encodedQueryParameters);

}

std::string Url::GetAbsoluteUrl() const {

  return GetUrlWithoutQuery(false) +

         FormatEncodedUrlQueryParameters(m_encodedQueryParameters);

}

} // namespace utils

} // namespace qcor
 
 No newline at end of file

mlir/tools/azure/azure-utils.hpp

0 → 100644
+52 −0
Original line number Diff line number Diff line 
#pragma once

#include <map>

#include <string>

#include <unordered_set>

#include <vector>

#include <memory>

namespace qcor {

namespace utils {

class Url {

private:

  std::string m_scheme;

  std::string m_host;

  uint16_t m_port{0};

  std::string m_encodedPath;

  std::map<std::string, std::string> m_encodedQueryParameters;

  void AppendQueryParameters(const std::string &encodedQueryParameters);



public:

  explicit Url(const std::string &encodedUrl);

  void SetScheme(const std::string &scheme) { m_scheme = scheme; }

  void SetHost(const std::string &encodedHost) { m_host = encodedHost; }

  void SetPort(uint16_t port) { m_port = port; }

  void SetPath(const std::string &encodedPath) { m_encodedPath = encodedPath; }

  void SetQueryParameters(std::map<std::string, std::string> queryParameters) {

    m_encodedQueryParameters = std::move(queryParameters);

  }

  void AppendPath(const std::string &encodedPath) {

    if (!m_encodedPath.empty() && m_encodedPath.back() != '/') {

      m_encodedPath += '/';

    }

    m_encodedPath += encodedPath;

  }

  void AppendQueryParameter(const std::string &encodedKey,

                            const std::string &encodedValue) {

    m_encodedQueryParameters[encodedKey] = encodedValue;

  }

  void RemoveQueryParameter(const std::string &encodedKey) {

    m_encodedQueryParameters.erase(encodedKey);

  }

  const std::string &GetHost() const { return m_host; }

  const std::string &GetPath() const { return m_encodedPath; }

  uint16_t GetPort() const { return m_port; }

  std::map<std::string, std::string> GetQueryParameters() const {

    return m_encodedQueryParameters;

  }

  const std::string &GetScheme() const { return m_scheme; }

  std::string GetRelativeUrl() const;

  std::string GetAbsoluteUrl() const;

  std::string GetUrlWithoutQuery(bool relative) const;

};

} // namespace utils

} // namespace qcor
 
 No newline at end of file