In the middle of a refactor (bb407c4f) · Commits · olcf / container-builder

Builder/include/Builder.h

+24 −22

Original line number	Diff line number	Diff line
		@@ -22,21 +22,25 @@ public:
		// Full build connection - this will not run until the io_service is started
		asio::spawn(io_service,
		[&](asio::yield_context yield) {
		Messenger client(io_service, "8080", yield);
		boost::system::error_code error;
		Messenger client(io_service, "8080", yield, error);
		if(error) {
		throw std::runtime_error("Error connecting to client: " + error.message());
		}

		// Receive client data
		ClientData client_data = client.async_receive_client_data();
		if (client.error) {
		throw std::runtime_error("Error receiving client data: " + client.error.message());
		ClientData client_data = client.async_read_client_data(yield, error);
		if (error) {
		throw std::runtime_error("Error receiving client data: " + error.message());
		}

		// Receive the definition file from the client
		client.async_receive_file("container.def");
		if (client.error) {
		throw std::runtime_error("Error receiving definition file: " + client.error.message());
		client.async_read_file("container.def", yield, error);
		if (error) {
		throw std::runtime_error("Error receiving definition file: " + error.message());
		}

		logger::write(client.socket, "Received container.def");
		logger::write("Received container.def");

		if(client_data.arch == Architecture::ppc64le) {
		// A dirty hack but the ppc64le qemu executable must be in the place the kernel expects it
		@@ -52,8 +56,7 @@ public:
		// Launch our build as a subprocess
		// We use "unbuffer" to fake the build into thinking it has a real TTY, which the command output eventually will
		// This causes things like wget and color ls to work nicely
		std::string build_command(
		"/usr/bin/sudo ");
		std::string build_command("/usr/bin/sudo ");
		// If the cleint has a tty trick the subprocess into thinking that as well
		if(client_data.tty) {
		build_command += "/usr/bin/unbuffer ";
		@@ -68,29 +71,28 @@ public:
		throw std::runtime_error("subprocess error: " + build_ec.message());
		}

		logger::write(client.socket, "launched build process: " + build_command);
		logger::write("launched build process: " + build_command);

		// Read process pipe output and write it to the client
		// line buffer(ish) by reading from the pipe until we hit \n, \r
		// NOTE: read_until will fill buffer until line_matcher is satisfied but generally will contain additional data.
		// This is fine as all we care about is dumping everything from std_pipe to our buffer and don't require exact line buffering
		// TODO: just call read_some perhaps?
		// stream from the async pipe process output to the socket
		asio::streambuf buffer;
		boost::regex line_matcher{"\\r\|\\n"};
		std::size_t read_size = 0;
		boost::system::error_code stream_error;
		bool fin = false;
		do {
		// Read from the pipe into a buffer
		read_size = asio::async_read_until(std_pipe, buffer, line_matcher, yield[stream_error]);
		std_pipe.async_read_some(buffer, yield[stream_error]);
		if (stream_error != boost::system::errc::success && stream_error != boost::asio::error::eof) {
		throw std::runtime_error("reading process pipe failed: " + stream_error.message());
		}
		if(read_size == 0 \|\| stream_error) {
		fin = true;
		}
		// Write the buffer to our socket
		client.async_send(buffer);
		if (client.error) {
		throw std::runtime_error("sending process pipe failed: " + client.error.message());
		client.async_write_some_streambuf(fin, buffer, yield, error);
		if (error) {
		throw std::runtime_error("sending process pipe failed: " + error.message());
		}
		} while (read_size > 0 && !stream_error);
		} while (!fin);

		// Get the return value from the build subprocess
		logger::write("Waiting on build process to exit");

Builder/src/main.cpp

+0 −1

Original line number	Diff line number	Diff line
		@@ -5,7 +5,6 @@ int main(int argc, char *argv[]) {
		try {
		Builder builder;
		builder.run();

		}
		catch (const std::exception& ex) {
		logger::write(std::string() + "Builder exception encountered: " + ex.what(), logger::severity_level::fatal);

Common/include/Messenger.h

+32 −29

Original line number	Diff line number	Diff line
		@@ -5,6 +5,7 @@
		#include <boost/asio/io_service.hpp>
		#include <boost/asio/ip/tcp.hpp>
		#include <boost/filesystem.hpp>
		#include <boost/asio/spawn.hpp>
		#include <boost/beast/core.hpp>
		#include <boost/beast/websocket.hpp>
		#include <iostream>
		@@ -19,7 +20,8 @@

		namespace asio = boost::asio;
		using asio::ip::tcp;
		namespace websocket = boost::beast::websocket;
		namespace beast = boost::beast;
		namespace websocket = beast::websocket;

		class Messenger {

		@@ -29,22 +31,24 @@ public:
		const std::string &host,
		const std::string &port,
		asio::yield_context yield,
		boost::system::error_code error) : web_socket(io_service) {
		boost::system::error_code error) : stream(io_service) {
		do {
		tcp::resolver queue_resolver(io_service);
		asio::async_connect(web_socket.next_layer(), queue_resolver.resolve({host, port}), yield[error]);
		asio::async_connect(stream.next_layer(), queue_resolver.resolve({host, port}), yield[error]);
		} while (error);

		web_socket.async_handshake(host, "/", yield[error]);
		stream.async_handshake(host, "/", yield[error]);
		stream.binary(true);
		}

		// Create a server messenger by doing an async block listen on the specified port
		explicit Messenger(asio::io_service &io_service,
		const std::string &port,
		asio::yield_context yield,
		boost::system::error_code error) : web_socket(io_service) {
		boost::system::error_code error) : stream(io_service) {
		tcp::acceptor acceptor(io_service, tcp::endpoint(tcp::v4(), std::stoi(port)));
		acceptor.async_accept(web_socket.next_layer(), yield[error]);
		acceptor.async_accept(stream.next_layer(), yield[error]);
		stream.binary(true);
		}

		// Create a server messenger by doing an async block give the socket
		@@ -52,46 +56,45 @@ public:
		explicit Messenger(tcp::acceptor& acceptor,
		asio::io_service &io_service,
		asio::yield_context yield,
		boost::system::error_code error) : web_socket(io_service) {
		acceptor.async_accept(web_socket.next_layer(), yield[error]);
		boost::system::error_code error) : stream(io_service) {
		acceptor.async_accept(stream.next_layer(), yield[error]);
		stream.binary(true);
		}

		std::string async_receive();
		std::string async_read_string(asio::yield_context yield,
		boost::system::error_code& error);

		void async_send(const std::string &message,
		void async_write_string(const std::string &message,
		asio::yield_context yield,
		boost::system::error_code error);
		boost::system::error_code& error);

		void async_send(asio::streambuf &message_body,
		void async_write_streambuf(asio::streambuf &message_body,
		asio::yield_context yield,
		boost::system::error_code error);
		boost::system::error_code& error);

		void async_receive_file(boost::filesystem::path file_path,
		void async_read_file(boost::filesystem::path file_path,
		asio::yield_context yield,
		boost::system::error_code error,
		const bool print_progress = false
		);
		boost::system::error_code& error);

		void async_send_file(boost::filesystem::path file_path,
		void async_write_file(boost::filesystem::path file_path,
		asio::yield_context yield,
		boost::system::error_code error,
		const bool print_progress = false
		boost::system::error_code& error,
		);

		BuilderData async_receive_builder(asio::yield_context yield,
		boost::system::error_code error);
		BuilderData async_read_builder(asio::yield_context yield,
		boost::system::error_code& error);

		void async_send(BuilderData builder,
		void async_write_builder(BuilderData builder,
		asio::yield_context yield,
		boost::system::error_code error);
		boost::system::error_code& error);

		ClientData async_receive_client_data(asio::yield_context yield,
		boost::system::error_code error);
		ClientData async_read_client_data(asio::yield_context yield,
		boost::system::error_code& error);

		void async_send(ClientData client_data,
		void async_write_client_data(ClientData client_data,
		asio::yield_context yield,
		boost::system::error_code error);
		boost::system::error_code& error);

		private:
		websocket::stream<tcp::socket> web_socket;
		websocket::stream<tcp::socket> stream;
		};
		No newline at end of file

Common/src/Messenger.cpp

+80 −207

File changed.

Preview size limit exceeded, changes collapsed.

Scripts/ProvisionBuilder

+1 −1

Original line number	Diff line number	Diff line
		@@ -19,7 +19,7 @@ chmod 0440 /etc/sudoers.d/builder

		apt-get -y update
		apt-get -y install expect
		apt-get -y install yum rpm
		apt-get -y install yum rpm dnf

		#####################################
		# begin ppc64le QUEMU stuff