Unverified Commit c55f328d authored by Joshua S Brown's avatar Joshua S Brown Committed by GitHub
Browse files

Merge pull request #29 from ORNL/JoshuaSBrown-git

Joshua s brown git
parents 63d4a26f 191d7a2e
Pipeline #266992 failed with stage
......@@ -19,8 +19,10 @@ agent_app = typer.Typer()
def start(
log_path: str = typer.Option("", help="Path to logs on disk."),
debug: bool = typer.Option(False, help="If debug logs are enabled."),
zmq_activity_port: int = typer.Option(0, help="Port at which activities are received."),
zmq_heartbeat_port: int = typer.Option(0, help="Port for agent heartbeat communication."),
zmq_activity_port: int = typer.Option(
0, help="Port at which activities are received."),
zmq_heartbeat_port: int = typer.Option(
0, help="Port for agent heartbeat communication."),
config_path: str = typer.Option("", help="Path to config.")):
"""
Start the agent (set logger and ZMQ ports).
......
......@@ -11,5 +11,6 @@ Developer API
:hidden:
abstract_plugin_api_reference
git_plugin_api_reference
globus_plugin_api_reference
rsync_plugin_api_reference
GitHub Plugin API
================
.. automodule:: zambeze.orchestration.plugin_modules.git
:members:
......@@ -283,7 +283,9 @@ def test_globus_move_check():
}
]
assert globus_plugin.check(package)
checked_actions = globus_plugin.check(package)
print(checked_actions)
assert checked_actions["move_to_globus_collection"][0]
@pytest.mark.globus
......
# Local imports
import zambeze.orchestration.plugin_modules.git as git
# Standard imports
import os
import pytest
import random
@pytest.mark.gitlab_runner
def test_git_checkCommitSuccess():
"""Tests that the Git Plugin correctly verifies the json dict used to make
a commit to a file.
Test will create a file and then upload it to the GitHub Repo. The file
will contain a single random number.
"""
access_token = os.getenv("ZAMBEZE84_GITHUB_ACCESS_TOKEN")
current_dir = os.getcwd()
file_name = "demofile_for_git_commit.txt"
f = open(current_dir + "/" + file_name, "w")
original_number = random.randint(0, 100000000000)
f.write(str(original_number))
f.close()
package = [
{
"commit": {
"repo": "TestRepo",
"owner": "Zambeze84",
"branch": "main",
"source": {
"path": current_dir + "/" + file_name,
"type": "posix absolute",
},
"destination": {"path": file_name, "type": "GitHub repository root"},
"commit_message": "Adding a file",
"credentials": {
"user_name": "zambeze84",
"access_token": access_token,
"email": "zambeze84@gmail.com",
},
}
}
]
git_plugin = git.Git()
git_plugin.configure({})
checked_actions = git_plugin.check(package)
print(checked_actions)
assert checked_actions["commit"][0]
@pytest.mark.gitlab_runner
def test_git_checkCommitFailure1():
"""This test should fail because the commit package is missing the repo
key"""
access_token = os.getenv("ZAMBEZE84_GITHUB_ACCESS_TOKEN")
current_dir = os.getcwd()
file_name = "demofile_for_git_commit.txt"
f = open(current_dir + "/" + file_name, "w")
original_number = random.randint(0, 100000000000)
f.write(str(original_number))
f.close()
package = [
{
"commit": {
"owner": "Zambeze84",
"branch": "main",
"source": {
"path": current_dir + "/" + file_name,
"type": "posix absolute",
},
"destination": {"path": file_name, "type": "GitHub repository root"},
"commit_message": "Adding a file",
"credentials": {
"user_name": "zambeze84",
"access_token": access_token,
"email": "zambeze84@gmail.com",
},
}
}
]
git_plugin = git.Git()
git_plugin.configure({})
checked_actions = git_plugin.check(package)
assert not checked_actions["commit"][0]
@pytest.mark.gitlab_runner
def test_git_checkCommitFailure2():
"""This test should fail because the commit package is missing the
destination type key"""
access_token = os.getenv("ZAMBEZE84_GITHUB_ACCESS_TOKEN")
current_dir = os.getcwd()
file_name = "demofile_for_git_commit.txt"
f = open(current_dir + "/" + file_name, "w")
original_number = random.randint(0, 100000000000)
f.write(str(original_number))
f.close()
package = [
{
"commit": {
"repo": "TestRepo",
"owner": "Zambeze84",
"branch": "main",
"source": {
"path": current_dir + "/" + file_name,
"type": "posix absolute",
},
"destination": {
"path": file_name,
},
"commit_message": "Adding a file",
"credentials": {
"user_name": "zambeze84",
"access_token": access_token,
"email": "zambeze84@gmail.com",
},
}
}
]
git_plugin = git.Git()
git_plugin.configure({})
checked_actions = git_plugin.check(package)
assert not checked_actions["commit"][0]
@pytest.mark.gitlab_runner
def test_git_checkCommitFailure3():
"""This test should fail because the commit package is missing the
source key"""
access_token = os.getenv("ZAMBEZE84_GITHUB_ACCESS_TOKEN")
current_dir = os.getcwd()
file_name = "demofile_for_git_commit.txt"
f = open(current_dir + "/" + file_name, "w")
original_number = random.randint(0, 100000000000)
f.write(str(original_number))
f.close()
package = [
{
"commit": {
"repo": "TestRepo",
"owner": "Zambeze84",
"branch": "main",
"destination": {"path": file_name, "type": "GitHub repository root"},
"commit_message": "Adding a file",
"credentials": {
"user_name": "zambeze84",
"access_token": access_token,
"email": "zambeze84@gmail.com",
},
}
}
]
git_plugin = git.Git()
git_plugin.configure({})
checked_actions = git_plugin.check(package)
assert not checked_actions["commit"][0]
@pytest.mark.gitlab_runner
def test_git_checkCommitFailure4():
"""This test should fail because the commit package is missing the
credentials key"""
current_dir = os.getcwd()
file_name = "demofile_for_git_commit.txt"
f = open(current_dir + "/" + file_name, "w")
original_number = random.randint(0, 100000000000)
f.write(str(original_number))
f.close()
package = [
{
"commit": {
"repo": "TestRepo",
"owner": "Zambeze84",
"branch": "main",
"source": {
"path": current_dir + "/" + file_name,
"type": "posix absolute",
},
"destination": {"path": file_name, "type": "GitHub repository root"},
"commit_message": "Adding a file",
}
}
]
git_plugin = git.Git()
git_plugin.configure({})
checked_actions = git_plugin.check(package)
assert not checked_actions["commit"][0]
@pytest.mark.gitlab_runner
def test_git_processCommitAndDownload():
"""Tests that the Git Plugin correctly verifies the json dict used to make
a commit to a file.
Test will create a file and then upload it to the GitHub Repo. The file
will contain a single random number.
"""
access_token = os.getenv("ZAMBEZE84_GITHUB_ACCESS_TOKEN")
current_dir = os.getcwd()
file_name = "demofile_for_git_commit.txt"
f = open(current_dir + "/" + file_name, "w")
original_number = random.randint(0, 100000000000)
f.write(str(original_number))
f.close()
package = [
{
"commit": {
"repo": "TestRepo",
"owner": "Zambeze84",
"branch": "main",
"source": {
"path": current_dir + "/" + file_name,
"type": "posix absolute",
},
"destination": {"path": file_name, "type": "GitHub repository root"},
"commit_message": "Adding a file",
"credentials": {
"user_name": "zambeze84",
"access_token": access_token,
"email": "zambeze84@gmail.com",
},
}
}
]
git_plugin = git.Git()
git_plugin.configure({})
git_plugin.check(package)
git_plugin.process(package)
file_name2 = "demofile_for_git_commit_download.txt"
package = [
{
"download": {
"repo": "TestRepo",
"owner": "Zambeze84",
"branch": "main",
"destination": {
"path": current_dir + "/" + file_name2,
"type": "posix absolute",
},
"source": {"path": file_name, "type": "GitHub repository root"},
"credentials": {
"access_token": access_token,
},
}
}
]
git_plugin = git.Git()
git_plugin.configure({})
git_plugin.check(package)
git_plugin.process(package)
with open(file_name2) as f:
number_from_repo = f.read()
assert number_from_repo == str(original_number)
This diff is collapsed.
......@@ -198,7 +198,8 @@ def checkAllItemsHaveValidEndpoints(
class Globus(Plugin):
def __init__(self, logger: Optional[logging.Logger] = None) -> None:
super().__init__("globus", logger=logger)
self.__name = "globus"
super().__init__(self.__name, logger=logger)
# Client id is specific to Zambeze project it was created by registering
# at developers.globus.org
self.__access_to_globus_cloud = False
......@@ -208,13 +209,11 @@ class Globus(Plugin):
self.__configured = False
self.__flow = "client credential"
self.__hostname = None
self.__name = "globus"
self.__supported_actions = {
"transfer": False,
"move_to_globus_collection": False,
"move_from_globus_collection": False,
}
pass
###################################################################################
......@@ -452,56 +451,76 @@ class Globus(Plugin):
print(f"Destination is: {destination}")
shutil.copyfile(source, destination)
def __runTransferSanityCheck(self, action_package: dict) -> bool:
def __runTransferSanityCheck(self, action_package: dict) -> (bool, str):
# Any agent with the globus plugin can submit a job to globus if it
# has access to the globus cloud
if not self.__access_to_globus_cloud:
return False
return False, "No access to Globus Service to conduct 'transfer'."
if "source_collection_UUID" not in action_package:
return False
return False, "'source_collection_UUID' missing from 'transfer'."
if "destination_collection_UUID" not in action_package:
return False
return False, "'destination_collection_UUID' missing from 'transfer'."
if "items" not in action_package:
return False
return False, "'items' missing from 'transfer'"
else:
for item in action_package["items"]:
if "source" not in item:
return False
return False, "'source' missing from 'items' in 'transfer'"
else:
if not checkEndpoint(item["source"], ["globus relative"]):
return False
return (
False,
"'source' in ['transfer']['items'] must \
have a value of 'globus relative'",
)
if "destination" not in item:
return False
return False, "'destination' missing from 'items' in 'transfer'"
else:
if not checkEndpoint(item["destination"], ["globus relative"]):
return False
return (
False,
"'destination' in ['transfer']['items'] must \
have a value of 'globus relative'",
)
return True
def __runMoveToGlobusSanityCheck(self, action_package: dict) -> bool:
def __runMoveToGlobusSanityCheck(self, action_package: dict) -> (bool, str):
supported_source_path_types = ["posix absolute", "posix user home"]
supported_destination_path_types = ["globus relative"]
# This is needed in case there is more than a single collection on
# the machine
if not validUUID(action_package["destination_collection_UUID"]):
return False
return (
False,
f"Invalid 'destination_collection_UUID' detected in \
'move_to_globus_collection': \
{action_package['destination_collection_UUID']}",
)
# This is needed so the correct orchestrator picks executes the task
# a bit redundant though becuase the globus collection UUID should
# be unique
if self.__hostname != action_package["source_host_name"]:
return False
return (
False,
f"{self.__hostname} is not equal to provided \
{action_package['source_host_name']} in 'move_to_globus_collection'",
)
return checkAllItemsHaveValidEndpoints(
action_package["items"],
supported_source_path_types,
supported_destination_path_types,
return (
checkAllItemsHaveValidEndpoints(
action_package["items"],
supported_source_path_types,
supported_destination_path_types,
),
"",
)
def __runMoveFromGlobusSanityCheck(self, action_package: dict) -> bool:
def __runMoveFromGlobusSanityCheck(self, action_package: dict) -> (bool, str):
"""Run a sanity check for the action "move_from_globus_collection"
return: Will return true if the sanity check passes false otherwise
......@@ -541,14 +560,27 @@ class Globus(Plugin):
supported_source_path_types = ["globus relative"]
supported_destination_path_types = ["posix absolute", "posix user home"]
if not validUUID(action_package["source_collection_UUID"]):
return False
return (
False,
f"Invalid 'source_collection_UUID' dectected in \
'move_from_globus_collection': \
{action_package['source_collection_UUID']}",
)
# Check that the UUID is associated with this machine
if not action_package["source_collection_UUID"] in self.__collections:
return False
return (
False,
"Missing 'source_collection_UUID' in \
'move_from_globus_collection'",
)
if self.__hostname != action_package["destination_host_name"]:
return False
return (
False,
f"{self.__hostname} is not equal to provided \
{action_package['source_host_name']} in 'move_from_globus_collection'",
)
return checkAllItemsHaveValidEndpoints(
action_package["items"],
......@@ -719,13 +751,11 @@ class Globus(Plugin):
checks = {}
# Here we are cycling a list of dicts
for index in range(len(package)):
print(f"Index is {index}")
print(f"Package length is {len(package)}")
for action in package[index]:
# Check if the action is supported
if self.__supported_actions[action] is False:
checks[action] = False
checks[action] = (False, "action is not supported.")
continue
if action == "transfers":
......
......@@ -327,9 +327,8 @@ class Rsync(Plugin):
message = message + f"{action} unsupported action\n"
continue
supported_actions[action_key] = True
supported_actions[action_key] = (True, message)
print(message)
return supported_actions
def process(self, arguments: list[dict]):
......
......@@ -48,7 +48,7 @@ class Shell(Plugin):
def check(self, arguments: list[dict]) -> dict:
print("Checking shell plugin")
return {"run": False}
return {"run": (False, "Checks not implemented.")}
def process(self, arguments: list[dict]):
"""
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment