Unverified Commit 6681ed5e authored by Joshua S Brown's avatar Joshua S Brown Committed by GitHub
Browse files

Merge pull request #34 from ORNL/JoshuaSBrown-globus-plugin-test-transfer-from-esnet

Joshua s brown globus plugin test transfer from esnet
parents 74463dca 17833123
Pipeline #267549 failed with stage
in 0 seconds
......@@ -112,9 +112,7 @@ def test_git_checkCommitFailure2():
"path": current_dir + "/" + file_name,
"type": "posix absolute",
},
"destination": {
"path": file_name,
},
"destination": {"path": file_name},
"commit_message": "Adding a file",
"credentials": {
"user_name": "zambeze84",
......@@ -253,9 +251,7 @@ def test_git_processCommitAndDownload():
"type": "posix absolute",
},
"source": {"path": file_name, "type": "GitHub repository root"},
"credentials": {
"access_token": access_token,
},
"credentials": {"access_token": access_token},
}
}
]
......
......@@ -6,6 +6,7 @@ import os
import pytest
import random
import socket
import time
GITLAB_RUNNER_UUIDs = ["f4e5e85c-3a35-455f-9d91-1ee3a0564935"]
......@@ -98,9 +99,10 @@ def test_checkAllItemsHaveValidEndpoints():
supported_source_path_types = ["posix absolute", "posix user home"]
supported_destination_path_types = ["globus relative"]
assert globus.checkAllItemsHaveValidEndpoints(
output = globus.checkAllItemsHaveValidEndpoints(
items, supported_source_path_types, supported_destination_path_types
)
assert output[0]
items2 = [
{
......@@ -111,9 +113,10 @@ def test_checkAllItemsHaveValidEndpoints():
# This should be false because in this case "globus relative" is not in the
# supported_source_path_types
assert not globus.checkAllItemsHaveValidEndpoints(
output = globus.checkAllItemsHaveValidEndpoints(
items2, supported_source_path_types, supported_destination_path_types
)
assert not output[0]
items3 = [
{
......@@ -124,9 +127,10 @@ def test_checkAllItemsHaveValidEndpoints():
# This should be false because in this case "destination" is missing a
# "path"
assert not globus.checkAllItemsHaveValidEndpoints(
output = globus.checkAllItemsHaveValidEndpoints(
items3, supported_source_path_types, supported_destination_path_types
)
assert not output[0]
@pytest.mark.globus
......@@ -164,7 +168,7 @@ def test_globus_basic1():
assert globus_plugin.configured
assert len(globus_plugin.supportedActions) == 1
assert len(globus_plugin.supportedActions) == 2
# We assume that we at least have access to the globus cloud
assert "transfer" in globus_plugin.supportedActions
......@@ -211,7 +215,7 @@ def test_globus_basic2():
globus_plugin.configure(configuration)
assert len(globus_plugin.supportedActions) == 3
assert len(globus_plugin.supportedActions) == 4
@pytest.mark.globus
......@@ -262,6 +266,7 @@ def test_globus_move_check():
destination_path = "/"
sub_folder = ""
# This is so that we can run the test both as a runner and as user
if os.getenv(required_env_variables[0]) in GITLAB_RUNNER_UUIDs:
sub_folder = "runner/"
......@@ -284,7 +289,6 @@ def test_globus_move_check():
]
checked_actions = globus_plugin.check(package)
print(checked_actions)
assert checked_actions["move_to_globus_collection"][0]
......@@ -337,6 +341,7 @@ def test_globus_transfer_check():
destination_path = "/"
sub_folder = ""
# This is so that we can run the test both as a runner and as user
if os.getenv(required_env_variables[0]) in GITLAB_RUNNER_UUIDs:
sub_folder = "runner/"
......@@ -376,7 +381,11 @@ def test_globus_transfer_check():
}
},
]
assert globus_plugin.check(package)
output = globus_plugin.check(package)
for item in output:
assert output[item][0]
@pytest.mark.globus
......@@ -436,6 +445,7 @@ def test_globus_process():
relative_destination_file_path = "/"
sub_folder = ""
# This is so that we can run the test both as a runner and as user
if os.getenv(required_env_variables[0]) in GITLAB_RUNNER_UUIDs:
sub_folder = "runner/"
# action items in the list should be executed in order
......@@ -487,7 +497,7 @@ def test_globus_process():
+ sub_folder
+ os.path.basename(file_path)
)
# After it gets transferred using globus it should end up moving to the subfolder
# After it gets transferred using globus it should end up moving to the sub_folder
abs_path_destination_shared = (
path_to_endpoint
+ relative_destination_file_path
......@@ -500,8 +510,255 @@ def test_globus_process():
if os.path.exists(abs_path_destination_shared):
os.remove(abs_path_destination_shared)
checked_items = globus_plugin.check(package)
all_checks_pass = True
for item in checked_items:
if not checked_items[item][0]:
print("Something went wrong.")
print(checked_items[item][1])
all_checks_pass = False
if all_checks_pass:
globus_plugin.process(package)
# After processing we should verify that the file exists at the final location
assert os.path.exists(abs_path_destination_shared)
@pytest.mark.globus
def test_globus_process_from_esnet():
"""NOTE
We cannot simply download to a collection, we have to download to
a folder that is owned by the user running the this test. I.e. if
the GitLab Runner is being used then we have to specify a folder that
the GitLab Runner can actually access from the POSIX side.
"""
required_env_variables = [
"ZAMBEZE_CI_TEST_GLOBUS_CLIENT_ID",
"ZAMBEZE_CI_TEST_GLOBUS_APP_KEY",
"ZAMBEZE_CI_TEST_GLOBUS_COLLECTION_UUID",
"ZAMBEZE_CI_TEST_GLOBUS_COLLECTION_SHARED_UUID",
]
for env_var in required_env_variables:
if env_var not in os.environ:
raise Exception(
"Globus test cannot be run if the env variable"
f" {env_var} is not defined and a local "
"globus-connect-server and endpoint have not been"
" set up."
)
path_to_endpoint = "/home/cades/Collections/default"
path_to_endpoint_shared = "/home/cades/Collections/default/shared"
configuration = {
"client_id": os.getenv(required_env_variables[0]),
"authentication_flow": {
"type": "client credential",
"secret": os.getenv(required_env_variables[1]),
},
"collections": [
{
"UUID": os.getenv(required_env_variables[2]),
"path": path_to_endpoint,
"type": "mapped",
},
{
"UUID": os.getenv(required_env_variables[3]),
"path": path_to_endpoint_shared,
"type": "guest",
},
],
}
globus_plugin = globus.Globus()
globus_plugin.configure(configuration)
# action items in the list should be executed in order
ESNET_GLOBUS_ENDPOINT_UUID = "ece400da-0182-4777-91d6-27a1808f8371"
# This is so that we can run the test both as a runner and as user
sub_folder = ""
if os.getenv(required_env_variables[0]) in GITLAB_RUNNER_UUIDs:
sub_folder = "/runner"
package = [
{
"transfer": {
"source_collection_UUID": ESNET_GLOBUS_ENDPOINT_UUID,
"destination_collection_UUID": os.getenv(required_env_variables[2]),
"type": "synchronous",
"items": [
{
"source": {"type": "globus relative", "path": "/1M.dat"},
"destination": {
"type": "globus relative",
"path": sub_folder + "/1M.dat",
},
}
],
}
}
]
# This test is designed to move a file to the globus endpoint
# So before we get started we are going to make sure that a file
# does not already exist at that location
abs_path_destination = path_to_endpoint + sub_folder + "/1M.dat"
if os.path.exists(abs_path_destination):
os.remove(abs_path_destination)
if globus_plugin.check(package):
globus_plugin.process(package)
# After processing we should verify that the file exists at the final location
assert os.path.exists(abs_path_destination)
def test_globus_process_async():
required_env_variables = [
"ZAMBEZE_CI_TEST_GLOBUS_CLIENT_ID",
"ZAMBEZE_CI_TEST_GLOBUS_APP_KEY",
"ZAMBEZE_CI_TEST_GLOBUS_COLLECTION_UUID",
"ZAMBEZE_CI_TEST_GLOBUS_COLLECTION_SHARED_UUID",
]
for env_var in required_env_variables:
if env_var not in os.environ:
raise Exception(
"Globus test cannot be run if the env variable"
f" {env_var} is not defined and a local "
"globus-connect-server and endpoint have not been"
" set up."
)
path_to_endpoint = "/home/cades/Collections/default"
path_to_endpoint_shared = "/home/cades/Collections/default/shared"
configuration = {
"client_id": os.getenv(required_env_variables[0]),
"authentication_flow": {
"type": "client credential",
"secret": os.getenv(required_env_variables[1]),
},
"collections": [
{
"UUID": os.getenv(required_env_variables[2]),
"path": path_to_endpoint,
"type": "mapped",
},
{
"UUID": os.getenv(required_env_variables[3]),
"path": path_to_endpoint_shared,
"type": "guest",
},
],
}
globus_plugin = globus.Globus()
globus_plugin.configure(configuration)
# Create a file on the local posix system
file_name = "demofile_for_globus1.txt"
f = open(file_name, "w")
original_number = random.randint(0, 100000000000)
f.write(str(original_number))
f.close()
current_valid_path = os.getcwd()
file_path = current_valid_path + "/" + file_name
relative_destination_file_path = "/"
sub_folder = ""
if os.getenv(required_env_variables[0]) in GITLAB_RUNNER_UUIDs:
sub_folder = "runner/"
# action items in the list should be executed in order
package = [
{
"move_to_globus_collection": {
"destination_collection_UUID": os.getenv(required_env_variables[2]),
"source_host_name": socket.gethostname(),
"items": [
{
"source": {"type": "posix absolute", "path": file_path},
"destination": {
"type": "globus relative",
"path": relative_destination_file_path + sub_folder,
},
}
],
}
},
{
"transfer": {
"source_collection_UUID": os.getenv(required_env_variables[2]),
"destination_collection_UUID": os.getenv(required_env_variables[3]),
"type": "asynchronous",
"items": [
{
"source": {
"type": "globus relative",
"path": "/" + sub_folder + file_name,
},
"destination": {
"type": "globus relative",
"path": relative_destination_file_path
+ sub_folder
+ file_name,
},
}
],
}
},
]
abs_path_destination = path_to_endpoint + sub_folder + file_name
if os.path.exists(abs_path_destination):
os.remove(abs_path_destination)
output = globus_plugin.check(package)
if output["transfer"][0]:
globus_plugin.process(package)
else:
print("Check failed no transfer was conducted.")
print(output["transfer"][1])
# After processing we should verify that the file exists at the final location
assert os.path.exists(abs_path_destination)
abs_path_destination = (
path_to_endpoint
+ relative_destination_file_path
+ sub_folder
+ os.path.basename(file_path)
)
# After it gets transferred using globus it should end up moving to the subfolder
abs_path_destination_shared = (
path_to_endpoint
+ relative_destination_file_path
+ "shared/"
+ sub_folder
+ os.path.basename(file_path)
)
if os.path.exists(abs_path_destination):
os.remove(abs_path_destination)
if os.path.exists(abs_path_destination_shared):
os.remove(abs_path_destination_shared)
if globus_plugin.check(package):
result = globus_plugin.process(package)
result = globus_plugin.process([result["transfer"]["callback"]])
while result["get_task_status"]["result"]["status"] != "SUCCEEDED":
print("waiting...")
time.sleep(1)
result = globus_plugin.process([result["get_task_status"]["callback"]])
print("complete")
print(result["get_task_status"])
# After processing we should verify that the file exists at the final location
assert os.path.exists(abs_path_destination_shared)
......@@ -80,7 +80,7 @@ class Plugin(ABC):
:type arguments: list[dict]
:return: Returns the list of actions that are vaid
:rtype: dict with the actions valid actions listed with bool set to
True and invalid ones False
True and invalid ones False, along with a message.
:Example:
......@@ -93,8 +93,8 @@ class Plugin(ABC):
>>> for action in checked_actions:
>>> print(f"{action}: {checked_actions[action]}")
>>> # Should print
>>> # action1 True
>>> # action2 False
>>> # action1 True, ""
>>> # action2 False, "This was the problem"
"""
@abstractmethod
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment