test_plugin_globus.py 24.7 KB
Newer Older
Brown, Joshua's avatar
Brown, Joshua committed
1
2
3
4
5
6
7
8
# Local imports
import zambeze.orchestration.plugin_modules.globus as globus

# Standard imports
import os
import pytest
import random
import socket
9
import time
Brown, Joshua's avatar
Brown, Joshua committed
10

Brown, Joshua's avatar
Brown, Joshua committed
11
GITLAB_RUNNER_UUIDs = ["f4e5e85c-3a35-455f-9d91-1ee3a0564935"]
12

JoshuaSBrown's avatar
JoshuaSBrown committed
13

14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
@pytest.mark.unit
def test_getMappedCollections():
    config = {
        "collections": [
            {
                "UUID": "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX",
                "path": "/here/file",
                "type": "guest",
            },
            {
                "UUID": "YYYYYYYY-YYYY-YYYY-YYYY-YYYYYYYYYYYY",
                "path": "/there/file2",
                "type": "mapped",
            },
        ]
    }

    mapped_coll_UUIDs = globus.getMappedCollections(config)

    assert len(mapped_coll_UUIDs) == 1
    assert mapped_coll_UUIDs[0] == "YYYYYYYY-YYYY-YYYY-YYYY-YYYYYYYYYYYY"


@pytest.mark.unit
def test_getGlobusScopes():
    mapped_collections = []
    scopes = globus.getGlobusScopes(mapped_collections)
    assert scopes == "urn:globus:auth:scope:transfer.api.globus.org:all"

    # These are invalid entries so they will be ignored
    mapped_collections = ["", "XXXX"]
    scopes = globus.getGlobusScopes(mapped_collections)
    assert scopes == "urn:globus:auth:scope:transfer.api.globus.org:all"

    # Though the middle entry is not really a valid UUID it is 36 chars so
    # it should be passed as a valid scope
    mapped_collections = [
        "",
        "YYYYYYYY-YYYY-YYYY-YYYY-YYYYYYYYYYYY",
        "XXXX",
        "ZZZZZZZZ-ZZZZ-ZZZZ-ZZZZ-ZZZZZZZZZZZZ",
    ]

    scopes = globus.getGlobusScopes(mapped_collections)

    correct_scopes = "urn:globus:auth:scope:transfer.api.globus.org:all\
[*https://auth.globus.org/scopes/\
YYYYYYYY-YYYY-YYYY-YYYY-YYYYYYYYYYYY/data_access\
 *https://auth.globus.org/scopes/\
ZZZZZZZZ-ZZZZ-ZZZZ-ZZZZ-ZZZZZZZZZZZZ/data_access]"

    assert scopes == correct_scopes


@pytest.mark.unit
def test_checkAllItemsHaveValidEndpoints():
    file_name = "demofile_checkAllItemsHaveValidEndpoints.txt"
    f = open(file_name, "w")
    f.write("Testing for valid source file for test_checkAllItemsHaveValidEndpoints")
    f.close()

    current_valid_path = os.getcwd()
    file_path = current_valid_path + "/" + file_name

    file_name2 = "demofile2_checkAllItemsHaveValidEndpoints.txt"
    f = open(file_name2, "w")
    f.write("Testing for valid source file for test_checkAllItemsHaveValidEndpoints")
    f.close()

    file_path2 = current_valid_path + "/" + file_name2

    items = [
        {
            "source": {"type": "posix absolute", "path": file_path},
            "destination": {"type": "globus relative", "path": "/"},
        },
        {
            "source": {"type": "posix absolute", "path": file_path2},
            "destination": {
                "type": "globus relative",
                "path": "/sub_folder/file2.jpeg",
            },
        },
    ]

    supported_source_path_types = ["posix absolute", "posix user home"]
    supported_destination_path_types = ["globus relative"]

102
    output = globus.checkAllItemsHaveValidEndpoints(
103
104
        items, supported_source_path_types, supported_destination_path_types
    )
105
    assert output[0]
106
107
108
109
110
111
112
113
114
115

    items2 = [
        {
            "source": {"type": "globus relative", "path": "/home/cades/file.txt"},
            "destination": {"type": "globus relative", "path": "/"},
        }
    ]

    # This should be false because in this case "globus relative" is not in the
    # supported_source_path_types
116
    output = globus.checkAllItemsHaveValidEndpoints(
117
118
        items2, supported_source_path_types, supported_destination_path_types
    )
119
    assert not output[0]
120
121
122
123
124
125
126
127
128
129

    items3 = [
        {
            "source": {"type": "posix absolute", "path": "/home/cades/file.txt"},
            "destination": {"type": "globus relative"},
        }
    ]

    # This should be false because in this case "destination" is missing a
    # "path"
130
    output = globus.checkAllItemsHaveValidEndpoints(
131
132
        items3, supported_source_path_types, supported_destination_path_types
    )
133
    assert not output[0]
134
135


Brown, Joshua's avatar
Brown, Joshua committed
136
137
@pytest.mark.globus
def test_globus_basic1():
JoshuaSBrown's avatar
JoshuaSBrown committed
138

Brown, Joshua's avatar
Brown, Joshua committed
139
    required_env_variables = [
JoshuaSBrown's avatar
JoshuaSBrown committed
140
141
142
        "ZAMBEZE_CI_TEST_GLOBUS_CLIENT_ID",
        "ZAMBEZE_CI_TEST_GLOBUS_APP_KEY",
    ]
JoshuaSBrown's avatar
JoshuaSBrown committed
143

Brown, Joshua's avatar
Brown, Joshua committed
144
145
    for env_var in required_env_variables:
        if env_var not in os.environ:
JoshuaSBrown's avatar
JoshuaSBrown committed
146
147
148
149
150
151
152
            raise Exception(
                "Globus test cannot be run if the env variable"
                f" {env_var} is not defined and a local "
                "globus-connect-server and endpoint have not been"
                " set up."
            )

Brown, Joshua's avatar
Brown, Joshua committed
153
154
155
    globus_plugin = globus.Globus()

    assert globus_plugin.name == "globus"
156
    assert globus_plugin.configured is False
Brown, Joshua's avatar
Brown, Joshua committed
157
158
159

    """Requires that the env variable is provided"""
    configuration = {
Brown, Joshua's avatar
Brown, Joshua committed
160
        "client_id": os.getenv(required_env_variables[0]),
JoshuaSBrown's avatar
JoshuaSBrown committed
161
        "authentication_flow": {
JoshuaSBrown's avatar
JoshuaSBrown committed
162
            "type": "client credential",
Brown, Joshua's avatar
Brown, Joshua committed
163
            "secret": os.getenv(required_env_variables[1]),
JoshuaSBrown's avatar
JoshuaSBrown committed
164
165
        },
    }
Brown, Joshua's avatar
Brown, Joshua committed
166
167
168
169
170

    globus_plugin.configure(configuration)

    assert globus_plugin.configured

171
    assert len(globus_plugin.supportedActions) == 2
Brown, Joshua's avatar
Brown, Joshua committed
172
173
174
    # We assume that we at least have access to the globus cloud
    assert "transfer" in globus_plugin.supportedActions

175

Brown, Joshua's avatar
Brown, Joshua committed
176
177
@pytest.mark.globus
def test_globus_basic2():
JoshuaSBrown's avatar
JoshuaSBrown committed
178

Brown, Joshua's avatar
Brown, Joshua committed
179
    required_env_variables = [
JoshuaSBrown's avatar
JoshuaSBrown committed
180
181
182
183
        "ZAMBEZE_CI_TEST_GLOBUS_CLIENT_ID",
        "ZAMBEZE_CI_TEST_GLOBUS_APP_KEY",
        "ZAMBEZE_CI_TEST_GLOBUS_COLLECTION_UUID",
    ]
JoshuaSBrown's avatar
JoshuaSBrown committed
184

Brown, Joshua's avatar
Brown, Joshua committed
185
186
    for env_var in required_env_variables:
        if env_var not in os.environ:
JoshuaSBrown's avatar
JoshuaSBrown committed
187
188
189
190
191
192
193
            raise Exception(
                "Globus test cannot be run if the env variable"
                f" {env_var} is not defined and a local "
                "globus-connect-server and endpoint have not been"
                " set up."
            )

Brown, Joshua's avatar
Brown, Joshua committed
194
195
196
197
198
199
200
201
    globus_plugin = globus.Globus()

    """Requires that the env variable is provided

    A valid globus collection must also exist on this machine
    in order for the move to globus endpoint and move from
    globus endpoint to local posix file system to work."""
    configuration = {
Brown, Joshua's avatar
Brown, Joshua committed
202
        "client_id": os.getenv(required_env_variables[0]),
JoshuaSBrown's avatar
JoshuaSBrown committed
203
        "authentication_flow": {
JoshuaSBrown's avatar
JoshuaSBrown committed
204
            "type": "client credential",
Brown, Joshua's avatar
Brown, Joshua committed
205
            "secret": os.getenv(required_env_variables[1]),
JoshuaSBrown's avatar
JoshuaSBrown committed
206
207
208
        },
        "collections": [
            {
Brown, Joshua's avatar
Brown, Joshua committed
209
                "UUID": os.getenv(required_env_variables[2]),
JoshuaSBrown's avatar
JoshuaSBrown committed
210
211
                "path": "/home/cades/Collections/default",
                "type": "mapped",
Brown, Joshua's avatar
Brown, Joshua committed
212
            }
JoshuaSBrown's avatar
JoshuaSBrown committed
213
214
        ],
    }
Brown, Joshua's avatar
Brown, Joshua committed
215
216
217

    globus_plugin.configure(configuration)

218
    assert len(globus_plugin.supportedActions) == 4
Brown, Joshua's avatar
Brown, Joshua committed
219

220

Brown, Joshua's avatar
Brown, Joshua committed
221
222
@pytest.mark.globus
def test_globus_move_check():
JoshuaSBrown's avatar
JoshuaSBrown committed
223

Brown, Joshua's avatar
Brown, Joshua committed
224
    required_env_variables = [
JoshuaSBrown's avatar
JoshuaSBrown committed
225
226
227
228
        "ZAMBEZE_CI_TEST_GLOBUS_CLIENT_ID",
        "ZAMBEZE_CI_TEST_GLOBUS_APP_KEY",
        "ZAMBEZE_CI_TEST_GLOBUS_COLLECTION_UUID",
    ]
JoshuaSBrown's avatar
JoshuaSBrown committed
229

Brown, Joshua's avatar
Brown, Joshua committed
230
231
    for env_var in required_env_variables:
        if env_var not in os.environ:
JoshuaSBrown's avatar
JoshuaSBrown committed
232
233
234
235
236
237
238
            raise Exception(
                "Globus test cannot be run if the env variable"
                f" {env_var} is not defined and a local "
                "globus-connect-server and endpoint have not been"
                " set up."
            )

Brown, Joshua's avatar
Brown, Joshua committed
239
    configuration = {
Brown, Joshua's avatar
Brown, Joshua committed
240
        "client_id": os.getenv(required_env_variables[0]),
JoshuaSBrown's avatar
JoshuaSBrown committed
241
        "authentication_flow": {
JoshuaSBrown's avatar
JoshuaSBrown committed
242
            "type": "client credential",
Brown, Joshua's avatar
Brown, Joshua committed
243
            "secret": os.getenv(required_env_variables[1]),
JoshuaSBrown's avatar
JoshuaSBrown committed
244
245
246
        },
        "collections": [
            {
Brown, Joshua's avatar
Brown, Joshua committed
247
                "UUID": os.getenv(required_env_variables[2]),
JoshuaSBrown's avatar
JoshuaSBrown committed
248
249
                "path": "/home/cades/Collections/default",
                "type": "mapped",
Brown, Joshua's avatar
Brown, Joshua committed
250
            }
JoshuaSBrown's avatar
JoshuaSBrown committed
251
252
        ],
    }
Brown, Joshua's avatar
Brown, Joshua committed
253
254
255
256
257
258
259
260
261
262
263
264
265
266

    globus_plugin = globus.Globus()
    globus_plugin.configure(configuration)

    # Create a file on the local posix system
    file_name = "demofile_for_globus1.txt"
    f = open(file_name, "w")
    original_number = random.randint(0, 100000000000)
    f.write(str(original_number))
    f.close()

    current_valid_path = os.getcwd()
    file_path = current_valid_path + "/" + file_name

Brown, Joshua's avatar
Brown, Joshua committed
267
268
    destination_path = "/"
    sub_folder = ""
Brown, Joshua's avatar
Brown, Joshua committed
269
    # This is so that we can run the test both as a runner and as user
Brown, Joshua's avatar
Brown, Joshua committed
270
271
272
    if os.getenv(required_env_variables[0]) in GITLAB_RUNNER_UUIDs:
        sub_folder = "runner/"

JoshuaSBrown's avatar
JoshuaSBrown committed
273
    package = [
JoshuaSBrown's avatar
JoshuaSBrown committed
274
275
        {
            "move_to_globus_collection": {
Brown, Joshua's avatar
Brown, Joshua committed
276
                "destination_collection_UUID": os.getenv(required_env_variables[2]),
JoshuaSBrown's avatar
JoshuaSBrown committed
277
278
279
                "source_host_name": socket.gethostname(),
                "items": [
                    {
JoshuaSBrown's avatar
JoshuaSBrown committed
280
                        "source": {"type": "posix absolute", "path": file_path},
Brown, Joshua's avatar
Brown, Joshua committed
281
282
                        "destination": {
                            "type": "globus relative",
JoshuaSBrown's avatar
JoshuaSBrown committed
283
284
                            "path": destination_path + sub_folder,
                        },
JoshuaSBrown's avatar
JoshuaSBrown committed
285
                    }
JoshuaSBrown's avatar
JoshuaSBrown committed
286
287
288
289
                ],
            }
        }
    ]
290

JoshuaSBrown's avatar
JoshuaSBrown committed
291
292
    checked_actions = globus_plugin.check(package)
    assert checked_actions["move_to_globus_collection"][0]
293
294
295
296


@pytest.mark.globus
def test_globus_transfer_check():
JoshuaSBrown's avatar
JoshuaSBrown committed
297
298

    required_env_variables = [
JoshuaSBrown's avatar
JoshuaSBrown committed
299
300
301
302
303
        "ZAMBEZE_CI_TEST_GLOBUS_CLIENT_ID",
        "ZAMBEZE_CI_TEST_GLOBUS_APP_KEY",
        "ZAMBEZE_CI_TEST_GLOBUS_COLLECTION_UUID",
        "ZAMBEZE_CI_TEST_GLOBUS_COLLECTION_SHARED_UUID",
    ]
JoshuaSBrown's avatar
JoshuaSBrown committed
304
305
306

    for env_var in required_env_variables:
        if env_var not in os.environ:
JoshuaSBrown's avatar
JoshuaSBrown committed
307
308
309
310
311
312
            raise Exception(
                "Globus test cannot be run if the env variable"
                f" {env_var} is not defined and a local "
                "globus-connect-server and endpoint have not been"
                " set up."
            )
JoshuaSBrown's avatar
JoshuaSBrown committed
313

314
    configuration = {
Brown, Joshua's avatar
Brown, Joshua committed
315
        "client_id": os.getenv(required_env_variables[0]),
JoshuaSBrown's avatar
JoshuaSBrown committed
316
        "authentication_flow": {
JoshuaSBrown's avatar
JoshuaSBrown committed
317
            "type": "client credential",
Brown, Joshua's avatar
Brown, Joshua committed
318
            "secret": os.getenv(required_env_variables[1]),
JoshuaSBrown's avatar
JoshuaSBrown committed
319
320
321
        },
        "collections": [
            {
Brown, Joshua's avatar
Brown, Joshua committed
322
                "UUID": os.getenv(required_env_variables[2]),
JoshuaSBrown's avatar
JoshuaSBrown committed
323
324
                "path": "/home/cades/Collections/default",
                "type": "mapped",
325
            }
JoshuaSBrown's avatar
JoshuaSBrown committed
326
327
        ],
    }
Brown, Joshua's avatar
Brown, Joshua committed
328

329
330
331
332
333
334
335
336
337
338
339
340
341
    globus_plugin = globus.Globus()
    globus_plugin.configure(configuration)

    # Create a file on the local posix system
    file_name = "demofile_for_globus1.txt"
    f = open(file_name, "w")
    original_number = random.randint(0, 100000000000)
    f.write(str(original_number))
    f.close()

    current_valid_path = os.getcwd()
    file_path = current_valid_path + "/" + file_name

Brown, Joshua's avatar
Brown, Joshua committed
342
343
    destination_path = "/"
    sub_folder = ""
Brown, Joshua's avatar
Brown, Joshua committed
344
    # This is so that we can run the test both as a runner and as user
Brown, Joshua's avatar
Brown, Joshua committed
345
346
347
    if os.getenv(required_env_variables[0]) in GITLAB_RUNNER_UUIDs:
        sub_folder = "runner/"

JoshuaSBrown's avatar
JoshuaSBrown committed
348
349
350
    package = [
        {
            "move_to_globus_collection": {
JoshuaSBrown's avatar
JoshuaSBrown committed
351
                "destination_collection_UUID": os.getenv(required_env_variables[2]),
JoshuaSBrown's avatar
JoshuaSBrown committed
352
353
354
                "source_host_name": socket.gethostname(),
                "items": [
                    {
JoshuaSBrown's avatar
JoshuaSBrown committed
355
                        "source": {"type": "posix absolute", "path": file_path},
Brown, Joshua's avatar
Brown, Joshua committed
356
357
                        "destination": {
                            "type": "globus relative",
JoshuaSBrown's avatar
JoshuaSBrown committed
358
                            "path": destination_path + sub_folder,
Brown, Joshua's avatar
Brown, Joshua committed
359
                        },
JoshuaSBrown's avatar
Format    
JoshuaSBrown committed
360
361
362
363
364
                    }
                ],
            }
        },
        {
JoshuaSBrown's avatar
JoshuaSBrown committed
365
            "transfer": {
JoshuaSBrown's avatar
JoshuaSBrown committed
366
367
                "source_collection_UUID": os.getenv(required_env_variables[2]),
                "destination_collection_UUID": os.getenv(required_env_variables[3]),
JoshuaSBrown's avatar
JoshuaSBrown committed
368
369
370
                "type": "synchronous",
                "items": [
                    {
Brown, Joshua's avatar
Brown, Joshua committed
371
                        "source": {
JoshuaSBrown's avatar
JoshuaSBrown committed
372
                            "type": "globus relative",
JoshuaSBrown's avatar
JoshuaSBrown committed
373
                            "path": "/" + sub_folder + file_name,
JoshuaSBrown's avatar
Format    
JoshuaSBrown committed
374
                        },
Brown, Joshua's avatar
Brown, Joshua committed
375
376
377
                        "destination": {
                            "type": "globus relative",
                            "path": destination_path + sub_folder + file_name,
JoshuaSBrown's avatar
JoshuaSBrown committed
378
                        },
JoshuaSBrown's avatar
Format    
JoshuaSBrown committed
379
                    }
JoshuaSBrown's avatar
JoshuaSBrown committed
380
                ],
JoshuaSBrown's avatar
Format    
JoshuaSBrown committed
381
            }
JoshuaSBrown's avatar
JoshuaSBrown committed
382
        },
JoshuaSBrown's avatar
Format    
JoshuaSBrown committed
383
    ]
384
385
386
387
388

    output = globus_plugin.check(package)

    for item in output:
        assert output[item][0]
389
390
391
392
393


@pytest.mark.globus
def test_globus_process():

JoshuaSBrown's avatar
JoshuaSBrown committed
394
    required_env_variables = [
JoshuaSBrown's avatar
JoshuaSBrown committed
395
396
397
398
399
        "ZAMBEZE_CI_TEST_GLOBUS_CLIENT_ID",
        "ZAMBEZE_CI_TEST_GLOBUS_APP_KEY",
        "ZAMBEZE_CI_TEST_GLOBUS_COLLECTION_UUID",
        "ZAMBEZE_CI_TEST_GLOBUS_COLLECTION_SHARED_UUID",
    ]
JoshuaSBrown's avatar
JoshuaSBrown committed
400
401
402

    for env_var in required_env_variables:
        if env_var not in os.environ:
JoshuaSBrown's avatar
JoshuaSBrown committed
403
404
405
406
407
408
            raise Exception(
                "Globus test cannot be run if the env variable"
                f" {env_var} is not defined and a local "
                "globus-connect-server and endpoint have not been"
                " set up."
            )
JoshuaSBrown's avatar
JoshuaSBrown committed
409

410
411
412
413
    path_to_endpoint = "/home/cades/Collections/default"
    path_to_endpoint_shared = "/home/cades/Collections/default/shared"

    configuration = {
Brown, Joshua's avatar
Brown, Joshua committed
414
        "client_id": os.getenv(required_env_variables[0]),
JoshuaSBrown's avatar
JoshuaSBrown committed
415
        "authentication_flow": {
JoshuaSBrown's avatar
JoshuaSBrown committed
416
            "type": "client credential",
Brown, Joshua's avatar
Brown, Joshua committed
417
            "secret": os.getenv(required_env_variables[1]),
JoshuaSBrown's avatar
JoshuaSBrown committed
418
419
420
        },
        "collections": [
            {
Brown, Joshua's avatar
Brown, Joshua committed
421
                "UUID": os.getenv(required_env_variables[2]),
JoshuaSBrown's avatar
JoshuaSBrown committed
422
423
424
425
                "path": path_to_endpoint,
                "type": "mapped",
            },
            {
Brown, Joshua's avatar
Brown, Joshua committed
426
                "UUID": os.getenv(required_env_variables[3]),
JoshuaSBrown's avatar
JoshuaSBrown committed
427
428
429
430
431
                "path": path_to_endpoint_shared,
                "type": "guest",
            },
        ],
    }
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446

    globus_plugin = globus.Globus()
    globus_plugin.configure(configuration)

    # Create a file on the local posix system
    file_name = "demofile_for_globus1.txt"
    f = open(file_name, "w")
    original_number = random.randint(0, 100000000000)
    f.write(str(original_number))
    f.close()

    current_valid_path = os.getcwd()
    file_path = current_valid_path + "/" + file_name

    relative_destination_file_path = "/"
Brown, Joshua's avatar
Brown, Joshua committed
447
    sub_folder = ""
Brown, Joshua's avatar
Brown, Joshua committed
448
    # This is so that we can run the test both as a runner and as user
Brown, Joshua's avatar
Brown, Joshua committed
449
    if os.getenv(required_env_variables[0]) in GITLAB_RUNNER_UUIDs:
JoshuaSBrown's avatar
JoshuaSBrown committed
450
        sub_folder = "runner/"
451
    # action items in the list should be executed in order
JoshuaSBrown's avatar
JoshuaSBrown committed
452
453
454
    package = [
        {
            "move_to_globus_collection": {
JoshuaSBrown's avatar
JoshuaSBrown committed
455
                "destination_collection_UUID": os.getenv(required_env_variables[2]),
JoshuaSBrown's avatar
JoshuaSBrown committed
456
457
458
459
460
461
                "source_host_name": socket.gethostname(),
                "items": [
                    {
                        "source": {"type": "posix absolute", "path": file_path},
                        "destination": {
                            "type": "globus relative",
Brown, Joshua's avatar
Brown, Joshua committed
462
                            "path": relative_destination_file_path + sub_folder,
JoshuaSBrown's avatar
JoshuaSBrown committed
463
                        },
JoshuaSBrown's avatar
JoshuaSBrown committed
464
465
466
                    }
                ],
            }
467
        },
JoshuaSBrown's avatar
Format    
JoshuaSBrown committed
468
469
        {
            "transfer": {
JoshuaSBrown's avatar
JoshuaSBrown committed
470
471
                "source_collection_UUID": os.getenv(required_env_variables[2]),
                "destination_collection_UUID": os.getenv(required_env_variables[3]),
JoshuaSBrown's avatar
Format    
JoshuaSBrown committed
472
473
474
                "type": "synchronous",
                "items": [
                    {
Brown, Joshua's avatar
Brown, Joshua committed
475
                        "source": {
JoshuaSBrown's avatar
Format    
JoshuaSBrown committed
476
                            "type": "globus relative",
JoshuaSBrown's avatar
JoshuaSBrown committed
477
                            "path": "/" + sub_folder + file_name,
JoshuaSBrown's avatar
JoshuaSBrown committed
478
                        },
Brown, Joshua's avatar
Brown, Joshua committed
479
480
                        "destination": {
                            "type": "globus relative",
JoshuaSBrown's avatar
JoshuaSBrown committed
481
482
483
484
                            "path": relative_destination_file_path
                            + sub_folder
                            + file_name,
                        },
JoshuaSBrown's avatar
Format    
JoshuaSBrown committed
485
                    }
JoshuaSBrown's avatar
JoshuaSBrown committed
486
                ],
JoshuaSBrown's avatar
Format    
JoshuaSBrown committed
487
            }
JoshuaSBrown's avatar
JoshuaSBrown committed
488
        },
JoshuaSBrown's avatar
Format    
JoshuaSBrown committed
489
    ]
490
491
492
493

    # This test is designed to move a file to the globus endpoint
    # So before we get started we are going to make sure that a file
    # does not already exist at that location
JoshuaSBrown's avatar
JoshuaSBrown committed
494
    abs_path_destination = (
JoshuaSBrown's avatar
JoshuaSBrown committed
495
496
497
498
        path_to_endpoint
        + relative_destination_file_path
        + sub_folder
        + os.path.basename(file_path)
JoshuaSBrown's avatar
JoshuaSBrown committed
499
    )
Brown, Joshua's avatar
Brown, Joshua committed
500
    # After it gets transferred using globus it should end up moving to the sub_folder
JoshuaSBrown's avatar
JoshuaSBrown committed
501
    abs_path_destination_shared = (
JoshuaSBrown's avatar
JoshuaSBrown committed
502
503
504
        path_to_endpoint
        + relative_destination_file_path
        + "shared/"
Brown, Joshua's avatar
Brown, Joshua committed
505
        + sub_folder
JoshuaSBrown's avatar
JoshuaSBrown committed
506
507
        + os.path.basename(file_path)
    )
508
509
510
511
512
    if os.path.exists(abs_path_destination):
        os.remove(abs_path_destination)
    if os.path.exists(abs_path_destination_shared):
        os.remove(abs_path_destination_shared)

513
514
515
516
517
518
519
520
521
    checked_items = globus_plugin.check(package)
    all_checks_pass = True
    for item in checked_items:
        if not checked_items[item][0]:
            print("Something went wrong.")
            print(checked_items[item][1])
            all_checks_pass = False

    if all_checks_pass:
522
523
524
525
        globus_plugin.process(package)

    # After processing we should verify that the file exists at the final location
    assert os.path.exists(abs_path_destination_shared)
JoshuaSBrown's avatar
JoshuaSBrown committed
526
527
528
529


@pytest.mark.globus
def test_globus_process_from_esnet():
Brown, Joshua's avatar
Brown, Joshua committed
530
    """NOTE
JoshuaSBrown's avatar
JoshuaSBrown committed
531

JoshuaSBrown's avatar
JoshuaSBrown committed
532
    We cannot simply download to a collection, we have to download to
Brown, Joshua's avatar
Brown, Joshua committed
533
534
535
536
    a folder that is owned by the user running the this test. I.e. if
    the GitLab Runner is being used then we have to specify a folder that
    the GitLab Runner can actually access from the POSIX side.
    """
JoshuaSBrown's avatar
JoshuaSBrown committed
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
    required_env_variables = [
        "ZAMBEZE_CI_TEST_GLOBUS_CLIENT_ID",
        "ZAMBEZE_CI_TEST_GLOBUS_APP_KEY",
        "ZAMBEZE_CI_TEST_GLOBUS_COLLECTION_UUID",
        "ZAMBEZE_CI_TEST_GLOBUS_COLLECTION_SHARED_UUID",
    ]

    for env_var in required_env_variables:
        if env_var not in os.environ:
            raise Exception(
                "Globus test cannot be run if the env variable"
                f" {env_var} is not defined and a local "
                "globus-connect-server and endpoint have not been"
                " set up."
            )

    path_to_endpoint = "/home/cades/Collections/default"
    path_to_endpoint_shared = "/home/cades/Collections/default/shared"

    configuration = {
        "client_id": os.getenv(required_env_variables[0]),
        "authentication_flow": {
            "type": "client credential",
            "secret": os.getenv(required_env_variables[1]),
        },
        "collections": [
            {
                "UUID": os.getenv(required_env_variables[2]),
                "path": path_to_endpoint,
                "type": "mapped",
            },
            {
                "UUID": os.getenv(required_env_variables[3]),
                "path": path_to_endpoint_shared,
                "type": "guest",
            },
        ],
    }

    globus_plugin = globus.Globus()
    globus_plugin.configure(configuration)

    # action items in the list should be executed in order
    ESNET_GLOBUS_ENDPOINT_UUID = "ece400da-0182-4777-91d6-27a1808f8371"

Brown, Joshua's avatar
Brown, Joshua committed
582
583
584
    # This is so that we can run the test both as a runner and as user
    sub_folder = ""
    if os.getenv(required_env_variables[0]) in GITLAB_RUNNER_UUIDs:
585
        sub_folder = "/runner"
Brown, Joshua's avatar
Brown, Joshua committed
586

JoshuaSBrown's avatar
JoshuaSBrown committed
587
588
589
590
    package = [
        {
            "transfer": {
                "source_collection_UUID": ESNET_GLOBUS_ENDPOINT_UUID,
591
                "destination_collection_UUID": os.getenv(required_env_variables[2]),
JoshuaSBrown's avatar
JoshuaSBrown committed
592
593
594
                "type": "synchronous",
                "items": [
                    {
595
                        "source": {"type": "globus relative", "path": "/1M.dat"},
Brown, Joshua's avatar
Brown, Joshua committed
596
597
                        "destination": {
                            "type": "globus relative",
Brown, Joshua's avatar
Brown, Joshua committed
598
                            "path": sub_folder + "/1M.dat",
Brown, Joshua's avatar
Brown, Joshua committed
599
                        },
JoshuaSBrown's avatar
JoshuaSBrown committed
600
601
602
                    }
                ],
            }
603
        }
JoshuaSBrown's avatar
JoshuaSBrown committed
604
605
606
607
608
    ]

    # This test is designed to move a file to the globus endpoint
    # So before we get started we are going to make sure that a file
    # does not already exist at that location
Brown, Joshua's avatar
Merge    
Brown, Joshua committed
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
    abs_path_destination = (
        path_to_endpoint
        + sub_folder
        + '/1M.dat'
    )

    if os.path.exists(abs_path_destination):
        os.remove(abs_path_destination)

    if globus_plugin.check(package):
        result = globus_plugin.process(package)

    # After processing we should verify that the file exists at the final location
    assert os.path.exists(abs_path_destination)


625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
def test_globus_process_async():

    required_env_variables = [
        "ZAMBEZE_CI_TEST_GLOBUS_CLIENT_ID",
        "ZAMBEZE_CI_TEST_GLOBUS_APP_KEY",
        "ZAMBEZE_CI_TEST_GLOBUS_COLLECTION_UUID",
        "ZAMBEZE_CI_TEST_GLOBUS_COLLECTION_SHARED_UUID",
    ]

    for env_var in required_env_variables:
        if env_var not in os.environ:
            raise Exception(
                "Globus test cannot be run if the env variable"
                f" {env_var} is not defined and a local "
                "globus-connect-server and endpoint have not been"
                " set up."
            )

    path_to_endpoint = "/home/cades/Collections/default"
    path_to_endpoint_shared = "/home/cades/Collections/default/shared"

    configuration = {
        "client_id": os.getenv(required_env_variables[0]),
        "authentication_flow": {
            "type": "client credential",
            "secret": os.getenv(required_env_variables[1]),
        },
        "collections": [
            {
                "UUID": os.getenv(required_env_variables[2]),
                "path": path_to_endpoint,
                "type": "mapped",
            },
            {
                "UUID": os.getenv(required_env_variables[3]),
                "path": path_to_endpoint_shared,
                "type": "guest",
            },
        ],
    }

    globus_plugin = globus.Globus()
    globus_plugin.configure(configuration)

    # Create a file on the local posix system
    file_name = "demofile_for_globus1.txt"
    f = open(file_name, "w")
    original_number = random.randint(0, 100000000000)
    f.write(str(original_number))
    f.close()

    current_valid_path = os.getcwd()
    file_path = current_valid_path + "/" + file_name

    relative_destination_file_path = "/"
    sub_folder = ""
    if os.getenv(required_env_variables[0]) in GITLAB_RUNNER_UUIDs:
        sub_folder = "runner/"
    # action items in the list should be executed in order
    package = [
        {
            "move_to_globus_collection": {
                "destination_collection_UUID": os.getenv(required_env_variables[2]),
                "source_host_name": socket.gethostname(),
                "items": [
                    {
                        "source": {"type": "posix absolute", "path": file_path},
                        "destination": {
                            "type": "globus relative",
                            "path": relative_destination_file_path + sub_folder,
                        },
                    }
                ],
            }
        },
        {
            "transfer": {
                "source_collection_UUID": os.getenv(required_env_variables[2]),
                "destination_collection_UUID": os.getenv(required_env_variables[3]),
                "type": "asynchronous",
                "items": [
                    {
                        "source": {
                            "type": "globus relative",
                            "path": "/" + sub_folder + file_name,
                        },
                        "destination": {
                            "type": "globus relative",
                            "path": relative_destination_file_path
                            + sub_folder
                            + file_name,
                        },
                    }
                ],
            }
        },
    ]

Brown, Joshua's avatar
Merge    
Brown, Joshua committed
723
    abs_path_destination = path_to_endpoint + sub_folder + file_name
JoshuaSBrown's avatar
JoshuaSBrown committed
724

JoshuaSBrown's avatar
JoshuaSBrown committed
725
726
727
    if os.path.exists(abs_path_destination):
        os.remove(abs_path_destination)

728
729
    output = globus_plugin.check(package)
    if output["transfer"][0]:
JoshuaSBrown's avatar
JoshuaSBrown committed
730
        globus_plugin.process(package)
731
732
733
    else:
        print("Check failed no transfer was conducted.")
        print(output["transfer"][1])
JoshuaSBrown's avatar
JoshuaSBrown committed
734
735

    # After processing we should verify that the file exists at the final location
736
    assert os.path.exists(abs_path_destination)
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
    abs_path_destination = (
        path_to_endpoint
        + relative_destination_file_path
        + sub_folder
        + os.path.basename(file_path)
    )
    # After it gets transferred using globus it should end up moving to the subfolder
    abs_path_destination_shared = (
        path_to_endpoint
        + relative_destination_file_path
        + "shared/"
        + sub_folder
        + os.path.basename(file_path)
    )
    if os.path.exists(abs_path_destination):
        os.remove(abs_path_destination)
    if os.path.exists(abs_path_destination_shared):
        os.remove(abs_path_destination_shared)

    if globus_plugin.check(package):
        result = globus_plugin.process(package)

        result = globus_plugin.process([result["transfer"]["callback"]])
        while result["get_task_status"]["result"]["status"] != "SUCCEEDED":
            print("waiting...")
            time.sleep(1)
            result = globus_plugin.process([result["get_task_status"]["callback"]])

        print("complete")
        print(result["get_task_status"])
    # After processing we should verify that the file exists at the final location
    assert os.path.exists(abs_path_destination_shared)