Merge branch 'sm/issue_45' into 'development'

Legion fix for issue #45

See merge request mtip/spinifel!69

settings/multiple_conform.toml

+# This toml file contains settings for multiple conformations hdf5 file/Frontier/Crusher
+[debug]
+test = "CI settings for multiple conformations on Frontier/Crusher"
+verbosity = 0
+use_callmonitor = false
+[data]
+in_dir = "${test_data_dir}"
+out_dir = "${out_dir}"
+name = "3iyf_3j03_128x128_1M.h5"
+[detector]
+shape = [1, 128, 128]
+[psana]
+enable = false
+ps_smd_n_events = 1000
+ps_eb_nodes = 1
+ps_srv_nodes = 0
+ps_batch_size=1000
+ps_dir = "${test_data_dir}"
+exp = "xpptut15"
+runnum = 1
+ps_parallel = "legion"
+detname = "amopnccd"
+[runtime]
+N_images_per_rank = 1000
+use_cuda = true
+use_cufinufft = true
+use_cupy = true
+use_single_prec = false
+chk_convergence = true
+must_converge = false
+[gpu]
+devices_per_node = 1
+[algorithm]
+nER = 50
+nHIO = 25
+beta = 0.3
+cutoff = 0.05
+N_phase_loops = 10
+N_clipping = 0
+N_binning = 0
+N_orientations = 10000 # model_slices
+N_batch_size = 1000         
+[fsc]
+pdb_path = "${test_data_dir}/mult_conf_data"
+fsc_zoom = 0.6
+fsc_nsearch = 420

setup/reconfigure_legion.sh

@@ -33,6 +33,8 @@ cmake -DCMAKE_PREFIX_PATH="$CONDA_ENV_DIR" \
     -DLegion_USE_GASNet=$([ $LEGION_USE_GASNET -eq 1 ] && echo ON || echo OFF) \
     -DGASNet_ROOT_DIR="$GASNET_ROOT" \
     -DGASNet_CONDUITS=${LEGION_GASNET_CONDUIT:-$GASNET_CONDUIT} \
+    -DLegion_NETWORKS=gasnetex \
+    -DLegion_MAX_NUM_NODES=8192 \
     -DLegion_USE_HDF5=ON \
     -DLegion_MAX_DIM=4 \
     -DCMAKE_INSTALL_PREFIX="$LEGION_INSTALL_DIR" \

spinifel/legion/autocorrelation.py

@@ -418,8 +418,12 @@ def solve_ac_merge(
     results_r = solve_ac_dict["results_r"]
 
     alambda = 1
-    rlambdas = Mtot / N * 2 ** (np.arange(N_procs) - N_procs / 2).astype(np.float64)
-    flambdas = 1e5 * 10 ** (np.arange(N_procs) - N_procs // 2).astype(np.float64)
+    # temp fix for flambdas/rlambdas, for single precision - maxval=8, for double precision - maxval=32
+    max_val = 32
+    if settings.use_single_prec:
+        max_val =  8
+    flambdas = 1e5 * 10 ** np.fmod((np.arange(N_procs) - N_procs // 2), max_val).astype(np.float64)
+    rlambdas = Mtot / N * 2 ** np.fmod((np.arange(N_procs) - N_procs / 2), max_val).astype(np.float64)
     summary_p = solve_ac_dict["summary_p"]
     summary = solve_ac_dict["summary"]
     reciprocal_extent = solve_ac_dict["reciprocal_extent"]

spinifel/legion/orientation_matching.py

@@ -219,12 +219,12 @@ def match_conf(phased, orientations_p, slices_p, min_dist_p, min_dist_proc, conf
         for i in range(settings.N_conformations):
             # check fsc future values -> if convergence has failed then continue with match
             if len(fsc) > 0 and check_convergence_single_conf(fsc[i]):
-                logger.log(f"conformation {i} HAS converged in orientation_matching check")
+                logger.log(f"conformation {i} HAS converged in orientation_matching check",level=1)
                 update_min_dist(conf_p, min_dist_p, i, settings.N_conformations, n_images_per_rank)
                 #fill_min_dist(min_dist_p, i, settings.N_conformations)
             else:
                 if len(fsc) > 0:
-                    logger.log(f"conformation {i} has NOT converged in orientation_matching check")
+                    logger.log(f"conformation {i} has NOT converged in orientation_matching check",level=1)
                 match_single_conf(phased[i], orientations_p[i], slices_p, min_dist_p, n_images_per_rank, i, settings.N_conformations, ready_objs)
         N_procs = Tunable.select(Tunable.GLOBAL_PYS).get()
         # TODO initialize min_dist_proc to zero for those conformations that are completed

spinifel/legion/phasing.py

@@ -267,10 +267,10 @@ def new_phase_conf(generation, solved, fsc, phased_regions_dict=None):
         if len(fsc) > 0 and check_convergence_single_conf(fsc[i]):
             assert create_regions is False
             phased_conf.append(phased_regions_dict[i]["phased"])
-            logger.log(f"conformation {i} HAS converged in new_phase_conf")
+            logger.log(f"conformation {i} HAS converged in new_phase_conf", level=1)
         else:
             if len(fsc) > 0:
-                logger.log(f"conformation {i} has NOT converged in new_phase_conf")
+                logger.log(f"conformation {i} has NOT converged in new_phase_conf",level=1)
             if create_regions is False:
                 phased, phased_regions_dict[i] = new_phase(generation, solved[i], phased_regions_dict[i],i)
                 phased_conf.append(phased)

spinifel/sequential/autocorrelation.py

@@ -110,7 +110,7 @@ class Merge:
         else:
             rotmat = np.zeros((0, 3, 3))
             logger.log(
-                "WARNING: gen_nonuniform_positions got empty orientation - returning h,k,l for Null rotation"
+                "WARNING: gen_nonuniform_positions got empty orientation - returning h,k,l for Null rotation", level=1
             )
 
         # TODO : How to ensure we support all formats of pixel_position reciprocal
@@ -341,7 +341,7 @@ class Merge:
 
         ret, info = cg(W, d, x0=x0, maxiter=self.maxiter, callback=self.callback)
         if info != 0:
-            print(f"WARNING: CG did not converge at rlambda = {self.rlambda}")
+            logger.log(f"WARNING: CG did not converge at rlambda = {self.rlambda}",level=1)
         ac = ret.reshape((self.M,) * 3).get()
         if self.use_reciprocal_symmetry:
             assert np.all(np.isreal(ac))