Commit 98989693 authored by mohammad's avatar mohammad
Browse files

addressed Jareds comments

parent bc56e4a5

megatron/arguments.py

+9 −4
Original line number Diff line number Diff line
@@ -136,14 +136,16 @@ def parse_args(extra_args_provider=None, defaults={}, 
def _print_args(args):

    """Print arguments."""

    if args.rank == 0:

        print('-------------------- arguments --------------------', flush=True)

        print('------------------------ arguments ------------------------',

              flush=True)

        str_list = []

        for arg in vars(args):

            dots = '.' * (32 - len(arg))

            dots = '.' * (48 - len(arg))

            str_list.append('  {} {} {}'.format(arg, dots, getattr(args, arg)))

        for arg in sorted(str_list, key=lambda x: x.lower()):

            print(arg, flush=True)

        print('---------------- end of arguments ----------------', flush=True)

        print('-------------------- end of arguments ---------------------',

              flush=True)





def _check_arg_is_not_none(args, arg):
@@ -401,7 +403,10 @@ def _add_data_args(parser): 
    group = parser.add_argument_group(title='data and dataloader')



    group.add_argument('--data-path', nargs='*', default=None,

                       help='Path to combined dataset to split.')

                       help='Path to the training dataset. Accepted format:'

                       '1) a single data path, 2) multiple datasets in the'

                       'form: dataset1-weight dataset1-path dataset2-weight '

                       'dataset2-path ...')

    group.add_argument('--split', type=str, default='969, 30, 1',

                       help='Comma-separated list of proportions for training,'

                       ' validation, and test split. For example the split '

megatron/data/helpers.cpp

+2 −2
Original line number Diff line number Diff line
@@ -60,7 +60,7 @@ void build_blending_indices(py::array_t<uint8_t>& dataset_index, 
  for(int64_t sample_idx = 0; sample_idx < size; ++sample_idx) {



    // Determine where the max error in sampling is happening.

    double sample_idx_double = std::max(static_cast<double>(sample_idx), 1.0);

    auto sample_idx_double = std::max(static_cast<double>(sample_idx), 1.0);

    int64_t max_error_index = 0;

    double max_error = weights_ptr[0] * sample_idx_double -

      static_cast<double>(current_samples[0]);
@@ -86,7 +86,7 @@ void build_blending_indices(py::array_t<uint8_t>& dataset_index, 
  if (verbose) {

    std::cout << " > sample ratios:" << std::endl;

    for (int64_t dataset_idx = 0; dataset_idx < num_datasets; ++dataset_idx) {

      double ratio = static_cast<double>(current_samples[dataset_idx]) /

      auto ratio = static_cast<double>(current_samples[dataset_idx]) /

	static_cast<double>(size);

      std::cout << "   dataset " << dataset_idx << ", input: " <<

	weights_ptr[dataset_idx] << ", achieved: " << ratio << std::endl;