Unverified Commit 3356ef6f authored by John Chilton's avatar John Chilton Committed by GitHub
Browse files

Merge pull request #13697 from bernt-matthias/topic/backport13688pt2 

[21.09] backport of #13688 part 2 (Fix sorting of recursively discovered datasets)
parents 7275b2f8 4bc56038

lib/galaxy/job_execution/output_collect.py

+20 −16
Original line number Diff line number Diff line
@@ -512,8 +512,9 @@ def walk_over_extra_files(target_dir, extra_file_collector, job_working_director 
    match the given collector's match criteria. If the collector has the

    recurse flag enabled, will also recursively descend into child folders.

    """

    matches = []

    parent_paths = parent_paths or []



    def _walk(target_dir, extra_file_collector, job_working_directory, matchable, parent_paths):

        directory = discover_target_directory(target_dir, job_working_directory)

        if os.path.isdir(directory):

            for filename in os.listdir(directory):
@@ -523,14 +524,17 @@ def walk_over_extra_files(target_dir, extra_file_collector, job_working_director 
                        new_parent_paths = parent_paths[:]

                        new_parent_paths.append(filename)

                        # The current directory is already validated, so use that as the next job_working_directory when recursing

                    for match in walk_over_extra_files(filename, extra_file_collector, directory, matchable, parent_paths=new_parent_paths):

                        yield match

                        yield from _walk(

                            filename, extra_file_collector, directory, matchable, parent_paths=new_parent_paths

                        )

                else:

                    match = extra_file_collector.match(matchable, filename, path=path, parent_paths=parent_paths)

                    if match:

                    matches.append(match)

                        yield match



    for match in extra_file_collector.sort(matches):

    for match in extra_file_collector.sort(

        _walk(target_dir, extra_file_collector, job_working_directory, matchable, parent_paths)

    ):

        yield match

test/functional/tools/multi_output_recurse_collection.xml

0 → 100644
+57 −0
Original line number Diff line number Diff line 
<!--essentially the same as multi_output_recurse_collection, but:

  - test collection and generated order in the collection

-->

<tool id="multi_output_recurse_collection" name="multi_output_recurse_collection" version="0.1.0" profile="22.01">

  <macros>

    <xml name="outelem" tokens="number" token_path="">

      <element name="@PATH@@NUMBER@" ftype="txt">

        <assert_contents>

          <has_text text="@NUMBER@"/>

        </assert_contents>

      </element>

    </xml>

  </macros>

  <command><![CDATA[

    mkdir 1;

    echo "3" > 1/3.txt;

    echo "4" > 1/4.txt;

    mkdir 2;

    echo "1" > 2/1.txt;

    echo "2" > 2/2.txt;

    mkdir 3;

    echo "5" > 3/5.txt;

    mkdir 3/1;

    echo "0" > 3/1/0.txt;

    ]]></command>

  <inputs>

    <param type="boolean" name="match_relative"/>

  </inputs>

  <outputs>

    <collection type="list" name="output">

      <discover_datasets directory="./" recurse="true" pattern="__designation_and_ext__"/>

    </collection>

    <collection type="list" name="output_relative">

      <discover_datasets directory="./" recurse="true" pattern="__designation_and_ext__" match_relative_path="true"/>

    </collection>

  </outputs>

  <tests>

    <test>

      <output_collection name="output" count="6">

        <expand macro="outelem" number="0"/>

        <expand macro="outelem" number="1"/>

        <expand macro="outelem" number="2"/>

        <expand macro="outelem" number="3"/>

        <expand macro="outelem" number="4"/>

        <expand macro="outelem" number="5"/>

      </output_collection>

      <output_collection name="output_relative" count="6">

        <expand macro="outelem" path="1/" number="3"/>

        <expand macro="outelem" path="1/" number="4"/>

        <expand macro="outelem" path="2/" number="1"/>

        <expand macro="outelem" path="2/" number="2"/>

        <expand macro="outelem" path="3/" number="5"/>

        <expand macro="outelem" path="3/1/" number="0"/>

      </output_collection>

    </test>

  </tests>

</tool>

test/functional/tools/samples_tool_conf.xml

+1 −0
Original line number Diff line number Diff line
@@ -28,6 +28,7 @@ 
  <tool file="multi_output_assign_primary.xml" />

  <tool file="multi_output_assign_primary_ext_dbkey.xml" />

  <tool file="multi_output_recurse.xml" />

  <tool file="multi_output_recurse_collection.xml" />

  <tool file="tool_provided_metadata_1.xml" />

  <tool file="tool_provided_metadata_2.xml" />

  <tool file="tool_provided_metadata_3.xml" />