Commit f280699e authored by Cage, Gregory's avatar Cage, Gregory
Browse files

Handle soft_kill more appropriately for pulsar runner

parent 79a371fa

.gitlab-ci.yml

+11 −11
Original line number Diff line number Diff line
@@ -6,7 +6,7 @@ stages: 


variables:

  GIT_STRATEGY: clone

  CONTAINER_GALAXY_URL: "${CI_REGISTRY_IMAGE}"

  CONTAINER_GALAXY_URL: "${NDIP_DOCKER_REPOSITORY}/${CI_PROJECT_PATH}"

  CONTAINER_GALAXY_BASE_URL: "${CONTAINER_GALAXY_URL}/base"

  CONTAINER_GALAXY_COMMIT_URL: "${CONTAINER_GALAXY_URL}/commit"

  GALAXY_VERSION_PYTHON: 24.1.dev3+ornl
@@ -15,7 +15,7 @@ variables: 
# This import is for the func_rse_docker_* functions

before_script:

  - curl https://code.ornl.gov/rse-deployment/rse-sharables/raw/master/rse-bash-modules.sh -O

  - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY

  - docker login -u $NDIP_DOCKER_USER -p $NDIP_DOCKER_PASSWORD $NDIP_DOCKER_REPOSITORY

  - source rse-bash-modules.sh

  - func_rse_docker_cleanup
@@ -33,8 +33,8 @@ service-build: 
      docker build

      -f dockerfiles/Dockerfile 

      -t galaxy .

    - docker tag galaxy $CONTAINER_GALAXY_COMMIT_URL:$CI_COMMIT_SHORT_SHA

    - docker push $CONTAINER_GALAXY_COMMIT_URL:$CI_COMMIT_SHORT_SHA

    - docker tag galaxy $CONTAINER_GALAXY_COMMIT_URL:build-$CI_COMMIT_SHORT_SHA

    - docker push $CONTAINER_GALAXY_COMMIT_URL:build-$CI_COMMIT_SHORT_SHA

  tags:

    - rse-multi-builder

  except:
@@ -46,7 +46,7 @@ client-tests: 
    - >    

      docker run

      --entrypoint=""

      $CONTAINER_GALAXY_COMMIT_URL:$CI_COMMIT_SHORT_SHA

      $CONTAINER_GALAXY_COMMIT_URL:build-$CI_COMMIT_SHORT_SHA

      bash -c ". .venv/bin/activate && cd client && yarn run jest"

  tags:

    - rse-multi-builder
@@ -63,7 +63,7 @@ client-tests: 
#      docker run

#      --entrypoint=""

#      --network=host

#      $CONTAINER_GALAXY_COMMIT_URL:$CI_COMMIT_SHORT_SHA

#      $CONTAINER_GALAXY_COMMIT_URL:build-$CI_COMMIT_SHORT_SHA

#      bash -c "GALAXY_TEST_DBURI=postgresql://postgres:psql@127.0.0.1:5432/galaxy_test ./run_tests.sh -api"

#    - docker rm -f postgres

#  tags:
@@ -78,7 +78,7 @@ unit-tests: 
    - >    

      docker run

      --entrypoint=""

      $CONTAINER_GALAXY_COMMIT_URL:$CI_COMMIT_SHORT_SHA

      $CONTAINER_GALAXY_COMMIT_URL:build-$CI_COMMIT_SHORT_SHA

      bash -c "GALAXY_SKIP_EXTERNAL_DEPENDENCY_MANAGEMENT=1  ./run_tests.sh -unit"

  tags:

    - rse-multi-builder
@@ -108,7 +108,7 @@ package: 
  script:

    - >

      docker run

      $CONTAINER_GALAXY_COMMIT_URL:$CI_COMMIT_SHORT_SHA

      $CONTAINER_GALAXY_COMMIT_URL:build-$CI_COMMIT_SHORT_SHA

      bash -c ". .venv/bin/activate . && pip install build twine && cd packages/objectstore &&

      sed -i -e '/version =/ s/= .*/= '$GALAXY_VERSION_PYTHON'/' setup.cfg &&

      make dist &&
@@ -123,8 +123,8 @@ package: 
tag-docker:

  stage: deploy

  script:

    - docker pull $CONTAINER_GALAXY_COMMIT_URL:$CI_COMMIT_SHORT_SHA

    - docker tag $CONTAINER_GALAXY_COMMIT_URL:$CI_COMMIT_SHORT_SHA $CONTAINER_GALAXY_URL:$GALAXY_VERSION_DOCKER

    - docker pull $CONTAINER_GALAXY_COMMIT_URL:build-$CI_COMMIT_SHORT_SHA

    - docker tag $CONTAINER_GALAXY_COMMIT_URL:build-$CI_COMMIT_SHORT_SHA $CONTAINER_GALAXY_URL:$GALAXY_VERSION_DOCKER

    - docker push $CONTAINER_GALAXY_URL:$GALAXY_VERSION_DOCKER

  when: manual

  except:

lib/galaxy/jobs/handler.py

+5 −2
Original line number Diff line number Diff line
@@ -1240,7 +1240,7 @@ class DefaultJobDispatcher: 
            log.debug(f"({job_wrapper.job_id}) Dispatching to {job_wrapper.job_destination.runner} runner")

        runner.put(job_wrapper)



    def stop(self, job, job_wrapper):

    def stop(self, job, job_wrapper, soft_kill=False):

        """

        Stop the given job. The input variable job may be either a Job or a Task.

        """
@@ -1260,6 +1260,9 @@ class DefaultJobDispatcher: 
            runner_name = job_runner_name.split(":", 1)[0]

            log.debug(f"Stopping job {job_wrapper.get_id_tag()} in {runner_name} runner")

            try:

                try:

                    self.job_runners[runner_name].stop_job(job_wrapper, soft_kill=soft_kill)

                except:

                    self.job_runners[runner_name].stop_job(job_wrapper)

            except KeyError:

                log.error(f"stop(): ({job_wrapper.get_id_tag()}) Invalid job runner: {runner_name}")

lib/galaxy/jobs/runners/pulsar.py

+1 −1
Original line number Diff line number Diff line
@@ -762,7 +762,7 @@ class PulsarJobRunner(AsynchronousJobRunner): 
                )

            return False



    def stop_job(self, job_wrapper, soft_kill=True):

    def stop_job(self, job_wrapper, soft_kill=False):

        job = job_wrapper.get_job()

        if not job.job_runner_external_id:

            return

lib/galaxy/managers/jobs.py

+3 −1
Original line number Diff line number Diff line
@@ -337,12 +337,14 @@ class JobManager: 
        if not job.finished:

            try:

                job.mark_stopped(self.app.config.track_jobs_in_database)

                job_wrapper = self.app.job_manager.job_handler.job_queue.job_wrapper(job)

                self.app.job_manager.job_handler.dispatcher.stop(job, job_wrapper, soft_kill=True)

                session = self.app.model.session

                with transaction(session):

                    session.commit()

                self.app.job_manager.stop(job, message="")

                return True

            except Exception:

            except Exception as e:

                log.error("Job Runner does not support stopping job early.")

        return False