Skip to content
Snippets Groups Projects
Commit 8fcda2e3 authored by John Chilton's avatar John Chilton
Browse files

Attempt to merge Pulsar Docker images.

parent 4a13e057
No related branches found
No related tags found
No related merge requests found
......@@ -8,17 +8,31 @@ ENV PULSAR_CONFIG_CONDA_PREFIX /usr/local
# bzip2 for Miniconda.
# TODO: pycurl stuff...
RUN apt-get update \
# Install CVMFS client
&& apt-get install -y --no-install-recommends lsb-release wget \
gcc python-setuptools \
&& wget https://ecsft.cern.ch/dist/cvmfs/cvmfs-release/cvmfs-release-latest_all.deb \
&& dpkg -i cvmfs-release-latest_all.deb \
&& rm -f cvmfs-release-latest_all.deb \
# Install packages
&& apt-get update \
&& apt-get install -y --no-install-recommends gcc python-setuptools \
python-dev python-pip \
cvmfs cvmfs-config-default \
slurm-llnl slurm-drmaa-dev \
bzip2 \
# Install Pulsar Python requirements
&& pip install --no-cache-dir -U pip \
&& pip install --no-cache-dir drmaa wheel kombu pykube poster \
webob psutil PasteDeploy six pyyaml paramiko \
# Remove build deps and cleanup
&& apt-get -y remove python-dev gcc wget lsb-release \
&& apt-get -y autoremove \
&& apt-get autoclean \
&& rm -rf /var/lib/apt/lists/* /var/log/dpkg.log
RUN pip install -U pip && pip install wheel kombu pykube poster
&& rm -rf /var/lib/apt/lists/* /var/log/dpkg.log \
&& /usr/sbin/create-munge-key
ADD pulsar_app-*.dev0-py2.py3-none-any.whl /pulsar_app-*.dev0-py2.py3-none-any.whl
RUN pip install /pulsar_app-*.dev0-py2.py3-none-any.whl[galaxy_extended_metadata]
RUN pip install --no-cache-dir /pulsar_app-*.dev0-py2.py3-none-any.whl[galaxy_extended_metadata]
RUN _pulsar-configure-galaxy-cvmfs
RUN _pulsar-conda-init --conda_prefix=/pulsar_dependencies/conda
......@@ -41,12 +41,9 @@ RUN python setup.py install \
&& pulsar-config --auto_conda --host 0.0.0.0 \
&& echo "export DRMAA_LIBRARY_PATH=/usr/lib/slurm-drmaa/lib/libdrmaa.so" >> local_env.sh \
&& cp docker/cvmfs/app.yml . \
&& cp ./docker/testing/configure_slurm.py /usr/sbin/configure_slurm.py \
&& chown -R pulsar:pulsar /pulsar \
&& chmod +x /usr/local/bin/pulsar \
&& cp /pulsar/docker/cvmfs/default.local /etc/cvmfs/ \
&& cp /pulsar/docker/cvmfs/galaxyproject.org.conf /etc/cvmfs/domain.d/ \
&& cp /pulsar/docker/cvmfs/data.galaxyproject.org.pub /etc/cvmfs/keys/ \
&& _pulsar-configure-galaxy-cvmfs \
&& chown pulsar -R /usr/local
# Pulsar will listen on this port
......@@ -55,5 +52,5 @@ EXPOSE 8913
# Must run CVMFS setup otherwise autofs does not get configured nor automount
# starts. Then start Slurm and switch to a less-priviledged user for Pulsar.
CMD /usr/bin/cvmfs_config setup; service munge start; \
python /usr/sbin/configure_slurm.py; service slurmd start; \
_pulsar-configure-slurm; service slurmd start; \
service slurmctld start; su pulsar -c "/usr/local/bin/pulsar"
......@@ -11,7 +11,7 @@ init_pulsar
cd pulsar
echo "Running pulsar-config with default arguments"
pulsar-config
pulsar-config --auto_conda
cd ..
check_pulsar
......
......@@ -11,11 +11,11 @@ from pulsar import messaging
from galaxy.objectstore import build_object_store_from_config
try:
# If galaxy-tool-util or Galaxy >=19.09 present.
from galaxy.tools.deps import build_dependency_manager
from galaxy.tool_util.deps import build_dependency_manager
DependencyManager = None
except ImportError:
# If galaxy-lib or Galaxy <19.05 present.
from galaxy.tool_util.deps import DependencyManager
from galaxy.tools.deps import DependencyManager
build_dependency_manager = None
try:
# If galaxy-lib or Galaxy <19.05 present.
......@@ -142,10 +142,11 @@ class PulsarApp(object):
self.object_store = build_object_store_from_config(object_store_config, config_dict=config_dict)
def __setup_dependency_manager(self, conf):
default_tool_dependency_dir = "dependencies"
if build_dependency_manager is not None:
self.dependency_manager = build_dependency_manager(app_config_dict=conf)
self.dependency_manager = build_dependency_manager(app_config_dict=conf, default_tool_dependency_dir=default_tool_dependency_dir)
else:
dependencies_dir = conf.get("tool_dependency_dir", "dependencies")
dependencies_dir = conf.get("tool_dependency_dir", default_tool_dependency_dir)
resolvers_config_file = conf.get("dependency_resolvers_config_file", "dependency_resolvers_conf.xml")
conda_config = {k: v for k, v in conf.items() if k.startswith("conda_")}
self.dependency_manager = DependencyManager(dependencies_dir, resolvers_config_file, app_config=conda_config)
......
import pkg_resources
import shutil
def main():
config_path = pkg_resources.resource_filename('pulsar.scripts', 'cvmfs_data/default.local')
key_path = pkg_resources.resource_filename('pulsar.scripts', 'cvmfs_data/data.galaxyproject.org.pub')
domain_path = pkg_resources.resource_filename('pulsar.scripts', 'cvmfs_data/galaxyproject.org.conf')
# copy2?
shutil.copy(config_path, "/etc/cvmfs/")
shutil.copy(key_path, "/etc/cvmfs/keys/")
shutil.copy(domain_path, "/etc/cvmfs/domain.d/")
webob
psutil
PasteDeploy
six
six==1.11.0
pyyaml
galaxy-job-metrics>=19.9.0.dev1
galaxy-objectstore>=19.9.0.dev1
galaxy-tool-util>=19.9.0.dev1
galaxy-tool-util>=19.9.0.dev3
paramiko
# Problematic for Python 3.
......
......@@ -101,18 +101,21 @@ setup(
pulsar-submit=pulsar.scripts.submit:main
pulsar-run=pulsar.scripts.run:main
_pulsar-conda-init=pulsar.scripts._conda_init:main
_pulsar-configure-slurm=pulsar.scripts._configure_slurm:main
_pulsar-configure-galaxy-cvmfs=pulsar.scripts._configure_galaxy_cvmfs:main
''',
scripts=scripts,
package_data={'pulsar': [
'managers/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh',
'managers/util/job_script/CLUSTER_SLOTS_STATEMENT.sh',
'scripts/cvmfs_data/*',
]},
package_dir={'pulsar': 'pulsar'},
include_package_data=True,
install_requires=requirements,
extras_require={
':python_version=="2.7"': py27_requirements,
'galaxy_extended_metadata': ['galaxy-job-execution>=19.9.0.dev0'],
'galaxy_extended_metadata': ['galaxy-job-execution>=19.9.0.dev0', 'galaxy-util[template]'],
},
license="Apache License 2.0",
zip_safe=False,
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment