Loading configure/Dockerfile +2 −2 Original line number Diff line number Diff line Loading @@ -7,10 +7,10 @@ FROM alpine # Install gomplate (for templating) into the bin directory ARG GOMPLATE_VERSION=2.8.0 ARG GOMPLATE_VERSION=3.1.0 RUN apk --no-cache add curl && \ mkdir -p /esg/bin && \ curl -o /esg/bin/gomplate -fsSL https://github.com/hairyhenderson/gomplate/releases/download/${GOMPLATE_VERSION}/gomplate_linux-amd64-slim && \ curl -o /esg/bin/gomplate -fsSL https://github.com/hairyhenderson/gomplate/releases/download/v${GOMPLATE_VERSION}/gomplate_linux-amd64-slim && \ chmod 755 /esg/bin/gomplate # Copy the scripts into the bin directory COPY ./scripts /esg/bin Loading docs/_docs/compose/publishing.md +1 −3 Original line number Diff line number Diff line Loading @@ -52,15 +52,13 @@ fetch-certificate ### Generate map files ```sh esgprep mapfile --project test /esg/data/test ls -l mapfiles esgmapfile --project test /esg/data/test ``` ### Publish to the PostgreSQL database ```sh esgpublish --project test --map mapfiles/test.test.map --service fileservice esglist_datasets test ``` ### Publish to the TDS Loading docs/_docs/compose/testing.md +1 −1 Original line number Diff line number Diff line Loading @@ -51,7 +51,7 @@ The following procedure can be used to manually test an ESGF node installation u * username=rootAdmin * password=<value from cat $ESGF_CONFIG/secrets/rootadmin-password> * Test publishing a test dataset following the instructions at [Data Publishing](/usage/publishing/) * Test publishing a test dataset following the instructions at [Data Publishing](../publishing/) ### Automatic Testing Loading publisher/Dockerfile +33 −27 Original line number Diff line number Diff line Loading @@ -6,58 +6,62 @@ ARG ESGF_HUB=esgfhub ARG ESGF_PREFIX= ARG ESGF_VERSION=latest # This build stage is required because COPY --from=$ARG is not supported # https://github.com/moby/moby/issues/34482 # This build stage is required because COPY --from=$ARG is not supported # https://github.com/moby/moby/issues/34482 FROM ${ESGF_HUB}/${ESGF_PREFIX}configure:${ESGF_VERSION} as configuration FROM continuumio/miniconda:4.3.27 FROM continuumio/miniconda # Create an unprivileged user to use ENV PUBLISH_USER publish ENV PUBLISH_GROUP publish # It is IMPORTANT that the UID is the same as the tomcat user in the TDS container # It is IMPORTANT that the UID is the same as the tomcat user in the TDS container ENV PUBLISH_UID 1001 ENV PUBLISH_GID 1001 RUN groupadd -g $PUBLISH_GID $PUBLISH_GROUP && \ useradd -M -g $PUBLISH_GID -s /sbin/nologin -u $PUBLISH_UID $PUBLISH_USER # Set environment vars that control the behaviour # Set environment vars that control the behaviour ENV ESGINI /esg/config/esgcet/esg.ini ENV UVCDAT_ANONYMOUS_LOG no # Install packages using conda and pip RUN conda install -y -c conda-forge -c uvcdat \ gcc_linux-64 \ cdutil \ lxml \ requests \ # Install dependencies using conda # NOTE: esgprep has very specific version requirements ARG CDAT_LABEL_CHANNEL=cdat/label/80 RUN conda install -y -c $CDAT_LABEL_CHANNEL -c conda-forge -c cdat \ psycopg2 \ decorator \ Tempita \ myproxyclient && \ pip install \ 'SQLAlchemy==0.7.10' \ sqlalchemy_migrate \ ESGConfigParser==0.1.10 \ esgprep==2.7.40 lxml \ cdutil \ gcc_linux-64 \ 'requests==2.20.0' \ 'netCDF4==1.4.0' # Make the conda-installed gcc available as gcc # See https://conda.io/docs/user-guide/tasks/build-packages/compiler-tools.html RUN ln -s /opt/conda/bin/x86_64-conda_cos6-linux-gnu-gcc /usr/local/bin/gcc # Install the publisher client from GitHub ARG ESG_PUBLISHER_VERSION=v3.2.7 # Install the publisher client from GitHub # TODO: REMOVE PATCH # Before installing, patch the publisher to fix this issue - https://github.com/ESGF/esg-publisher/issues/133 ARG ESG_PUBLISHER_VERSION=v3.5.5 RUN git clone https://github.com/ESGF/esg-publisher.git /application && \ cd /application && \ git checkout $ESG_PUBLISHER_VERSION && \ cd src/python/esgcet && \ sed -i \ "s/config.get('DEFAULT', 'hessian_service_certs_location')/getConfig().get('DEFAULT', 'hessian_service_certs_location')/g" \ ./esgcet/publish/utility.py && \ python setup.py install && \ cd # Install entrypoint script # Install entrypoint script COPY scripts/* /usr/local/bin/ # Install configuration files and templates # Start with the /esg/bin directory from esgf-configure # Install configuration files and templates # Start with the /esg/bin directory from esgf-configure COPY --chown=1001:0 --from=configuration /esg/bin /esg/bin # Make the configuration files owned by the publish user and root group # This means the container can be run by any user in the root group (e.g. on OpenShift) # Make the configuration files owned by the publish user and root group # This means the container can be run by any user in the root group (e.g. on OpenShift) COPY --chown=1001:0 ./conf /esg/config/esgcet/.defaults RUN chown 1001:0 /esg/config/esgcet && chmod 775 /esg/config/esgcet # Create the thredds content root owned by publish user and root group Loading @@ -67,7 +71,9 @@ RUN mkdir -p /esg/content/thredds/esgcet && \ WORKDIR /esg/config/esgcet # Run as the publish user # Run as the publish user USER 1001 # esgprep requires that the USER environment variable is set ENV USER $PUBLISH_USER ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] CMD ["run-publish-scripts"] publisher/conf/esg.ini +2 −2 Original line number Diff line number Diff line Loading @@ -5,6 +5,7 @@ gateway_options = ESG-PCMDI, ESG-NCAR, ESG-ORNL, ESG-BADC, ESG-NCI, ESG-DKRZ hessian_service_debug = false hessian_service_keyfile = /esg/config/esgcet/.globus/certificate-file hessian_service_certfile = /esg/config/esgcet/.globus/certificate-file hessian_service_certs_location = /esg/config/esgcet/.globus/certificates hessian_service_polling_delay = 3 hessian_service_polling_iterations = 10 hessian_service_port = 443 Loading Loading @@ -43,6 +44,7 @@ thredds_reinit_url = {{ .Env.ESGF_TDS_REINIT_URL }} thredds_restrict_access = esg-user thredds_root = /esg/content/thredds/esgcet thredds_root_catalog_name = Earth System Root catalog esg_certificates_bundle = /esg/config/esgcet/trust-bundle.pem [initialize] initial_models_table = /esg/config/esgcet/esgcet_models_table.txt Loading @@ -61,5 +63,3 @@ srmls = /usr/local/esg/bin/srm-ls [hsi] hsi = /usr/local/bin/hsi offline_lister_executable = %(home)s/work/Esgcet/esgcet/scripts/hsils.py [config:test] Loading
configure/Dockerfile +2 −2 Original line number Diff line number Diff line Loading @@ -7,10 +7,10 @@ FROM alpine # Install gomplate (for templating) into the bin directory ARG GOMPLATE_VERSION=2.8.0 ARG GOMPLATE_VERSION=3.1.0 RUN apk --no-cache add curl && \ mkdir -p /esg/bin && \ curl -o /esg/bin/gomplate -fsSL https://github.com/hairyhenderson/gomplate/releases/download/${GOMPLATE_VERSION}/gomplate_linux-amd64-slim && \ curl -o /esg/bin/gomplate -fsSL https://github.com/hairyhenderson/gomplate/releases/download/v${GOMPLATE_VERSION}/gomplate_linux-amd64-slim && \ chmod 755 /esg/bin/gomplate # Copy the scripts into the bin directory COPY ./scripts /esg/bin Loading
docs/_docs/compose/publishing.md +1 −3 Original line number Diff line number Diff line Loading @@ -52,15 +52,13 @@ fetch-certificate ### Generate map files ```sh esgprep mapfile --project test /esg/data/test ls -l mapfiles esgmapfile --project test /esg/data/test ``` ### Publish to the PostgreSQL database ```sh esgpublish --project test --map mapfiles/test.test.map --service fileservice esglist_datasets test ``` ### Publish to the TDS Loading
docs/_docs/compose/testing.md +1 −1 Original line number Diff line number Diff line Loading @@ -51,7 +51,7 @@ The following procedure can be used to manually test an ESGF node installation u * username=rootAdmin * password=<value from cat $ESGF_CONFIG/secrets/rootadmin-password> * Test publishing a test dataset following the instructions at [Data Publishing](/usage/publishing/) * Test publishing a test dataset following the instructions at [Data Publishing](../publishing/) ### Automatic Testing Loading
publisher/Dockerfile +33 −27 Original line number Diff line number Diff line Loading @@ -6,58 +6,62 @@ ARG ESGF_HUB=esgfhub ARG ESGF_PREFIX= ARG ESGF_VERSION=latest # This build stage is required because COPY --from=$ARG is not supported # https://github.com/moby/moby/issues/34482 # This build stage is required because COPY --from=$ARG is not supported # https://github.com/moby/moby/issues/34482 FROM ${ESGF_HUB}/${ESGF_PREFIX}configure:${ESGF_VERSION} as configuration FROM continuumio/miniconda:4.3.27 FROM continuumio/miniconda # Create an unprivileged user to use ENV PUBLISH_USER publish ENV PUBLISH_GROUP publish # It is IMPORTANT that the UID is the same as the tomcat user in the TDS container # It is IMPORTANT that the UID is the same as the tomcat user in the TDS container ENV PUBLISH_UID 1001 ENV PUBLISH_GID 1001 RUN groupadd -g $PUBLISH_GID $PUBLISH_GROUP && \ useradd -M -g $PUBLISH_GID -s /sbin/nologin -u $PUBLISH_UID $PUBLISH_USER # Set environment vars that control the behaviour # Set environment vars that control the behaviour ENV ESGINI /esg/config/esgcet/esg.ini ENV UVCDAT_ANONYMOUS_LOG no # Install packages using conda and pip RUN conda install -y -c conda-forge -c uvcdat \ gcc_linux-64 \ cdutil \ lxml \ requests \ # Install dependencies using conda # NOTE: esgprep has very specific version requirements ARG CDAT_LABEL_CHANNEL=cdat/label/80 RUN conda install -y -c $CDAT_LABEL_CHANNEL -c conda-forge -c cdat \ psycopg2 \ decorator \ Tempita \ myproxyclient && \ pip install \ 'SQLAlchemy==0.7.10' \ sqlalchemy_migrate \ ESGConfigParser==0.1.10 \ esgprep==2.7.40 lxml \ cdutil \ gcc_linux-64 \ 'requests==2.20.0' \ 'netCDF4==1.4.0' # Make the conda-installed gcc available as gcc # See https://conda.io/docs/user-guide/tasks/build-packages/compiler-tools.html RUN ln -s /opt/conda/bin/x86_64-conda_cos6-linux-gnu-gcc /usr/local/bin/gcc # Install the publisher client from GitHub ARG ESG_PUBLISHER_VERSION=v3.2.7 # Install the publisher client from GitHub # TODO: REMOVE PATCH # Before installing, patch the publisher to fix this issue - https://github.com/ESGF/esg-publisher/issues/133 ARG ESG_PUBLISHER_VERSION=v3.5.5 RUN git clone https://github.com/ESGF/esg-publisher.git /application && \ cd /application && \ git checkout $ESG_PUBLISHER_VERSION && \ cd src/python/esgcet && \ sed -i \ "s/config.get('DEFAULT', 'hessian_service_certs_location')/getConfig().get('DEFAULT', 'hessian_service_certs_location')/g" \ ./esgcet/publish/utility.py && \ python setup.py install && \ cd # Install entrypoint script # Install entrypoint script COPY scripts/* /usr/local/bin/ # Install configuration files and templates # Start with the /esg/bin directory from esgf-configure # Install configuration files and templates # Start with the /esg/bin directory from esgf-configure COPY --chown=1001:0 --from=configuration /esg/bin /esg/bin # Make the configuration files owned by the publish user and root group # This means the container can be run by any user in the root group (e.g. on OpenShift) # Make the configuration files owned by the publish user and root group # This means the container can be run by any user in the root group (e.g. on OpenShift) COPY --chown=1001:0 ./conf /esg/config/esgcet/.defaults RUN chown 1001:0 /esg/config/esgcet && chmod 775 /esg/config/esgcet # Create the thredds content root owned by publish user and root group Loading @@ -67,7 +71,9 @@ RUN mkdir -p /esg/content/thredds/esgcet && \ WORKDIR /esg/config/esgcet # Run as the publish user # Run as the publish user USER 1001 # esgprep requires that the USER environment variable is set ENV USER $PUBLISH_USER ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] CMD ["run-publish-scripts"]
publisher/conf/esg.ini +2 −2 Original line number Diff line number Diff line Loading @@ -5,6 +5,7 @@ gateway_options = ESG-PCMDI, ESG-NCAR, ESG-ORNL, ESG-BADC, ESG-NCI, ESG-DKRZ hessian_service_debug = false hessian_service_keyfile = /esg/config/esgcet/.globus/certificate-file hessian_service_certfile = /esg/config/esgcet/.globus/certificate-file hessian_service_certs_location = /esg/config/esgcet/.globus/certificates hessian_service_polling_delay = 3 hessian_service_polling_iterations = 10 hessian_service_port = 443 Loading Loading @@ -43,6 +44,7 @@ thredds_reinit_url = {{ .Env.ESGF_TDS_REINIT_URL }} thredds_restrict_access = esg-user thredds_root = /esg/content/thredds/esgcet thredds_root_catalog_name = Earth System Root catalog esg_certificates_bundle = /esg/config/esgcet/trust-bundle.pem [initialize] initial_models_table = /esg/config/esgcet/esgcet_models_table.txt Loading @@ -61,5 +63,3 @@ srmls = /usr/local/esg/bin/srm-ls [hsi] hsi = /usr/local/bin/hsi offline_lister_executable = %(home)s/work/Esgcet/esgcet/scripts/hsils.py [config:test]
