Commit 6e6e88ab authored by Luca Cinquini's avatar Luca Cinquini
Browse files

Checkpoint where the publisher is working.

parent 4ace7b93

data-node/Dockerfile

+8 −1
Original line number Diff line number Diff line
@@ -32,6 +32,10 @@ RUN cd /usr/local/tomcat/webapps/thredds && \ 
    rm thredds.war && \

    chown -R tomcat:tomcat /usr/local/tomcat/webapps/thredds



# TDS configuration root

RUN mkdir /esg/content/thredds

VOLUME /esg/content/thredds



# TDS memory configuration

COPY thredds/conf/threddsConfig.xml /esg/content/thredds/threddsConfig.xml
@@ -51,8 +55,11 @@ RUN chmod +x /usr/local/bin/change_data_node_password.sh 
# TDS test ESGF catalogs

# COPY esgf-content/thredds/ /esg/content/thredds/



# TDS catalogs location

RUN mkdir -p /esg/content/thredds/esgcet



# change ownership of content directory

RUN chown -R tomcat:tomcat /esg/content/thredds

RUN chown -R tomcat:tomcat  /esg/content/thredds/



# start container as non-privilged user

USER tomcat

docker-compose.yml

+17 −2
Original line number Diff line number Diff line
@@ -103,7 +103,6 @@ services: 
      - esgf-config

    networks:

      - default

    #  - dbnetwork



  # ESGF postgres database

  esgf-postgres:
@@ -118,6 +117,23 @@ services: 
     volumes:

       - pg_data:/var/lib/pgsql/data



  # ESGF publisher application

  esgf-publisher:

    image: esgfhub/esgf-publisher:${ESGF_VERSION}

    container_name: publisher

    depends_on:

       - esgf-postgres

    networks:

       - default

       - dbnetwork

    volumes:

      - $ESGF_DATA_DIR:/esg/data

      - $ESGF_CONFIG/httpd/certs/:/etc/certs/

      - $ESGF_CONFIG/grid-security/certificates/:/etc/grid-security/certificates/

      - $ESGF_CONFIG/esg/config/esgf.properties:/esg/config/esgf.properties

    volumes_from:

      - esgf-data-node



  # data container to hold CoG source code and python virtual environment

  # this container stops after the volume 'cog_data' is initialized

  esgf-cog:
@@ -142,7 +158,6 @@ services: 
       - COG_INSTALL_DIR=/usr/local/cog/cog_install

       - COG_CONFIG_DIR=/usr/local/cog/cog_config





  esgf-httpd:

   image: esgfhub/esgf-httpd:${ESGF_VERSION}

   container_name: httpd

esg-publisher/Dockerfile

0 → 100644
+52 −0
Original line number Diff line number Diff line 
# ESGF node with esgf-publisher application



FROM esgfhub/esgf-node:1.0



MAINTAINER Luca Cinquini <luca.cinquini@jpl.nasa.gov>



# install library pre-requisites

RUN yum -y update; yum -y install libxml2-devel libxslt-devel; yum clean all



# install Anaconda

ENV CDAT_HOME=/usr/local/conda

RUN cd /tmp && rm -rf $CDAT_HOME && \

    wget --no-check-certificate https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh && \

    bash Miniconda2-latest-Linux-x86_64.sh -b -p $CDAT_HOME



# create CDAT virtual environment with Anaconda

ENV PATH=${CDAT_HOME}/bin:$PATH

RUN conda create -y -n esgf-pub -c conda-forge -c uvcdat cdutil 



# activate virtual env and fetch some pre-requisites

RUN source ${CDAT_HOME}/bin/activate esgf-pub && \

    conda install -y -c conda-forge lxml requests psycopg2 decorator Tempita myproxyclient



# install other python pre-requisites

RUN source ${CDAT_HOME}/bin/activate esgf-pub && \ 

    pip install SQLAlchemy==0.7.10 && \

    pip install sqlalchemy_migrate && \

    pip install esgprep



# install ESGF publisher

# location: /usr/local/conda/lib/python2.7/site-packages/esgcet-3.0.1-py2.7.egg

ENV ESG_PUBLISHER_VERSION=v3.0.1

RUN source ${CDAT_HOME}/bin/activate esgf-pub && \

    cd /tmp && \

    git clone https://github.com/ESGF/esg-publisher.git && \

    cd esg-publisher && \

    git checkout $ESG_PUBLISHER_VERSION && \

    cd src/python/esgcet && \

    python setup.py install



# copy configuration files

COPY config/esgcet /esg/config/esgcet

ENV ESGINI /esg/config/esgcet/esg.ini



# temporary: copy pre-generated Globus certificate

COPY config/globus/certificate-file /tmp/certificate-file



# activate publisher environment

RUN source ${CDAT_HOME}/bin/activate esgf-pub



# keep container running

CMD ["tail", "-f", "/dev/null"]

esg-publisher/config/esgcet/cf-standard-name-table.xml

0 → 100644
+9503 −0

File added.

Preview size limit exceeded, changes collapsed.

esg-publisher/config/esgcet/esg.ini

0 → 100644
+64 −0
Original line number Diff line number Diff line 
[DEFAULT]

checksum = sha256sum | SHA256

dburl = postgresql://dbsuper:changeit@esgf-postgres:5432/esgcet

gateway_options = ESG-PCMDI, ESG-NCAR, ESG-ORNL, ESG-BADC, ESG-NCI, ESG-DKRZ

hessian_service_debug = false

hessian_service_keyfile = %(home)s/.globus/certificate-file

hessian_service_certfile = %(home)s/.globus/certificate-file

hessian_service_polling_delay = 3

hessian_service_polling_iterations = 10

hessian_service_port = 443

hessian_service_remote_metadata_url = http://my-node.esgf.org/esgcet/remote/hessian/guest/remoteMetadataService

hessian_service_url = https://my-node.esgf.org/esg-search/remote/secure/client-cert/hessian/publishingService

log_format = %(levelname)-10s %(asctime)s %(message)s

log_level = WARNING

offline_lister =

        HRMatPCMDI | hsi

project_options =

        test | Test | 1

root_id = esgf

thredds_dataset_roots =

        test | /esg/data/test

thredds_aggregation_services =

        OpenDAP | /thredds/dodsC/                 | gridded          | aggregationservice

        LAS     | http://my-node.esgf.org/las/getUI.do | LASatESGFdataNode | aggregationservice

thredds_file_services =

        HTTPServer | /thredds/fileServer/       | HTTPServer    | fileservice

        GridFTP    | gsiftp://my-node.esgf.org:2811/ | GRIDFTP       | fileservice

        OpenDAP    | /thredds/dodsC/            | OpenDAPServer | fileservice

thredds_url = http://my-node.esgf.org/thredds/catalog/esgcet

thredds_username = dnode_user

thredds_password = changeit

thredds_authentication_realm = THREDDS Data Server

thredds_catalog_basename = %(dataset_id)s.v%(version)s.xml

thredds_master_catalog_name = Earth System Grid catalog

thredds_max_catalogs_per_directory = 500

thredds_offline_services =

#        SRM | srm://esgf-data-node:6288/srm/v2/server?SFN=/archive.sample.gov | HRMatPCMDI

thredds_reinit_error_url = https://esgf-data-node:8443/thredds/admin/content/logs/catalogInit.log

thredds_error_pattern = Catalog init

thredds_reinit_success_pattern = reinit ok

thredds_fatal_error_pattern = **Fatal

thredds_reinit_url = https://esgf-data-node:8443/thredds/admin/debug?Catalogs/recheck

thredds_restrict_access = esg-user

thredds_root = /esg/content/thredds/esgcet

thredds_root_catalog_name = Earth System Root catalog



[initialize]

initial_models_table = /esg/config/esgcet/esgcet_models_table.txt

log_level = INFO



[extract]

log_level = INFO

validate_standard_names = True



[srmls]

offline_lister_executable = %(home)s/work/Esgcet/esgcet/scripts/srmls.py

srm_archive = /garchive.nersc.gov

srm_server = srm://somehost.llnl.gov:6288/srm/v2/server

srmls = /usr/local/esg/bin/srm-ls



[hsi]

hsi = /usr/local/bin/hsi

offline_lister_executable = %(home)s/work/Esgcet/esgcet/scripts/hsils.py