Commit 401deb61 authored by Luca Cinquini's avatar Luca Cinquini
Browse files

Working on installing esg-publisher.

parent 6e6e88ab

data-node/Dockerfile

+53 −3
Original line number Diff line number Diff line 
# ESGF Data Node:

# ESGF Data Node: Docker image containing:

# o TDS customized with ESGF access control filters

# o ESGF publisher client (which needs access to the TDS file system)

# o ORP



FROM esgfhub/esgf-tomcat:1.0
@@ -9,6 +10,50 @@ MAINTAINER Luca Cinquini <luca.cinquini@jpl.nasa.gov> 
# default ESGF repository

ARG ESGF_REPO=http://distrib-coffee.ipsl.jussieu.fr/pub/esgf



#============================================



# ESGF publisher client



# install library pre-requisites

RUN yum -y update; yum -y install libxml2-devel libxslt-devel; yum clean all



# install Anaconda

ENV CDAT_HOME=/usr/local/conda

RUN cd /tmp && rm -rf $CDAT_HOME && \

    wget --no-check-certificate https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh && \

    bash Miniconda2-latest-Linux-x86_64.sh -b -p $CDAT_HOME



# create CDAT virtual environment with Anaconda

ENV PATH=${CDAT_HOME}/bin:$PATH

RUN conda create -y -n esgf-pub -c conda-forge -c uvcdat cdutil



# activate virtual env and fetch some pre-requisites

RUN source ${CDAT_HOME}/bin/activate esgf-pub && \

    conda install -y -c conda-forge lxml requests psycopg2 decorator Tempita myproxyclient



# install other python pre-requisites

RUN source ${CDAT_HOME}/bin/activate esgf-pub && \

    pip install SQLAlchemy==0.7.10 && \

    pip install sqlalchemy_migrate && \

    pip install esgprep



# install ESGF publisher

# location: /usr/local/conda/lib/python2.7/site-packages/esgcet-3.0.1-py2.7.egg

ENV ESG_PUBLISHER_VERSION=v3.0.1

RUN source ${CDAT_HOME}/bin/activate esgf-pub && \

    cd /tmp && \

    git clone https://github.com/ESGF/esg-publisher.git && \

    cd esg-publisher && \

    git checkout $ESG_PUBLISHER_VERSION && \

    cd src/python/esgcet && \

    python setup.py install



# env needed by Python client to trust the data node server certicate

ENV SSL_CERT_DIR /etc/grid-security/certificates

ENV ESGINI /esg/config/esgcet/esg.ini



#==========================================================



# ESGF ORP

RUN mkdir -p /usr/local/tomcat/webapps/esg-orp
@@ -23,7 +68,10 @@ RUN cd /usr/local/tomcat/webapps/esg-orp && \ 
# these values are the same for all ESGF nodes

COPY esgf-orp/conf/esg-orp.properties /usr/local/tomcat/webapps/esg-orp/WEB-INF/classes/esg-orp.properties



#=============================================



# TDS



RUN mkdir -p /usr/local/tomcat/webapps/thredds

#COPY thredds/thredds.war /usr/local/tomcat/webapps/thredds/thredds.war

ADD $ESGF_REPO/dist/devel/thredds/5.0/5.0.0/thredds.war /usr/local/tomcat/webapps/thredds/
@@ -33,12 +81,14 @@ RUN cd /usr/local/tomcat/webapps/thredds && \ 
    chown -R tomcat:tomcat /usr/local/tomcat/webapps/thredds



# TDS configuration root

RUN mkdir /esg/content/thredds

VOLUME /esg/content/thredds

RUN mkdir -p /esg/content/thredds



# TDS memory configuration

COPY thredds/conf/threddsConfig.xml /esg/content/thredds/threddsConfig.xml



# ESGF root catalog

COPY thredds/conf/catalog.xml /esg/content/thredds/catalog.xml



# TDS customized applicationContext.xml file with ESGF authorizer

COPY thredds/conf/applicationContext.xml /usr/local/tomcat/webapps/thredds/WEB-INF/applicationContext.xml

data-node/catalog.xml

0 → 100644
+11 −0
Original line number Diff line number Diff line 
<?xml version="1.0" encoding="UTF-8"?>

<catalog name="THREDDS Server Default Catalog : You must change this to fit your server!"

         xmlns="http://www.unidata.ucar.edu/namespaces/thredds/InvCatalog/v1.0"

         xmlns:xlink="http://www.w3.org/1999/xlink"

         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

         xsi:schemaLocation="http://www.unidata.ucar.edu/namespaces/thredds/InvCatalog/v1.0

           http://www.unidata.ucar.edu/schemas/thredds/InvCatalog.1.0.6.xsd">



  <catalogRef name="Earth System Root catalog" xlink:title="Earth System Root catalog" xlink:href="esgcet/catalog.xml"/>



</catalog>

data-node/globus/certificate-file

0 → 100644
+87 −0
Original line number Diff line number Diff line 
-----BEGIN CERTIFICATE-----

MIIFADCCAuigAwIBAgICAoIwDQYJKoZIhvcNAQELBQAwRjENMAsGA1UECgwERVNH

RjERMA8GA1UECwwIRVNHRi5PUkcxIjAgBgNVBAMMGWVzZ2Ytbm9kZS5qcGwubmFz

YS5nb3YtQ0EwHhcNMTcwMjA4MTAzNzA4WhcNMTcwMjEwMTA0MjA4WjBlMQ0wCwYD

VQQKEwRFU0dGMREwDwYDVQQLEwhFU0dGLk9SRzFBMD8GA1UEAxM4aHR0cHM6Ly9l

c2dmLW5vZGUuanBsLm5hc2EuZ292L2VzZ2YtaWRwL29wZW5pZC9yb290QWRtaW4w

ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDbFYysdInKz3eOWwOE5lxw

okjuCiXwcH58+n+CJJ7YDlTKN6oAwuEkgPTNHan3Hp41UvJDZuIS81e/np7jpyBl

qH3V+uFsB+vUIH6A+rEZjGSlJB4lvfq8pHz8avm/O79dWa29iLBHRFcgf9tk2htR

DapF8+Fx4VhS+R//gIg3vhC/KomgRbfo59i8pUSRNFJipvE9bs1XWad0/qP3rCe4

eZKC/B2ZCm4EDNwT6lCtXrxIUzMQofNDOmZVya6pJYxaBTNhL2bp+D/JsdRAbBRl

Cg0Az9TitNbHKCV3NbZWOW1B0PwoPOEpPMA7qTnTpuNa51k5fZad2KnrcZIBmeDr

AgMBAAGjgdgwgdUwgdIGCSoDBAQDAgEHCASBxAyBwWVzZy52by5ncm91cC5yb2xl

cz1ncm91cF9OQVNBIE9CU19yb2xlX3VzZXI7Z3JvdXBfTkFTQSBPQlNfcm9sZV9z

dXBlcjtncm91cF9OQVNBIE9CU19yb2xlX3B1Ymxpc2hlcjtncm91cF93aGVlbF9y

b2xlX3N1cGVyOmVzZy52by5vcGVuaWQ9aHR0cHM6Ly9lc2dmLW5vZGUuanBsLm5h

c2EuZ292L2VzZ2YtaWRwL29wZW5pZC9yb290QWRtaW4wDQYJKoZIhvcNAQELBQAD

ggIBALVCmXyhWjreCVrPBnixdsEpEgb7wuBcjtSQZTymMLesiZBUDBYySyuNPQMe

tRKehLgPLK3r9lJ6eh4arrBypLNQ5dhnGqPujBFaV19VLnTFrAY2hSWWSracLAWG

bakX569zKD8caIh0fBrsZf43ICcUHs8hWnQi/xcZv2Uz5L6DWDnAYI1mkBEfY6pd

v/0CxWknoAUbJ1Q2rWQ0uFiB+BI8G7TToVDU9tMfaNEorrj+ircgbFMP5+7/R1yA

Tb5GpxG8qVCS+pvR+zL/KligPFLtNiINa4zvzS9397BEDnnHmJrBly0KjeaaLgqW

dVsQhau5/yUDT5n/vn0jKual17AlSmX6HY4QXh+/hFIjOXlHGgdUXGEkxSgEO/Zc

WNvVFO4K5m9f7O2rRh4MIh+fY6JcKCyHNkCEBC0zmIpC5jaM0N0bujV1aHQB2ubV

lhGzWaUfDFhBph26UrFma/oh9Gok5fouiGc+M7c4y3A8ObBZaFO4RUnjkBJN0d2V

tVQjhmiPWTWERmUA6XxApIWaKoWfH6VvdL+Zl2r9xDSHZPK2zpxUmm3jPR4j/43k

DG8GAh65Gean6hUVrbHLWlZ7z9HDToWdlQoOFkbusXbqQnDEhFGM9ZGLgkUPCNKc

mPTcNB6Y84fzGKcuxOg7DQGAykutq0bZpGPQxy0SREwRteea

-----END CERTIFICATE-----

-----BEGIN RSA PRIVATE KEY-----

MIIEowIBAAKCAQEA2xWMrHSJys93jlsDhOZccKJI7gol8HB+fPp/giSe2A5Uyjeq

AMLhJID0zR2p9x6eNVLyQ2biEvNXv56e46cgZah91frhbAfr1CB+gPqxGYxkpSQe

Jb36vKR8/Gr5vzu/XVmtvYiwR0RXIH/bZNobUQ2qRfPhceFYUvkf/4CIN74QvyqJ

oEW36OfYvKVEkTRSYqbxPW7NV1mndP6j96wnuHmSgvwdmQpuBAzcE+pQrV68SFMz

EKHzQzpmVcmuqSWMWgUzYS9m6fg/ybHUQGwUZQoNAM/U4rTWxygldzW2VjltQdD8

KDzhKTzAO6k506bjWudZOX2Wndip63GSAZng6wIDAQABAoIBAQCOeK4ZvFObY13R

YpigRZpZ2P9qZYkUmmN3bQOco6n0wSUcuCRDCHiXX5FTDnZocS3baYyIjpB4WG14

/HNBgn7L/U2P6O3Nbj1LnrIFuIGRJr6QsjM0wh4brTBmij7NwNIKp/YXapuQTP4H

8X3AUSIwjJ1tjlLO1we0NbQ8WN68uBW7DgBakwtLcZG5YLzv3gmyflKto6Xcbu35

XjP1lprpTZSP48r0rlgLYfXEDh2p9fNWZ70tAmW9P7oZ7eDciUDmJkO++R6//lj+

TDuFQPLQdC3hxSjyRKLgFA6Zk9DOqxOdgtIgJ1AbRhUVleOoPbvxdXgkBicAPfX8

ShOIxuwpAoGBAP11ha3UlxsjNFRIk/2P93wN3XZGNZL1YJyoMI0LCoxsTsm2dSRY

EJ/p++6P7fAvLcFMWdl5gsyIDZZYApjwH2nGGTfs4BINmHDlKw8sufn4bzPIDPYU

l1KGzQd15AglKNbscU2/1WhDgdOR6AO+hp7equOUqhStUJ07ytPHnEVPAoGBAN1H

zrqZXhNAb+q3y9+6UpQAfR2xmQnAe9FeOffVuWuUrVvrIRnLmlYJCwc/i80EBe2z

iFqvX5Mkc61y5ACeYi3l5oE3YnbrD2DesUxmMfAIT5UZlIMiWAoEPDPdov85rFDF

slX3Fb/YWiLdGCB81F2ZI1Lo+4rKhW3kJ9chmDulAoGAfTkiCacd3NeA1B9jloQZ

dt32y6eWcnvu9pcWHakV1hjwuDIAICuuUwOsZ8oOeXsltPm+7nZdTCXv6Jak+eCw

9HZeIUZT0jeLWEOxCXdEjx7R+hhdZvr3ZFubUpNJTUjmq4D33bzod/HaY6gTpD3u

jvFeJsBdY4fZa0WOSSG2sHsCgYBJ8LOwgI0B69c53D6gvLbZQ6yLHowfyRTWqiC+

osj5nM2s2E3QO8uPkigAPQiu8fg1LHeUreNG3LhWrfxoBISUB61VYjdhuzwBR4iB

Gej35x8ImGqawwAeILopLpdHp3Lf7p+fwBehal78JVrpiCzCNeiqzLU49GFPjhyN

82jWDQKBgB57nBAKoBlonnQQOgKI62T4WQo5L5IdgdFnsI9Ev1FlexCTjA9e5j+e

+gZHX3xgzi+KvXEuV6hcePLLpnuRce3DX4ypILyDp68mhJUR+yXJzIy000QFPZNQ

uGltFF+ViWo0wmDzg50yDgR+jLBJP00JT/MPdBjG1WVsHQVXVmFy

-----END RSA PRIVATE KEY-----

-----BEGIN CERTIFICATE-----

MIIFXjCCA0agAwIBAgIBbjANBgkqhkiG9w0BAQsFADBYMQ0wCwYDVQQKDARFU0dG

MREwDwYDVQQLDAhFU0dGLk9SRzEcMBoGA1UECwwTc2ltcGxlY2EubnNjLmxpdS5z

ZTEWMBQGA1UEAwwNTlNDIFNpbXBsZSBDQTAeFw0xNjEyMTUxMTAwMTRaFw0xNzEy

MTUxMTAwMTRaMEYxDTALBgNVBAoMBEVTR0YxETAPBgNVBAsMCEVTR0YuT1JHMSIw

IAYDVQQDDBllc2dmLW5vZGUuanBsLm5hc2EuZ292LUNBMIICIjANBgkqhkiG9w0B

AQEFAAOCAg8AMIICCgKCAgEAw8f1Mel6ujolxiefzwJYaf23E6wc4yWNiOj/2fpK

gO+Hl18rH6e2mt5aEjDBL5bieS6Fq4M9IvUpUrVWINaa+8W4QDLZAmousErd3DCL

zmF8Mnqn27FHrGrVRcYNTrTZNKug/yBELmEeLHJUiU9h9mp6Gni1tsTArd3mK4WH

RFDaIhFNc7r3GOwoLvo2IBfsKlikjJEYWVVV2vkMXFs9jTxYEkSjd88/p279+bWC

QhSWQEBTz/LpLFsV+15Rai+Eechqb2AZuOEX4tR0yUERVt0OJUo5ENjfAzLxL3UH

pL1gCVABh3U7Rp8QNvLmn6H364XG9gvM5SSo5QS5Hb5YkrPYeVr6cXix5NPszMOs

EEf2xsFwQ4g94Aa22AqzliX46DPpDK8+KkSluSRNH9ykvnWs8LU1ejhX4uTmCz8E

KQd9abR5V2L2HEWOxn82RfuGSk0sT0u5AEAVotJAtAp5SZm4dNRtye7uyCcCf2JJ

g74geh4hUaXV6RSne1yyULjAkPXzxNt274pq2UR7CkpWii+vtBB7+lV6jQ93hWH8

5IIym85mhIvYfXLdOydiTPAWdJ76uwCPzoAJ8Wby/7zXJHmIl57KR9j7ZZr8Mj2y

T8o8YJOWPXrdzXfzKMhAiotF25AvpGA7Ql2GUW/FU99KoJRsUYXqW9W/ZLBLZi28

KTUCAwEAAaNFMEMwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUBiC9xevZmQPD

b/fOmZcZogPn4Y0wEQYJYIZIAYb4QgEBBAQDAgAHMA0GCSqGSIb3DQEBCwUAA4IC

AQCClg7rsYzPV99V3+3OLETDaHKV3ndv/pW98ZSLQJ4yDiVD8OzEGe/wPBDOQV6R

/fcFaR5vvvmbnhhfwJv3dHaNj+d+W1RZt5FOnNpZY9NBPZeEln+tt/lTOdefEelZ

nZthpeVOndOwFHEpg/faowjcWESOvBAr7Apvla/Rq5bFgJrCszrNsG5OnOxepQXl

lJv5iFEYPr5xhlUOcKUcayHfK64M2h9/9Dg2g6AkKxjiuN458IhiXOBjX0IdhP3O

GT5mlpaVaQHnt/aYqluCqC4ekvIKYUxt1eh6G997clV8vOYFP1xJ9HTnK+Lb1D7a

axiu+NBVyLvNrXzdsKaeu1KY5Cau5LGa20BChSDVg0+8szWZegHPof3Aa1qydNSr

S6RAAGLZulMqApA+JFA5Zy/bnO34ISrHxKqzbYN90EgMS4TjiOox/y7xI5Z4WRcm

2KApq4NmIf/194kKGORXAfkD78v19NcC7iOeCqThp1gq+IcHiL+bgB9MrXr4E21/

GP67Y2RV/K+SU2odw9a25H3aLPzaiHTTRod2GoaSGY+ZXN5c7EaWRBQnSNCTBCn2

/SlLBVE98MYSeZwv82CoJW8Y/oIIfPoz54/CTJCwsJB/g9JexF6OnTiLWDo4TnJN

+OoGyvkRDIQSDCcCV5/0RdUKDwwte2GvEJwz3RqSOUNisw==

-----END CERTIFICATE-----

data-node/publisher/esgcet/cf-standard-name-table.xml

0 → 100644
+9503 −0

File added.

Preview size limit exceeded, changes collapsed.

data-node/publisher/esgcet/esg.ini

0 → 100644
+64 −0
Original line number Diff line number Diff line 
[DEFAULT]

checksum = sha256sum | SHA256

dburl = postgresql://dbsuper:changeit@esgf-postgres:5432/esgcet

gateway_options = ESG-PCMDI, ESG-NCAR, ESG-ORNL, ESG-BADC, ESG-NCI, ESG-DKRZ

hessian_service_debug = false

hessian_service_keyfile = %(home)s/.globus/certificate-file

hessian_service_certfile = %(home)s/.globus/certificate-file

hessian_service_polling_delay = 3

hessian_service_polling_iterations = 10

hessian_service_port = 443

hessian_service_remote_metadata_url = http://my-node.esgf.org/esgcet/remote/hessian/guest/remoteMetadataService

hessian_service_url = https://my-node.esgf.org/esg-search/remote/secure/client-cert/hessian/publishingService

log_format = %(levelname)-10s %(asctime)s %(message)s

log_level = WARNING

offline_lister =

        HRMatPCMDI | hsi

project_options =

        test | Test | 1

root_id = esgf

thredds_dataset_roots =

        test | /esg/data/test

thredds_aggregation_services =

        OpenDAP | /thredds/dodsC/                 | gridded          | aggregationservice

        LAS     | http://my-node.esgf.org/las/getUI.do | LASatESGFdataNode | aggregationservice

thredds_file_services =

        HTTPServer | /thredds/fileServer/       | HTTPServer    | fileservice

        GridFTP    | gsiftp://my-node.esgf.org:2811/ | GRIDFTP       | fileservice

        OpenDAP    | /thredds/dodsC/            | OpenDAPServer | fileservice

thredds_url = http://my-node.esgf.org/thredds/catalog/esgcet

thredds_username = dnode_user

thredds_password = changeit

thredds_authentication_realm = THREDDS Data Server

thredds_catalog_basename = %(dataset_id)s.v%(version)s.xml

thredds_master_catalog_name = Earth System Grid catalog

thredds_max_catalogs_per_directory = 500

thredds_offline_services =

#        SRM | srm://esgf-data-node:6288/srm/v2/server?SFN=/archive.sample.gov | HRMatPCMDI

thredds_reinit_error_url = https://esgf-data-node:8443/thredds/admin/content/logs/catalogInit.log

thredds_error_pattern = Catalog init

thredds_reinit_success_pattern = reinit ok

thredds_fatal_error_pattern = **Fatal

thredds_reinit_url = https://esgf-data-node:8443/thredds/admin/debug?Catalogs/recheck

thredds_restrict_access = esg-user

thredds_root = /esg/content/thredds/esgcet

thredds_root_catalog_name = Earth System Root catalog



[initialize]

initial_models_table = /esg/config/esgcet/esgcet_models_table.txt

log_level = INFO



[extract]

log_level = INFO

validate_standard_names = True



[srmls]

offline_lister_executable = %(home)s/work/Esgcet/esgcet/scripts/srmls.py

srm_archive = /garchive.nersc.gov

srm_server = srm://somehost.llnl.gov:6288/srm/v2/server

srmls = /usr/local/esg/bin/srm-ls



[hsi]

hsi = /usr/local/bin/hsi

offline_lister_executable = %(home)s/work/Esgcet/esgcet/scripts/hsils.py