Commit f9b9dd65 authored by Luca Cinquini's avatar Luca Cinquini Committed by GitHub
Browse files

Merge branch 'devel' into slcs-server

parents 7d0cdd3e a81ae413

.gitignore

+6 −0
Original line number Diff line number Diff line 
.DS_Store

**/*.war

**/*.tar

**/*.tgz

**/*.swp

**/*.jar

**/*.gz
 
 No newline at end of file

aws/EsgfCluster.config

0 → 100644
+34 −0
Original line number Diff line number Diff line 
Content-Type: multipart/mixed; boundary="==BOUNDARY=="

MIME-Version: 1.0



--==BOUNDARY==

Content-Type: text/cloud-boothook; charset="us-ascii"



# Set Docker daemon options

cloud-init-per once docker_options echo 'OPTIONS="${OPTIONS} --storage-opt dm.basesize=10G"' >> /etc/sysconfig/docker



--==BOUNDARY==

Content-Type: text/x-shellscript; charset="us-ascii"



#!/bin/bash

# Set any ECS agent configuration options

echo "ECS_CLUSTER=EsgfCluster" >> /etc/ecs/ecs.config

echo "ECS_INSTANCE_ATTRIBUTES={\"esgf_type\":\"master_node\"}" >> /etc/ecs/ecs.config



# mount the EFS file system

# must also restart docker and docker agent for the file system to become available to docker containers

# TODO: parametrize the EFS endpoint

sudo yum install -y nfs-utils

sudo stop ecs

sudo mkdir -p /esgf_data

#sudo chmod 777 /esgf_data

#sudo chown ec2-user:ec2-user /esgf_data

sudo echo "fs-2ca17785.efs.us-west-2.amazonaws.com:/ /esgf_data nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 0 0" >> /etc/fstab

sudo mount -a -t nfs4

sudo service docker restart

sudo start ecs



# extract the ESGF host configuration to the standard location

tar xvf /esgf_data/esgf_config.tar -C /home/ec2-user



--==BOUNDARY==--

aws/EsgfMasterNodeService.json

0 → 100644
+36 −0
Original line number Diff line number Diff line 
{

    "cluster": "EsgfCluster", 

    "serviceName": "EsgfMasterNodeService", 

    "taskDefinition": "EsgfMasterNodeTask", 

    "loadBalancers": [

        {

            "targetGroupArn": "",

            "loadBalancerName": "EsgfClassicLoadBalancer",

            "containerName": "httpd",

            "containerPort": 80

        }

    ],

    "desiredCount": 1, 

    "clientToken": "", 

    "role": "ecsServiceRole",

    "deploymentConfiguration": {

        "maximumPercent": 200, 

        "minimumHealthyPercent": 50

    }, 

    "placementConstraints": [

        {

            "type": "memberOf", 

            "expression": "attribute:esgf_type==master_node"

        }

    ], 

    "placementStrategy": [

        {

            "field": "attribute:ecs.availability-zone",

            "type": "spread"

        },

        {

             "field": "instanceId",

             "type": "spread"

        }

    ]

}

aws/EsgfMasterNodeTask.json

0 → 100644
+518 −0
Original line number Diff line number Diff line 
{

  "requiresAttributes": [],

  "taskDefinitionArn": "arn:aws:ecs:us-west-2:031003439974:task-definition/EsgfMasterNodeTask:37",

  "networkMode": "bridge",

  "status": "ACTIVE",

  "revision": 37,

  "taskRoleArn": null,

  "containerDefinitions": [

    {

      "volumesFrom": [],

      "memory": 128,

      "extraHosts": null,

      "dnsServers": null,

      "disableNetworking": null,

      "dnsSearchDomains": null,

      "portMappings": [],

      "hostname": null,

      "essential": true,

      "entryPoint": [],

      "mountPoints": [

        {

          "containerPath": "/etc/certs",

          "sourceVolume": "esgf_certs",

          "readOnly": null

        },

        {

          "containerPath": "/etc/httpd/conf.d",

          "sourceVolume": "esgf_httpd_conf",

          "readOnly": null

        },

        {

          "containerPath": "/etc/grid-security/certificates",

          "sourceVolume": "esgf_grid_security_certs",

          "readOnly": null

        },

        {

          "containerPath": "/esg/config",

          "sourceVolume": "esgf_config",

          "readOnly": null

        },

        {

          "containerPath": "/esg/config/tomcat",

          "sourceVolume": "esgf_tomcat_config",

          "readOnly": null

        }

      ],

      "name": "config",

      "ulimits": null,

      "dockerSecurityOptions": null,

      "environment": [],

      "links": null,

      "workingDirectory": null,

      "readonlyRootFilesystem": null,

      "image": "centos:6",

      "command": [

        "tail",

        "-f",

        "/dev/null"

      ],

      "user": null,

      "dockerLabels": null,

      "logConfiguration": null,

      "cpu": 0,

      "privileged": null,

      "memoryReservation": null

    },

    {

      "volumesFrom": [

        {

          "readOnly": null,

          "sourceContainer": "config"

        }

      ],

      "memory": 1024,

      "extraHosts": null,

      "dnsServers": null,

      "disableNetworking": null,

      "dnsSearchDomains": null,

      "portMappings": [

        {

          "hostPort": 8000,

          "containerPort": 8000,

          "protocol": "tcp"

        }

      ],

      "hostname": null,

      "essential": false,

      "entryPoint": null,

      "mountPoints": [

        {

          "containerPath": "/usr/local/cog/cog_config",

          "sourceVolume": "cog_data",

          "readOnly": null

        }

      ],

      "name": "cog",

      "ulimits": null,

      "dockerSecurityOptions": null,

      "environment": [

        {

          "name": "COG_CONFIG_DIR",

          "value": "/usr/local/cog/cog_config"

        },

        {

          "name": "COG_DIR",

          "value": "/usr/local/cog"

        },

        {

          "name": "COG_INSTALL_DIR",

          "value": "/usr/local/cog/cog_install"

        },

        {

          "name": "ESGF_HOSTNAME",

          "value": "airborne-elb1.dyndns.org"

        }

      ],

      "links": [

        "postgres:esgf-postgres"

      ],

      "workingDirectory": null,

      "readonlyRootFilesystem": null,

      "image": "esgfhub/esgf-cog:1.2",

      "command": [

        "airborne-elb1.dyndns.org",

        "true",

        "false"

      ],

      "user": null,

      "dockerLabels": null,

      "logConfiguration": null,

      "cpu": 0,

      "privileged": null,

      "memoryReservation": null

    },

    {

      "volumesFrom": [],

      "memory": 512,

      "extraHosts": null,

      "dnsServers": null,

      "disableNetworking": null,

      "dnsSearchDomains": null,

      "portMappings": [

        {

          "hostPort": 5432,

          "containerPort": 5432,

          "protocol": "tcp"

        }

      ],

      "hostname": null,

      "essential": true,

      "entryPoint": null,

      "mountPoints": [],

      "name": "postgres",

      "ulimits": null,

      "dockerSecurityOptions": null,

      "environment": [],

      "links": null,

      "workingDirectory": null,

      "readonlyRootFilesystem": null,

      "image": "esgfhub/esgf-postgres:1.2",

      "command": null,

      "user": null,

      "dockerLabels": null,

      "logConfiguration": null,

      "cpu": 0,

      "privileged": null,

      "memoryReservation": null

    },

    {

      "volumesFrom": [

        {

          "readOnly": null,

          "sourceContainer": "cog"

        },

        {

          "readOnly": null,

          "sourceContainer": "config"

        }

      ],

      "memory": 2048,

      "extraHosts": null,

      "dnsServers": null,

      "disableNetworking": null,

      "dnsSearchDomains": null,

      "portMappings": [

        {

          "hostPort": 80,

          "containerPort": 80,

          "protocol": "tcp"

        },

        {

          "hostPort": 443,

          "containerPort": 443,

          "protocol": "tcp"

        }

      ],

      "hostname": null,

      "essential": true,

      "entryPoint": [

        "/usr/local/bin/docker-entrypoint.sh"

      ],

      "mountPoints": [

        {

          "containerPath": "/usr/local/cog/cog_config",

          "sourceVolume": "cog_data",

          "readOnly": null

        }

      ],

      "name": "httpd",

      "ulimits": null,

      "dockerSecurityOptions": null,

      "environment": [

        {

          "name": "COG_CONFIG_DIR",

          "value": "/usr/local/cog/cog_config"

        },

        {

          "name": "ESGF_HOSTNAME",

          "value": "airborne-elb1.dyndns.org"

        },

        {

          "name": "SSL_CERT_DIR",

          "value": "/etc/grid-security/certificates"

        }

      ],

      "links": [

        "postgres:esgf-postgres",

        "solr:esgf-solr",

        "index-node:esgf-index-node",

        "data-node:esgf-data-node",

        "idp-node:esgf-idp-node"

      ],

      "workingDirectory": null,

      "readonlyRootFilesystem": null,

      "image": "esgfhub/esgf-httpd:1.2",

      "command": null,

      "user": null,

      "dockerLabels": null,

      "logConfiguration": null,

      "cpu": 0,

      "privileged": null,

      "memoryReservation": null

    },

    {

      "volumesFrom": [],

      "memory": 4096,

      "extraHosts": null,

      "dnsServers": null,

      "disableNetworking": null,

      "dnsSearchDomains": null,

      "portMappings": [

        {

          "hostPort": 8983,

          "containerPort": 8983,

          "protocol": "tcp"

        },

        {

          "hostPort": 8984,

          "containerPort": 8984,

          "protocol": "tcp"

        }

      ],

      "hostname": null,

      "essential": true,

      "entryPoint": null,

      "mountPoints": [

        {

          "containerPath": "/esg/solr-index",

          "sourceVolume": "solr_data",

          "readOnly": null

        }

      ],

      "name": "solr",

      "ulimits": null,

      "dockerSecurityOptions": null,

      "environment": [],

      "links": null,

      "workingDirectory": null,

      "readonlyRootFilesystem": null,

      "image": "esgfhub/esgf-solr:1.2",

      "command": null,

      "user": null,

      "dockerLabels": null,

      "logConfiguration": null,

      "cpu": 0,

      "privileged": null,

      "memoryReservation": null

    },

    {

      "volumesFrom": [

        {

          "readOnly": null,

          "sourceContainer": "config"

        }

      ],

      "memory": 2048,

      "extraHosts": null,

      "dnsServers": null,

      "disableNetworking": null,

      "dnsSearchDomains": null,

      "portMappings": [

        {

          "hostPort": 8081,

          "containerPort": 8080,

          "protocol": "tcp"

        },

        {

          "hostPort": 8444,

          "containerPort": 8443,

          "protocol": "tcp"

        }

      ],

      "hostname": null,

      "essential": true,

      "entryPoint": [],

      "mountPoints": [],

      "name": "index-node",

      "ulimits": null,

      "dockerSecurityOptions": null,

      "environment": [],

      "links": [

        "solr:esgf-solr"

      ],

      "workingDirectory": null,

      "readonlyRootFilesystem": null,

      "image": "esgfhub/esgf-index-node:1.2",

      "command": null,

      "user": null,

      "dockerLabels": null,

      "logConfiguration": null,

      "cpu": 0,

      "privileged": null,

      "memoryReservation": null

    },

    {

      "volumesFrom": [

        {

          "readOnly": null,

          "sourceContainer": "config"

        }

      ],

      "memory": 2048,

      "extraHosts": null,

      "dnsServers": null,

      "disableNetworking": null,

      "dnsSearchDomains": null,

      "portMappings": [

        {

          "hostPort": 8080,

          "containerPort": 8080,

          "protocol": "tcp"

        },

        {

          "hostPort": 8443,

          "containerPort": 8443,

          "protocol": "tcp"

        }

      ],

      "hostname": null,

      "essential": true,

      "entryPoint": null,

      "mountPoints": [

        {

          "containerPath": "/esg/content/thredds",

          "sourceVolume": "tds_data",

          "readOnly": null

        },

        {

          "containerPath": "/esg/data",

          "sourceVolume": "esgf_data",

          "readOnly": null

        },

        {

          "containerPath": "/usr/local/tomcat/webapps/thredds/WEB-INF/web.xml",

          "sourceVolume": "tds_web_xml",

          "readOnly": null

        }

      ],

      "name": "data-node",

      "ulimits": null,

      "dockerSecurityOptions": null,

      "environment": [

        {

          "name": "ESGF_HOSTNAME",

          "value": "airborne-elb1.dyndns.org"

        }

      ],

      "links": [

        "postgres:esgf-postgres"

      ],

      "workingDirectory": null,

      "readonlyRootFilesystem": null,

      "image": "esgfhub/esgf-data-node:1.2",

      "command": null,

      "user": null,

      "dockerLabels": null,

      "logConfiguration": null,

      "cpu": 0,

      "privileged": null,

      "memoryReservation": null

    },

    {

      "volumesFrom": [

        {

          "readOnly": null,

          "sourceContainer": "config"

        }

      ],

      "memory": 1024,

      "extraHosts": null,

      "dnsServers": null,

      "disableNetworking": null,

      "dnsSearchDomains": null,

      "portMappings": [

        {

          "hostPort": 8082,

          "containerPort": 8080,

          "protocol": "tcp"

        },

        {

          "hostPort": 8445,

          "containerPort": 8443,

          "protocol": "tcp"

        }

      ],

      "hostname": null,

      "essential": true,

      "entryPoint": null,

      "mountPoints": [],

      "name": "idp-node",

      "ulimits": null,

      "dockerSecurityOptions": null,

      "environment": [],

      "links": [

        "postgres:esgf-postgres"

      ],

      "workingDirectory": null,

      "readonlyRootFilesystem": null,

      "image": "esgfhub/esgf-idp-node:1.2",

      "command": null,

      "user": null,

      "dockerLabels": null,

      "logConfiguration": null,

      "cpu": 0,

      "privileged": null,

      "memoryReservation": null

    }

  ],

  "placementConstraints": [

    {

      "expression": "attribute:esgf_type == master_node",

      "type": "memberOf"

    }

  ],

  "volumes": [

    {

      "host": {

        "sourcePath": "/home/ec2-user/esgf_config/httpd/certs"

      },

      "name": "esgf_certs"

    },

    {

      "host": {

        "sourcePath": "/home/ec2-user/esgf_config/grid-security/certificates"

      },

      "name": "esgf_grid_security_certs"

    },

    {

      "host": {

        "sourcePath": "/home/ec2-user/esgf_config/esg/config"

      },

      "name": "esgf_config"

    },

    {

      "host": {

        "sourcePath": "/home/ec2-user/esgf_config/httpd/conf"

      },

      "name": "esgf_httpd_conf"

    },

    {

      "host": {

        "sourcePath": "/home/ec2-user/esgf_config/esg/config/tomcat"

      },

      "name": "esgf_tomcat_config"

    },

    {

      "host": {

        "sourcePath": "/esgf_data/cog_data"

      },

      "name": "cog_data"

    },

    {

      "host": {

        "sourcePath": "/esgf_data/solr_data"

      },

      "name": "solr_data"

    },

    {

      "host": {

        "sourcePath": "/esgf_data/tds_data"

      },

      "name": "tds_data"

    },

    {

      "host": {

        "sourcePath": "/esgf_data/esgf_data"

      },

      "name": "esgf_data"

    },

    {

      "host": {

        "sourcePath": "/home/ec2-user/esgf_config/webapps/thredds/WEB-INF/web.xml"

      },

      "name": "tds_web_xml"

    }

  ],

  "family": "EsgfMasterNodeTask"

}

cog/Dockerfile

+25 −10
Original line number Diff line number Diff line 
# ESGF node with CoG web application

# When running a container, must have $ESGF_HOSTNAME env set,

#

# To run on localhost:

# docker run -ti -p 8000:8000 --name cog esgfhub/esgf-cog:${ESGF_VERSION}

# To run with a specific hostname:

# docker run -ti -p 8000:8000 --name cog -e ESGF_HOSTNAME=${ESGF_HOSTNAME} esgfhub/esgf-cog:${ESGF_VERSION} ${ESGF_HOSTNAME} false true

# To map the local source directory:

# docker run -ti -p 8000:8000 --name cog -v /Users/cinquini/Documents/workspace/cog:/usr/local/cog/cog_install esgfhub/esgf-cog:${ESGF_VERSION}



FROM esgfhub/esgf-node:1.1

FROM esgfhub/esgf-node:1.2



MAINTAINER Luca Cinquini <luca.cinquini@jpl.nasa.gov>



# choose CoG version

ENV COG_TAG=v3.8.0

ENV COG_TAG=v3.9.1



# setup CoG environment

ENV COG_DIR=/usr/local/cog
@@ -25,8 +31,8 @@ RUN cd $COG_DIR && \ 
RUN cd $COG_DIR && \

    git clone https://github.com/EarthSystemCoG/COG cog_install && \

    cd cog_install && \

    git checkout -b devel origin/devel

    #git checkout $COG_TAG

    #git checkout -b devel origin/devel

    git checkout $COG_TAG



# install CoG dependencies

RUN cd $COG_INSTALL_DIR && \
@@ -55,6 +61,15 @@ RUN cd $COG_DIR && \ 
    make  && \

    cp mkproxy $COG_DIR/venv/lib/python2.7/site-packages/globusonline/transfer/api_client/x509_proxy/.



# collect static files to ./static directory

# must use a minimal settings file (configured with sqllite3 database)

COPY conf/cog_settings.cfg /usr/local/cog/cog_config/cog_settings.cfg

RUN cd $COG_INSTALL_DIR && \

    source $COG_DIR/venv/bin/activate && \

    #python setup.py -q setup_cog --esgf=false

    python manage.py collectstatic --no-input && \

    rm /usr/local/cog/cog_config/cog_settings.cfg



# for some unknown reason, must reinstall captcha

#RUN source $COG_DIR/venv/bin/activate && \

#    pip uninstall -y django-simple-captcha && \
@@ -80,9 +95,9 @@ RUN chown -R cogadmin:cogadmin $COG_DIR 
VOLUME $COG_DIR/venv

VOLUME $COG_INSTALL_DIR



# scripts

# startup

COPY  scripts/ /usr/local/bin/

RUN chmod +x /usr/local/bin/*.sh



COPY conf/supervisord.cog.conf /etc/supervisor/conf.d/supervisord.cog.conf

ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]

CMD ["my-node.esgf.org", "false", "true"]
 
# will override these CMD options at run time

CMD ["localhost", "false", "true"]