Merge pull request #136 from ESGF/issue/115/esg-search (218a03ac) · Commits · ESGF / mirrors / Esgf Docker

.gitlab-ci.yml

+21 −0

Original line number	Diff line number	Diff line
		@@ -109,6 +109,20 @@ build:nginx:
		CONTEXT_DIR: $CI_PROJECT_DIR/images/nginx
		needs: ["build:base"]

		build:search-builder:
		extends: .docker-build
		stage: build-3
		variables:
		CONTEXT_DIR: $CI_PROJECT_DIR/images/search-builder
		needs: ["build:jdk"]

		build:solr:
		extends: .docker-build
		stage: build-3
		variables:
		CONTEXT_DIR: $CI_PROJECT_DIR/images/solr
		needs: ["build:jre"]

		build:tomcat:
		extends: .docker-build
		stage: build-3
		@@ -116,6 +130,13 @@ build:tomcat:
		CONTEXT_DIR: $CI_PROJECT_DIR/images/tomcat
		needs: ["build:jre"]

		build:search:
		extends: .docker-build
		stage: build-4
		variables:
		CONTEXT_DIR: $CI_PROJECT_DIR/images/search
		needs: ["build:search-builder", "build:tomcat"]

		build:thredds:
		extends: .docker-build
		stage: build-4

README.md

+9 −9

Original line number	Diff line number	Diff line
		@@ -25,15 +25,16 @@ will not benefit from many features provided by Kubernetes, including:
		This project is under heavy active development, with the implementation depending on the ESGF
		Future Architecture discussions.

		Currently, only an unauthenticated data node is implemented. The data node uses THREDDS to serve
		catalog and OPeNDAP endpoints, but uses Nginx to do direct file serving which should by more
		performant than THREDDS.
		Currently, data and index nodes are implemented but without authentication.

		The data node uses THREDDS to serve catalog and OPeNDAP endpoints, but uses Nginx for direct file
		serving which should be more performant than THREDDS.

		The data node is capable of using existing catalogs from the current publisher to specify the
		available data, however it can also use a catalog-free configuration which utilises
		available data, however it is designed primarily to use a catalog-free configuration which utilises
		[datasetScan elements](https://www.unidata.ucar.edu/software/tds/current/reference/DatasetScan.html),
		to serve all files under a given dataset root. This is designed to work with the next-generation
		publisher being developed at LLNL that does not rely on THREDDS catalogs for publishing metadata.
		to serve all files under a given dataset root. This will work with the next-generation publisher
		being developed at LLNL that does not rely on THREDDS catalogs for publishing metadata.

		## Image tags

		@@ -59,8 +60,7 @@ release or a particular commit, in order to avoid unexpected code changes or dif
		the container image between load-balanced nodes.

		You can check the [available tags on Docker Hub](https://hub.docker.com/r/esgfdeploy/thredds/tags).
		All the ESGF Docker images are built together, so any given tag will always be available for all
		images.
		All the ESGF Docker images are built together, so any given tag will be available for all images.

		## Making a deployment

		@@ -80,4 +80,4 @@ git checkout future-architecture
		Then follow the deployment guide for your chosen deployment method:

		* [Deploy ESGF using Ansible](./docs/deploy-ansible.md)
		* [Deploy ESGF to Kubernetes using Helm](./docs/deploy-kubernetes.md)
		* [Deploy ESGF to Kubernetes using Helm](./docs/kubernetes/deploy.md)

Vagrantfile

+14 −2

Original line number	Diff line number	Diff line
		@@ -55,13 +55,25 @@ Vagrant.configure(2) do \|config\|
		location: "/test_data/group_workspaces/jasmin2/cp4cds1/data/c3s-cordex"
		}
		]
		solr_replicas = [
		{
		name: "llnl",
		master_url: "https://esgf-node.llnl.gov/solr"
		}
		]
		ansible.groups = {
		"data" => ["default"],
		"data:vars" => {
		"index" => ["default"],
		"all:vars" => {
		"hostname" => "192.168.100.100.nip.io",
		"image_tag" => "issue-123-existing-catalogs",
		"image_tag" => "issue-115-esg-search",
		},
		"data:vars" => {
		"data_mounts" => "#{data_mounts.to_json}",
		"data_datasets" => "#{data_datasets.to_json}"
		},
		"index:vars" => {
		"solr_replicas" => "#{solr_replicas.to_json}"
		}
		}
		end

deploy/ansible/playbook.yml

+2 −1

Original line number	Diff line number	Diff line
		@@ -8,5 +8,6 @@
		become: true
		roles:
		- docker
		- { name: data, when: "'data' in group_names", tags: [data] }
		- { name: data, tags: [data] }
		- { name: index, tags: [index] }
		- proxy

deploy/ansible/roles/data/tasks/fileserver_install.yml

0 → 100644

+41 −0

Original line number	Diff line number	Diff line
		---

		- name: Create Docker network
		docker_network:
		name: esgf

		- name: Make fileserver config directory
		file:
		path: /esg/config/fileserver
		state: directory

		- name: Write fileserver configuration
		template:
		src: fileserver.conf.j2
		dest: /esg/config/fileserver/fileserver.conf

		- name: Start fileserver container
		docker_container:
		name: fileserver
		image: "{{ fileserver_image_prefix }}/{{ fileserver_image_repository }}:{{ fileserver_image_tag }}"
		pull: "{{ fileserver_image_pull }}"
		detach: yes
		restart_policy: unless-stopped
		exposed_ports:
		- "8080"
		networks:
		- name: esgf
		networks_cli_compatible: yes
		user: "{{ data_security_context_user }}"
		groups: "{{ data_security_context_groups }}"
		# Append the Nginx config volume to the data mounts
		# Append the catalog volume to the data mounts
		volumes: >-
		[
		"/esg/config/fileserver:/etc/nginx/conf.d:ro",
		{% for mount in data_mounts %}
		"{{ mount.host_path }}:{{ mount.mount_path }}:ro",
		{% endfor %}
		]
		state: started
		restart: yes