Unverified Commit a0611bb0 authored by Matt Pryor's avatar Matt Pryor Committed by GitHub
Browse files

Merge pull request #127 from ESGF/issue/122/log-streams 

Separate log streams for Kubernetes
parents b37849b4 94cc8aad

deploy/kubernetes/chart/templates/fileServer/deployment.yaml

+38 −0
Original line number Diff line number Diff line
@@ -23,6 +23,26 @@ spec: 
      {{- with .Values.data.podSecurityContext }}

      securityContext: {{ toYaml . | nindent 8 }}

      {{- end }}

      initContainers:

        # Create a named pipe for the access log

        # This allows us to separate the access log from the error log, so that it is tagged differently when forwarded

        # This makes processing the access log for statistics easier

        - name: make-log-pipes

          {{ include "esgf.deployment.image" (list . $fileServer.image) }}

          args:

            # Continue to send the error log to stderr, but send the access log to a named pipe

            - bash

            - -c

            - |

              set -ex

              ln -s /dev/stderr /var/log/nginx/error.log

              mkfifo /var/log/nginx/access.log

          {{- with .Values.data.securityContext }}

          securityContext: {{ toYaml . | nindent 12 }}

          {{- end }}

          volumeMounts:

            - name: nginx-logs

              mountPath: /var/log/nginx

      containers:

        - name: file-server

          {{ include "esgf.deployment.image" (list . $fileServer.image) }}
@@ -50,7 +70,22 @@ spec: 
            - name: nginx-conf

              mountPath: /etc/nginx/conf.d

              readOnly: true

            - name: nginx-logs

              mountPath: /var/log/nginx

            {{- include "esgf.data.volumeMounts" . | nindent 12 }}

        # Tail the access log separately

        - name: file-server-access-log

          {{ include "esgf.deployment.image" (list . $fileServer.image) }}

          args:

            # Just cat the access log

            - cat

            - /var/log/nginx/access.log

          {{- with .Values.data.securityContext }}

          securityContext: {{ toYaml . | nindent 12 }}

          {{- end }}

          volumeMounts:

            - name: nginx-logs

              mountPath: /var/log/nginx

      {{- with $fileServer.nodeSelector }}

      nodeSelector: {{ toYaml . | nindent 8 }}

      {{- end }}
@@ -64,5 +99,8 @@ spec: 
        - name: nginx-conf

          configMap:

            name: {{ include "esgf.component.fullname" (list . "fileServer") }}

        # Each pod gets a directory to hold the named pipes for the logs

        - name: nginx-logs

          emptyDir: {}

        {{- include "esgf.data.volumes" . | nindent 8 }}

{{- end -}}

deploy/kubernetes/chart/templates/thredds/deployment.yaml

+45 −3
Original line number Diff line number Diff line
@@ -27,9 +27,9 @@ spec: 
      {{- with .Values.data.podSecurityContext }}

      securityContext: {{ toYaml . | nindent 8 }}

      {{- end }}

      initContainers:

        {{- if (and $thredds.catalogVolume $thredds.localCache.enabled) }}

        # If the local cache is enabled, wait for the node we have landed on to be initialised before starting

      initContainers:

        - name: wait-for-cache

          {{ include "esgf.deployment.image" (list . $thredds.image) }}

          env:
@@ -68,6 +68,27 @@ spec: 
            - name: thredds-cache

              mountPath: /thredds/cache

        {{- end }}

        # Create named pipes for the log files

        - name: make-log-pipes

          {{ include "esgf.deployment.image" (list . $thredds.image) }}

          args:

            - bash

            - -c

            - |

              set -ex

              mkfifo /thredds/logs/serverStartup.log

              mkfifo /thredds/logs/catalogInit.log

              mkfifo /thredds/logs/httpout.log

              mkfifo /thredds/logs/featureCollectionScan.log

              mkfifo /thredds/logs/fmrc.log

              mkfifo /thredds/logs/threddsServlet.log

              mkfifo /thredds/logs/cache.log

          {{- with .Values.data.securityContext }}

          securityContext: {{ toYaml . | nindent 12 }}

          {{- end }}

          volumeMounts:

            - name: thredds-logs

              mountPath: /thredds/logs

      containers:

        - name: thredds

          {{ include "esgf.deployment.image" (list . $thredds.image) }}
@@ -106,7 +127,25 @@ spec: 
              {{- end }}

            - name: thredds-cache

              mountPath: /opt/tomcat/content/thredds/cache

            - name: thredds-logs

              mountPath: /opt/tomcat/content/thredds/logs

            {{- include "esgf.data.volumeMounts" . | nindent 12 }}

        # Tail the log pipes

        {{- $topContext := . }}

        {{- range (list "serverStartup.log" "catalogInit.log" "httpout.log" "featureCollectionScan.log" "fmrc.log" "threddsServlet.log" "cache.log") }}

        - name: thredds-log-{{ trimSuffix ".log" . | lower }}

          {{ include "esgf.deployment.image" (list $topContext $thredds.image) }}

          args:

            # Just cat the log file

            - cat

            - /thredds/logs/{{ . }}

          {{- with $topContext.Values.data.securityContext }}

          securityContext: {{ toYaml . | nindent 12 }}

          {{- end }}

          volumeMounts:

            - name: thredds-logs

              mountPath: /thredds/logs

        {{- end }}

      {{- with $thredds.nodeSelector }}

      nodeSelector: {{ toYaml . | nindent 8 }}

      {{- end }}
@@ -135,5 +174,8 @@ spec: 
        # Each pod gets its own cache directory on the local disk

        - name: thredds-cache

          emptyDir: {}

        # Each pod gets a directory to hold the named pipes for log files

        - name: thredds-logs

          emptyDir: {}

        {{- include "esgf.data.volumes" . | nindent 8 }}

{{- end -}}

images/thredds/Dockerfile

+13 −3
Original line number Diff line number Diff line
@@ -25,9 +25,19 @@ USER root 


# Make the content root directory, fix permissions so that any user in the

# esgf group can use it and point tomcat at it

# We also make the cache directory with correct permissions and declare it

# as a volume - this ensures that Docker named volumes will work correctly

RUN mkdir -p ./content/thredds/cache && \

#

# Make the cache directory with correct permissions and declare it as a volume

# This ensures that Docker named volumes will work correctly

#

# Make the log directory and create symlinks for log files to stdout

RUN mkdir -p ./content/thredds/{cache,logs} && \

    ln -s /dev/stdout ./content/thredds/logs/serverStartup.log && \

    ln -s /dev/stdout ./content/thredds/logs/catalogInit.log && \

    ln -s /dev/stdout ./content/thredds/logs/httpout.log && \

    ln -s /dev/stdout ./content/thredds/logs/featureCollectionScan.log && \

    ln -s /dev/stdout ./content/thredds/logs/fmrc.log && \

    ln -s /dev/stdout ./content/thredds/logs/threddsServlet.log && \

    ln -s /dev/stdout ./content/thredds/logs/cache.log && \

    chown -R $ESGF_USER:$ESGF_GROUP ./content/thredds && \

    chmod -R u+w,g+w,o= ./content/thredds

ENV CATALINA_EXTRA_OPTS "-Dtds.content.root.path=$CATALINA_HOME/content"

images/thredds/log4j2.xml

+187 −15
Original line number Diff line number Diff line 
<?xml version="1.0" encoding="UTF-8"?>

<Configuration monitorInterval="30">

<configuration monitorInterval="30">

  <properties>

    <property name="tds.content.root.path">../content</property>

    <property name="tds.log.dir">${sys:tds.content.root.path}/thredds/logs</property>

  </properties>



  <Appenders>

        <Console name="console" target="SYSTEM_OUT">



    <File name="serverStartupAppender" fileName="${tds.log.dir}/serverStartup.log" immediateFlush="true">

      <PatternLayout pattern="%d{yyyy-MM-dd'T'HH:mm:ss.SSSZ} [%10r][%8X{ID}] %-5p %c: %m%n"/>

        </Console>

        <Async name="async">

            <AppenderRef ref="console" />

        </Async>

    </File>



    <File name="catalogInit" fileName="${tds.log.dir}/catalogInit.log" immediateFlush="true">

      <PatternLayout pattern="%m%n"/>

    </File>



    <File name="httpout" fileName="${tds.log.dir}/httpout.log" immediateFlush="true">

      <PatternLayout pattern="%d %c: %m%n"/>

    </File>



    <File name="fcScan" fileName="${tds.log.dir}/featureCollectionScan.log" immediateFlush="true">

      <PatternLayout pattern="[%d{yyyy-MM-dd'T'HH:mm:ss.SSSZ}] %-5p %c: %m%n"/>

    </File>



    <File name="fmrc" fileName="${tds.log.dir}/fmrc.log" immediateFlush="true">

      <PatternLayout pattern="[%d{yyyy-MM-dd'T'HH:mm:ss.SSSZ}] %-5p %c: %m%n"/>

    </File>



    <File name="threddsServlet" fileName="${tds.log.dir}/threddsServlet.log" immediateFlush="true">

      <PatternLayout pattern="%d{yyyy-MM-dd'T'HH:mm:ss.SSS Z} [%10r][%8X{ID}] %-5p - %c - %m%n"/>

    </File>



    <File name="cacheLog" fileName="${tds.log.dir}/cache.log" immediateFlush="true">

      <PatternLayout pattern="[%d{yyyy-MM-dd'T'HH:mm:ss.SSSZ}] %m%n"/>

    </File>



  </Appenders>

    <Loggers>

        <Root level="info">

            <AppenderRef ref="async" />

        </Root>

    </Loggers>

</Configuration>
 


  <loggers>



    <logger name="serverStartup" level="info" additivity="false">

      <appender-ref ref="serverStartupAppender"/>

    </logger>



    <logger name="thredds.core.ConfigCatalogInitialization" level="debug" additivity="false">

      <appender-ref ref="catalogInit"/>

    </logger>



    <logger name="thredds.server.catalog.tracker" level="info" additivity="false">

      <appender-ref ref="catalogInit"/>

    </logger>



    <logger name="thredds.featurecollection.FeatureCollectionConfig" level="info" additivity="false">

      <appender-ref ref="catalogInit"/>

    </logger>



    <logger name="ucar.nc2.thredds.MetadataExtractorAcdd" level="info" additivity="false">

      <appender-ref ref="catalogInit"/>

    </logger>



    <logger name="thredds.featurecollection.InvDatasetFeatureCollection.catalogInit" level="info" additivity="false">

      <appender-ref ref="catalogInit"/>

    </logger>



    <logger name="catalogInit" level="info" additivity="false">

      <appender-ref ref="catalogInit"/>

    </logger>



    <logger name="thredds.server.config.TdsContext.catalogInit" level="info" additivity="false">

      <appender-ref ref="catalogInit"/>

    </logger>



    <logger name="cacheLogger" level="info" additivity="false">

      <appender-ref ref="cacheLog"/>

    </logger>



    <logger name="thredds.filesystem.CacheManager" level="info" additivity="false">

      <appender-ref ref="cacheLog"/>

    </logger>



    <!-- fmrc -->

    <logger name="ucar.nc2.ft.fmrc" level="info" additivity="false">

      <appender-ref ref="fmrc"/>

    </logger>



    <logger name="thredds.featurecollection.InvDatasetFcFmrc" level="info" additivity="false">

      <appender-ref ref="fmrc"/>

    </logger>



    <logger name="thredds.servlet.DataRootHandler.fcScan" level="info" additivity="false">

      <appender-ref ref="fmrc"/>

    </logger>



    <logger name="thredds.inventory.CollectionManagerAbstract" level="info" additivity="false">

      <appender-ref ref="fmrc"/>

    </logger>



    <logger name="thredds.inventory.bdb.MetadataManager" level="warn" additivity="false">

      <appender-ref ref="fmrc"/>

    </logger>



    <!-- featureCollection -->

    <logger name="featureCollectionScan" level="info" additivity="false">

      <appender-ref ref="fcScan"/>

    </logger>



    <logger name="thredds.inventory.MFileCollectionManager" level="info" additivity="false">

      <appender-ref ref="fcScan"/>

    </logger>



    <logger name="thredds.featurecollection.CollectionUpdater" level="info" additivity="false">

      <appender-ref ref="fcScan"/>

    </logger>



    <logger name="thredds.server.admin.AdminCollectionController" level="info" additivity="false">

      <appender-ref ref="fcScan"/>

    </logger>



    <logger name="ucar.nc2.grib" level="info" additivity="false">

      <appender-ref ref="fcScan"/>

    </logger>



    <logger name="ucar.nc2.grib.grib1.builder.Grib1CollectionBuilder" level="info" additivity="false">

      <appender-ref ref="fcScan"/>

    </logger>



    <logger name="ucar.nc2.grib.grib2.builder.Grib2CollectionBuilder" level="info" additivity="false">

      <appender-ref ref="fcScan"/>

    </logger>



    <logger name="thredds.featurecollection.InvDatasetFeatureCollection" level="info" additivity="false">

      <appender-ref ref="fcScan"/>

    </logger>



    <logger name="thredds.featurecollection.InvDatasetFcGrib" level="info" additivity="false">

      <appender-ref ref="fcScan"/>

    </logger>



    <logger name="org.quartz" level="warn" additivity="false">

      <appender-ref ref="fcScan"/>

    </logger>



    <!-- WMS stuff has routine higher level of messages -->

    <logger name="uk.ac.rdg.resc.ncwms" level="warn" additivity="false">

      <appender-ref ref="threddsServlet"/>

    </logger>



    <logger name="thredds.server.wms.responses" level="warn" additivity="false">

      <appender-ref ref="threddsServlet"/>

    </logger>



    <logger name="thredds.server.metadata" level="warn" additivity="false">

      <appender-ref ref="threddsServlet"/>

    </logger>



    <!-- spring -->



    <logger name="org.springframework.http" level="info" additivity="false">

      <appender-ref ref="threddsServlet"/>

    </logger>



    <logger name="org.springframework.beans.factory" level="info" additivity="false">

      <appender-ref ref="serverStartupAppender"/>

    </logger>



    <logger name="org.springframework.web" level="info" additivity="false">

      <appender-ref ref="serverStartupAppender"/>

    </logger>



    <logger name="org.springframework" level="info" additivity="false">

      <appender-ref ref="serverStartupAppender"/>

    </logger>



    <!-- =========================================== -->



    <logger name="org.apache.http.impl.conn" level="info" additivity="false">

       <appender-ref ref="httpout"/>

     </logger>



    <!-- All normal logging messages go to threddsServlet.log -->

    <root level="debug" additivity="false">

      <appender-ref ref="threddsServlet"/>

    </root>



  </loggers>



</configuration>