[telegraf] optional alternative config templates (#441)

.github/workflows/helm-charts-test.yaml

@@ -60,9 +60,22 @@ jobs:
         if: steps.list-changed.outputs.changed == 'true'
 
       - name: Run chart-testing (install)
-        run: ct install --namespace=default
+        id: install
+        run: |
+          ct install --namespace=default 2>&1 | tee $RUNNER_TEMP/$GITHUB_RUN_NUMBER.txt
 # When https://github.com/helm/chart-testing/issues/212 is fixed, this can be used to set the license key instead of using env from secret
 #        run: ct install --namespace=default --helm-extra-args="--set license.key=${INFLUXDB_ENTERPRISE_LICENSE_KEY}"
 #        env:
 #          INFLUXDB_ENTERPRISE_LICENSE_KEY: "${{ secrets.INFLUXDB_ENTERPRISE_LICENSE_KEY }}"
         if: steps.list-changed.outputs.changed == 'true'
+
+      - name: Check logs for errors
+        run: |
+          set +e
+          patterns="Error loading config file"
+          grep -f $GITHUB_WORKSPACE/.github/workflows/patterns.txt $RUNNER_TEMP/$GITHUB_RUN_NUMBER.txt
+          if [[ $? -eq 0 ]]; then
+            echo "::error:: Errors found in logs"
+            exit 1
+          fi
+        if: steps.list-changed.outputs.changed == 'true'

.github/workflows/patterns.txt

+Error loading config file

charts/telegraf/Chart.yaml

 apiVersion: v2
 name: telegraf
-version: 1.8.14
-appVersion: 1.21.3
+version: 1.8.15
+appVersion: 1.21.4
 deprecated: false
 description: Telegraf is an agent written in Go for collecting, processing, aggregating, and writing metrics.
 keywords:

charts/telegraf/README.md

@@ -64,3 +64,17 @@ To configure the chart, do either of the following:
   ```
 
   > **Tip**: Use the default [values.yaml](values.yaml).
+
+## Telegraf configuration
+
+Telegraf configuration is generated by YAML to TOML conversion of `.Values.config` entry.
+In some rare cases, Telegraf may not be able to parse the generated file.
+
+There is an alternative config conversion template that may produce better results.
+To use it, set `tplVersion` value to `2` when installing/upgrading the chart:
+
+  ```bash
+  helm upgrade --install my-release --set tplVersion=2 influxdata/telegraf
+  ```
+
+Please note this is experimental feature.

charts/telegraf/ci/complex-values.yaml

+## this config requires v2 config templates to render valid configuration (tplVersion, see at the end)
+config:
+  processors:
+    - enum:
+        # this is wrong mapping copied from values.yaml to test compatibility hack in v2 code
+        mapping:
+          field: "status"
+          dest: "status_code"
+          value_mappings:
+            healthy: 1
+            problem: 2
+            critical: 3
+    # issue 119
+    - enum:
+        order: 1
+        mapping:
+          - field: system_location_section
+            dest: system_location_sectionFloorNumber
+            value_mappings:
+              2nd: 2
+              1st: 1
+              Ground: 0
+              Basement: -1
+    - converter:
+        order: 2
+        fields:
+          integer:
+            - system_location_sectionFloorNumber
+    # issue 119
+    - enum:
+        mapping:
+          field: state
+          dest: status_code
+          default: -1
+          value_mappings:
+            RUNNING: 1
+            RUNNABLE: 1
+            SUSPENDED: 2
+            PENDING_CREATE: 3
+            MAINTENANCE: 4
+            UNKNOWN_STATE: 0
+    - converter:
+        fields:
+          integer: ["check_passed", "content_mismatch"]
+          tag: ["error_code", "http_status"]
+  aggregators:
+    # issue 375
+    - basicstats:
+        period: "30s"
+        drop_original: true
+        stats:
+          - "sum"
+        taginclude:
+          - "counter"
+          - "sql_instance"
+        tagpass:
+          counter:
+            - "Blocked tasks"
+            - "CPU usage %"
+            - "Used memory (KB)"
+  inputs:
+    - cpu:
+    # issue 382
+    - statsd:
+        service_address: ":8125"
+        percentiles:
+          - 50
+          - 95
+          - 99
+          - 99.5
+        metric_separator: "_"
+        allowed_pending_messages: 10000
+        percentile_limit: 1000
+    # issue 111
+    - postgresql_extensible:
+        address: "host=localhost user=postgres sslmode=disable"
+        query:
+          - sqlquery: "SELECT count(*) as total_user_count FROM users"
+            version: 801
+          - sqlquery: "SELECT count(*) as total_post_count FROM posts"
+            version: 801
+    # issue 433
+    - mqtt_consumer:
+        qos: 0
+        servers:
+          - "tcp://mqtt.eclipseprojects.io:1883"
+        topics:
+          - "telegraf/host01/cpu"
+          - "telegraf/+/mem"
+          - "sensors/#"
+        topic_parsing:
+          - topic: "telegraf/one/cpu/23"
+            measurement: "_/_/measurement/_"
+            types:
+              test: "int"
+        json_v2:
+          - measurement_name: ""
+            timestamp_path: ""
+            object:
+              - path: "@this"
+                tags:
+                  - "device"
+                  - "data.type"
+                  - "owner"
+                field:
+                  - path: ""
+                    type: "int"
+    # issue 429
+    - modbus:
+        name: "PS MAIN ENGINE"
+        controller: 'tcp://127.0.0.1:502'
+        slave_id: 1
+        holding_registers:
+          - name: "Coolant Level"
+            byte_order: CDAB
+            data_type: FLOAT32
+            scale: 0.001
+            address: [51410, 51411]
+  outputs:
+    - file:
+        files:
+          - "stdout"
+
+## Use tpl v2 templates
+tplVersion: 2

charts/telegraf/templates/_helpers.tpl

@@ -83,7 +83,7 @@ app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end }}
 {{- end -}}
 
-{{- define "outputs" -}}
+{{- define "outputs.v1" -}}
 {{- range $outputIdx, $configObject := . -}}
     {{- range $output, $config := . -}}
 
@@ -178,7 +178,7 @@ app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end }}
 {{- end -}}
 
-{{- define "inputs" -}}
+{{- define "inputs.v1" -}}
 {{- range $inputIdx, $configObject := . -}}
     {{- range $input, $config := . -}}
 
@@ -288,7 +288,7 @@ app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end }}
 {{- end -}}
 
-{{- define "processors" -}}
+{{- define "processors.v1" -}}
 {{- range $processorIdx, $configObject := . -}}
     {{- range $processor, $config := . -}}
 
@@ -410,8 +410,7 @@ app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end }}
 {{- end -}}
 
-
-{{- define "aggregators" -}}
+{{- define "aggregators.v1" -}}
 {{- range $aggregatorIdx, $configObject := . -}}
     {{- range $aggregator, $config := . -}}
 
@@ -541,6 +540,162 @@ app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end -}}
 
 {{/*
+Backward compatible version support.
+*/}}
+
+{{- define "outputs" -}}
+{{- include "section" (prepend . "outputs") -}}
+{{- end -}}
+
+{{- define "inputs" -}}
+{{- include "section" (prepend . "inputs") -}}
+{{- end -}}
+
+{{- define "processors" -}}
+{{- include "section" (prepend . "processors") -}}
+{{- end -}}
+
+{{- define "aggregators" -}}
+{{- include "section" (prepend . "aggregators") -}}
+{{- end -}}
+
+{{- define "detect.version" -}}
+{{ (default 1 .Values.tplVersion) | int64 }}
+{{- end -}}
+
+{{- define "section" -}}
+{{- $name := index . 0 -}}
+{{- $version := index . 1 | int64 -}}
+{{- $suffix := ternary "v2" "v1" (eq 2 $version) -}}
+{{- $templateName := printf "%s.%s" $name $suffix -}}
+{{- with index . 2 -}}
+  {{ include $templateName . }}
+{{- end }}
+{{- end -}}
+
+{{- define "processors.v2" -}}
+  {{ include "section.v2" (list "processors" .) }}
+{{- end -}}
+
+{{- define "aggregators.v2" -}}
+  {{ include "section.v2" (list "aggregators" .) }}
+{{- end -}}
+
+{{- define "inputs.v2" -}}
+  {{ include "section.v2" (list "inputs" .) }}
+{{- end -}}
+
+{{- define "outputs.v2" -}}
+  {{ include "section.v2" (list "outputs" .) }}
+{{- end -}}
+
+{{- define "section.v2" -}}
+{{- $name := index . 0 -}}
+{{- with index . 1 -}}
+{{- range $itemIdx, $configObject := . -}}
+    {{- range $item, $config := . }}
+    [[{{ $name }}.{{- $item }}]]
+    {{- if $config -}}
+    {{- $tp := typeOf $config -}}
+    {{- if eq $tp "map[string]interface {}" -}}
+      {{- $args := dict "key" $item "value" $config "level" 1 "type" $name -}}
+      {{ include "any.table" $args }}
+    {{- end }}
+    {{- end }}
+    {{ end }}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Renders indented table.
+*/}}
+
+{{- define "any.table" -}}
+{{- $n := (mul .level 2) | add 2 | int }}
+{{- $args := dict "key" .key "value" .value "type" .type -}}
+{{- include "any.table.raw" $args | indent $n }}
+{{- end }}
+
+{{/*
+Renders a table.
+Renders primitive and arrays of primitive types first, then nested tables and arrays of nested tables.
+*/}}
+
+{{- define "any.table.raw" -}}
+{{- $key := .key }}
+{{- $type := .type }}
+{{- range $k, $v := .value }}
+  {{- $tps := typeOf $v }}
+  {{- if eq $tps "string" }}
+  {{ $k }} = {{ $v | quote }}
+  {{- else if eq $tps "float64" }}
+    {{- $rv := float64 (int64 $v) }}
+    {{- if eq $rv $v }}
+  {{ $k }} = {{ $v | int64 }}
+    {{- else }}
+  {{ $k }} = {{ $v }}
+    {{- end }}
+  {{- else if eq $tps "int64" }}
+  {{ $k }} = {{ $v }}
+  {{- else if eq $tps "bool" }}
+  {{ $k }} = {{ $v }}
+  {{- else if eq $tps "[]interface {}" }}
+    {{- if ne (index $v 0 | typeOf) "map[string]interface {}" }}
+  {{ $k }} = [
+      {{- $numOut := len $v }}
+      {{- $numOut := sub $numOut 1 }}
+      {{- range $b, $xv := $v }}
+        {{- $i := int64 $b }}
+        {{- $xtps := typeOf $xv }}
+        {{- if eq $xtps "string" }}
+    {{ $xv | quote }}
+        {{- else if eq $xtps "float64" }}
+          {{- $rxv := float64 (int64 $xv) }}
+          {{- if eq $rxv $xv }}
+            {{- if eq $k "percentiles" }}
+    {{ $xv | int64 }}.0
+            {{- else }}
+    {{ $xv | int64 }}
+            {{- end }}
+          {{- else }}
+    {{ $xv }}
+          {{- end }}
+        {{- else if eq $xtps "int64" }}
+    {{ $xv }}
+        {{- end }}
+        {{- if ne $i $numOut -}}
+        ,
+        {{- end -}}
+      {{- end }}
+  ]
+    {{- end }}
+  {{- end }}
+{{- end }}
+{{- range $k, $v := .value }}
+  {{- $tps := typeOf $v }}
+  {{- if eq $tps "map[string]interface {}" }}
+      {{- $args := dict "key" (printf "%s.%s" $key $k) "value" $v "type" $type }}
+      {{- /* hack for existing incorrect mapping in values.yaml */ -}}
+      {{- if eq "processors.enum.mapping" (printf "%s.%s" $type $args.key) }}
+  [[{{ $type }}.{{ $args.key }}]]
+      {{- else }}
+  [{{ $type }}.{{ $args.key }}]
+      {{- end }}
+      {{- include "any.table" $args -}}
+  {{- else if eq $tps "[]interface {}" }}
+    {{- if eq (index $v 0 | typeOf) "map[string]interface {}" }}
+      {{- range $b, $xv := $v }}
+        {{- $args := dict "key" (printf "%s.%s" $key $k) "value" $xv  "type" $type }}
+  [[{{ $type }}.{{ $args.key }}]]
+        {{- include "any.table" $args -}}
+      {{- end }}
+    {{- end }}
+  {{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
 Create the name of the service account to use
 */}}
 {{- define "telegraf.serviceAccountName" -}}

charts/telegraf/templates/configmap.yaml

@@ -6,11 +6,12 @@ metadata:
     {{- include "telegraf.labels" . | nindent 4 }}
 data:
   telegraf.conf: |+
+    {{- $tplVersion := include "detect.version" . -}}
     {{ template "global_tags" .Values.config.global_tags }}
     {{ template "agent" .Values.config.agent }}
-    {{ template "processors" .Values.config.processors }}
-    {{ template "aggregators" .Values.config.aggregators }}
-    {{ template "outputs" .Values.config.outputs }}
+    {{ template "processors" (list $tplVersion .Values.config.processors) }}
+    {{ template "aggregators" (list $tplVersion .Values.config.aggregators) }}
+    {{ template "outputs" (list $tplVersion .Values.config.outputs) }}
     {{- if .Values.metrics.health.enabled }}
     [[outputs.health]]
       service_address = "{{ .Values.metrics.health.service_address }}"
@@ -21,7 +22,7 @@ data:
       [[outputs.health.contains]]
         field = "buffer_size"
     {{- end }}
-    {{ template "inputs" .Values.config.inputs -}}
+    {{ template "inputs" (list $tplVersion .Values.config.inputs) }}
     {{- if .Values.metrics.internal.enabled }}
     [[inputs.internal]]
       collect_memstats = {{ or .Values.metrics.internal.collect_memstats (.Values.metrics.collect_memstats | default false) }}