Parseable Docs

OpenTelemetry Logs

GitHubView on GitHub

Parseable enables ingesting the OpenTelemetry logs sent from the OTEL collectors in the JSON format. In order to do so, Parseable exposes default endpoint /v1/logs at which the OTEL collector sends the log data.

Below is the sample configuration of the OTEL collector to send the logs to Parseable

exporters:
  otlphttp/parseablelogs:
    endpoint: "<parseable-endpoint> eg. http://localhost:8000"
    headers:
      Authorization: "Basic <base64 encoded string of username:password>"
      X-P-Stream: otel-logs
      X-P-Log-Source: otel-logs
      Content-Type: application/json
    encoding: json
    tls:
      insecure: true

service:
  pipelines:
    logs:
      exporters: [otlphttp/parseablelogs]

Data Flattening

The OpenTelemetry logs that are in the form of a nested JSON. When the log event is received at /v1/logs endpoint, Parseable will flatten the nested JSON to a flat JSON object. This is done to make the data more queryable and filterable. Here's a quick comparison of the nested JSON and the flattened JSON:

OTEL JSON format sample:

{
  "resourceLogs": [
    {
      "resource": {
        "attributes": [
          {
            "key": "container.id",
            "value": {
              "stringValue": "61625241a33e64c6c34de99b1827a713b630f5548e715a87c6f8d052f1e3a2ab"
            }
          },
          {
            "key": "docker.cli.cobra.command_path",
            "value": {
              "stringValue": "docker compose"
            }
          },
          {
            "key": "host.arch",
            "value": {
              "stringValue": "amd64"
            }
          },
          {
            "key": "host.name",
            "value": {
              "stringValue": "61625241a33e"
            }
          },
          {
            "key": "os.description",
            "value": {
              "stringValue": "Linux 5.15.167.4-microsoft-standard-WSL2"
            }
          },
          {
            "key": "os.type",
            "value": {
              "stringValue": "linux"
            }
          },
          {
            "key": "process.command_line",
            "value": {
              "stringValue": "/opt/java/openjdk/bin/java -Xmx400m -Xms400m -XX:SharedArchiveFile=/opt/kafka/kafka.jsa -Xlog:gc*:file=/opt/kafka/bin/../logs/kafkaServer-gc.log:time,tags:filecount=10,filesize=100M -Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dkafka.logs.dir=/opt/kafka/bin/../logs -Dlog4j.configuration=file:/opt/kafka/bin/../config/log4j.properties -javaagent:/tmp/opentelemetry-javaagent.jar -Dotel.jmx.target.system=kafka-broker kafka.Kafka /opt/kafka/config/server.properties"
            }
          },
          {
            "key": "process.executable.path",
            "value": {
              "stringValue": "/opt/java/openjdk/bin/java"
            }
          },
          {
            "key": "process.pid",
            "value": {
              "intValue": "1"
            }
          },
          {
            "key": "process.runtime.description",
            "value": {
              "stringValue": "Eclipse Adoptium OpenJDK 64-Bit Server VM 21.0.2+13-LTS"
            }
          },
          {
            "key": "process.runtime.name",
            "value": {
              "stringValue": "OpenJDK Runtime Environment"
            }
          },
          {
            "key": "process.runtime.version",
            "value": {
              "stringValue": "21.0.2+13-LTS"
            }
          },
          {
            "key": "service.instance.id",
            "value": {
              "stringValue": "eff8a3d7-15f2-4718-94e5-6b984a9c18d7"
            }
          },
          {
            "key": "service.name",
            "value": {
              "stringValue": "kafka"
            }
          },
          {
            "key": "telemetry.distro.name",
            "value": {
              "stringValue": "opentelemetry-java-instrumentation"
            }
          },
          {
            "key": "telemetry.distro.version",
            "value": {
              "stringValue": "2.9.0"
            }
          },
          {
            "key": "telemetry.sdk.language",
            "value": {
              "stringValue": "java"
            }
          },
          {
            "key": "telemetry.sdk.name",
            "value": {
              "stringValue": "opentelemetry"
            }
          },
          {
            "key": "telemetry.sdk.version",
            "value": {
              "stringValue": "1.43.0"
            }
          }
        ]
      },
      "scopeLogs": [
        {
          "scope": {
            "name": "kafka.server.BrokerServer"
          },
          "logRecords": [
            {
              "timeUnixNano": "1735735228333091540",
              "observedTimeUnixNano": "1735735228333101306",
              "severityNumber": 9,
              "severityText": "INFO",
              "body": {
                "stringValue": "[BrokerServer id=1] Transition from STARTED to SHUTTING_DOWN"
              },
              "traceId": "",
              "spanId": ""
            }
          ]
        }
      ]
    }
  ]
}

Flattened JSON:

{
  "body": "[BrokerServer id=1] Transition from STARTED to SHUTTING_DOWN",
  "container.id": "61625241a33e64c6c34de99b1827a713b630f5548e715a87c6f8d052f1e3a2ab",
  "docker.cli.cobra.command_path": "docker compose",
  "flags": 0,
  "host.arch": "amd64",
  "host.name": "61625241a33e",
  "log_record_dropped_attributes_count": 0,
  "observable_time_unix_nano": 1735735228333101300,
  "os.description": "Linux 5.15.167.4-microsoft-standard-WSL2",
  "os.type": "linux",
  "p_metadata": "",
  "p_tags": "",
  "p_timestamp": "2025-01-01T23:15:01.661+05:30",
  "process.command_line": "/opt/java/openjdk/bin/java -Xmx400m -Xms400m -XX:SharedArchiveFile=/opt/kafka/kafka.jsa -Xlog:gc*:file=/opt/kafka/bin/../logs/kafkaServer-gc.log:time,tags:filecount=10,filesize=100M -Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dkafka.logs.dir=/opt/kafka/bin/../logs -Dlog4j.configuration=file:/opt/kafka/bin/../config/log4j.properties -javaagent:/tmp/opentelemetry-javaagent.jar -Dotel.jmx.target.system=kafka-broker kafka.Kafka /opt/kafka/config/server.properties",
  "process.executable.path": "/opt/java/openjdk/bin/java",
  "process.pid": "1",
  "process.runtime.description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 21.0.2+13-LTS",
  "process.runtime.name": "OpenJDK Runtime Environment",
  "process.runtime.version": "21.0.2+13-LTS",
  "resource_dropped_attributes_count": 0,
  "schema_url": "",
  "scope_dropped_attributes_count": 0,
  "scope_log_schema_url": "",
  "scope_name": "kafka.server.BrokerServer",
  "scope_version": "",
  "service.instance.id": "eff8a3d7-15f2-4718-94e5-6b984a9c18d7",
  "service.name": "kafka",
  "severity_number": 9,
  "severity_text": "SEVERITY_NUMBER_INFO",
  "span_id": "",
  "telemetry.distro.name": "opentelemetry-java-instrumentation",
  "telemetry.distro.version": "2.9.0",
  "telemetry.sdk.language": "java",
  "telemetry.sdk.name": "opentelemetry",
  "telemetry.sdk.version": "1.43.0",
  "time_unix_nano": 1735735228333091600,
  "trace_id": ""
}

All the resource attributes are stored as key-value pair where each attribute key becomes the field and the value, which may be of one of stringValue/intValue/doubleValue etc, becomes the value for that particular field.

Each log record in the incoming log event is stored as individual record in Parseable and all resource attributes, scope name, version etc which are at the scope level (parent of logRecords) are copied in each ingested record.

On this page