Merge pull request #110 from zhaixiaojuan/main

Add logstash:8.7.0

library/logstash/8.7.0/Dockerfile

@@ -0,0 +1,77 @@
+# This Dockerfile was generated from templates/Dockerfile.j2
+FROM cr.loongnix.cn/library/debian:buster-slim
+
+RUN for iter in {1..10}; do \
+export DEBIAN_FRONTEND=noninteractive && \
+apt-get update -y && \
+apt-get upgrade -y && \
+apt-get install -y procps findutils tar gzip curl && \
+apt-get install -y locales && \
+apt-get clean all && \
+locale-gen 'en_US.UTF-8' && \
+    apt-get clean metadata && \
+exit_code=0 && break || exit_code=$? && \
+    echo "packaging error: retry $iter in 10s" && \
+    apt-get clean all && \
+apt-get clean metadata && \
+sleep 10; done; \
+    (exit $exit_code)
+
+# Provide a non-root user to run the process.
+RUN groupadd --gid 1000 logstash && \
+    adduser --uid 1000 --gid 1000 \
+       --home /usr/share/logstash --no-create-home \
+      logstash
+
+# Add Logstash itself.
+RUN \
+curl -Lo - https://github.com/Loongson-Cloud-Community/logstash/releases/download/v8.7.0/logstash-8.7.0-loong64.tar.gz | \
+    tar zxf - -C /usr/share && \
+    mv /usr/share/logstash-8.7.0 /usr/share/logstash && \
+chown --recursive logstash:logstash /usr/share/logstash/ && \
+    chown -R logstash:root /usr/share/logstash && \
+    chmod -R g=u /usr/share/logstash && \
+    mkdir /licenses/ && \
+    mv /usr/share/logstash/NOTICE.TXT /licenses/NOTICE.TXT && \
+    mv /usr/share/logstash/LICENSE.txt /licenses/LICENSE.txt && \
+find /usr/share/logstash -type d -exec chmod g+s {} \; && \
+ln -s /usr/share/logstash /opt/logstash
+
+WORKDIR /usr/share/logstash
+ENV ELASTIC_CONTAINER true
+ENV PATH=/usr/share/logstash/bin:$PATH
+
+# Provide a minimal configuration, so that simple invocations will provide
+# a good experience.
+COPY config/pipelines.yml config/pipelines.yml
+COPY config/logstash-full.yml config/logstash.yml
+COPY config/log4j2.properties config/
+COPY config/log4j2.file.properties config/
+COPY pipeline/default.conf pipeline/logstash.conf
+RUN chown --recursive logstash:root config/ pipeline/
+# Ensure Logstash gets the correct locale by default.
+ENV LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8
+COPY env2yaml/env2yaml /usr/local/bin/
+# Place the startup wrapper script.
+COPY bin/docker-entrypoint /usr/local/bin/
+RUN chmod 0755 /usr/local/bin/docker-entrypoint
+
+USER 1000
+
+EXPOSE 9600 5044
+
+LABEL  org.label-schema.schema-version="1.0" \
+  org.label-schema.vendor="Elastic" \
+  org.opencontainers.image.vendor="Elastic" \
+  org.label-schema.name="logstash" \
+  org.opencontainers.image.title="logstash" \
+  org.label-schema.version="8.7.0" \
+  org.opencontainers.image.version="8.7.0" \
+  org.label-schema.url="https://www.elastic.co/products/logstash" \
+  org.label-schema.vcs-url="https://github.com/elastic/logstash" \
+  org.label-schema.license="Elastic License" \
+  org.opencontainers.image.licenses="Elastic License" \
+  org.opencontainers.image.description="Logstash is a free and open server-side data processing pipeline that ingests data from a multitude of sources, transforms it, and then sends it to your favorite 'stash.'" \
+  org.label-schema.build-date=2023-03-26T04:15:39+00:00 \
+org.opencontainers.image.created=2023-03-26T04:15:39+00:00
+ENTRYPOINT ["/usr/local/bin/docker-entrypoint"]

library/logstash/8.7.0/Makefile

@@ -0,0 +1,41 @@
+# This file is generated by the template.
+
+REGISTRY?=cr.loongnix.cn
+ORGANIZATION?=library
+REPOSITORY?=logstash
+TAG?=8.7.0
+LATEST?=true
+
+IMAGE=$(REGISTRY)/$(ORGANIZATION)/$(REPOSITORY):$(TAG)
+LATEST_IMAGE=$(REGISTRY)/$(ORGANIZATION)/$(REPOSITORY):latest
+LOGSTASH_REPOSITORY?=https://github.com/elastic/logstash.git
+
+default: image
+
+image: download
+	docker build \
+		--build-arg http_proxy=$(http_proxy) \
+		--build-arg https_proxy=$(https_proxy) \
+		-t $(IMAGE) \
+		.
+
+push:
+	docker push $(IMAGE)
+	#latest image
+	@if [ $(LATEST) = "true" ]; \
+		then \
+		docker tag $(IMAGE) $(LATEST_IMAGE); \
+		docker push $(LATEST_IMAGE); \
+	fi
+
+download:
+	if [ ! -d env2yaml ]; then \
+		mkdir env2yaml; \
+		git clone --branch v$(TAG) --depth=1 $(LOGSTASH_REPOSITORY); \
+		cd logstash/docker/data/logstash/env2yaml ; \
+		go env -w GO111MODULE=off && go get gopkg.in/yaml.v2 && go build; \
+		cp env2yaml ../../../../../env2yaml; \
+	fi
+
+clean:
+	rm -rf logstash

library/logstash/8.7.0/bin/docker-entrypoint

@@ -0,0 +1,31 @@
+#!/bin/bash -e
+
+# Map environment variables to entries in logstash.yml.
+# Note that this will mutate logstash.yml in place if any such settings are found.
+# This may be undesirable, especially if logstash.yml is bind-mounted from the
+# host system.
+env2yaml /usr/share/logstash/config/logstash.yml
+
+if [[ -n "$LOG_STYLE" ]]; then
+  case "$LOG_STYLE" in
+    console)
+      # This is the default. Nothing to do.
+      ;;
+    file)
+      # Overwrite the default config with the stack config. Do this as a
+      # copy, not a move, in case the container is restarted.
+      cp -f /usr/share/logstash/config/log4j2.file.properties /usr/share/logstash/config/log4j2.properties
+      ;;
+    *)
+      echo "ERROR: LOG_STYLE set to [$LOG_STYLE]. Expected [console] or [file]" >&2
+      exit 1 ;;
+  esac
+fi
+
+export LS_JAVA_OPTS="-Dls.cgroup.cpuacct.path.override=/ -Dls.cgroup.cpu.path.override=/ $LS_JAVA_OPTS"
+
+if [[ -z $1 ]] || [[ ${1:0:1} == '-' ]] ; then
+  exec logstash "$@"
+else
+  exec "$@"
+fi

library/logstash/8.7.0/config/log4j2.file.properties

@@ -0,0 +1,147 @@
+status = error
+name = LogstashPropertiesConfig
+
+appender.console.type = Console
+appender.console.name = plain_console
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c]%notEmpty{[%X{pipeline.id}]}%notEmpty{[%X{plugin.id}]} %m%n
+
+appender.json_console.type = Console
+appender.json_console.name = json_console
+appender.json_console.layout.type = JSONLayout
+appender.json_console.layout.compact = true
+appender.json_console.layout.eventEol = true
+
+appender.rolling.type = RollingFile
+appender.rolling.name = plain_rolling
+appender.rolling.fileName = ${sys:ls.logs}/logstash-plain.log
+appender.rolling.filePattern = ${sys:ls.logs}/logstash-plain-%d{yyyy-MM-dd}-%i.log.gz
+appender.rolling.policies.type = Policies
+appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
+appender.rolling.policies.time.interval = 1
+appender.rolling.policies.time.modulate = true
+appender.rolling.layout.type = PatternLayout
+appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c]%notEmpty{[%X{pipeline.id}]}%notEmpty{[%X{plugin.id}]} %m%n
+appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
+appender.rolling.policies.size.size = 100MB
+appender.rolling.strategy.type = DefaultRolloverStrategy
+appender.rolling.strategy.max = 30
+appender.rolling.avoid_pipelined_filter.type = PipelineRoutingFilter
+
+appender.json_rolling.type = RollingFile
+appender.json_rolling.name = json_rolling
+appender.json_rolling.fileName = ${sys:ls.logs}/logstash-json.log
+appender.json_rolling.filePattern = ${sys:ls.logs}/logstash-json-%d{yyyy-MM-dd}-%i.log.gz
+appender.json_rolling.policies.type = Policies
+appender.json_rolling.policies.time.type = TimeBasedTriggeringPolicy
+appender.json_rolling.policies.time.interval = 1
+appender.json_rolling.policies.time.modulate = true
+appender.json_rolling.layout.type = JSONLayout
+appender.json_rolling.layout.compact = true
+appender.json_rolling.layout.eventEol = true
+appender.json_rolling.policies.size.type = SizeBasedTriggeringPolicy
+appender.json_rolling.policies.size.size = 100MB
+appender.json_rolling.strategy.type = DefaultRolloverStrategy
+appender.json_rolling.strategy.max = 30
+appender.json_rolling.avoid_pipelined_filter.type = PipelineRoutingFilter
+
+appender.routing.type = PipelineRouting
+appender.routing.name = pipeline_routing_appender
+appender.routing.pipeline.type = RollingFile
+appender.routing.pipeline.name = appender-${ctx:pipeline.id}
+appender.routing.pipeline.fileName = ${sys:ls.logs}/pipeline_${ctx:pipeline.id}.log
+appender.routing.pipeline.filePattern = ${sys:ls.logs}/pipeline_${ctx:pipeline.id}.%i.log.gz
+appender.routing.pipeline.layout.type = PatternLayout
+appender.routing.pipeline.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %m%n
+appender.routing.pipeline.policy.type = SizeBasedTriggeringPolicy
+appender.routing.pipeline.policy.size = 100MB
+appender.routing.pipeline.strategy.type = DefaultRolloverStrategy
+appender.routing.pipeline.strategy.max = 30
+
+rootLogger.level = ${sys:ls.log.level}
+rootLogger.appenderRef.console.ref = ${sys:ls.log.format}_console
+rootLogger.appenderRef.rolling.ref = ${sys:ls.log.format}_rolling
+rootLogger.appenderRef.routing.ref = pipeline_routing_appender
+
+# Slowlog
+
+appender.console_slowlog.type = Console
+appender.console_slowlog.name = plain_console_slowlog
+appender.console_slowlog.layout.type = PatternLayout
+appender.console_slowlog.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %m%n
+
+appender.json_console_slowlog.type = Console
+appender.json_console_slowlog.name = json_console_slowlog
+appender.json_console_slowlog.layout.type = JSONLayout
+appender.json_console_slowlog.layout.compact = true
+appender.json_console_slowlog.layout.eventEol = true
+
+appender.rolling_slowlog.type = RollingFile
+appender.rolling_slowlog.name = plain_rolling_slowlog
+appender.rolling_slowlog.fileName = ${sys:ls.logs}/logstash-slowlog-plain.log
+appender.rolling_slowlog.filePattern = ${sys:ls.logs}/logstash-slowlog-plain-%d{yyyy-MM-dd}-%i.log.gz
+appender.rolling_slowlog.policies.type = Policies
+appender.rolling_slowlog.policies.time.type = TimeBasedTriggeringPolicy
+appender.rolling_slowlog.policies.time.interval = 1
+appender.rolling_slowlog.policies.time.modulate = true
+appender.rolling_slowlog.layout.type = PatternLayout
+appender.rolling_slowlog.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %m%n
+appender.rolling_slowlog.policies.size.type = SizeBasedTriggeringPolicy
+appender.rolling_slowlog.policies.size.size = 100MB
+appender.rolling_slowlog.strategy.type = DefaultRolloverStrategy
+appender.rolling_slowlog.strategy.max = 30
+
+appender.json_rolling_slowlog.type = RollingFile
+appender.json_rolling_slowlog.name = json_rolling_slowlog
+appender.json_rolling_slowlog.fileName = ${sys:ls.logs}/logstash-slowlog-json.log
+appender.json_rolling_slowlog.filePattern = ${sys:ls.logs}/logstash-slowlog-json-%d{yyyy-MM-dd}-%i.log.gz
+appender.json_rolling_slowlog.policies.type = Policies
+appender.json_rolling_slowlog.policies.time.type = TimeBasedTriggeringPolicy
+appender.json_rolling_slowlog.policies.time.interval = 1
+appender.json_rolling_slowlog.policies.time.modulate = true
+appender.json_rolling_slowlog.layout.type = JSONLayout
+appender.json_rolling_slowlog.layout.compact = true
+appender.json_rolling_slowlog.layout.eventEol = true
+appender.json_rolling_slowlog.policies.size.type = SizeBasedTriggeringPolicy
+appender.json_rolling_slowlog.policies.size.size = 100MB
+appender.json_rolling_slowlog.strategy.type = DefaultRolloverStrategy
+appender.json_rolling_slowlog.strategy.max = 30
+
+logger.slowlog.name = slowlog
+logger.slowlog.level = trace
+logger.slowlog.appenderRef.console_slowlog.ref = ${sys:ls.log.format}_console_slowlog
+logger.slowlog.appenderRef.rolling_slowlog.ref = ${sys:ls.log.format}_rolling_slowlog
+logger.slowlog.additivity = false
+
+logger.licensereader.name = logstash.licensechecker.licensereader
+logger.licensereader.level = error
+
+# Silence http-client by default
+logger.apache_http_client.name = org.apache.http
+logger.apache_http_client.level = fatal
+
+# Deprecation log
+appender.deprecation_rolling.type = RollingFile
+appender.deprecation_rolling.name = deprecation_plain_rolling
+appender.deprecation_rolling.fileName = ${sys:ls.logs}/logstash-deprecation.log
+appender.deprecation_rolling.filePattern = ${sys:ls.logs}/logstash-deprecation-%d{yyyy-MM-dd}-%i.log.gz
+appender.deprecation_rolling.policies.type = Policies
+appender.deprecation_rolling.policies.time.type = TimeBasedTriggeringPolicy
+appender.deprecation_rolling.policies.time.interval = 1
+appender.deprecation_rolling.policies.time.modulate = true
+appender.deprecation_rolling.layout.type = PatternLayout
+appender.deprecation_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c]%notEmpty{[%X{pipeline.id}]}%notEmpty{[%X{plugin.id}]} %m%n
+appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy
+appender.deprecation_rolling.policies.size.size = 100MB
+appender.deprecation_rolling.strategy.type = DefaultRolloverStrategy
+appender.deprecation_rolling.strategy.max = 30
+
+logger.deprecation.name = org.logstash.deprecation, deprecation
+logger.deprecation.level = WARN
+logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_plain_rolling
+logger.deprecation.additivity = false
+
+logger.deprecation_root.name = deprecation
+logger.deprecation_root.level = WARN
+logger.deprecation_root.appenderRef.deprecation_rolling.ref = deprecation_plain_rolling
+logger.deprecation_root.additivity = false

library/logstash/8.7.0/config/log4j2.properties

@@ -0,0 +1,16 @@
+status = error
+name = LogstashPropertiesConfig
+
+appender.console.type = Console
+appender.console.name = plain_console
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c]%notEmpty{[%X{pipeline.id}]}%notEmpty{[%X{plugin.id}]} %m%n
+
+appender.json_console.type = Console
+appender.json_console.name = json_console
+appender.json_console.layout.type = JSONLayout
+appender.json_console.layout.compact = true
+appender.json_console.layout.eventEol = true
+
+rootLogger.level = ${sys:ls.log.level}
+rootLogger.appenderRef.console.ref = ${sys:ls.log.format}_console

library/logstash/8.7.0/config/logstash-full.yml

@@ -0,0 +1,2 @@
+http.host: "0.0.0.0"
+xpack.monitoring.elasticsearch.hosts: [ "http://elasticsearch:9200" ]

library/logstash/8.7.0/config/logstash-oss.yml

@@ -0,0 +1 @@
+http.host: "0.0.0.0"

library/logstash/8.7.0/config/pipelines.yml

@@ -0,0 +1,6 @@
+# This file is where you define your pipelines. You can define multiple.
+# For more information on multiple pipelines, see the documentation:
+#   https://www.elastic.co/guide/en/logstash/current/multiple-pipelines.html
+
+- pipeline.id: main
+  path.config: "/usr/share/logstash/pipeline"

library/logstash/8.7.0/pipeline/default.conf

@@ -0,0 +1,12 @@
+input {
+  beats {
+    port => 5044
+  }
+}
+
+output {
+  stdout {
+    codec => rubydebug
+  }
+}
+