Skip to content

Commit

Permalink
Merge branch 'fluent:master' into feature/out_es_upstream_support_ext…
Browse files Browse the repository at this point in the history
…ended
  • Loading branch information
mabrarov authored Dec 20, 2024
2 parents 7ee017f + 44a2d0d commit c4d46ec
Show file tree
Hide file tree
Showing 2,058 changed files with 58,239 additions and 28,270 deletions.
6 changes: 3 additions & 3 deletions .github/workflows/call-build-linux-packages.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ jobs:
call-build-capture-source:
# Capture source tarball and generate checksum for it
name: Extract any supporting metadata
runs-on: ubuntu-latest
runs-on: ubuntu-22.04
environment: ${{ inputs.environment }}
permissions:
contents: read
Expand Down Expand Up @@ -105,8 +105,8 @@ jobs:
call-build-linux-packages:
name: ${{ matrix.distro }} package build and stage to S3
environment: ${{ inputs.environment }}
# Ensure for OSS Fluent Bit repo we enable usage of Actuated runners for ARM builds, for forks it should keep existing ubuntu-latest usage.
runs-on: ${{ (contains(matrix.distro, 'arm' ) && (github.repository == 'fluent/fluent-bit') && 'actuated-arm64-8cpu-16gb') || 'ubuntu-latest' }}
# Ensure for OSS Fluent Bit repo we enable usage of Actuated runners for ARM builds, for forks it should keep existing ubuntu-22.04 usage.
runs-on: ${{ (contains(matrix.distro, 'arm' ) && (github.repository == 'fluent/fluent-bit') && 'actuated-arm64-8cpu-16gb') || 'ubuntu-22.04' }}
permissions:
contents: read
strategy:
Expand Down
4 changes: 0 additions & 4 deletions .github/workflows/call-build-macos.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -74,8 +74,6 @@ jobs:
fail-fast: false
matrix:
config:
- name: "Normal macOS-latest runner (Intel)"
runner: macos-12
- name: "Apple Silicon macOS runner"
runner: macos-14

Expand Down Expand Up @@ -121,8 +119,6 @@ jobs:
fail-fast: false
matrix:
config:
- name: "Normal macOS-latest package (Intel)"
os: macos-12
- name: "Apple Silicon macOS package"
os: macos-14

Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/call-run-integration-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -192,7 +192,7 @@ jobs:
bats-version: 1.9.0

- name: Create k8s Kind Cluster
uses: helm/kind-action@v1.10.0
uses: helm/kind-action@v1.11.0
with:
node_image: kindest/node:${{ matrix.k8s-release }}
cluster_name: kind
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/call-test-images.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@ jobs:
ref: ${{ inputs.ref }}

- name: Create k8s Kind Cluster
uses: helm/kind-action@v1.10.0
uses: helm/kind-action@v1.11.0

- name: Set up Helm
uses: azure/setup-helm@v4
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/staging-release.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@ jobs:

staging-release-apt-packages:
name: S3 - update APT packages bucket
runs-on: ubuntu-latest
runs-on: ubuntu-22.04
environment: release
needs:
- staging-release-version-check
Expand Down Expand Up @@ -327,7 +327,7 @@ jobs:

staging-release-source-s3:
name: S3 - update source bucket
runs-on: ubuntu-latest
runs-on: ubuntu-22.04
environment: release
needs:
- staging-release-version-check
Expand Down
11 changes: 5 additions & 6 deletions .github/workflows/unit-tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,7 @@ jobs:
matrix:
arch:
- s390x
- riscv64
steps:
- name: Checkout Fluent Bit code
uses: actions/checkout@v4
Expand All @@ -188,19 +189,17 @@ jobs:
id: build-and-test-on-qemu
with:
arch: ${{ matrix.arch }}
distro: ubuntu20.04
distro: ubuntu22.04
shell: /bin/bash
dockerRunArgs: |
--volume "/var/lib/dbus/machine-id:/var/lib/dbus/machine-id"
--volume "/etc/machine-id:/etc/machine-id"
install: |
apt-get update
apt-get install -y gcc-7 g++-7 clang-6.0 libyaml-dev cmake flex bison libssl-dev libbpf-dev linux-tools-common#libsystemd-dev
ln -s /usr/bin/llvm-symbolizer-6.0 /usr/bin/llvm-symbolizer || true
apt-get install -y gcc-12 g++-12 libyaml-dev cmake flex bison libssl-dev libbpf-dev linux-tools-common
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 90
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 90
update-alternatives --install /usr/bin/clang clang /usr/bin/clang-6.0 90
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 90
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 90
run: |
cd build
export nparallel=$(( $(getconf _NPROCESSORS_ONLN) > 8 ? 8 : $(getconf _NPROCESSORS_ONLN) ))
Expand Down
5 changes: 5 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -321,6 +321,11 @@ if (FLB_SYSTEM_LINUX)
include(cmake/s390x.cmake)
endif ()

# Build for Linux - riscv64 arch
if (FLB_SYSTEM_LINUX)
include(cmake/riscv64.cmake)
endif ()

# Enable signed char support on Linux AARCH64 if specified
if (FLB_LINUX_ON_AARCH64)
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsigned-char")
Expand Down
8 changes: 3 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,14 @@

![](documentation/fluentbit_ecosystem.png)

[Fluent Bit](http://fluentbit.io) is a fast Log Processor and Forwarder for Linux, Windows, Embedded Linux, MacOS and BSD family operating systems. It's part of the Graduated [Fluentd](http://fluentd.org) Ecosystem and a [CNCF](https://cncf.io) sub-project.
[Fluent Bit](http://fluentbit.io) is a fast Log, Metrics and Traces Processor and Forwarder for Linux, Windows, Embedded Linux, MacOS and BSD family operating systems. It's part of the Graduated [Fluentd](http://fluentd.org) Ecosystem and a [CNCF](https://cncf.io) sub-project.

Fluent Bit allows to collect log events or metrics from different sources, process them and deliver them to different backends such as [Fluentd](http://fluentd.org), Elasticsearch, Splunk, DataDog, Kafka, New Relic, Azure services, AWS services, Google services, NATS, InfluxDB or any custom HTTP end-point.
Fluent Bit allows to collect different signal types such as logs, metrics and traces from different sources, process them and deliver them to different backends such as [Fluentd](http://fluentd.org), Elasticsearch, Splunk, DataDog, Kafka, New Relic, Azure services, AWS services, Google services, NATS, InfluxDB or any custom HTTP end-point.

Fluent Bit comes with full SQL [Stream Processing](https://docs.fluentbit.io/manual/stream-processing/introduction) capabilities: data manipulation and analytics using SQL queries.

Fluent Bit runs on x86_64, x86, arm32v7, and arm64v8 architectures.



## Features

- High Performance at low CPU and Memory footprint
Expand All @@ -48,7 +46,7 @@ Fluent Bit runs on x86_64, x86, arm32v7, and arm64v8 architectures.

## Fluent Bit in Production

[Fluent Bit](https://fluentbit.io) is used widely in production environments. As of 2022, [Fluent Bit surpasses 3 Billion downloads](https://www.cncf.io/blog/2022/10/13/fluent-bit-surpasses-three-billion-downloads/) and continues to be deployed over **10 million times a day**. The following is a preview of who uses Fluent Bit heavily in production:
Fluent Bit is a widely adopted solution in production environments. As of 2024, Fluent Bit has surpassed 15 billion downloads and continues to be deployed over 10 million times daily. Below is a preview of some of the organizations that rely heavily on Fluent Bit in their production systems:

> If your company uses Fluent Bit and is not listed, feel free to open a GitHub issue and we will add the logo.
Expand Down
1 change: 1 addition & 0 deletions cmake/headers.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ include_directories(

${CMAKE_CURRENT_BINARY_DIR}/${FLB_PATH_LIB_JANSSON}/include
${CMAKE_CURRENT_BINARY_DIR}/lib/cmetrics
${CMAKE_CURRENT_BINARY_DIR}/lib/cprofiles/include
${CMAKE_CURRENT_BINARY_DIR}/include

${CMAKE_CURRENT_BINARY_DIR}/lib/monkey/include/
Expand Down
4 changes: 2 additions & 2 deletions cmake/libraries.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ set(FLB_PATH_LIB_ONIGMO "lib/onigmo")
set(FLB_PATH_LIB_MPACK "lib/mpack-amalgamation-1.1.1")
set(FLB_PATH_LIB_MINIZ "lib/miniz")
set(FLB_PATH_LIB_TUTF8E "lib/tutf8e")
set(FLB_PATH_LIB_CARES "lib/c-ares-1.33.1")
set(FLB_PATH_LIB_CARES "lib/c-ares-1.34.4")
set(FLB_PATH_LIB_SNAPPY "lib/snappy-fef67ac")
set(FLB_PATH_LIB_RDKAFKA "lib/librdkafka-2.4.0")
set(FLB_PATH_LIB_RING_BUFFER "lib/lwrb")
set(FLB_PATH_LIB_WASM_MICRO_RUNTIME "lib/wasm-micro-runtime-WAMR-1.3.0")
set(FLB_PATH_LIB_WASM_MICRO_RUNTIME "lib/wasm-micro-runtime-WAMR-1.3.3")
8 changes: 8 additions & 0 deletions cmake/riscv64.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(riscv64)")
message(STATUS "Forcing characters to be signed, as on x86_64.")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsigned-char")
if(FLB_LUAJIT)
message(WARNING "LuaJIT is disabled, this platform does not support built-in LuaJIT and system provided one neither.")
set(FLB_LUAJIT OFF)
endif()
endif ()
Binary file modified documentation/fluentbit_users.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
74 changes: 74 additions & 0 deletions include/fluent-bit/flb_cfl_ra_key.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */

/* Fluent Bit
* ==========
* Copyright (C) 2015-2024 The Fluent Bit Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#ifndef FLB_CFL_RA_KEY_H
#define FLB_CFL_RA_KEY_H

#include <fluent-bit/flb_info.h>
#include <fluent-bit/flb_sds.h>
#include <fluent-bit/flb_regex.h>
#include <fluent-bit/record_accessor/flb_ra_parser.h>
#include <monkey/mk_core.h>
#include <cfl/cfl.h>

#include <stdbool.h>

enum cfl_ra_types {
FLB_CFL_RA_BOOL = 0,
FLB_CFL_RA_INT,
FLB_CFL_RA_FLOAT,
FLB_CFL_RA_STRING,
FLB_CFL_RA_NULL
};

/* condition value types */
typedef union {
bool boolean;
int64_t i64;
double f64;
flb_sds_t string;
} cfl_ra_val;

/* Represent any value object */
struct flb_cfl_ra_value {
int type;
struct cfl_variant v;
cfl_ra_val val;
};

struct flb_cfl_ra_value *flb_cfl_ra_key_to_value(flb_sds_t ckey,
struct cfl_variant vobj,
struct mk_list *subkeys);
void flb_cfl_ra_key_value_destroy(struct flb_cfl_ra_value *v);

int flb_cfl_ra_key_value_get(flb_sds_t ckey, struct cfl_variant vobj,
struct mk_list *subkeys,
cfl_sds_t *start_key,
cfl_sds_t *out_key, struct cfl_variant **out_val);

int flb_cfl_ra_key_strcmp(flb_sds_t ckey, struct cfl_variant vobj,
struct mk_list *subkeys, char *str, int len);
int flb_cfl_ra_key_regex_match(flb_sds_t ckey, struct cfl_variant vobj,
struct mk_list *subkeys, struct flb_regex *regex,
struct flb_regex_search *result);
int flb_cfl_ra_key_value_append(struct flb_ra_parser *rp, struct cfl_variant *vobj,
struct cfl_variant *in_val);
int flb_cfl_ra_key_value_update(struct flb_ra_parser *rp, struct cfl_variant *vobj,
cfl_sds_t in_key, struct cfl_variant *in_val);
#endif
64 changes: 64 additions & 0 deletions include/fluent-bit/flb_cfl_record_accessor.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */

/* Fluent Bit
* ==========
* Copyright (C) 2015-2024 The Fluent Bit Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#ifndef FLB_CFL_RECORD_ACCESSOR_H
#define FLB_CFL_RECORD_ACCESSOR_H

#include <fluent-bit/flb_info.h>
#include <fluent-bit/flb_regex.h>
#include <fluent-bit/flb_sds.h>
#include <fluent-bit/flb_sds_list.h>
#include <monkey/mk_core.h>
#include <cfl/cfl.h>

struct flb_cfl_record_accessor {
size_t size_hint;
flb_sds_t pattern;
struct mk_list list; /* List of parsed strings */
struct mk_list _head; /* Head to custom list (only used by flb_mp.h) */
};
void flb_cfl_ra_destroy(struct flb_cfl_record_accessor *cra);
int flb_cfl_ra_subkey_count(struct flb_cfl_record_accessor *cra);
struct flb_cfl_record_accessor *flb_cfl_ra_create(char *str, int translate_env);
flb_sds_t flb_cfl_ra_create_str_from_list(struct flb_sds_list *str_list);
struct flb_cfl_record_accessor *flb_cfl_ra_create_from_list(struct flb_sds_list *str_list, int translate_env);
flb_sds_t flb_cfl_ra_translate(struct flb_cfl_record_accessor *cra,
char *tag, int tag_len,
struct cfl_variant var, struct flb_regex_search *result);
flb_sds_t flb_cfl_ra_translate_check(struct flb_cfl_record_accessor *cra,
char *tag, int tag_len,
struct cfl_variant var, struct flb_regex_search *result,
int check);
void flb_cfl_ra_dump(struct flb_cfl_record_accessor *cra);
int flb_cfl_ra_is_static(struct flb_cfl_record_accessor *cra);
int flb_cfl_ra_strcmp(struct flb_cfl_record_accessor *ra, struct cfl_variant var,
char *str, int len);
int flb_cfl_ra_regex_match(struct flb_cfl_record_accessor *cra, struct cfl_variant var,
struct flb_regex *regex, struct flb_regex_search *result);
int flb_cfl_ra_get_kv_pair(struct flb_cfl_record_accessor *ra,
struct cfl_variant var,
cfl_sds_t *start_key,
cfl_sds_t *out_key, struct cfl_variant **out_val);
struct flb_cfl_ra_value *flb_cfl_ra_get_value_object(struct flb_cfl_record_accessor *cra,
struct cfl_variant var);
int flb_cfl_ra_update_kv_pair(struct flb_cfl_record_accessor *cra, struct cfl_variant var,
cfl_sds_t in_key, struct cfl_variant *in_val);
int flb_cfl_ra_append_kv_pair(struct flb_cfl_record_accessor *cra, struct cfl_variant var,
struct cfl_variant *in_val);
#endif
Loading

0 comments on commit c4d46ec

Please sign in to comment.