log_format.hh

/**
 * Copyright (c) 2007-2012, Timothy Stack
 *
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * * Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer.
 * * Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 * * Neither the name of Timothy Stack nor the names of its contributors
 * may be used to endorse or promote products derived from this software
 * without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * @file log_format.hh
 */

#ifndef log_format_hh
#define log_format_hh

#include <stdint.h>
#include <sys/time.h>
#include <time.h>
#define __STDC_FORMAT_MACROS
#include <limits>
#include <memory>
#include <set>
#include <string>
#include <utility>
#include <vector>

#include <inttypes.h>
#include <sys/types.h>

#include "base/date_time_scanner.hh"
#include "base/intern_string.hh"
#include "base/lnav_log.hh"
#include "base/log_level_enum.hh"
#include "highlighter.hh"
#include "line_buffer.hh"
#include "log_format_fwd.hh"
#include "logfile.hh"
#include "pcrepp/pcre2pp.hh"
#include "robin_hood/robin_hood.h"
#include "shared_buffer.hh"

struct sqlite3;
class logfile;
class log_vtab_manager;
struct exec_context;

struct logline_value_name_cmp {
    explicit logline_value_name_cmp(const intern_string_t* name)
        : lvc_name(name)
    {
    }

    bool operator()(const logline_value& lv) const
    {
        return (*this->lvc_name) == lv.lv_meta.lvm_name;
    }

    const intern_string_t* lvc_name;
};

struct logline_value_col_eq {
    explicit logline_value_col_eq(const logline_value_meta::table_column& col)
        : lvc_column(col)
    {
    }

    bool operator()(const logline_value& lv) const
    {
        if (lv.lv_meta.lvm_column.is<logline_value_meta::table_column>()) {
            return this->lvc_column
                == lv.lv_meta.lvm_column
                       .get<logline_value_meta::table_column>();
        }
        return false;
    }

    const logline_value_meta::table_column& lvc_column;
};

class log_vtab_impl;

/**
 * Base class for implementations of log format parsers.
 */
class log_format : public std::enable_shared_from_this<log_format> {
public:
    /**
     * @return The collection of builtin log formats.
     */
    static std::vector<std::shared_ptr<log_format>>& get_root_formats();

    static std::shared_ptr<log_format> find_root_format(const char* name);

    struct sample_t {
        positioned_property<std::string> s_line;
        std::string s_description;
        log_level_t s_level{LEVEL_UNKNOWN};
        std::set<std::string> s_matched_regexes;
    };

    struct action_def {
        std::string ad_name;
        std::string ad_label;
        std::vector<std::string> ad_cmdline;
        bool ad_capture_output{false};

        bool operator<(const action_def& rhs) const
        {
            return this->ad_name < rhs.ad_name;
        }
    };

    virtual ~log_format() = default;

    /**
     * Reset whatever a previous file left on this object, before it is tried
     * as a candidate for another one.  Only the state that has not been moved
     * onto scan_batch_context yet needs this -- the fields a subclass
     * discovers from the file it is reading, such as the column names in a
     * header.
     */
    virtual void clear() {}

    /**
     * Get the name of this log format.
     *
     * @return The log format name.
     */
    virtual const intern_string_t get_name() const = 0;

    struct name_matched {};
    struct name_mismatched {
        size_t nm_partial;
        std::string nm_pattern;
    };

    using match_name_result
        = mapbox::util::variant<name_matched, name_mismatched>;

    virtual match_name_result match_name(const std::string& filename)
    {
        return name_matched{};
    }

    using scan_match = log_format_scan_match;

    struct scan_error {
        std::string se_message;
    };

    struct scan_no_match {
        const char* snm_reason{nullptr};
    };

    struct scan_incomplete {};

    using scan_result_t = mapbox::util::
        variant<scan_match, scan_no_match, scan_error, scan_incomplete>;

    virtual scan_result_t test_line(
        sample_t& sample, std::vector<lnav::console::user_message>& msgs);

    /**
     * Scan a log line to see if it matches this log format.
     *
     * @param dst The vector of loglines that the formatter should append to
     *   if it detected a match.
     * @param offset The offset in the file where this line is located.
     * @param prefix The contents of the line.
     * @param len The length of the prefix string.
     */
    virtual scan_result_t scan(logfile& lf,
                               std::vector<logline>& dst,
                               const line_info& li,
                               shared_buffer_ref& sbr,
                               scan_batch_context& sbc) = 0;

    virtual bool scan_for_partial(const log_format_file_state& lffs,
                                  shared_buffer_ref& sbr,
                                  size_t& len_out) const
    {
        return false;
    }

    /**
     * Remove redundant data from the log line string.
     *
     * XXX We should probably also add some attributes to the line here, so we
     * can highlight things like the date.
     *
     * @param line The log line to edit.
     */
    virtual void scrub(std::string& line) {}

    virtual void annotate(logfile* lf,
                          uint64_t line_number,
                          string_attrs_t& sa,
                          logline_value_vector& values) const;

    virtual void rewrite(exec_context& ec,
                         shared_buffer_ref& line,
                         string_attrs_t& sa,
                         std::string& value_out)
    {
        value_out.assign(line.get_data(), line.length());
    }

    virtual std::optional<size_t> stats_index_for_value(
        const intern_string_t& name) const
    {
        return std::nullopt;
    }

    virtual std::shared_ptr<log_format> specialized(scan_batch_context& sbc,
                                                    int fmt_lock = -1) = 0;

    /**
     * @return Scratch for this format to discover into while it is a
     * candidate, or null if it discovers nothing from the file.
     * @see format_scan_state
     */
    virtual std::unique_ptr<format_scan_state> make_scan_state() const
    {
        return nullptr;
    }

    /**
     * Take over what was discovered while this format's root was the winning
     * candidate.  Called on the specialized copy, just after specialized().
     */
    virtual void adopt_scan_state(format_scan_state& fss) {}

    /**
     * The timestamp flags this scan should accumulate into.  A specialized
     * copy belongs to one file and keeps its own; a root is shared by every
     * file being probed against it, so a candidate accumulates into the
     * batch and process_prefix hands the winner's flags to the copy.
     */
    uint32_t& timestamp_flags_for(scan_batch_context& sbc)
    {
        return this->lf_specialized ? this->lf_timestamp_flags
                                    : sbc.sbc_timestamp_flags;
    }

    virtual std::shared_ptr<log_vtab_impl> get_vtab_impl() const
    {
        return nullptr;
    }

    /**
     * Rewrites sbr, the message containing ll, to the line that ll is:
     * its sub_line'th (logfile::sub_line_index()) of the lines the format
     * renders the message into.
     */
    virtual void get_subline(const log_format_file_state& lffs,
                             const logline& ll,
                             size_t sub_line,
                             shared_buffer_ref& sbr,
                             subline_options opts = subline_options{})
    {
    }

    /**
     * @return True if get_subline() already has the rendered form of the
     * message containing the given line, so the caller can skip reading the
     * raw message from the file.
     */
    virtual bool has_cached_subline(const logline& ll,
                                    subline_options opts) const
    {
        return false;
    }

    virtual const std::vector<std::string>* get_actions(
        const logline_value& lv) const
    {
        return nullptr;
    }

    virtual std::set<std::string> get_source_path() const
    {
        std::set<std::string> retval;

        retval.insert("default");

        return retval;
    }

    virtual bool hide_field(const intern_string_t field_name, bool val)
    {
        return false;
    }

    virtual std::map<intern_string_t, logline_value_meta> get_field_states()
    {
        return {};
    }

    // Drop user-controlled field visibility state (set via
    // `:hide-fields`/`:show-fields`) so a fresh session starts
    // without inherited hide state.  Called from `reset_session()`.
    // The default implementation walks every known field and marks it
    // visible; formats with additional hide state (e.g. the static
    // registry `metric_log_format` uses for cross-instance columns)
    // are already covered because `get_field_states` includes those
    // and `hide_field` routes the update through the same channel.
    virtual void reset_user_field_state()
    {
        for (const auto& [name, _] : this->get_field_states()) {
            this->hide_field(name, false);
        }
    }

    const char* const* get_timestamp_formats() const
    {
        if (this->lf_timestamp_format.empty()) {
            return nullptr;
        }

        return &this->lf_timestamp_format[0];
    }

    /**
     * @param file_dts The scanner belonging to the file the value came from,
     * i.e. logfile::get_time_scanner().
     * @return A scratch scanner carrying that file's base time and zone, for
     * re-parsing a timestamp-valued field.  It has to come from the file: the
     * format's own lf_date_time only holds what the format definition set.
     */
    date_time_scanner build_time_scanner(
        const date_time_scanner& file_dts) const;

    void check_for_new_year(std::vector<logline>& dst,
                            exttm log_tv,
                            timeval timeval1,
                            scan_batch_context& sbc) const;

    virtual std::string get_pattern_path(const pattern_locks& pl,
                                         uint64_t line_number) const;

    virtual intern_string_t get_pattern_name(const pattern_locks& pl,
                                             uint64_t line_number) const;

    virtual std::string get_pattern_regex(const pattern_locks& pl,
                                          uint64_t line_number) const
    {
        return "";
    }

    // The number of parsed-value columns a row of this format
    // produces.  Default defers to `get_value_metadata()` (and eats
    // the vector copy); formats that know the count cheaply — e.g.
    // `metric_log_format` — should override so hot callers can skip
    // loading the row just to size its columns.
    virtual size_t get_value_metadata_count() const
    {
        return this->get_value_metadata().size();
    }

    virtual std::vector<logline_value_meta> get_value_metadata() const
    {
        return {};
    }

    virtual bool format_changed() { return false; }

    bool operator<(const log_format& rhs) const
    {
        return this->get_name() < rhs.get_name();
    }

    static bool name_lt(const std::shared_ptr<const log_format>& lhs,
                        const std::shared_ptr<const log_format>& rhs)
    {
        return intern_string_t::case_lt(lhs->get_name(), rhs->get_name());
    }

    exttm tm_for_display(logfile::iterator ll,
                         string_fragment sf,
                         date_time_scanner& dts);

    enum class subsecond_unit {
        milli,
        micro,
        nano,
    };

    /**
     * The shape of the file that contains the log messages.
     */
    enum class file_type_t {
        TEXT,
        JSON,
        TABULAR,
    };

    /**
     * The bit for a file type, for use in a set of file types.
     */
    static constexpr uint8_t file_type_bit(file_type_t ft)
    {
        return 1 << static_cast<uint8_t>(ft);
    }

    std::string lf_description;
    log_format* lf_root_format{this};
    bool lf_multiline{true};
    bool lf_structured{false};
    bool lf_formatted_lines{false};
    file_type_t lf_file_type{file_type_t::TEXT};
    date_time_scanner lf_date_time;
    intern_string_t lf_timestamp_field{intern_string::lookup("timestamp", -1)};
    intern_string_t lf_start_timestamp_field;
    intern_string_t lf_subsecond_field;
    std::optional<subsecond_unit> lf_subsecond_unit;
    intern_string_t lf_time_field;
    std::vector<const char*> lf_timestamp_format;
    uint32_t lf_timestamp_flags{0};
    timestamp_point_of_reference_t lf_timestamp_point_of_reference{
        timestamp_point_of_reference_t::end};
    factory_container<lnav::pcre2pp::code> lf_timestamp_anchor_pattern;
    intern_string_t lf_timestamp_anchor_format;
    std::map<std::string, action_def> lf_action_defs;
    std::vector<highlighter> lf_highlighters;
    bool lf_is_self_describing{false};
    bool lf_time_ordered{true};
    bool lf_specialized{false};
    bool lf_level_hideable{true};
    // Flags a format whose loglines are metric samples (one numeric
    // value per column, timestamped).  The logfile_sub_source uses
    // this to compose a cross-file `col=value` rendering in the LOG
    // view, fold sibling rows that share a timestamp, and drive the
    // stem-labeled overlay above the focused row.
    bool lf_is_metric{false};
    /**
     * Whether the format declares both a source file and a source line
     * field, which give an unmatched message's log_msg_schema
     * (all_logs_vtab).  Without them, it's the message's text.
     */
    bool lf_has_src_location{false};
    /**
     * Whether the format's messages can have a body (SA_BODY): text that
     * isn't parsed into fields.  Without one, the format has parsed every
     * part of the message, so there's no text to match to a log statement
     * or to parse for a message format.
     */
    bool lf_has_body{false};
    std::optional<uint64_t> lf_max_unrecognized_lines;
    std::map<const intern_string_t, std::shared_ptr<format_tag_def>>
        lf_tag_defs;

    std::map<const intern_string_t, std::shared_ptr<format_partition_def>>
        lf_partition_defs;

    struct opid_descriptor {
        positioned_property<intern_string_t> od_field;
        factory_container<lnav::pcre2pp::code> od_extractor;
        std::optional<std::string> od_prefix;
        std::string od_suffix;
        std::string od_joiner{", "};

        std::optional<std::string> matches(const string_fragment& sf) const;
    };

    struct opid_descriptors {
        intern_string_t od_name;
        std::shared_ptr<std::vector<opid_descriptor>> od_descriptors;
        uint16_t od_index{0};

        std::string to_string(
            const lnav::map::small<size_t, std::string>& lod) const;
    };

    enum class opid_source_t {
        from_field,
        from_description,
        from_whole_msg,
    };

    std::optional<opid_source_t> lf_opid_source{};

    std::shared_ptr<std::map<intern_string_t, opid_descriptors>>
        lf_opid_description_def{
            std::make_shared<std::map<intern_string_t, opid_descriptors>>()};

    std::shared_ptr<std::map<intern_string_t, opid_descriptors>>
        lf_subid_description_def{
            std::make_shared<std::map<intern_string_t, opid_descriptors>>()};

    std::shared_ptr<std::vector<opid_descriptors*>> lf_opid_description_def_vec{
        std::make_shared<std::vector<opid_descriptors*>>()};

    std::shared_ptr<std::vector<opid_descriptors*>>
        lf_subid_description_def_vec{
            std::make_shared<std::vector<opid_descriptors*>>()};

    ArenaAlloc::Alloc<char> lf_desc_allocator{2 * 1024};

    using desc_field_set
        = robin_hood::unordered_set<intern_string_t,
                                    intern_hasher,
                                    std::equal_to<intern_string_t>>;

    desc_field_set lf_desc_fields;

    using desc_cap_map
        = robin_hood::unordered_map<intern_string_t,
                                    string_fragment,
                                    intern_hasher,
                                    std::equal_to<intern_string_t>>;
    desc_cap_map lf_desc_captures;

    static const intern_string_t LOG_TIME_STR;
    static const intern_string_t LOG_LEVEL_STR;
    static const intern_string_t LOG_OPID_STR;
    static const intern_string_t LOG_THREAD_ID_STR;
    static const intern_string_t LOG_RAW_TEXT_STR;
    static const intern_string_t LOG_EXTRA_FIELDS_STR;

protected:
    static std::vector<std::shared_ptr<log_format>> lf_root_formats;

    struct pcre_format {
        template<typename T, std::size_t N>
        explicit pcre_format(const T (&regex)[N])
            : name(regex), pcre(lnav::pcre2pp::code::from_const(
                                    regex, PCRE2_CASELESS | PCRE2_DOTALL)
                                    .to_shared()),
              pf_timestamp_index(this->pcre->name_index("timestamp"))
        {
        }

        pcre_format() = default;

        const char* name{nullptr};
        std::shared_ptr<lnav::pcre2pp::code> pcre;
        int pf_timestamp_index{-1};
    };

    static bool next_format(const pcre_format* fmt,
                            int& index,
                            int& locked_index);

    const char* log_scanf(scan_batch_context& sbc,
                          uint32_t line_number,
                          string_fragment line,
                          const pcre_format* fmt,
                          const char* time_fmt[],
                          struct exttm* tm_out,
                          struct timeval* tv_out,

                          string_fragment* ts_out,
                          std::optional<string_fragment>* level_out);
};

#endif