Merge remote-tracking branch 'origin/dev' into travis_docker_build

.travis.yml

@@ -1,3 +1,4 @@
+sudo: false
 language: go
 go:
     - 1.4

CHANGES.txt

@@ -1,4 +1,104 @@
-0.11.0 (2015-??-??)
+0.11.0 (2016-??-??)
+===================
+
+Backwards Incompatibilities
+---------------------------
+
+* StatAccumInput `percent_threshold` param type convert to slice.
+
+* HttpInput `user` param changed to `username` to match other HTTP plugins.
+
+* DockerLogInput changed to use `logs` API endpoint instead of `attach`. This
+  helps prevent data loss by allowing use of the `since` parameter to fetch
+  records that were generated while Heka was down, but it means this input will
+  now only work with containers using the `json-file` or `journald` logging
+  drivers.
+
+* Added PatternGroupingSplitter.
+
+Bug Handling
+------------
+
+* Updated Sarama dependency from pre-1.0 release fork to fork (with only test
+  code changes) of Sarama 1.5.0 release.
+
+* Fixed ESJsonEncoder generating invalid JSON when `DynamicFields` is first of
+  multiple specified fields but the message contains no dynamic fields.
+
+* Fixed bug where DockerLogInput would not reconnect when a Docker daemon
+  was down for some time (#1843).
+
+* More verbose logging from the DockerLogInput plugin (#1843).
+
+Features
+--------
+
+* Added TLS support to KafkaInput and KafkaOutput plugins.
+
+* Added decoder support to StatAccumInput.
+
+* Added `git_clone_to_path` to the cmake build to allow git repos to be cloned
+  into alternate locations; useful for relocating forks of Go packages into
+  their original import paths.
+
+* Added `initial_tail` config option to LogstreamerInput (#1482, #1801).
+
+* Added Splitter support to DockerLogInput plugin (#1843).
+
+* Added `fields_from_labels` config option to DockerLogInput (#1843).
+
+* Added DockerStatsInput plugin.
+
+* Added iowait percentage output field in filter procstat (#1888).
+
+0.10.1 (2016-??-??)
+===================
+
+Features
+--------
+
+* Added `check_data_interval` setting to LogstreamerInput.
+
+* Added `separator` and `maximum_depth` settings to JSON decoder (#1809).
+
+* Added `status_field` setting to http_status filter (#1876)
+
+Bug Handling
+------------
+
+* Fixed conditional which was causing InfluxDB line protocol encoder to not
+  correctly honor the `source_value_field` config setting.
+
+* Don't use `os.Exit` in the pipeline's main `Run` function or else any
+  wrapping deffered functions (such as those that output the cpu and mem
+  profiles) won't get called.
+
+* Stop zero-length records with no error ending the splitter if there's
+  more data to read (#1561)
+
+* Fixed gzipped file seeking in logstreamer may cause an OOM exception.
+
+* Fixed LogstreamerInput file rotation missing newly created files with new
+  names by always triggering LogstreamSet rescan before checking location in
+  stream (#1452).
+
+* JSON decoder no longer exits when flattening the field structure fails.
+
+* Fixed bug where HttpOutput would discard the response body before trying to
+  use the response body when response returned an error status code.
+
+0.10.0 (2015-12-30)
+===================
+
+Bug Handling
+------------
+
+* Fixed issue where LogstreamerInput doesn't notice when a zero-length file is
+  deleted and replaced by a new zero-length file before any data is appended
+  (#1199).
+
+0.10.0b2 (2015-11-20)
+=====================
 
 Backwards Incompatibilities
 ---------------------------
@@ -8,23 +108,62 @@ Backwards Incompatibilities
 Bug Handling
 ------------
 
+* Updated Sarama dependency from pre-1.0 release fork to fork (with only test
+  code changes) of Sarama 1.5.0 release.
+
 Features
 --------
 
-* Allow TcpOutput to re-establish the connection after a configurable number of
-  successfully delivered messages.
+* Added decoder support to StatAccumInput.
+
+* Added `git_clone_to_path` to the cmake build to allow git repos to be cloned
+  into alternate locations; useful for relocating forks of Go packages into
+  their original import paths.
 
 0.10.0 (2015-??-??)
 =====================
 
+Backwards Incompatibilities
+---------------------------
+
+* `DecoderRunner` interface changed `SetSendFailure` to `SetFailureHandling`.
+
 Features
 --------
 
+* Allow TcpOutput to re-establish the connection after a configurable number
+  of successfully delivered messages.
+
 * Added `set_hostname` option to UdpInput configuration (#1668).
 
+* Added `linux_netdev` and `linux_netstat` decoders (#1712).
+
+* Improved error output for ElasticSearchOutput HTTP failures.
+
+* Added `log_decode_failures` config value to decoders.
+
+* Adder `log_flags` to hekad config, to control the prefix for STDOUT and
+  STDERR logs.
+
 Bug Handling
 ------------
 
+* Fixed issue where ElasticSearchOutput was trying to send zero length requests
+  (#1783).
+
+* Fixed race condition in ElasticSearchOutput (#1786).
+
+* AMQPInput `Run` method now returns an error when the input channel closes but
+  `Stop` hasn't been called to successfully trigger restarts (#1757).
+
+* Fixed error where restarting plugins were losing specified configuration
+  (#1756).
+
+* Fixed config error where global `max_pack_idle` setting was the wrong type
+  and was being ignored (#1778).
+
+* Fixed race condition in InputRunner's default deliverer initialization.
+
 * Set hostname correctly in the Graylog decoder (#1663).
 
 * Add sample rate support for gauge type in StatsdInput.
@@ -36,6 +175,34 @@ Bug Handling
 
 * Force ElasticSearch index name to lower case, as required by ElasticSearch.
 
+* Fixed default buffer config values.
+
+* Updated docker configuration so docker build works again (#1634).
+
+* Add generic json lua sandbox decoder (#1651).
+
+* Fixed bug where a SandboxInput configured with a `ticker_interval` would
+  get stuck in an infinite loop on shutdown (#1705).
+
+* Fixed race condition in the BufferReader's queue file lookup code (#1639).
+
+* Changes in StatAccumInput, FileOutput, TcpInput, ProcessDirectoryInput tests
+  to minimize intermittent Travis failures.
+
+* Ensure the response is read until it is complete in the HttpOutput plugin.
+  Previously, the deferred Body.Close() may have been called on incomplete
+  responses. This resulted in the connections not returning to the client's
+  Transport connection pool.
+
+* Fixed panic that was occurring when loading a config file or directory that
+  exists but which registers no plugins (#1597).
+
+* Delay start up when a buffered plugin's buffer is at capacity to give the
+  back-pressure time to resolve (#1738).
+
+* Fixed bug where LogStreamerInput would sometimes loop infinitely reading the
+  same file over and over when reading gzipped log files.
+
 0.10.0b1 (2015-08-07)
 =====================
 
@@ -50,6 +217,8 @@ Features
 
 * Allow multiple sandbox module directories to be specified (#1525).
 
+* Add Nginx stub status lua sandbox decoder
+
 * Add Nginx stub status lua sandbox decoder.
 
 * Added support for SIGUSR2 signal for use when Heka is wedged, triggers a Heka
@@ -87,6 +256,10 @@ Backwards Incompatibilities
 * Major overhaul of filter and output plugin APIs to support disk buffering
   (#1378).
 
+* `queue_max_buffer_size` and `queue_full_action` config options for TcpOutput
+  and ElasticSearchOutput are now moved to `max_buffer_size` and `full_action`
+  in the `buffering` subsection (#1378).
+
 * Go 1.4 now required for building.
 
 * Deprecated the read_next_field() sandbox API; it is targeted for removal in

CMakeLists.txt

@@ -19,7 +19,9 @@ find_package(Go 1.4 REQUIRED)
 find_package(Git REQUIRED)
 find_package(Protobuf 2.3 QUIET)
 set(CPACK_PACKAGE_FILE_NAME ${CMAKE_PROJECT_NAME}-${CPACK_PACKAGE_VERSION_MAJOR}_${CPACK_PACKAGE_VERSION_MINOR}_${CPACK_PACKAGE_VERSION_PATCH}-${GO_PLATFORM}-${GO_ARCH})
-set(CMAKE_INSTALL_PREFIX ${CMAKE_PROJECT_NAME})
+if (NOT DEFINED CMAKE_INSTALL_PREFIX)
+    set(CMAKE_INSTALL_PREFIX ${CMAKE_PROJECT_NAME})
+endif()
 set(PROJECT_PATH "${CMAKE_BINARY_DIR}/heka")
 set(HEKA_PATH "${PROJECT_PATH}/src/github.com/mozilla-services/heka")
 set(LUA_INCLUDE_PATH "${PROJECT_PATH}/include")
@@ -95,9 +97,13 @@ elseif(UNIX)
     if (RPMBUILD_EXECUTABLE)
         set(CPACK_GENERATOR ${CPACK_GENERATOR} "RPM")
         set(CPACK_RPM_PACKAGE_LICENSE "MPLv2.0")
-		# todo we should pull lua_sandbox out of the heka/build and packaging
-		set(CPACK_RPM_PACKAGE_PROVIDES "libluasb.so.0()(64bit), libluasandbox.so.0()(64bit)")
-		set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION /usr/share/man)
+        set(CPACK_RPM_PACKAGE_DESCRIPTION "Heka is a tool for collecting and collating data from a number of different sources, performing 'in-flight' processing of collected data, and delivering the results to any number of destinations for further analysis.")
+	# todo we should pull lua_sandbox out of the heka/build and packaging
+	set(CPACK_RPM_PACKAGE_PROVIDES "libluasb.so.0()(64bit), libluasandbox.so.0()(64bit)")
+	set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION /usr/share/man)
+        set(CPACK_RPM_PACKAGE_REQUIRES_PRE "shadow-utils")
+        set(CPACK_RPM_PRE_INSTALL_SCRIPT_FILE "${CMAKE_SOURCE_DIR}/rpm/heka.preinst.in")
+        set(CPACK_RPM_POST_INSTALL_SCRIPT_FILE "${CMAKE_SOURCE_DIR}/rpm/heka.postinst.in")
     endif()
 endif()
 
@@ -177,18 +183,20 @@ if (INCLUDE_DOCUMENTATION)
     install(DIRECTORY "${HEKA_PATH}/docs/build/man/" DESTINATION share/man/man5 FILES_MATCHING PATTERN "*.5")
 endif()
 
-if (MINGW AND INCLUDE_SANDBOX)
-    add_custom_command(TARGET heka_source POST_BUILD
-    COMMAND ${CMAKE_COMMAND} -E copy_directory "${PROJECT_PATH}/lib/" "${HEKA_PATH}"
-    COMMAND ${CMAKE_COMMAND} -E copy_directory "${PROJECT_PATH}/lib/" "${HEKA_PATH}/pipeline"
-    COMMAND ${CMAKE_COMMAND} -E copy_directory "${PROJECT_PATH}/lib/" "${HEKA_PATH}/plugins"
-    COMMAND ${CMAKE_COMMAND} -E copy_directory "${PROJECT_PATH}/lib/" "${HEKA_PATH}/sandbox/lua"
-    COMMAND ${CMAKE_COMMAND} -E copy_directory "${PROJECT_PATH}/lib/" "${HEKA_PATH}/sandbox/plugins"
-    COMMAND ${CMAKE_COMMAND} -E copy_directory "${PROJECT_PATH}/lib/" "${HEKA_PATH}/cmd/hekad"
-    COMMENT "Install dll's for the mock generation and unit tests")
-    install(DIRECTORY "${PROJECT_PATH}/lib/" DESTINATION bin PATTERN "*.dll" PATTERN "luasandbox" EXCLUDE PATTERN "luasandbox/*" EXCLUDE)
-else()
-    install(DIRECTORY "${PROJECT_PATH}/lib/" DESTINATION lib PATTERN "*${CMAKE_SHARED_LIBRARY_SUFFIX}" PATTERN "luasandbox" EXCLUDE PATTERN "luasandbox/*" EXCLUDE)
+if (INCLUDE_SANDBOX)
+    if (MINGW)
+        add_custom_command(TARGET heka_source POST_BUILD
+        COMMAND ${CMAKE_COMMAND} -E copy_directory "${PROJECT_PATH}/lib/" "${HEKA_PATH}"
+        COMMAND ${CMAKE_COMMAND} -E copy_directory "${PROJECT_PATH}/lib/" "${HEKA_PATH}/pipeline"
+        COMMAND ${CMAKE_COMMAND} -E copy_directory "${PROJECT_PATH}/lib/" "${HEKA_PATH}/plugins"
+        COMMAND ${CMAKE_COMMAND} -E copy_directory "${PROJECT_PATH}/lib/" "${HEKA_PATH}/sandbox/lua"
+        COMMAND ${CMAKE_COMMAND} -E copy_directory "${PROJECT_PATH}/lib/" "${HEKA_PATH}/sandbox/plugins"
+        COMMAND ${CMAKE_COMMAND} -E copy_directory "${PROJECT_PATH}/lib/" "${HEKA_PATH}/cmd/hekad"
+        COMMENT "Install dll's for the mock generation and unit tests")
+        install(DIRECTORY "${PROJECT_PATH}/lib/" DESTINATION bin PATTERN "*.dll" PATTERN "luasandbox" EXCLUDE PATTERN "luasandbox/*" EXCLUDE)
+    else()
+        install(DIRECTORY "${PROJECT_PATH}/lib/" DESTINATION lib PATTERN "*${CMAKE_SHARED_LIBRARY_SUFFIX}" PATTERN "luasandbox" EXCLUDE PATTERN "luasandbox/*" EXCLUDE)
+    endif()
 endif()
 
 install(PROGRAMS "${HEKA_EXE}" DESTINATION bin)
@@ -246,7 +254,7 @@ if (UNIX AND DPKG_EXECUTABLE)
         DEPENDS hekad
         COMMENT "Custom deb target")
 
-    add_subdirectory(debian)
+    add_subdirectory(packaging/debian)
 endif()
 
 # MOVING INCLUSION OF CPACK DOWN HERE SO IT ACTUALLY GETS THE VARIABLES WE SET
@@ -285,16 +293,17 @@ endif()
 if (INCLUDE_MOZSVC)
     add_test(mozsvc ${GO_EXECUTABLE} test ${BENCHMARK_FLAG} ${COVERAGE_FLAG} github.com/mozilla-services/heka-mozsvc-plugins)
 endif()
-install(FILES "${HEKA_PATH}/cmd/heka-sbmgr/sbmgr.toml" "${HEKA_PATH}/cmd/heka-sbmgr/hekad.toml.sbmgr" DESTINATION bin)
-if(INCLUDE_SANDBOX)
-install(FILES "${HEKA_PATH}/sandbox/lua/testsupport/hekabench_cbuf_counter.lua" DESTINATION bin)
-endif()
+
+install(FILES "${HEKA_PATH}/cmd/heka-sbmgr/sbmgr.toml" "${HEKA_PATH}/cmd/heka-sbmgr/hekad.toml.sbmgr" DESTINATION "share/${CMAKE_PROJECT_NAME}/examples")
 install(FILES "${CMAKE_SOURCE_DIR}/LICENSE.txt" DESTINATION "share/${CMAKE_PROJECT_NAME}")
 install(DIRECTORY "${CMAKE_SOURCE_DIR}/dasher" DESTINATION "share/${CMAKE_PROJECT_NAME}")
-install(DIRECTORY "${PROJECT_PATH}/lib/luasandbox/modules/" DESTINATION "share/${CMAKE_PROJECT_NAME}/lua_modules")
-install(DIRECTORY "${PROJECT_PATH}/lib/luasandbox/io_modules/" DESTINATION "share/${CMAKE_PROJECT_NAME}/lua_io_modules")
 install(DIRECTORY "${CMAKE_SOURCE_DIR}/examples/conf/" DESTINATION "share/${CMAKE_PROJECT_NAME}/examples")
-install(DIRECTORY "${CMAKE_SOURCE_DIR}/sandbox/lua/modules/" DESTINATION "share/${CMAKE_PROJECT_NAME}/lua_modules")
-install(DIRECTORY "${CMAKE_SOURCE_DIR}/sandbox/lua/decoders/" DESTINATION "share/${CMAKE_PROJECT_NAME}/lua_decoders")
-install(DIRECTORY "${CMAKE_SOURCE_DIR}/sandbox/lua/filters/" DESTINATION "share/${CMAKE_PROJECT_NAME}/lua_filters")
-install(DIRECTORY "${CMAKE_SOURCE_DIR}/sandbox/lua/encoders/" DESTINATION "share/${CMAKE_PROJECT_NAME}/lua_encoders")
+if (INCLUDE_SANDBOX)
+    install(FILES "${HEKA_PATH}/sandbox/lua/testsupport/hekabench_cbuf_counter.lua" DESTINATION "share/${CMAKE_PROJECT_NAME}/examples")
+    install(DIRECTORY "${PROJECT_PATH}/lib/luasandbox/modules/" DESTINATION "share/${CMAKE_PROJECT_NAME}/lua_modules")
+    install(DIRECTORY "${PROJECT_PATH}/lib/luasandbox/io_modules/" DESTINATION "share/${CMAKE_PROJECT_NAME}/lua_io_modules")
+    install(DIRECTORY "${CMAKE_SOURCE_DIR}/sandbox/lua/modules/" DESTINATION "share/${CMAKE_PROJECT_NAME}/lua_modules")
+    install(DIRECTORY "${CMAKE_SOURCE_DIR}/sandbox/lua/decoders/" DESTINATION "share/${CMAKE_PROJECT_NAME}/lua_decoders")
+    install(DIRECTORY "${CMAKE_SOURCE_DIR}/sandbox/lua/filters/" DESTINATION "share/${CMAKE_PROJECT_NAME}/lua_filters")
+    install(DIRECTORY "${CMAKE_SOURCE_DIR}/sandbox/lua/encoders/" DESTINATION "share/${CMAKE_PROJECT_NAME}/lua_encoders")
+endif()

Dockerfile

@@ -1,5 +1,5 @@
 # heka_base image
-FROM debian:jessie
+FROM golang:1.4
 
 MAINTAINER Chance Zibolski <[email protected]> (@chance)
 
@@ -18,27 +18,15 @@ RUN     apt-get update && \
         ruby-dev \
         protobuf-compiler \
         python-sphinx \
-        wget
-
-# Install Go 1.3.1
-RUN curl -s https://storage.googleapis.com/golang/go1.3.1.linux-amd64.tar.gz -o /tmp/go.tar.gz && \
-        echo "3af011cc19b21c7180f2604fd85fbc4ddde97143 /tmp/go.tar.gz" | sha1sum -c && \
-        tar -C /usr/local -xzf /tmp/go.tar.gz
+        wget \
+        debhelper \
+        fakeroot \
+        libgeoip-dev \
+        libgeoip1 \
+        golang-goprotobuf-dev
 
 WORKDIR /heka
 
-ENV GOROOT /usr/local/go
-ENV PATH $PATH:/usr/local/go/bin:/go/bin
-
-ENV CTEST_OUTPUT_ON_FAILURE 1
-ENV BUILD_DIR   /heka/build
-ENV GOPATH      $BUILD_DIR/heka
-ENV GOBIN       $GOPATH/bin
-ENV PATH        $PATH:$GOBIN
-# Build faster
-ENV NUM_JOBS    10
-
 EXPOSE 4352
 
 COPY . /heka
-RUN ./build.sh

build.sh

@@ -8,5 +8,5 @@ NUM_JOBS=${NUM_JOBS:-1}
 # build heka
 mkdir -p $BUILD_DIR
 cd $BUILD_DIR
-cmake -DCMAKE_BUILD_TYPE=release ..
+cmake -DCMAKE_BUILD_TYPE=release $@ ..
 make -j $NUM_JOBS

cmake/CMakeDebHelperInstall.cmake

@@ -8,7 +8,7 @@ endif()
 # Copy all generated files where the packing will happen,
 # exclude the DEBIAN-directory.
 file( COPY
-	"${CPACK_OUTPUT_FILE_PREFIX}/debian/${CPACK_DEBIAN_PACKAGE_NAME}/"
+	"${CPACK_OUTPUT_FILE_PREFIX}/packaging/debian/${CPACK_DEBIAN_PACKAGE_NAME}/"
 	DESTINATION "${CMAKE_CURRENT_BINARY_DIR}" 
 	PATTERN DEBIAN EXCLUDE
 )