diff --git a/docker/.env.example b/docker/.env.example index 24d827f..a0f050e 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -4,3 +4,4 @@ UI_VERSION=1.0 POST_VERSION=1.0 COMMENT_VERSION=1.0 UI_PORT=9292 +ZIPKIN_ENABLED=true diff --git a/docker/docker-compose-logging.yml b/docker/docker-compose-logging.yml new file mode 100644 index 0000000..ae0ed2e --- /dev/null +++ b/docker/docker-compose-logging.yml @@ -0,0 +1,33 @@ +version: '3.3' +services: + fluentd: + image: ${USERNAME}/fluentd + ports: + - "24224:24224" + - "24224:24224/udp" + + elasticsearch: + image: elasticsearch:6.8.0 + expose: + - 9200 + + ports: + - "9200:9200" + + + kibana: + image: kibana:6.8.0 + ports: + - "5601:5601" + + zipkin: + image: openzipkin/zipkin + ports: + - "9411:9411" + networks: + - front_net + - back_net + +networks: + front_net: + back_net: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 251f679..b663a3b 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -11,17 +11,33 @@ services: - comment_db ui: image: ${USERNAME}/ui:${UI_VERSION} + environment: + - ZIPKIN_ENABLED=${ZIPKIN_ENABLED} ports: - ${UI_PORT}:${UI_PORT}/tcp networks: front_net: + logging: + driver: "fluentd" + options: + fluentd-address: localhost:24224 + tag: service.ui post: image: ${USERNAME}/post:${POST_VERSION} + environment: + - ZIPKIN_ENABLED=${ZIPKIN_ENABLED} networks: - front_net - back_net + logging: + driver: "fluentd" + options: + fluentd-address: localhost:24224 + tag: service.post comment: image: ${USERNAME}/comment:${COMMENT_VERSION} + environment: + - ZIPKIN_ENABLED=${ZIPKIN_ENABLED} networks: - front_net - back_net diff --git a/logging/fluentd/Dockerfile b/logging/fluentd/Dockerfile new file mode 100644 index 0000000..329f139 --- /dev/null +++ b/logging/fluentd/Dockerfile @@ -0,0 +1,6 @@ +FROM fluent/fluentd:v0.12 + +RUN gem install fluent-plugin-elasticsearch --no-rdoc --no-ri --version 1.9.5 +RUN gem install fluent-plugin-grok-parser --no-rdoc --no-ri --version 1.0.0 + +ADD fluent.conf /fluentd/etc diff --git a/logging/fluentd/fluent.conf b/logging/fluentd/fluent.conf new file mode 100644 index 0000000..43e309e --- /dev/null +++ b/logging/fluentd/fluent.conf @@ -0,0 +1,52 @@ + + @type forward + port 24224 + bind 0.0.0.0 + + + + @type parser + format json + key_name log + + + + @type parser + key_name log + format grok + grok_pattern %{RUBY_LOGGER} + + + + @type parser + format grok + + pattern service=%{WORD:service} \| event=%{WORD:event} \| request_id=%{GREEDYDATA:request_id} \| message='%{GREEDYDATA:message}' + + + pattern service=%{WORD:service} \| event=%{WORD:event} \| path=%{URIPATH:path} \| request_id=%{GREEDYDATA:request_id} \| remote_addr=%{IPV4:remote_addr} \| method=%{GREEDYDATA:method} \| response_status=%{NUMBER:response_status} + + + pattern %{GREEDYDATA:message} + + key_name message + + + + @type copy + + @type elasticsearch + host elasticsearch + port 9200 + logstash_format true + logstash_prefix fluentd + logstash_dateformat %Y%m%d + include_tag_key true + type_name access_log + tag_key @log_name + flush_interval 1s + + + @type stdout + + diff --git a/reddit/views/layout.haml b/reddit/views/layout.haml index 48b881d..7bdcd46 100644 --- a/reddit/views/layout.haml +++ b/reddit/views/layout.haml @@ -1,4 +1,3 @@ -!!! 5 %html(lang="en") %head %meta(charset="utf-8") @@ -6,7 +5,7 @@ %meta(name="viewport" content="width=device-width, initial-scale=1.0") %title="Monolith Reddit :: #{@title}" %link{ href: 'https://maxcdn.bootstrapcdn.com/bootstrap/3.3.6/css/bootstrap.min.css', integrity: "sha384-1q8mTJOASx8j1Au+a5WDVnPi2lkFfwwEAa8hDDdjZlpLegxhjVME1fgjWPGmkzs7", type: 'text/css', rel: 'stylesheet', crossorigin: 'anonymous' } - %link{ href: 'https://maxcdn.bootstrapcdn.com/bootstrap/3.3.6/css/bootstrap-theme.min.css', integrity: 'sha384-fLW2N01lMqjakBkx3l/M9EahuwpSfeNvV63J5ezn3uZzapT0u7EYsXMjQV+0En5r', type: 'text/css', rel: 'stylesheet', crossorigin: 'anonymous' } + %link{ href: 'https://maxcdn.bootstrapcdn.com/bootstrap/3.3.6/css/bootstrap-theme.min.css', integrity: "sha384-fLW2N01lMqjakBkx3l/M9EahuwpSfeNvV63J5ezn3uZzapT0u7EYsXMjQV+0En5r", type: 'text/css', rel: 'stylesheet', crossorigin: 'anonymous' } %script{ href: 'https://maxcdn.bootstrapcdn.com/bootstrap/3.3.6/js/bootstrap.min.js', integrity: 'sha384-0mSbJDEHialfmuBBQP6A4Qrprq5OVfW37PRR3j5ELqxss1yVqOtnepnHVP9aJ7xS', crossorigin: 'anonymous'} %script{ src: 'https://ajax.googleapis.com/ajax/libs/jquery/1.12.4/jquery.min.js' } %body diff --git a/src/.env.example b/src/.env.example new file mode 100644 index 0000000..24d827f --- /dev/null +++ b/src/.env.example @@ -0,0 +1,6 @@ +USERNAME=akozhin +MONGO_VERSION=3.2 +UI_VERSION=1.0 +POST_VERSION=1.0 +COMMENT_VERSION=1.0 +UI_PORT=9292 diff --git a/src/comment/Gemfile b/src/comment/Gemfile index 82e3ac2..0eb06eb 100644 --- a/src/comment/Gemfile +++ b/src/comment/Gemfile @@ -5,6 +5,5 @@ gem 'bson_ext' gem 'mongo' gem 'puma' gem 'prometheus-client' -gem "rack", '>= 2.0.6' +gem 'rack' gem 'rufus-scheduler' -gem 'tzinfo-data' diff --git a/src/comment/Gemfile.lock b/src/comment/Gemfile.lock index e095838..639c2cf 100644 --- a/src/comment/Gemfile.lock +++ b/src/comment/Gemfile.lock @@ -1,37 +1,31 @@ GEM remote: https://rubygems.org/ specs: - bson (4.3.0) + bson (4.2.2) bson_ext (1.5.1) - et-orbi (1.1.6) + et-orbi (1.0.8) tzinfo - fugit (1.1.6) - et-orbi (~> 1.1, >= 1.1.6) - raabro (~> 1.1) - mongo (2.6.2) - bson (>= 4.3.0, < 5.0.0) - mustermann (1.0.3) - prometheus-client (0.8.0) - quantile (~> 0.2.1) - puma (3.12.0) - quantile (0.2.1) - raabro (1.1.6) - rack (2.0.6) - rack-protection (2.0.4) + mongo (2.4.3) + bson (>= 4.2.1, < 5.0.0) + mustermann (1.0.2) + prometheus-client (0.7.1) + quantile (~> 0.2.0) + puma (3.10.0) + quantile (0.2.0) + rack (2.0.5) + rack-protection (2.0.2) rack - rufus-scheduler (3.5.2) - fugit (~> 1.1, >= 1.1.5) - sinatra (2.0.4) + rufus-scheduler (3.4.2) + et-orbi (~> 1.0) + sinatra (2.0.2) mustermann (~> 1.0) rack (~> 2.0) - rack-protection (= 2.0.4) + rack-protection (= 2.0.2) tilt (~> 2.0) thread_safe (0.3.6) - tilt (2.0.9) - tzinfo (1.2.5) + tilt (2.0.8) + tzinfo (1.2.3) thread_safe (~> 0.1) - tzinfo-data (1.2018.7) - tzinfo (>= 1.0.0) PLATFORMS ruby @@ -41,10 +35,9 @@ DEPENDENCIES mongo prometheus-client puma - rack (>= 2.0.6) + rack rufus-scheduler sinatra (~> 2.0.2) - tzinfo-data BUNDLED WITH - 1.17.2 + 1.16.1 diff --git a/src/comment/build_info.txt b/src/comment/build_info.txt index 22fe61c..282d137 100644 --- a/src/comment/build_info.txt +++ b/src/comment/build_info.txt @@ -1,2 +1,2 @@ -770bca0 -monitoring-1 +c769a58 +logging-1 diff --git a/src/comment/comment_app.rb b/src/comment/comment_app.rb index 942e9a0..c173726 100644 --- a/src/comment/comment_app.rb +++ b/src/comment/comment_app.rb @@ -13,17 +13,8 @@ DB_URL ||= "mongodb://#{COMMENT_DATABASE_HOST}:#{COMMENT_DATABASE_PORT}" # App version and build info -if File.exist?('VERSION') - VERSION ||= File.read('VERSION').strip -else - VERSION ||= "version_missing" -end - -if File.exist?('build_info.txt') - BUILD_INFO = File.readlines('build_info.txt') -else - BUILD_INFO = Array.new(2, "build_info_missing") -end +VERSION ||= File.read('VERSION').strip +BUILD_INFO = File.readlines('build_info.txt') configure do Mongo::Logger.logger.level = Logger::WARN diff --git a/src/comment/docker_build.sh b/src/comment/docker_build.sh index a4c8e82..d935a50 100755 --- a/src/comment/docker_build.sh +++ b/src/comment/docker_build.sh @@ -3,4 +3,4 @@ echo `git show --format="%h" HEAD | head -1` > build_info.txt echo `git rev-parse --abbrev-ref HEAD` >> build_info.txt -docker build -t $USER_NAME/comment . +docker build -t $USER_NAME/comment:logging . diff --git a/src/docker-compose.yml b/src/docker-compose.yml new file mode 100644 index 0000000..73b98ac --- /dev/null +++ b/src/docker-compose.yml @@ -0,0 +1,37 @@ +version: '3.3' +services: + post_db: + image: mongo:${MONGO_VERSION} + volumes: + - post_db:/data/db + networks: + back_net: + aliases: + - post_db + - comment_db + ui: + build: ./ui + image: ${USERNAME}/ui:${UI_VERSION} + ports: + - ${UI_PORT}:${UI_PORT}/tcp + networks: + front_net: + post: + build: ./post-py + image: ${USERNAME}/post:${POST_VERSION} + networks: + - front_net + - back_net + comment: + build: ./comment + image: ${USERNAME}/comment:${COMMENT_VERSION} + networks: + - front_net + - back_net + +volumes: + post_db: + +networks: + front_net: + back_net: diff --git a/src/post-py/Dockerfile b/src/post-py/Dockerfile index 0df623b..51033f4 100644 --- a/src/post-py/Dockerfile +++ b/src/post-py/Dockerfile @@ -11,3 +11,4 @@ ENV POST_DATABASE_HOST post_db ENV POST_DATABASE posts CMD ["python3", "post_app.py"] + diff --git a/src/post-py/build_info.txt b/src/post-py/build_info.txt index 22fe61c..282d137 100644 --- a/src/post-py/build_info.txt +++ b/src/post-py/build_info.txt @@ -1,2 +1,2 @@ -770bca0 -monitoring-1 +c769a58 +logging-1 diff --git a/src/post-py/docker_build.sh b/src/post-py/docker_build.sh index ccce114..ed46dad 100755 --- a/src/post-py/docker_build.sh +++ b/src/post-py/docker_build.sh @@ -3,4 +3,4 @@ echo `git show --format="%h" HEAD | head -1` > build_info.txt echo `git rev-parse --abbrev-ref HEAD` >> build_info.txt -docker build -t $USER_NAME/post . +docker build -t $USER_NAME/post:logging . diff --git a/src/post-py/requirements.txt b/src/post-py/requirements.txt index d9bdc00..59d33e3 100644 --- a/src/post-py/requirements.txt +++ b/src/post-py/requirements.txt @@ -1,5 +1,5 @@ prometheus_client==0.0.21 -flask==0.12.3 +flask==0.12.2 pymongo==3.5.1 structlog==17.2.0 py-zipkin==0.13.0 diff --git a/src/ui/Gemfile b/src/ui/Gemfile index dfc92d5..cb3870d 100644 --- a/src/ui/Gemfile +++ b/src/ui/Gemfile @@ -7,7 +7,7 @@ gem 'bson_ext' gem 'faraday' gem 'puma' gem 'prometheus-client' -gem "rack", '>= 2.0.6' +gem 'rack' gem 'rufus-scheduler' gem 'tzinfo-data' gem 'zipkin-tracer' diff --git a/src/ui/build_info.txt b/src/ui/build_info.txt index 22fe61c..282d137 100644 --- a/src/ui/build_info.txt +++ b/src/ui/build_info.txt @@ -1,2 +1,2 @@ -770bca0 -monitoring-1 +c769a58 +logging-1 diff --git a/src/ui/docker_build.sh b/src/ui/docker_build.sh index 5dd1c0e..0153412 100755 --- a/src/ui/docker_build.sh +++ b/src/ui/docker_build.sh @@ -3,4 +3,4 @@ echo `git show --format="%h" HEAD | head -1` > build_info.txt echo `git rev-parse --abbrev-ref HEAD` >> build_info.txt -docker build -t $USER_NAME/ui . +docker build -t $USER_NAME/ui:logging . diff --git a/src/ui/ui_app.rb b/src/ui/ui_app.rb index 85befd6..2b1d6eb 100644 --- a/src/ui/ui_app.rb +++ b/src/ui/ui_app.rb @@ -20,18 +20,8 @@ COMMENT_URL ||= "http://#{COMMENT_SERVICE_HOST}:#{COMMENT_SERVICE_PORT}" # App version and build info -if File.exist?('VERSION') - VERSION ||= File.read('VERSION').strip -else - VERSION ||= "version_missing" -end - -if File.exist?('build_info.txt') - BUILD_INFO = File.readlines('build_info.txt') -else - BUILD_INFO = Array.new(2, "build_info_missing") -end - +VERSION ||= File.read('VERSION').strip +BUILD_INFO = File.readlines('build_info.txt') @@host_info=ENV['HOSTNAME'] @@env_info=ENV['ENV']