otus-devops-2019-02 · akozhin · Jul 18, 2019 · Jul 18, 2019 · Jul 21, 2019 · Jul 21, 2019
diff --git a/docker/.env.example b/docker/.env.example
@@ -4,3 +4,4 @@ UI_VERSION=1.0
 POST_VERSION=1.0
 COMMENT_VERSION=1.0
 UI_PORT=9292
+ZIPKIN_ENABLED=true
diff --git a/docker/docker-compose-logging.yml b/docker/docker-compose-logging.yml
@@ -0,0 +1,33 @@
+version: '3.3'
+services:
+  fluentd:
+    image: ${USERNAME}/fluentd
+    ports:
+      - "24224:24224"
+      - "24224:24224/udp"
+
+  elasticsearch:
+    image: elasticsearch:6.8.0
+    expose:
+      - 9200
+
+    ports:
+      - "9200:9200"
+
+
+  kibana:
+    image: kibana:6.8.0
+    ports:
+      - "5601:5601"
+
+  zipkin:
+    image: openzipkin/zipkin
+    ports:
+      - "9411:9411"
+    networks:
+      - front_net
+      - back_net
+
+networks:
+  front_net:
+  back_net:
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
@@ -11,17 +11,33 @@ services:
           - comment_db
   ui:
     image: ${USERNAME}/ui:${UI_VERSION}
+    environment:
+      - ZIPKIN_ENABLED=${ZIPKIN_ENABLED}
     ports:
       - ${UI_PORT}:${UI_PORT}/tcp
     networks:
       front_net:
+    logging:
+      driver: "fluentd"
+      options:
+        fluentd-address: localhost:24224
+        tag: service.ui
   post:
     image: ${USERNAME}/post:${POST_VERSION}
+    environment:
+      - ZIPKIN_ENABLED=${ZIPKIN_ENABLED}
     networks:
       - front_net
       - back_net
+    logging:
+      driver: "fluentd"
+      options:
+        fluentd-address: localhost:24224
+        tag: service.post
   comment:
     image: ${USERNAME}/comment:${COMMENT_VERSION}
+    environment:
+      - ZIPKIN_ENABLED=${ZIPKIN_ENABLED}
     networks:
       - front_net
       - back_net

diff --git a/logging/fluentd/Dockerfile b/logging/fluentd/Dockerfile
@@ -0,0 +1,6 @@
+FROM fluent/fluentd:v0.12
+
+RUN gem install fluent-plugin-elasticsearch --no-rdoc --no-ri --version 1.9.5
+RUN gem install fluent-plugin-grok-parser --no-rdoc --no-ri --version 1.0.0
+
+ADD fluent.conf /fluentd/etc
diff --git a/logging/fluentd/fluent.conf b/logging/fluentd/fluent.conf
@@ -0,0 +1,52 @@
+<source>
+  @type forward
+  port 24224
+  bind 0.0.0.0
+</source>
+
+<filter service.post>
+  @type parser
+  format json
+  key_name log
+</filter>
+
+<filter service.ui>
+  @type parser
+  key_name log
+  format grok
+  grok_pattern %{RUBY_LOGGER}
+</filter>
+
+<filter service.ui>
+  @type parser
+  format grok
+  <grok>
+    pattern service=%{WORD:service} \| event=%{WORD:event} \| request_id=%{GREEDYDATA:request_id} \| message='%{GREEDYDATA:message}'
+  </grok>
+  <grok>
+    pattern service=%{WORD:service} \| event=%{WORD:event} \| path=%{URIPATH:path} \| request_id=%{GREEDYDATA:request_id} \| remote_addr=%{IPV4:remote_addr} \| method=%{GREEDYDATA:method} \| response_status=%{NUMBER:response_status}
+  </grok>
+  <grok>
+    pattern %{GREEDYDATA:message}
+  </grok>
+  key_name message
+</filter>
+
+<match *.**>
+  @type copy
+  <store>
+    @type elasticsearch
+    host elasticsearch
+    port 9200
+    logstash_format true
+    logstash_prefix fluentd
+    logstash_dateformat %Y%m%d
+    include_tag_key true
+    type_name access_log
+    tag_key @log_name
+    flush_interval 1s
+  </store>
+  <store>
+    @type stdout
+  </store>
+</match>
diff --git a/reddit/views/layout.haml b/reddit/views/layout.haml
@@ -1,12 +1,11 @@
-!!! 5
 %html(lang="en")
   %head
     %meta(charset="utf-8")
     %meta(http-equiv="X-UA-Compatible" content="IE=Edge,chrome=1")
     %meta(name="viewport" content="width=device-width, initial-scale=1.0")
     %title="Monolith Reddit :: #{@title}"
     %link{ href: 'https://maxcdn.bootstrapcdn.com/bootstrap/3.3.6/css/bootstrap.min.css', integrity: "sha384-1q8mTJOASx8j1Au+a5WDVnPi2lkFfwwEAa8hDDdjZlpLegxhjVME1fgjWPGmkzs7", type: 'text/css', rel: 'stylesheet', crossorigin: 'anonymous' }
-    %link{ href: 'https://maxcdn.bootstrapcdn.com/bootstrap/3.3.6/css/bootstrap-theme.min.css', integrity: 'sha384-fLW2N01lMqjakBkx3l/M9EahuwpSfeNvV63J5ezn3uZzapT0u7EYsXMjQV+0En5r', type: 'text/css', rel: 'stylesheet', crossorigin: 'anonymous' }
+    %link{ href: 'https://maxcdn.bootstrapcdn.com/bootstrap/3.3.6/css/bootstrap-theme.min.css', integrity: "sha384-fLW2N01lMqjakBkx3l/M9EahuwpSfeNvV63J5ezn3uZzapT0u7EYsXMjQV+0En5r", type: 'text/css', rel: 'stylesheet', crossorigin: 'anonymous' }
     %script{ href: 'https://maxcdn.bootstrapcdn.com/bootstrap/3.3.6/js/bootstrap.min.js', integrity: 'sha384-0mSbJDEHialfmuBBQP6A4Qrprq5OVfW37PRR3j5ELqxss1yVqOtnepnHVP9aJ7xS', crossorigin: 'anonymous'}
     %script{ src: 'https://ajax.googleapis.com/ajax/libs/jquery/1.12.4/jquery.min.js' }
   %body

diff --git a/src/.env.example b/src/.env.example
@@ -0,0 +1,6 @@
+USERNAME=akozhin
+MONGO_VERSION=3.2
+UI_VERSION=1.0
+POST_VERSION=1.0
+COMMENT_VERSION=1.0
+UI_PORT=9292
diff --git a/src/comment/Gemfile b/src/comment/Gemfile
@@ -5,6 +5,5 @@ gem 'bson_ext'
 gem 'mongo'
 gem 'puma'
 gem 'prometheus-client'
-gem "rack", '>= 2.0.6'
+gem 'rack'
 gem 'rufus-scheduler'
-gem 'tzinfo-data'
diff --git a/src/comment/Gemfile.lock b/src/comment/Gemfile.lock
@@ -1,37 +1,31 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    bson (4.3.0)
+    bson (4.2.2)
     bson_ext (1.5.1)
-    et-orbi (1.1.6)
+    et-orbi (1.0.8)
       tzinfo
-    fugit (1.1.6)
-      et-orbi (~> 1.1, >= 1.1.6)
-      raabro (~> 1.1)
-    mongo (2.6.2)
-      bson (>= 4.3.0, < 5.0.0)
-    mustermann (1.0.3)
-    prometheus-client (0.8.0)
-      quantile (~> 0.2.1)
-    puma (3.12.0)
-    quantile (0.2.1)
-    raabro (1.1.6)
-    rack (2.0.6)
-    rack-protection (2.0.4)
+    mongo (2.4.3)
+      bson (>= 4.2.1, < 5.0.0)
+    mustermann (1.0.2)
+    prometheus-client (0.7.1)
+      quantile (~> 0.2.0)
+    puma (3.10.0)
+    quantile (0.2.0)
+    rack (2.0.5)
+    rack-protection (2.0.2)
       rack
-    rufus-scheduler (3.5.2)
-      fugit (~> 1.1, >= 1.1.5)
-    sinatra (2.0.4)
+    rufus-scheduler (3.4.2)
+      et-orbi (~> 1.0)
+    sinatra (2.0.2)
       mustermann (~> 1.0)
       rack (~> 2.0)
-      rack-protection (= 2.0.4)
+      rack-protection (= 2.0.2)
       tilt (~> 2.0)
     thread_safe (0.3.6)
-    tilt (2.0.9)
-    tzinfo (1.2.5)
+    tilt (2.0.8)
+    tzinfo (1.2.3)
       thread_safe (~> 0.1)
-    tzinfo-data (1.2018.7)
-      tzinfo (>= 1.0.0)
 
 PLATFORMS
   ruby
@@ -41,10 +35,9 @@ DEPENDENCIES
   mongo
   prometheus-client
   puma
-  rack (>= 2.0.6)
+  rack
   rufus-scheduler
   sinatra (~> 2.0.2)
-  tzinfo-data
 
 BUNDLED WITH
-   1.17.2
+   1.16.1
diff --git a/src/comment/build_info.txt b/src/comment/build_info.txt
@@ -1,2 +1,2 @@
-770bca0
-monitoring-1
+c769a58
+logging-1
diff --git a/src/comment/comment_app.rb b/src/comment/comment_app.rb
@@ -13,17 +13,8 @@
 DB_URL ||= "mongodb://#{COMMENT_DATABASE_HOST}:#{COMMENT_DATABASE_PORT}"
 
 # App version and build info
-if File.exist?('VERSION')
-  VERSION ||= File.read('VERSION').strip
-else
-  VERSION ||= "version_missing"
-end
-
-if File.exist?('build_info.txt')
-  BUILD_INFO = File.readlines('build_info.txt')
-else
-  BUILD_INFO = Array.new(2, "build_info_missing")
-end
+VERSION ||= File.read('VERSION').strip
+BUILD_INFO = File.readlines('build_info.txt')
 
 configure do
   Mongo::Logger.logger.level = Logger::WARN

diff --git a/src/comment/docker_build.sh b/src/comment/docker_build.sh
@@ -3,4 +3,4 @@
 echo `git show --format="%h" HEAD | head -1` > build_info.txt
 echo `git rev-parse --abbrev-ref HEAD` >> build_info.txt
 
-docker build -t $USER_NAME/comment .
+docker build -t $USER_NAME/comment:logging .
diff --git a/src/docker-compose.yml b/src/docker-compose.yml
@@ -0,0 +1,37 @@
+version: '3.3'
+services:
+  post_db:
+    image: mongo:${MONGO_VERSION}
+    volumes:
+      - post_db:/data/db
+    networks:
+      back_net:
+        aliases:
+          - post_db
+          - comment_db
+  ui:
+    build: ./ui
+    image: ${USERNAME}/ui:${UI_VERSION}
+    ports:
+      - ${UI_PORT}:${UI_PORT}/tcp
+    networks:
+      front_net:
+  post:
+    build: ./post-py
+    image: ${USERNAME}/post:${POST_VERSION}
+    networks:
+      - front_net
+      - back_net
+  comment:
+    build: ./comment
+    image: ${USERNAME}/comment:${COMMENT_VERSION}
+    networks:
+      - front_net
+      - back_net
+
+volumes:
+  post_db:
+
+networks:
+  front_net:
+  back_net:
diff --git a/src/post-py/Dockerfile b/src/post-py/Dockerfile
@@ -11,3 +11,4 @@ ENV POST_DATABASE_HOST post_db
 ENV POST_DATABASE posts
 
 CMD ["python3", "post_app.py"]
+
diff --git a/src/post-py/build_info.txt b/src/post-py/build_info.txt
@@ -1,2 +1,2 @@
-770bca0
-monitoring-1
+c769a58
+logging-1
diff --git a/src/post-py/docker_build.sh b/src/post-py/docker_build.sh
@@ -3,4 +3,4 @@
 echo `git show --format="%h" HEAD | head -1` > build_info.txt
 echo `git rev-parse --abbrev-ref HEAD` >> build_info.txt
 
-docker build -t $USER_NAME/post .
+docker build -t $USER_NAME/post:logging .
diff --git a/src/post-py/requirements.txt b/src/post-py/requirements.txt
@@ -1,5 +1,5 @@
 prometheus_client==0.0.21
-flask==0.12.3
+flask==0.12.2
 pymongo==3.5.1
 structlog==17.2.0
 py-zipkin==0.13.0

diff --git a/src/ui/Gemfile b/src/ui/Gemfile
@@ -7,7 +7,7 @@ gem 'bson_ext'
 gem 'faraday'
 gem 'puma'
 gem 'prometheus-client'
-gem "rack", '>= 2.0.6'
+gem 'rack'
 gem 'rufus-scheduler'
 gem 'tzinfo-data'
 gem 'zipkin-tracer'
diff --git a/src/ui/build_info.txt b/src/ui/build_info.txt
@@ -1,2 +1,2 @@
-770bca0
-monitoring-1
+c769a58
+logging-1
diff --git a/src/ui/docker_build.sh b/src/ui/docker_build.sh
@@ -3,4 +3,4 @@
 echo `git show --format="%h" HEAD | head -1` > build_info.txt
 echo `git rev-parse --abbrev-ref HEAD` >> build_info.txt
 
-docker build -t $USER_NAME/ui .
+docker build -t $USER_NAME/ui:logging .
diff --git a/src/ui/ui_app.rb b/src/ui/ui_app.rb
@@ -20,18 +20,8 @@
 COMMENT_URL ||= "http://#{COMMENT_SERVICE_HOST}:#{COMMENT_SERVICE_PORT}"
 
 # App version and build info
-if File.exist?('VERSION')
-  VERSION ||= File.read('VERSION').strip
-else
-  VERSION ||= "version_missing"
-end
-
-if File.exist?('build_info.txt')
-  BUILD_INFO = File.readlines('build_info.txt')
-else
-  BUILD_INFO = Array.new(2, "build_info_missing")
-end
-
+VERSION ||= File.read('VERSION').strip
+BUILD_INFO = File.readlines('build_info.txt')
 @@host_info=ENV['HOSTNAME']
 @@env_info=ENV['ENV']
Original file line number	Diff line number	Diff line change
Expand Up		@@ -11,3 +11,4 @@ ENV POST_DATABASE_HOST post_db
		ENV POST_DATABASE posts

		CMD ["python3", "post_app.py"]