From 4ac1eb92a4cc571896a2fedd3aa6acc0574e460a Mon Sep 17 00:00:00 2001 From: rulingcom Date: Tue, 1 Sep 2026 13:49:54 +0800 Subject: [PATCH] Added automatic detection and repair for 502 errors caused by excessive detection stations. --- .../admin/health_check_monitors_controller.rb | 15 +-- app/models/health_check_alert.rb | 1 + app/models/health_check_log.rb | 1 + app/models/health_check_monitor.rb | 116 +++++++++++++++--- .../health_check_monitors/_form.html.erb | 2 +- .../health_check_monitors/_index.html.erb | 9 +- .../admin/health_check_monitors/logs.html.erb | 6 + config/locales/en.yml | 5 +- config/locales/zh_tw.yml | 5 +- health_check.gemspec | 10 +- lib/health_check/engine.rb | 11 ++ 11 files changed, 139 insertions(+), 42 deletions(-) diff --git a/app/controllers/admin/health_check_monitors_controller.rb b/app/controllers/admin/health_check_monitors_controller.rb index 381b971..fbcfb9c 100644 --- a/app/controllers/admin/health_check_monitors_controller.rb +++ b/app/controllers/admin/health_check_monitors_controller.rb @@ -41,20 +41,7 @@ class Admin::HealthCheckMonitorsController < OrbitAdminController end def manual_check - @results = @monitor.check_all - alert_logs = [] - @results.each do |url, result| - HealthCheckLog.create( - :health_check_monitor_id => @monitor.id, - :url => url, - :status => result[:status], - :response_time => result[:response_time], - :error_message => result[:error] - ) - target = @monitor.alert_target(url) - alert_logs << target if target - end - @monitor.send_alert_email(alert_logs) if alert_logs.present? + @monitor.update_attributes(:check_now => true) redirect_to logs_admin_health_check_monitor_path(@monitor) end diff --git a/app/models/health_check_alert.rb b/app/models/health_check_alert.rb index f2360bb..8102af3 100644 --- a/app/models/health_check_alert.rb +++ b/app/models/health_check_alert.rb @@ -7,4 +7,5 @@ class HealthCheckAlert field :url, type: String index({ health_check_monitor_id: 1, url: 1, created_at: -1 }) + index({ created_at: 1 }, { expire_after_seconds: 2592000 }) end diff --git a/app/models/health_check_log.rb b/app/models/health_check_log.rb index a3cab38..0473d17 100644 --- a/app/models/health_check_log.rb +++ b/app/models/health_check_log.rb @@ -11,4 +11,5 @@ class HealthCheckLog belongs_to :health_check_monitor, class_name: "HealthCheckMonitor", optional: true index({ health_check_monitor_id: 1, created_at: -1 }) + index({ created_at: 1 }, { expire_after_seconds: 2592000 }) end diff --git a/app/models/health_check_monitor.rb b/app/models/health_check_monitor.rb index ad2e120..91f6536 100644 --- a/app/models/health_check_monitor.rb +++ b/app/models/health_check_monitor.rb @@ -13,21 +13,28 @@ class HealthCheckMonitor # 告警視窗。只看這段時間內的檢查紀錄，同一網址在這段時間內也只寄一封 ALERT_WINDOW = 4.hours - field :name, type: String - field :target_urls, type: Array, default: [] - field :notification_emails, type: Array, default: [] - field :check_interval, type: Integer, default: 300 - field :fail_threshold, type: Integer, default: 3 - field :timeout, type: Integer, default: 10 - field :enabled, type: Boolean, default: true - field :last_checked_at, type: DateTime + # 跨 unicorn worker 的互斥鎖，存在 multithreads collection + LOCK_KEY = 'health_check' + + field :name, type: String + field :target_urls, type: Array, default: [] + field :notification_emails, type: Array, default: [] + field :check_interval_minutes, type: Integer, default: 5 + field :fail_threshold, type: Integer, default: 3 + field :timeout, type: Integer, default: 10 + field :enabled, type: Boolean, default: true + field :check_now, type: Boolean, default: false + field :last_checked_at, type: DateTime field :create_user_id field :update_user_id validates :name, presence: true + validates :check_interval_minutes, numericality: { greater_than_or_equal_to: 1, only_integer: true } scope :is_enabled, -> { where(enabled: true) } + # ── 表單用的字串轉陣列 ────────────────────────────── + def target_urls_text (target_urls || []).join("\n") end @@ -46,20 +53,24 @@ class HealthCheckMonitor self.notification_emails = value.to_s.split(",").map(&:strip).reject(&:blank?) end + # ── 參數 ──────────────────────────────────────── + def threshold fail_threshold.to_i > 0 ? fail_threshold.to_i : 3 end - def check_all - results = {} - (target_urls || []).each do |url| - results[url] = check_url(url) - end - self.last_checked_at = Time.now - self.save - results + def interval_minutes + check_interval_minutes.to_i > 0 ? check_interval_minutes.to_i : 5 end + def due? + return true if check_now + return true if last_checked_at.nil? + last_checked_at + interval_minutes.minutes <= Time.now + end + + # ── 實際檢查 ───────────────────────────────────── + def check_url(url) uri = URI.parse(url) started = Time.now @@ -75,12 +86,44 @@ class HealthCheckMonitor { :status => 0, :response_time => 0, :error => e.message } end + def check_all + results = {} + (target_urls || []).each do |url| + results[url] = check_url(url) + end + results + end + + # 手動與自動共用的唯一入口：檢查 → 寫紀錄 → 判斷 → 寄信 + def run_check + results = check_all + alert_logs = [] + results.each do |url, result| + HealthCheckLog.create( + :health_check_monitor_id => self.id, + :url => url, + :status => result[:status], + :response_time => result[:response_time], + :error_message => result[:error] + ) + target = alert_target(url) + alert_logs << target if target + end + send_alert_email(alert_logs) if alert_logs.present? + self.check_now = false + self.last_checked_at = Time.now + self.save + results + end + + # ── 告警判斷 ───────────────────────────────────── + def failure?(status) FAILURE_STATUSES.include?(status.to_i) end - # 這次檢查寫完 log 之後呼叫。 - # 條件：4 小時內已檢查滿 threshold 次、最近 threshold 次全部失敗、且 4 小時內還沒寄過這個網址 + # 條件：ALERT_WINDOW 內已檢查滿 threshold 次、最近 threshold 次全部失敗、 + # 且 ALERT_WINDOW 內還沒寄過這個網址 def alert_target(url) n = threshold window_start = Time.now - ALERT_WINDOW @@ -137,4 +180,41 @@ class HealthCheckMonitor false end end + + # ── 自動檢測進入點，由 CronWorker 每分鐘呼叫一次 ────────── + + def self.run_due_checks + return unless claim_lock + begin + is_enabled.each do |monitor| + begin + monitor.run_check if monitor.due? + rescue => e + Rails.logger.error("[health_check] #{monitor.name} 檢查失敗: #{e}") + end + end + ensure + release_lock + end + end + + # 用 Mongo 的 findAndModify 做原子性搶鎖，只有搶到的 worker 會執行 + def self.claim_lock + Multithread.create(:key => LOCK_KEY, :status => {}) if Multithread.where(:key => LOCK_KEY).count == 0 + Multithread.where(:key => LOCK_KEY).to_a[1..-1].to_a.each { |m| m.destroy } + claimed = Multithread.where(:key => LOCK_KEY).any_of( + { 'status.locked_until' => nil }, + { :'status.locked_until'.lt => Time.now } + ).find_one_and_update({ '$set' => { 'status.locked_until' => Time.now + 10.minutes } }) + !claimed.nil? + rescue => e + Rails.logger.error("[health_check] 搶鎖失敗: #{e}") + false + end + + def self.release_lock + Multithread.where(:key => LOCK_KEY).update_all('$set' => { 'status.locked_until' => nil }) + rescue => e + Rails.logger.error("[health_check] 釋放鎖失敗: #{e}") + end end diff --git a/app/views/admin/health_check_monitors/_form.html.erb b/app/views/admin/health_check_monitors/_form.html.erb index 9e4402f..c28b3fb 100644 --- a/app/views/admin/health_check_monitors/_form.html.erb +++ b/app/views/admin/health_check_monitors/_form.html.erb @@ -24,7 +24,7 @@
- <%= f.number_field :check_interval %> + <%= f.number_field :check_interval_minutes, :min => 1 %>
diff --git a/app/views/admin/health_check_monitors/_index.html.erb b/app/views/admin/health_check_monitors/_index.html.erb index c962d10..e7cd885 100644 --- a/app/views/admin/health_check_monitors/_index.html.erb +++ b/app/views/admin/health_check_monitors/_index.html.erb @@ -23,9 +23,14 @@ <%= monitor.target_urls.join("
").html_safe %> - <%= monitor.check_interval %> + <%= monitor.interval_minutes %> <%= monitor.enabled ? t(:yes_) : t(:no_) %> - <%= monitor.last_checked_at.blank? ? "-" : monitor.last_checked_at.strftime("%Y-%m-%d %H:%M") %> + + <%= monitor.last_checked_at.blank? ? "-" : monitor.last_checked_at.strftime("%Y-%m-%d %H:%M") %> + <% if monitor.check_now %> + <%= t("health_check_monitor.pending") %> + <% end %> + <% end %> diff --git a/app/views/admin/health_check_monitors/logs.html.erb b/app/views/admin/health_check_monitors/logs.html.erb index d7ec4a2..c77881e 100644 --- a/app/views/admin/health_check_monitors/logs.html.erb +++ b/app/views/admin/health_check_monitors/logs.html.erb @@ -6,6 +6,12 @@ <%= link_to t(:back), admin_health_check_monitors_path, :class => "btn" %>

+<% if @monitor.check_now %> +
<%= t("health_check_monitor.pending_notice") %>
+<% else %> +

<%= t("health_check_monitor.manual_check_notice") %>

+<% end %> + diff --git a/config/locales/en.yml b/config/locales/en.yml index 3f4a640..0a805d8 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -3,7 +3,7 @@ en: health_check_monitor: name: Name target_urls: Target URLs - check_interval: Check Interval (sec) + check_interval: Check Interval (minutes) fail_threshold: Alert After N Consecutive Failures timeout: Timeout (sec) notification_emails: Notification Emails @@ -14,5 +14,8 @@ en: manual_check: Check Now logs: Logs clean_logs: Clear All Logs + pending: Queued + pending_notice: Queued for checking. Please wait at least 1 minute and refresh this page. + manual_check_notice: "Check Now does not run immediately. Please wait at least 1 minute and refresh this page." one_per_line: One URL per line, must include http:// or https:// (defaults to https://) comma_separated: Comma separated diff --git a/config/locales/zh_tw.yml b/config/locales/zh_tw.yml index eb455fe..a53c462 100644 --- a/config/locales/zh_tw.yml +++ b/config/locales/zh_tw.yml @@ -3,7 +3,7 @@ zh_tw: health_check_monitor: name: 監控名稱 target_urls: 監控網址 - check_interval: 檢查間隔(秒) + check_interval: 檢查間隔(分鐘) fail_threshold: 連續失敗幾次寄信 timeout: 逾時(秒) notification_emails: 通知信箱 @@ -14,5 +14,8 @@ zh_tw: manual_check: 立即檢查 logs: 檢查紀錄 clean_logs: 清除全部紀錄 + pending: 排隊中 + pending_notice: 已排入檢查佇列，請等候至少 1 分鐘後重新整理此頁查看結果。 + manual_check_notice: 按下「立即檢查」後不會馬上有結果，請等候至少 1 分鐘後重新整理此頁。 one_per_line: 一行一個網址，需含 http:// 或 https://（未填會自動補 https://） comma_separated: 以逗號分隔 diff --git a/health_check.gemspec b/health_check.gemspec index 1d14ac4..70aacaa 100644 --- a/health_check.gemspec +++ b/health_check.gemspec @@ -7,11 +7,11 @@ require "health_check/version" Gem::Specification.new do |s| s.name = "health_check" s.version = HealthCheck::VERSION - s.authors = ["Orbitek"] - s.email = ["service@orbitek.co"] - s.homepage = "https://orbitek.co" - s.summary = "Health Check for orbitek" - s.description = "Health Check for orbitek" + s.authors = ["Ruling Digital"] + s.email = ["orbit@rulingcom.com"] + s.homepage = "http://www.rulingcom.com" + s.summary = "Health Check for orbit" + s.description = "Health Check for orbit" s.license = "MIT" s.files = Dir["{app,config,db,lib}/**/*", "MIT-LICENSE", "Rakefile", "README.rdoc"] diff --git a/lib/health_check/engine.rb b/lib/health_check/engine.rb index 2f4dc21..82b42a2 100644 --- a/lib/health_check/engine.rb +++ b/lib/health_check/engine.rb @@ -29,5 +29,16 @@ module HealthCheck end end end + + # 自動檢測：用核心的 CronWorker 每分鐘叫一次，不使用 crontab + config.after_initialize do + if ENV['worker_num'] == '0' && File.basename($0) != 'rake' && !defined?(Rails::Console) + begin + CronWorker.cron_every_minute(HealthCheckMonitor.method(:run_due_checks)) + rescue => e + Rails.logger.error("[health_check] CronWorker 啟動失敗: #{e}") + end + end + end end end