diff --git a/app/controllers/admin/health_check_monitors_controller.rb b/app/controllers/admin/health_check_monitors_controller.rb index 381b971..fbcfb9c 100644 --- a/app/controllers/admin/health_check_monitors_controller.rb +++ b/app/controllers/admin/health_check_monitors_controller.rb @@ -41,20 +41,7 @@ class Admin::HealthCheckMonitorsController < OrbitAdminController end def manual_check - @results = @monitor.check_all - alert_logs = [] - @results.each do |url, result| - HealthCheckLog.create( - :health_check_monitor_id => @monitor.id, - :url => url, - :status => result[:status], - :response_time => result[:response_time], - :error_message => result[:error] - ) - target = @monitor.alert_target(url) - alert_logs << target if target - end - @monitor.send_alert_email(alert_logs) if alert_logs.present? + @monitor.update_attributes(:check_now => true) redirect_to logs_admin_health_check_monitor_path(@monitor) end diff --git a/app/models/health_check_alert.rb b/app/models/health_check_alert.rb index f2360bb..8102af3 100644 --- a/app/models/health_check_alert.rb +++ b/app/models/health_check_alert.rb @@ -7,4 +7,5 @@ class HealthCheckAlert field :url, type: String index({ health_check_monitor_id: 1, url: 1, created_at: -1 }) + index({ created_at: 1 }, { expire_after_seconds: 2592000 }) end diff --git a/app/models/health_check_log.rb b/app/models/health_check_log.rb index a3cab38..0473d17 100644 --- a/app/models/health_check_log.rb +++ b/app/models/health_check_log.rb @@ -11,4 +11,5 @@ class HealthCheckLog belongs_to :health_check_monitor, class_name: "HealthCheckMonitor", optional: true index({ health_check_monitor_id: 1, created_at: -1 }) + index({ created_at: 1 }, { expire_after_seconds: 2592000 }) end diff --git a/app/models/health_check_monitor.rb b/app/models/health_check_monitor.rb index ad2e120..91f6536 100644 --- a/app/models/health_check_monitor.rb +++ b/app/models/health_check_monitor.rb @@ -13,21 +13,28 @@ class HealthCheckMonitor # 告警視窗。只看這段時間內的檢查紀錄,同一網址在這段時間內也只寄一封 ALERT_WINDOW = 4.hours - field :name, type: String - field :target_urls, type: Array, default: [] - field :notification_emails, type: Array, default: [] - field :check_interval, type: Integer, default: 300 - field :fail_threshold, type: Integer, default: 3 - field :timeout, type: Integer, default: 10 - field :enabled, type: Boolean, default: true - field :last_checked_at, type: DateTime + # 跨 unicorn worker 的互斥鎖,存在 multithreads collection + LOCK_KEY = 'health_check' + + field :name, type: String + field :target_urls, type: Array, default: [] + field :notification_emails, type: Array, default: [] + field :check_interval_minutes, type: Integer, default: 5 + field :fail_threshold, type: Integer, default: 3 + field :timeout, type: Integer, default: 10 + field :enabled, type: Boolean, default: true + field :check_now, type: Boolean, default: false + field :last_checked_at, type: DateTime field :create_user_id field :update_user_id validates :name, presence: true + validates :check_interval_minutes, numericality: { greater_than_or_equal_to: 1, only_integer: true } scope :is_enabled, -> { where(enabled: true) } + # ── 表單用的字串轉陣列 ────────────────────────────── + def target_urls_text (target_urls || []).join("\n") end @@ -46,20 +53,24 @@ class HealthCheckMonitor self.notification_emails = value.to_s.split(",").map(&:strip).reject(&:blank?) end + # ── 參數 ──────────────────────────────────────── + def threshold fail_threshold.to_i > 0 ? fail_threshold.to_i : 3 end - def check_all - results = {} - (target_urls || []).each do |url| - results[url] = check_url(url) - end - self.last_checked_at = Time.now - self.save - results + def interval_minutes + check_interval_minutes.to_i > 0 ? check_interval_minutes.to_i : 5 end + def due? + return true if check_now + return true if last_checked_at.nil? + last_checked_at + interval_minutes.minutes <= Time.now + end + + # ── 實際檢查 ───────────────────────────────────── + def check_url(url) uri = URI.parse(url) started = Time.now @@ -75,12 +86,44 @@ class HealthCheckMonitor { :status => 0, :response_time => 0, :error => e.message } end + def check_all + results = {} + (target_urls || []).each do |url| + results[url] = check_url(url) + end + results + end + + # 手動與自動共用的唯一入口:檢查 → 寫紀錄 → 判斷 → 寄信 + def run_check + results = check_all + alert_logs = [] + results.each do |url, result| + HealthCheckLog.create( + :health_check_monitor_id => self.id, + :url => url, + :status => result[:status], + :response_time => result[:response_time], + :error_message => result[:error] + ) + target = alert_target(url) + alert_logs << target if target + end + send_alert_email(alert_logs) if alert_logs.present? + self.check_now = false + self.last_checked_at = Time.now + self.save + results + end + + # ── 告警判斷 ───────────────────────────────────── + def failure?(status) FAILURE_STATUSES.include?(status.to_i) end - # 這次檢查寫完 log 之後呼叫。 - # 條件:4 小時內已檢查滿 threshold 次、最近 threshold 次全部失敗、且 4 小時內還沒寄過這個網址 + # 條件:ALERT_WINDOW 內已檢查滿 threshold 次、最近 threshold 次全部失敗、 + # 且 ALERT_WINDOW 內還沒寄過這個網址 def alert_target(url) n = threshold window_start = Time.now - ALERT_WINDOW @@ -137,4 +180,41 @@ class HealthCheckMonitor false end end + + # ── 自動檢測進入點,由 CronWorker 每分鐘呼叫一次 ────────── + + def self.run_due_checks + return unless claim_lock + begin + is_enabled.each do |monitor| + begin + monitor.run_check if monitor.due? + rescue => e + Rails.logger.error("[health_check] #{monitor.name} 檢查失敗: #{e}") + end + end + ensure + release_lock + end + end + + # 用 Mongo 的 findAndModify 做原子性搶鎖,只有搶到的 worker 會執行 + def self.claim_lock + Multithread.create(:key => LOCK_KEY, :status => {}) if Multithread.where(:key => LOCK_KEY).count == 0 + Multithread.where(:key => LOCK_KEY).to_a[1..-1].to_a.each { |m| m.destroy } + claimed = Multithread.where(:key => LOCK_KEY).any_of( + { 'status.locked_until' => nil }, + { :'status.locked_until'.lt => Time.now } + ).find_one_and_update({ '$set' => { 'status.locked_until' => Time.now + 10.minutes } }) + !claimed.nil? + rescue => e + Rails.logger.error("[health_check] 搶鎖失敗: #{e}") + false + end + + def self.release_lock + Multithread.where(:key => LOCK_KEY).update_all('$set' => { 'status.locked_until' => nil }) + rescue => e + Rails.logger.error("[health_check] 釋放鎖失敗: #{e}") + end end diff --git a/app/views/admin/health_check_monitors/_form.html.erb b/app/views/admin/health_check_monitors/_form.html.erb index 9e4402f..c28b3fb 100644 --- a/app/views/admin/health_check_monitors/_form.html.erb +++ b/app/views/admin/health_check_monitors/_form.html.erb @@ -24,7 +24,7 @@
<%= t("health_check_monitor.manual_check_notice") %>
+<% end %> +