221 lines
7.2 KiB
Ruby
221 lines
7.2 KiB
Ruby
# encoding: utf-8
|
||
require 'net/http'
|
||
require 'uri'
|
||
|
||
class HealthCheckMonitor
|
||
include Mongoid::Document
|
||
include Mongoid::Timestamps
|
||
|
||
# 視為失敗的狀態碼。0 代表連不上（逾時、DNS 失敗、憑證錯誤等）
|
||
# 若只要 502 才算失敗，改成 [502]
|
||
FAILURE_STATUSES = [0, 500, 502, 503, 504]
|
||
|
||
# 告警視窗。只看這段時間內的檢查紀錄，同一網址在這段時間內也只寄一封
|
||
ALERT_WINDOW = 4.hours
|
||
|
||
# 跨 unicorn worker 的互斥鎖，存在 multithreads collection
|
||
LOCK_KEY = 'health_check'
|
||
|
||
field :name, type: String
|
||
field :target_urls, type: Array, default: []
|
||
field :notification_emails, type: Array, default: []
|
||
field :check_interval_minutes, type: Integer, default: 5
|
||
field :fail_threshold, type: Integer, default: 3
|
||
field :timeout, type: Integer, default: 10
|
||
field :enabled, type: Boolean, default: true
|
||
field :check_now, type: Boolean, default: false
|
||
field :last_checked_at, type: DateTime
|
||
field :create_user_id
|
||
field :update_user_id
|
||
|
||
validates :name, presence: true
|
||
validates :check_interval_minutes, numericality: { greater_than_or_equal_to: 1, only_integer: true }
|
||
|
||
scope :is_enabled, -> { where(enabled: true) }
|
||
|
||
# ── 表單用的字串轉陣列 ──────────────────────────────
|
||
|
||
def target_urls_text
|
||
(target_urls || []).join("\n")
|
||
end
|
||
|
||
def target_urls_text=(value)
|
||
self.target_urls = value.to_s.split(/\r?\n/).map(&:strip).reject(&:blank?).map do |u|
|
||
u.match(/\Ahttps?:\/\//) ? u : "https://#{u}"
|
||
end
|
||
end
|
||
|
||
def notification_emails_text
|
||
(notification_emails || []).join(", ")
|
||
end
|
||
|
||
def notification_emails_text=(value)
|
||
self.notification_emails = value.to_s.split(",").map(&:strip).reject(&:blank?)
|
||
end
|
||
|
||
# ── 參數 ────────────────────────────────────────
|
||
|
||
def threshold
|
||
fail_threshold.to_i > 0 ? fail_threshold.to_i : 3
|
||
end
|
||
|
||
def interval_minutes
|
||
check_interval_minutes.to_i > 0 ? check_interval_minutes.to_i : 5
|
||
end
|
||
|
||
def due?
|
||
return true if check_now
|
||
return true if last_checked_at.nil?
|
||
last_checked_at + interval_minutes.minutes <= Time.now
|
||
end
|
||
|
||
# ── 實際檢查 ─────────────────────────────────────
|
||
|
||
def check_url(url)
|
||
uri = URI.parse(url)
|
||
started = Time.now
|
||
http = Net::HTTP.new(uri.host, uri.port)
|
||
http.use_ssl = (uri.scheme == 'https')
|
||
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
||
http.open_timeout = self.timeout
|
||
http.read_timeout = self.timeout
|
||
response = http.request(Net::HTTP::Get.new(uri.request_uri))
|
||
elapsed = ((Time.now - started) * 1000).round(2)
|
||
{ :status => response.code.to_i, :response_time => elapsed, :error => nil }
|
||
rescue => e
|
||
{ :status => 0, :response_time => 0, :error => e.message }
|
||
end
|
||
|
||
def check_all
|
||
results = {}
|
||
(target_urls || []).each do |url|
|
||
results[url] = check_url(url)
|
||
end
|
||
results
|
||
end
|
||
|
||
# 手動與自動共用的唯一入口：檢查 → 寫紀錄 → 判斷 → 寄信
|
||
def run_check
|
||
results = check_all
|
||
alert_logs = []
|
||
results.each do |url, result|
|
||
HealthCheckLog.create(
|
||
:health_check_monitor_id => self.id,
|
||
:url => url,
|
||
:status => result[:status],
|
||
:response_time => result[:response_time],
|
||
:error_message => result[:error]
|
||
)
|
||
target = alert_target(url)
|
||
alert_logs << target if target
|
||
end
|
||
send_alert_email(alert_logs) if alert_logs.present?
|
||
self.check_now = false
|
||
self.last_checked_at = Time.now
|
||
self.save
|
||
results
|
||
end
|
||
|
||
# ── 告警判斷 ─────────────────────────────────────
|
||
|
||
def failure?(status)
|
||
FAILURE_STATUSES.include?(status.to_i)
|
||
end
|
||
|
||
# 條件：ALERT_WINDOW 內已檢查滿 threshold 次、最近 threshold 次全部失敗、
|
||
# 且 ALERT_WINDOW 內還沒寄過這個網址
|
||
def alert_target(url)
|
||
n = threshold
|
||
window_start = Time.now - ALERT_WINDOW
|
||
logs = HealthCheckLog.where(:health_check_monitor_id => self.id,
|
||
:url => url,
|
||
:created_at.gte => window_start)
|
||
.desc(:created_at).limit(n).to_a
|
||
return nil if logs.size < n
|
||
return nil unless logs.all? { |log| failure?(log.status) }
|
||
return nil if alerted_recently?(url)
|
||
logs.first
|
||
end
|
||
|
||
def alerted_recently?(url)
|
||
HealthCheckAlert.where(:health_check_monitor_id => self.id,
|
||
:url => url,
|
||
:created_at.gte => Time.now - ALERT_WINDOW).exists?
|
||
end
|
||
|
||
def mark_alerted(url)
|
||
HealthCheckAlert.create(:health_check_monitor_id => self.id, :url => url)
|
||
end
|
||
|
||
# logs 是這次剛好達標的 HealthCheckLog 陣列，全部合併成一封
|
||
def send_alert_email(logs)
|
||
return false if notification_emails.blank? || logs.blank?
|
||
rows = logs.map do |log|
|
||
{
|
||
'url' => log.url.to_s,
|
||
'status' => log.status.to_s,
|
||
'error' => log.error_message.to_s
|
||
}
|
||
end
|
||
email = Email.new(
|
||
:module_app_key => 'health_check',
|
||
:mail_to => notification_emails,
|
||
:mail_subject => "[網站監控] #{self.name} - #{logs.size} 個網址失敗",
|
||
:template => 'email/health_check_alert.html.erb',
|
||
:template_data => {
|
||
'monitor_name' => self.name.to_s,
|
||
'threshold' => threshold,
|
||
'rows' => rows,
|
||
'checked_at' => logs.first.created_at.strftime('%Y-%m-%d %H:%M:%S')
|
||
},
|
||
:mail_sentdate => Time.now - 1.minute
|
||
)
|
||
email.save
|
||
begin
|
||
email.deliver
|
||
logs.each { |log| mark_alerted(log.url) }
|
||
true
|
||
rescue => e
|
||
Rails.logger.error("[health_check] 寄信失敗: #{e}")
|
||
false
|
||
end
|
||
end
|
||
|
||
# ── 自動檢測進入點，由 CronWorker 每分鐘呼叫一次 ──────────
|
||
|
||
def self.run_due_checks
|
||
return unless claim_lock
|
||
begin
|
||
is_enabled.each do |monitor|
|
||
begin
|
||
monitor.run_check if monitor.due?
|
||
rescue => e
|
||
Rails.logger.error("[health_check] #{monitor.name} 檢查失敗: #{e}")
|
||
end
|
||
end
|
||
ensure
|
||
release_lock
|
||
end
|
||
end
|
||
|
||
# 用 Mongo 的 findAndModify 做原子性搶鎖，只有搶到的 worker 會執行
|
||
def self.claim_lock
|
||
Multithread.create(:key => LOCK_KEY, :status => {}) if Multithread.where(:key => LOCK_KEY).count == 0
|
||
Multithread.where(:key => LOCK_KEY).to_a[1..-1].to_a.each { |m| m.destroy }
|
||
claimed = Multithread.where(:key => LOCK_KEY).any_of(
|
||
{ 'status.locked_until' => nil },
|
||
{ :'status.locked_until'.lt => Time.now }
|
||
).find_one_and_update({ '$set' => { 'status.locked_until' => Time.now + 10.minutes } })
|
||
!claimed.nil?
|
||
rescue => e
|
||
Rails.logger.error("[health_check] 搶鎖失敗: #{e}")
|
||
false
|
||
end
|
||
|
||
def self.release_lock
|
||
Multithread.where(:key => LOCK_KEY).update_all('$set' => { 'status.locked_until' => nil })
|
||
rescue => e
|
||
Rails.logger.error("[health_check] 釋放鎖失敗: #{e}")
|
||
end
|
||
end
|