Added automatic detection and repair for 502 errors caused by excessive detection stations.

app/controllers/admin/health_check_monitors_controller.rb

@@ -41,20 +41,7 @@ class Admin::HealthCheckMonitorsController < OrbitAdminController
   end
 
   def manual_check
-    @results = @monitor.check_all
-    alert_logs = []
-    @results.each do |url, result|
-      HealthCheckLog.create(
-        :health_check_monitor_id => @monitor.id,
-        :url => url,
-        :status => result[:status],
-        :response_time => result[:response_time],
-        :error_message => result[:error]
-      )
-      target = @monitor.alert_target(url)
-      alert_logs << target if target
-    end
-    @monitor.send_alert_email(alert_logs) if alert_logs.present?
+    @monitor.update_attributes(:check_now => true)
     redirect_to logs_admin_health_check_monitor_path(@monitor)
   end
 

app/models/health_check_alert.rb

@@ -7,4 +7,5 @@ class HealthCheckAlert
   field :url, type: String
 
   index({ health_check_monitor_id: 1, url: 1, created_at: -1 })
+  index({ created_at: 1 }, { expire_after_seconds: 2592000 })
 end

app/models/health_check_log.rb

@@ -11,4 +11,5 @@ class HealthCheckLog
   belongs_to :health_check_monitor, class_name: "HealthCheckMonitor", optional: true
 
   index({ health_check_monitor_id: 1, created_at: -1 })
+  index({ created_at: 1 }, { expire_after_seconds: 2592000 })
 end

app/models/health_check_monitor.rb

@@ -13,21 +13,28 @@ class HealthCheckMonitor
   # 告警視窗。只看這段時間內的檢查紀錄，同一網址在這段時間內也只寄一封
   ALERT_WINDOW = 4.hours
 
-  field :name,                type: String
-  field :target_urls,         type: Array,   default: []
-  field :notification_emails, type: Array,   default: []
-  field :check_interval,      type: Integer, default: 300
-  field :fail_threshold,      type: Integer, default: 3
-  field :timeout,             type: Integer, default: 10
-  field :enabled,             type: Boolean, default: true
-  field :last_checked_at,     type: DateTime
+  # 跨 unicorn worker 的互斥鎖，存在 multithreads collection
+  LOCK_KEY = 'health_check'
+
+  field :name,                   type: String
+  field :target_urls,            type: Array,   default: []
+  field :notification_emails,    type: Array,   default: []
+  field :check_interval_minutes, type: Integer, default: 5
+  field :fail_threshold,         type: Integer, default: 3
+  field :timeout,                type: Integer, default: 10
+  field :enabled,                type: Boolean, default: true
+  field :check_now,              type: Boolean, default: false
+  field :last_checked_at,        type: DateTime
   field :create_user_id
   field :update_user_id
 
   validates :name, presence: true
+  validates :check_interval_minutes, numericality: { greater_than_or_equal_to: 1, only_integer: true }
 
   scope :is_enabled, -> { where(enabled: true) }
 
+  # ── 表單用的字串轉陣列 ──────────────────────────────
+
   def target_urls_text
     (target_urls || []).join("\n")
   end
@@ -46,20 +53,24 @@ class HealthCheckMonitor
     self.notification_emails = value.to_s.split(",").map(&:strip).reject(&:blank?)
   end
 
+  # ── 參數 ────────────────────────────────────────
+
   def threshold
     fail_threshold.to_i > 0 ? fail_threshold.to_i : 3
   end
 
-  def check_all
-    results = {}
-    (target_urls || []).each do |url|
-      results[url] = check_url(url)
-    end
-    self.last_checked_at = Time.now
-    self.save
-    results
+  def interval_minutes
+    check_interval_minutes.to_i > 0 ? check_interval_minutes.to_i : 5
   end
 
+  def due?
+    return true if check_now
+    return true if last_checked_at.nil?
+    last_checked_at + interval_minutes.minutes <= Time.now
+  end
+
+  # ── 實際檢查 ─────────────────────────────────────
+
   def check_url(url)
     uri = URI.parse(url)
     started = Time.now
@@ -75,12 +86,44 @@ class HealthCheckMonitor
     { :status => 0, :response_time => 0, :error => e.message }
   end
 
+  def check_all
+    results = {}
+    (target_urls || []).each do |url|
+      results[url] = check_url(url)
+    end
+    results
+  end
+
+  # 手動與自動共用的唯一入口：檢查 → 寫紀錄 → 判斷 → 寄信
+  def run_check
+    results = check_all
+    alert_logs = []
+    results.each do |url, result|
+      HealthCheckLog.create(
+        :health_check_monitor_id => self.id,
+        :url => url,
+        :status => result[:status],
+        :response_time => result[:response_time],
+        :error_message => result[:error]
+      )
+      target = alert_target(url)
+      alert_logs << target if target
+    end
+    send_alert_email(alert_logs) if alert_logs.present?
+    self.check_now = false
+    self.last_checked_at = Time.now
+    self.save
+    results
+  end
+
+  # ── 告警判斷 ─────────────────────────────────────
+
   def failure?(status)
     FAILURE_STATUSES.include?(status.to_i)
   end
 
-  # 這次檢查寫完 log 之後呼叫。
-  # 條件：4 小時內已檢查滿 threshold 次、最近 threshold 次全部失敗、且 4 小時內還沒寄過這個網址
+  # 條件：ALERT_WINDOW 內已檢查滿 threshold 次、最近 threshold 次全部失敗、
+  #       且 ALERT_WINDOW 內還沒寄過這個網址
   def alert_target(url)
     n = threshold
     window_start = Time.now - ALERT_WINDOW
@@ -137,4 +180,41 @@ class HealthCheckMonitor
       false
     end
   end
+
+  # ── 自動檢測進入點，由 CronWorker 每分鐘呼叫一次 ──────────
+
+  def self.run_due_checks
+    return unless claim_lock
+    begin
+      is_enabled.each do |monitor|
+        begin
+          monitor.run_check if monitor.due?
+        rescue => e
+          Rails.logger.error("[health_check] #{monitor.name} 檢查失敗: #{e}")
+        end
+      end
+    ensure
+      release_lock
+    end
+  end
+
+  # 用 Mongo 的 findAndModify 做原子性搶鎖，只有搶到的 worker 會執行
+  def self.claim_lock
+    Multithread.create(:key => LOCK_KEY, :status => {}) if Multithread.where(:key => LOCK_KEY).count == 0
+    Multithread.where(:key => LOCK_KEY).to_a[1..-1].to_a.each { |m| m.destroy }
+    claimed = Multithread.where(:key => LOCK_KEY).any_of(
+      { 'status.locked_until' => nil },
+      { :'status.locked_until'.lt => Time.now }
+    ).find_one_and_update({ '$set' => { 'status.locked_until' => Time.now + 10.minutes } })
+    !claimed.nil?
+  rescue => e
+    Rails.logger.error("[health_check] 搶鎖失敗: #{e}")
+    false
+  end
+
+  def self.release_lock
+    Multithread.where(:key => LOCK_KEY).update_all('$set' => { 'status.locked_until' => nil })
+  rescue => e
+    Rails.logger.error("[health_check] 釋放鎖失敗: #{e}")
+  end
 end

app/views/admin/health_check_monitors/_form.html.erb

@@ -24,7 +24,7 @@
   <div class="control-group">
     <label class="control-label muted"><%= t("health_check_monitor.check_interval") %></label>
     <div class="controls">
-      <%= f.number_field :check_interval %>
+      <%= f.number_field :check_interval_minutes, :min => 1 %>
     </div>
   </div>
 

app/views/admin/health_check_monitors/_index.html.erb

@@ -23,9 +23,14 @@
           </div>
         </td>
         <td><%= monitor.target_urls.join("<br/>").html_safe %></td>
-        <td><%= monitor.check_interval %></td>
+        <td><%= monitor.interval_minutes %></td>
         <td><%= monitor.enabled ? t(:yes_) : t(:no_) %></td>
-        <td><%= monitor.last_checked_at.blank? ? "-" : monitor.last_checked_at.strftime("%Y-%m-%d %H:%M") %></td>
+        <td>
+          <%= monitor.last_checked_at.blank? ? "-" : monitor.last_checked_at.strftime("%Y-%m-%d %H:%M") %>
+          <% if monitor.check_now %>
+            <span class="label label-info"><%= t("health_check_monitor.pending") %></span>
+          <% end %>
+        </td>
       </tr>
     <% end %>
   </tbody>

app/views/admin/health_check_monitors/logs.html.erb

@@ -6,6 +6,12 @@
   <%= link_to t(:back), admin_health_check_monitors_path, :class => "btn" %>
 </p>
 
+<% if @monitor.check_now %>
+  <div class="alert alert-info"><%= t("health_check_monitor.pending_notice") %></div>
+<% else %>
+  <p class="muted"><%= t("health_check_monitor.manual_check_notice") %></p>
+<% end %>
+
 <table class="table main-list">
   <thead>
     <tr>

config/locales/en.yml

@@ -3,7 +3,7 @@ en:
   health_check_monitor:
     name: Name
     target_urls: Target URLs
-    check_interval: Check Interval (sec)
+    check_interval: Check Interval (minutes)
     fail_threshold: Alert After N Consecutive Failures
     timeout: Timeout (sec)
     notification_emails: Notification Emails
@@ -14,5 +14,8 @@ en:
     manual_check: Check Now
     logs: Logs
     clean_logs: Clear All Logs
+    pending: Queued
+    pending_notice: Queued for checking. Please wait at least 1 minute and refresh this page.
+    manual_check_notice: "Check Now does not run immediately. Please wait at least 1 minute and refresh this page."
     one_per_line: One URL per line, must include http:// or https:// (defaults to https://)
     comma_separated: Comma separated

config/locales/zh_tw.yml

@@ -3,7 +3,7 @@ zh_tw:
   health_check_monitor:
     name: 監控名稱
     target_urls: 監控網址
-    check_interval: 檢查間隔(秒)
+    check_interval: 檢查間隔(分鐘)
     fail_threshold: 連續失敗幾次寄信
     timeout: 逾時(秒)
     notification_emails: 通知信箱
@@ -14,5 +14,8 @@ zh_tw:
     manual_check: 立即檢查
     logs: 檢查紀錄
     clean_logs: 清除全部紀錄
+    pending: 排隊中
+    pending_notice: 已排入檢查佇列，請等候至少 1 分鐘後重新整理此頁查看結果。
+    manual_check_notice: 按下「立即檢查」後不會馬上有結果，請等候至少 1 分鐘後重新整理此頁。
     one_per_line: 一行一個網址，需含 http:// 或 https://（未填會自動補 https://）
     comma_separated: 以逗號分隔

health_check.gemspec

@@ -7,11 +7,11 @@ require "health_check/version"
 Gem::Specification.new do |s|
   s.name        = "health_check"
   s.version     = HealthCheck::VERSION
-  s.authors     = ["Orbitek"]
-  s.email       = ["service@orbitek.co"]
-  s.homepage    = "https://orbitek.co"
-  s.summary     = "Health Check for orbitek"
-  s.description = "Health Check for orbitek"
+  s.authors     = ["Ruling Digital"]
+  s.email       = ["orbit@rulingcom.com"]
+  s.homepage    = "http://www.rulingcom.com"
+  s.summary     = "Health Check for orbit"
+  s.description = "Health Check for orbit"
   s.license     = "MIT"
 
   s.files = Dir["{app,config,db,lib}/**/*", "MIT-LICENSE", "Rakefile", "README.rdoc"]

lib/health_check/engine.rb

@@ -29,5 +29,16 @@ module HealthCheck
         end
       end
     end
+
+    # 自動檢測：用核心的 CronWorker 每分鐘叫一次，不使用 crontab
+    config.after_initialize do
+      if ENV['worker_num'] == '0' && File.basename($0) != 'rake' && !defined?(Rails::Console)
+        begin
+          CronWorker.cron_every_minute(HealthCheckMonitor.method(:run_due_checks))
+        rescue => e
+          Rails.logger.error("[health_check] CronWorker 啟動失敗: #{e}")
+        end
+      end
+    end
   end
 end