diff --git a/app/jobs/get_announcement_from_rss.rb b/app/jobs/get_announcement_from_rss.rb new file mode 100644 index 0000000..267c29a --- /dev/null +++ b/app/jobs/get_announcement_from_rss.rb @@ -0,0 +1,7 @@ +class GetAnnouncementFromRss + @queue = :high + + def self.perform() + %x(ruby "#{Rails.root}/lib/rss_ntu_job.rb") + end +end \ No newline at end of file diff --git a/config/resque_schedule.yml b/config/resque_schedule.yml index d6a7e83..c7953d4 100644 --- a/config/resque_schedule.yml +++ b/config/resque_schedule.yml @@ -15,3 +15,9 @@ email_cron: class: EmailCron args: description: EmailCron + +get_announcement_from_rss: + cron: 0 0 [2,12] * * * + class: GetAnnouncementFromRss + args: + description: Loop through the announcement RSS until 24h ago diff --git a/lib/rss_ntu_job.rb b/lib/rss_ntu_job.rb new file mode 100644 index 0000000..df77593 --- /dev/null +++ b/lib/rss_ntu_job.rb @@ -0,0 +1,117 @@ +# encoding: utf-8 + +require 'rss' +require 'mongo' + +AUTHORS = %w[ 學生事務處-學生住宿服務組 + 學生事務處-學生職業生涯發展中心 + 學生事務處-衛生保健及醫療中心 + 學生事務處-課外活動指導組 + 學生事務處-學生活動中心管理組 + 學生事務處-僑生及陸生輔導組 + 學生事務處-生活輔導組 + 學生事務處-軍訓組 + 學生事務處-學生心理輔導中心 + 學生事務處-學生事務處] + +DB_BASE_NAME = "site_new" + +all = [] +continue = true +i = 1 +yesterday = Time.now - 86400 + +while continue do + open("http://ann.cc.ntu.edu.tw/asp/rss.asp?page=#{i}") do |rss| + feed = RSS::Parser.parse(rss.read.encode('utf-8', 'big5', invalid: :replace, undef: :replace, replace: '').gsub('Wes,', 'Wed,').gsub(/(encoding=\"big5\")/, 'encoding="utf-8"')) + feed.items.each do |item| + if item.pubDate > yesterday + if AUTHORS.include?(item.author) + author = item.author.strip + category = item.category.to_s.gsub(/\<(\/)*category\>/, '') + all << {title: item.title.strip, author: author, link: item.link.strip, date: item.pubDate, category: category, description: item.description.gsub("\r\n", '
').strip} + end + else + continue = false + break + end + end + end + i += 1 +end + +# Get corresponding category_id or create a new one +def get_category_id(category, categories, coll_cat, bulletin_module_id) + if categories.keys.include? "rss_#{category}" + [categories["rss_#{category}"], categories] + else + cat = { + _type: "Category", + module_app_id: bulletin_module_id, + key: "rss_#{category}", + disable: false, + custom: false, + title: {:zh_tw => category}, + created_at: Time.now, + updated_at: Time.now + } + + categories["rss_#{category}"] = result = coll_cat.save(cat) + [result, categories] + end +end + +# Get categories and id based on a given site number +def get_mongo_and_categories(site_number="0") + db = Mongo::Connection.new("localhost", 27017).db("#{DB_BASE_NAME}_#{site_number}") + + bulletin_module_id = db["module_apps"].find(key: "announcement").first + + coll_bulletin = db["bulletins"] + coll_buffer_cat = db["buffer_categories"] + coll_cat = db["categories"] + coll_bulletin_cat = db["categories"].find(module_app_id: bulletin_module_id["_id"]) + + categories = coll_bulletin_cat.find().to_a.inject({}) do |categories, category| + categories[category['key']] = category['_id'] + categories + end + [categories, coll_bulletin, coll_cat, bulletin_module_id["_id"],coll_buffer_cat] +end + +# Get categories +categories, coll_bulletin, coll_cat, bulletin_module_id, coll_buffer_cat = get_mongo_and_categories + +all.each do |bul| # Loop through all the items + category_id, categories = get_category_id(bul[:category], categories, coll_cat, bulletin_module_id) + unless coll_bulletin.find_one(rss_link: bul[:link]) + bulletin = { _type: "Bulletin", + postdate: bul[:date], + created_at: bul[:date], + updated_at: bul[:date], + public: true, + is_checked: true, + is_pending: false, + is_rejected: false, + category_id: category_id, + title: {:zh_tw => bul[:title]}, + text: {:zh_tw => bul[:description]}, + available_for_en: false, + available_for_zh_tw: true, + rss_link: bul[:link], + is_top: false, + is_hot: false, + is_hidden: false } + bs = coll_bulletin.save(bulletin) + + buffer_cat = { _type: "BufferCategory", + category_id: category_id, + categorizable_type: "Bulletin", + categorizable_id: bs } + coll_buffer_cat.save(buffer_cat) + + end +end + + +